drivers/net/enic/enic_fm_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2008-2019 Cisco Systems, Inc.  All rights reserved.
   3  */
   4
   5 #include <errno.h>
   6 #include <stdint.h>
   7 #include <rte_log.h>
   8 #include <rte_ethdev_driver.h>
   9 #include <rte_flow_driver.h>
  10 #include <rte_ether.h>
  11 #include <rte_ip.h>
  12 #include <rte_udp.h>
  13 #include <rte_memzone.h>
  14
  15 #include "enic_compat.h"
  16 #include "enic.h"
  17 #include "vnic_dev.h"
  18 #include "vnic_nic.h"
  19
  20 #define IP_DEFTTL  64   /* from RFC 1340. */
  21 #define IP6_VTC_FLOW 0x60000000
  22
  23 /* Highest Item type supported by Flowman */
  24 #define FM_MAX_ITEM_TYPE RTE_FLOW_ITEM_TYPE_VXLAN
  25
  26 /* Up to 1024 TCAM entries */
  27 #define FM_MAX_TCAM_TABLE_SIZE 1024
  28
  29 /* Up to 4096 entries per exact match table */
  30 #define FM_MAX_EXACT_TABLE_SIZE 4096
  31
  32 /* Number of counters to increase on for each increment */
  33 #define FM_COUNTERS_EXPAND  100
  34
  35 #define FM_INVALID_HANDLE 0
  36
  37 /* Low priority used for implicit VF -> representor flow */
  38 #define FM_LOWEST_PRIORITY 100000
  39
  40 /* High priority used for implicit representor -> VF flow */
  41 #define FM_HIGHEST_PRIORITY 0
  42
  43 /* Tag used for implicit VF <-> representor flows */
  44 #define FM_VF_REP_TAG 1
  45
  46 /*
  47  * Flow exact match tables (FET) in the VIC and rte_flow groups.
  48  * Use a simple scheme to map groups to tables.
  49  * Group 0 uses the single TCAM tables, one for each direction.
  50  * Group 1, 2, ... uses its own exact match table.
  51  *
  52  * The TCAM tables are allocated upfront during init.
  53  *
  54  * Exact match tables are allocated on demand. 3 paths that lead allocations.
  55  *
  56  * 1. Add a flow that jumps from group 0 to group N.
  57  *
  58  * If N does not exist, we allocate an exact match table for it, using
  59  * a dummy key. A key is required for the table.
  60  *
  61  * 2. Add a flow that uses group N.
  62  *
  63  * If N does not exist, we allocate an exact match table for it, using
  64  * the flow's key. Subsequent flows to the same group all should have
  65  * the same key.
  66  *
  67  * Without a jump flow to N, N is not reachable in hardware. No packets
  68  * reach N and match.
  69  *
  70  * 3. Add a flow to an empty group N.
  71  *
  72  * N has been created via (1) and the dummy key. We free that table, allocate
  73  * a new table using the new flow's key. Also re-do the existing jump flow to
  74  * point to the new table.
  75  */
  76 #define FM_TCAM_RTE_GROUP 0
  77
  78 struct enic_fm_fet {
  79         TAILQ_ENTRY(enic_fm_fet) list;
  80         uint32_t group; /* rte_flow group ID */
  81         uint64_t handle; /* Exact match table handle from flowman */
  82         uint8_t ingress;
  83         uint8_t default_key;
  84         int ref; /* Reference count via get/put */
  85         struct fm_key_template key; /* Key associated with the table */
  86 };
  87
  88 struct enic_fm_counter {
  89         SLIST_ENTRY(enic_fm_counter) next;
  90         uint32_t handle;
  91 };
  92
  93 /* rte_flow.fm */
  94 struct enic_fm_flow {
  95         bool counter_valid;
  96         uint64_t entry_handle;
  97         uint64_t action_handle;
  98         struct enic_fm_counter *counter;
  99         struct enic_fm_fet *fet;
 100         /* Auto-added steer action for hairpin flows (e.g. vnic->vnic) */
 101         struct enic_fm_flow *hairpin_steer_flow;
 102 };
 103
 104 struct enic_fm_jump_flow {
 105         TAILQ_ENTRY(enic_fm_jump_flow) list;
 106         struct rte_flow *flow;
 107         uint32_t group;
 108         struct fm_tcam_match_entry match;
 109         struct fm_action action;
 110 };
 111
 112 /*
 113  * Flowman uses host memory for commands. This structure is allocated
 114  * in DMA-able memory.
 115  */
 116 union enic_flowman_cmd_mem {
 117         struct fm_tcam_match_table fm_tcam_match_table;
 118         struct fm_exact_match_table fm_exact_match_table;
 119         struct fm_tcam_match_entry fm_tcam_match_entry;
 120         struct fm_exact_match_entry fm_exact_match_entry;
 121         struct fm_action fm_action;
 122 };
 123
 124 /*
 125  * PF has a flowman instance, and VF representors share it with PF.
 126  * PF allocates this structure and owns it. VF representors borrow
 127  * the PF's structure during API calls (e.g. create, query).
 128  */
 129 struct enic_flowman {
 130         struct enic *owner_enic; /* PF */
 131         struct enic *user_enic;  /* API caller (PF or representor) */
 132         /*
 133          * Representors and PF share the same underlying flowman.
 134          * Lock API calls to serialize accesses from them. Only used
 135          * when VF representors are present.
 136          */
 137         rte_spinlock_t lock;
 138         /* Command buffer */
 139         struct {
 140                 union enic_flowman_cmd_mem *va;
 141                 dma_addr_t pa;
 142         } cmd;
 143         /* TCAM tables allocated upfront, used for group 0 */
 144         uint64_t ig_tcam_hndl;
 145         uint64_t eg_tcam_hndl;
 146         /* Counters */
 147         SLIST_HEAD(enic_free_counters, enic_fm_counter) counters;
 148         void *counter_stack;
 149         uint32_t counters_alloced;
 150         /* Exact match tables for groups != 0, dynamically allocated */
 151         TAILQ_HEAD(fet_list, enic_fm_fet) fet_list;
 152         /*
 153          * Default exact match tables used for jump actions to
 154          * non-existent groups.
 155          */
 156         struct enic_fm_fet *default_eg_fet;
 157         struct enic_fm_fet *default_ig_fet;
 158         /* Flows that jump to the default table above */
 159         TAILQ_HEAD(jump_flow_list, enic_fm_jump_flow) jump_list;
 160         /*
 161          * Scratch data used during each invocation of flow_create
 162          * and flow_validate.
 163          */
 164         struct enic_fm_fet *fet;
 165         struct fm_tcam_match_entry tcam_entry;
 166         struct fm_action action;
 167         struct fm_action action_tmp; /* enic_fm_reorder_action_op */
 168         int action_op_count;
 169         /* Tags used for representor flows */
 170         uint8_t vf_rep_tag;
 171         /* For auto-added steer action for hairpin */
 172         int need_hairpin_steer;
 173         uint64_t hairpin_steer_vnic_h;
 174 };
 175
 176 static int enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle);
 177 /*
 178  * API functions (create, destroy, validate, flush) call begin_fm()
 179  * upon entering to save the caller enic (PF or VF representor) and
 180  * lock. Upon exit, they call end_fm() to unlock.
 181  */
 182 static struct enic_flowman *begin_fm(struct enic *enic);
 183 static void end_fm(struct enic_flowman *fm);
 184 /* Delete internal flows created for representor paths */
 185 static void delete_rep_flows(struct enic *enic);
 186
 187 /*
 188  * Common arguments passed to copy_item functions. Use this structure
 189  * so we can easily add new arguments.
 190  * item: Item specification.
 191  * fm_tcam_entry: Flowman TCAM match entry.
 192  * header_level: 0 for outer header, 1 for inner header.
 193  */
 194 struct copy_item_args {
 195         const struct rte_flow_item *item;
 196         struct fm_tcam_match_entry *fm_tcam_entry;
 197         uint8_t header_level;
 198 };
 199
 200 /* functions for copying items into flowman match */
 201 typedef int (enic_copy_item_fn)(struct copy_item_args *arg);
 202
 203 /* Info about how to copy items into flowman match */
 204 struct enic_fm_items {
 205         /* Function for copying and validating an item. */
 206         enic_copy_item_fn * const copy_item;
 207         /* List of valid previous items. */
 208         const enum rte_flow_item_type * const prev_items;
 209         /*
 210          * True if it's OK for this item to be the first item. For some NIC
 211          * versions, it's invalid to start the stack above layer 3.
 212          */
 213         const uint8_t valid_start_item;
 214 };
 215
 216 static enic_copy_item_fn enic_fm_copy_item_eth;
 217 static enic_copy_item_fn enic_fm_copy_item_ipv4;
 218 static enic_copy_item_fn enic_fm_copy_item_ipv6;
 219 static enic_copy_item_fn enic_fm_copy_item_raw;
 220 static enic_copy_item_fn enic_fm_copy_item_sctp;
 221 static enic_copy_item_fn enic_fm_copy_item_tcp;
 222 static enic_copy_item_fn enic_fm_copy_item_udp;
 223 static enic_copy_item_fn enic_fm_copy_item_vlan;
 224 static enic_copy_item_fn enic_fm_copy_item_vxlan;
 225
 226 /* Ingress actions */
 227 static const enum rte_flow_action_type enic_fm_supported_ig_actions[] = {
 228         RTE_FLOW_ACTION_TYPE_COUNT,
 229         RTE_FLOW_ACTION_TYPE_DROP,
 230         RTE_FLOW_ACTION_TYPE_FLAG,
 231         RTE_FLOW_ACTION_TYPE_JUMP,
 232         RTE_FLOW_ACTION_TYPE_MARK,
 233         RTE_FLOW_ACTION_TYPE_OF_POP_VLAN,
 234         RTE_FLOW_ACTION_TYPE_PORT_ID,
 235         RTE_FLOW_ACTION_TYPE_PASSTHRU,
 236         RTE_FLOW_ACTION_TYPE_QUEUE,
 237         RTE_FLOW_ACTION_TYPE_RSS,
 238         RTE_FLOW_ACTION_TYPE_VOID,
 239         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
 240         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP,
 241         RTE_FLOW_ACTION_TYPE_END, /* END must be the last entry */
 242 };
 243
 244 /* Egress actions */
 245 static const enum rte_flow_action_type enic_fm_supported_eg_actions[] = {
 246         RTE_FLOW_ACTION_TYPE_COUNT,
 247         RTE_FLOW_ACTION_TYPE_DROP,
 248         RTE_FLOW_ACTION_TYPE_JUMP,
 249         RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN,
 250         RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP,
 251         RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID,
 252         RTE_FLOW_ACTION_TYPE_PORT_ID,
 253         RTE_FLOW_ACTION_TYPE_PASSTHRU,
 254         RTE_FLOW_ACTION_TYPE_VOID,
 255         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
 256         RTE_FLOW_ACTION_TYPE_END,
 257 };
 258
 259 static const struct enic_fm_items enic_fm_items[] = {
 260         [RTE_FLOW_ITEM_TYPE_RAW] = {
 261                 .copy_item = enic_fm_copy_item_raw,
 262                 .valid_start_item = 0,
 263                 .prev_items = (const enum rte_flow_item_type[]) {
 264                                RTE_FLOW_ITEM_TYPE_UDP,
 265                                RTE_FLOW_ITEM_TYPE_END,
 266                 },
 267         },
 268         [RTE_FLOW_ITEM_TYPE_ETH] = {
 269                 .copy_item = enic_fm_copy_item_eth,
 270                 .valid_start_item = 1,
 271                 .prev_items = (const enum rte_flow_item_type[]) {
 272                                RTE_FLOW_ITEM_TYPE_END,
 273                 },
 274         },
 275         [RTE_FLOW_ITEM_TYPE_VLAN] = {
 276                 .copy_item = enic_fm_copy_item_vlan,
 277                 .valid_start_item = 1,
 278                 .prev_items = (const enum rte_flow_item_type[]) {
 279                                RTE_FLOW_ITEM_TYPE_ETH,
 280                                RTE_FLOW_ITEM_TYPE_END,
 281                 },
 282         },
 283         [RTE_FLOW_ITEM_TYPE_IPV4] = {
 284                 .copy_item = enic_fm_copy_item_ipv4,
 285                 .valid_start_item = 1,
 286                 .prev_items = (const enum rte_flow_item_type[]) {
 287                                RTE_FLOW_ITEM_TYPE_ETH,
 288                                RTE_FLOW_ITEM_TYPE_VLAN,
 289                                RTE_FLOW_ITEM_TYPE_END,
 290                 },
 291         },
 292         [RTE_FLOW_ITEM_TYPE_IPV6] = {
 293                 .copy_item = enic_fm_copy_item_ipv6,
 294                 .valid_start_item = 1,
 295                 .prev_items = (const enum rte_flow_item_type[]) {
 296                                RTE_FLOW_ITEM_TYPE_ETH,
 297                                RTE_FLOW_ITEM_TYPE_VLAN,
 298                                RTE_FLOW_ITEM_TYPE_END,
 299                 },
 300         },
 301         [RTE_FLOW_ITEM_TYPE_UDP] = {
 302                 .copy_item = enic_fm_copy_item_udp,
 303                 .valid_start_item = 1,
 304                 .prev_items = (const enum rte_flow_item_type[]) {
 305                                RTE_FLOW_ITEM_TYPE_IPV4,
 306                                RTE_FLOW_ITEM_TYPE_IPV6,
 307                                RTE_FLOW_ITEM_TYPE_END,
 308                 },
 309         },
 310         [RTE_FLOW_ITEM_TYPE_TCP] = {
 311                 .copy_item = enic_fm_copy_item_tcp,
 312                 .valid_start_item = 1,
 313                 .prev_items = (const enum rte_flow_item_type[]) {
 314                                RTE_FLOW_ITEM_TYPE_IPV4,
 315                                RTE_FLOW_ITEM_TYPE_IPV6,
 316                                RTE_FLOW_ITEM_TYPE_END,
 317                 },
 318         },
 319         [RTE_FLOW_ITEM_TYPE_SCTP] = {
 320                 .copy_item = enic_fm_copy_item_sctp,
 321                 .valid_start_item = 0,
 322                 .prev_items = (const enum rte_flow_item_type[]) {
 323                                RTE_FLOW_ITEM_TYPE_IPV4,
 324                                RTE_FLOW_ITEM_TYPE_IPV6,
 325                                RTE_FLOW_ITEM_TYPE_END,
 326                 },
 327         },
 328         [RTE_FLOW_ITEM_TYPE_VXLAN] = {
 329                 .copy_item = enic_fm_copy_item_vxlan,
 330                 .valid_start_item = 1,
 331                 .prev_items = (const enum rte_flow_item_type[]) {
 332                                RTE_FLOW_ITEM_TYPE_UDP,
 333                                RTE_FLOW_ITEM_TYPE_END,
 334                 },
 335         },
 336 };
 337
 338 static int
 339 enic_fm_copy_item_eth(struct copy_item_args *arg)
 340 {
 341         const struct rte_flow_item *item = arg->item;
 342         const struct rte_flow_item_eth *spec = item->spec;
 343         const struct rte_flow_item_eth *mask = item->mask;
 344         const uint8_t lvl = arg->header_level;
 345         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 346         struct fm_header_set *fm_data, *fm_mask;
 347
 348         ENICPMD_FUNC_TRACE();
 349         /* Match all if no spec */
 350         if (!spec)
 351                 return 0;
 352         if (!mask)
 353                 mask = &rte_flow_item_eth_mask;
 354         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 355         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 356         fm_data->fk_header_select |= FKH_ETHER;
 357         fm_mask->fk_header_select |= FKH_ETHER;
 358         memcpy(&fm_data->l2.eth, spec, sizeof(*spec));
 359         memcpy(&fm_mask->l2.eth, mask, sizeof(*mask));
 360         return 0;
 361 }
 362
 363 static int
 364 enic_fm_copy_item_vlan(struct copy_item_args *arg)
 365 {
 366         const struct rte_flow_item *item = arg->item;
 367         const struct rte_flow_item_vlan *spec = item->spec;
 368         const struct rte_flow_item_vlan *mask = item->mask;
 369         const uint8_t lvl = arg->header_level;
 370         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 371         struct fm_header_set *fm_data, *fm_mask;
 372         struct rte_ether_hdr *eth_mask;
 373         struct rte_ether_hdr *eth_val;
 374         uint32_t meta;
 375
 376         ENICPMD_FUNC_TRACE();
 377         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 378         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 379         /* Outer and inner packet vlans need different flags */
 380         meta = FKM_VLAN_PRES;
 381         if (lvl > 0)
 382                 meta = FKM_QTAG;
 383         fm_data->fk_metadata |= meta;
 384         fm_mask->fk_metadata |= meta;
 385
 386         /* Match all if no spec */
 387         if (!spec)
 388                 return 0;
 389         if (!mask)
 390                 mask = &rte_flow_item_vlan_mask;
 391
 392         eth_mask = (void *)&fm_mask->l2.eth;
 393         eth_val = (void *)&fm_data->l2.eth;
 394
 395         /*
 396          * Outer TPID cannot be matched. If inner_type is 0, use what is
 397          * in the eth header.
 398          */
 399         if (eth_mask->ether_type && mask->inner_type)
 400                 return -ENOTSUP;
 401
 402         /*
 403          * When packet matching, the VIC always compares vlan-stripped
 404          * L2, regardless of vlan stripping settings. So, the inner type
 405          * from vlan becomes the ether type of the eth header.
 406          */
 407         if (mask->inner_type) {
 408                 eth_mask->ether_type = mask->inner_type;
 409                 eth_val->ether_type = spec->inner_type;
 410         }
 411         fm_data->fk_header_select |= FKH_ETHER | FKH_QTAG;
 412         fm_mask->fk_header_select |= FKH_ETHER | FKH_QTAG;
 413         fm_data->fk_vlan = rte_be_to_cpu_16(spec->tci);
 414         fm_mask->fk_vlan = rte_be_to_cpu_16(mask->tci);
 415         return 0;
 416 }
 417
 418 static int
 419 enic_fm_copy_item_ipv4(struct copy_item_args *arg)
 420 {
 421         const struct rte_flow_item *item = arg->item;
 422         const struct rte_flow_item_ipv4 *spec = item->spec;
 423         const struct rte_flow_item_ipv4 *mask = item->mask;
 424         const uint8_t lvl = arg->header_level;
 425         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 426         struct fm_header_set *fm_data, *fm_mask;
 427
 428         ENICPMD_FUNC_TRACE();
 429         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 430         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 431         fm_data->fk_metadata |= FKM_IPV4;
 432         fm_mask->fk_metadata |= FKM_IPV4;
 433
 434         if (!spec)
 435                 return 0;
 436         if (!mask)
 437                 mask = &rte_flow_item_ipv4_mask;
 438
 439         fm_data->fk_header_select |= FKH_IPV4;
 440         fm_mask->fk_header_select |= FKH_IPV4;
 441         memcpy(&fm_data->l3.ip4, spec, sizeof(*spec));
 442         memcpy(&fm_mask->l3.ip4, mask, sizeof(*mask));
 443         return 0;
 444 }
 445
 446 static int
 447 enic_fm_copy_item_ipv6(struct copy_item_args *arg)
 448 {
 449         const struct rte_flow_item *item = arg->item;
 450         const struct rte_flow_item_ipv6 *spec = item->spec;
 451         const struct rte_flow_item_ipv6 *mask = item->mask;
 452         const uint8_t lvl = arg->header_level;
 453         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 454         struct fm_header_set *fm_data, *fm_mask;
 455
 456         ENICPMD_FUNC_TRACE();
 457         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 458         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 459         fm_data->fk_metadata |= FKM_IPV6;
 460         fm_mask->fk_metadata |= FKM_IPV6;
 461
 462         if (!spec)
 463                 return 0;
 464         if (!mask)
 465                 mask = &rte_flow_item_ipv6_mask;
 466
 467         fm_data->fk_header_select |= FKH_IPV6;
 468         fm_mask->fk_header_select |= FKH_IPV6;
 469         memcpy(&fm_data->l3.ip6, spec, sizeof(*spec));
 470         memcpy(&fm_mask->l3.ip6, mask, sizeof(*mask));
 471         return 0;
 472 }
 473
 474 static int
 475 enic_fm_copy_item_udp(struct copy_item_args *arg)
 476 {
 477         const struct rte_flow_item *item = arg->item;
 478         const struct rte_flow_item_udp *spec = item->spec;
 479         const struct rte_flow_item_udp *mask = item->mask;
 480         const uint8_t lvl = arg->header_level;
 481         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 482         struct fm_header_set *fm_data, *fm_mask;
 483
 484         ENICPMD_FUNC_TRACE();
 485         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 486         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 487         fm_data->fk_metadata |= FKM_UDP;
 488         fm_mask->fk_metadata |= FKM_UDP;
 489
 490         if (!spec)
 491                 return 0;
 492         if (!mask)
 493                 mask = &rte_flow_item_udp_mask;
 494
 495         fm_data->fk_header_select |= FKH_UDP;
 496         fm_mask->fk_header_select |= FKH_UDP;
 497         memcpy(&fm_data->l4.udp, spec, sizeof(*spec));
 498         memcpy(&fm_mask->l4.udp, mask, sizeof(*mask));
 499         return 0;
 500 }
 501
 502 static int
 503 enic_fm_copy_item_tcp(struct copy_item_args *arg)
 504 {
 505         const struct rte_flow_item *item = arg->item;
 506         const struct rte_flow_item_tcp *spec = item->spec;
 507         const struct rte_flow_item_tcp *mask = item->mask;
 508         const uint8_t lvl = arg->header_level;
 509         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 510         struct fm_header_set *fm_data, *fm_mask;
 511
 512         ENICPMD_FUNC_TRACE();
 513         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 514         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 515         fm_data->fk_metadata |= FKM_TCP;
 516         fm_mask->fk_metadata |= FKM_TCP;
 517
 518         if (!spec)
 519                 return 0;
 520         if (!mask)
 521                 mask = &rte_flow_item_tcp_mask;
 522
 523         fm_data->fk_header_select |= FKH_TCP;
 524         fm_mask->fk_header_select |= FKH_TCP;
 525         memcpy(&fm_data->l4.tcp, spec, sizeof(*spec));
 526         memcpy(&fm_mask->l4.tcp, mask, sizeof(*mask));
 527         return 0;
 528 }
 529
 530 static int
 531 enic_fm_copy_item_sctp(struct copy_item_args *arg)
 532 {
 533         const struct rte_flow_item *item = arg->item;
 534         const struct rte_flow_item_sctp *spec = item->spec;
 535         const struct rte_flow_item_sctp *mask = item->mask;
 536         const uint8_t lvl = arg->header_level;
 537         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 538         struct fm_header_set *fm_data, *fm_mask;
 539         uint8_t *ip_proto_mask = NULL;
 540         uint8_t *ip_proto = NULL;
 541         uint32_t l3_fkh;
 542
 543         ENICPMD_FUNC_TRACE();
 544         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 545         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 546         /*
 547          * The NIC filter API has no flags for "match sctp", so explicitly
 548          * set the protocol number in the IP pattern.
 549          */
 550         if (fm_data->fk_metadata & FKM_IPV4) {
 551                 struct rte_ipv4_hdr *ip;
 552                 ip = (struct rte_ipv4_hdr *)&fm_mask->l3.ip4;
 553                 ip_proto_mask = &ip->next_proto_id;
 554                 ip = (struct rte_ipv4_hdr *)&fm_data->l3.ip4;
 555                 ip_proto = &ip->next_proto_id;
 556                 l3_fkh = FKH_IPV4;
 557         } else if (fm_data->fk_metadata & FKM_IPV6) {
 558                 struct rte_ipv6_hdr *ip;
 559                 ip = (struct rte_ipv6_hdr *)&fm_mask->l3.ip6;
 560                 ip_proto_mask = &ip->proto;
 561                 ip = (struct rte_ipv6_hdr *)&fm_data->l3.ip6;
 562                 ip_proto = &ip->proto;
 563                 l3_fkh = FKH_IPV6;
 564         } else {
 565                 /* Need IPv4/IPv6 pattern first */
 566                 return -EINVAL;
 567         }
 568         *ip_proto = IPPROTO_SCTP;
 569         *ip_proto_mask = 0xff;
 570         fm_data->fk_header_select |= l3_fkh;
 571         fm_mask->fk_header_select |= l3_fkh;
 572
 573         if (!spec)
 574                 return 0;
 575         if (!mask)
 576                 mask = &rte_flow_item_sctp_mask;
 577
 578         fm_data->fk_header_select |= FKH_L4RAW;
 579         fm_mask->fk_header_select |= FKH_L4RAW;
 580         memcpy(fm_data->l4.rawdata, spec, sizeof(*spec));
 581         memcpy(fm_mask->l4.rawdata, mask, sizeof(*mask));
 582         return 0;
 583 }
 584
 585 static int
 586 enic_fm_copy_item_vxlan(struct copy_item_args *arg)
 587 {
 588         const struct rte_flow_item *item = arg->item;
 589         const struct rte_flow_item_vxlan *spec = item->spec;
 590         const struct rte_flow_item_vxlan *mask = item->mask;
 591         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 592         struct fm_header_set *fm_data, *fm_mask;
 593
 594         ENICPMD_FUNC_TRACE();
 595         /* Only 2 header levels (outer and inner) allowed */
 596         if (arg->header_level > 0)
 597                 return -EINVAL;
 598
 599         fm_data = &entry->ftm_data.fk_hdrset[0];
 600         fm_mask = &entry->ftm_mask.fk_hdrset[0];
 601         fm_data->fk_metadata |= FKM_VXLAN;
 602         fm_mask->fk_metadata |= FKM_VXLAN;
 603         /* items from here on out are inner header items */
 604         arg->header_level = 1;
 605
 606         /* Match all if no spec */
 607         if (!spec)
 608                 return 0;
 609         if (!mask)
 610                 mask = &rte_flow_item_vxlan_mask;
 611
 612         fm_data->fk_header_select |= FKH_VXLAN;
 613         fm_mask->fk_header_select |= FKH_VXLAN;
 614         memcpy(&fm_data->vxlan, spec, sizeof(*spec));
 615         memcpy(&fm_mask->vxlan, mask, sizeof(*mask));
 616         return 0;
 617 }
 618
 619 /*
 620  * Currently, raw pattern match is very limited. It is intended for matching
 621  * UDP tunnel header (e.g. vxlan or geneve).
 622  */
 623 static int
 624 enic_fm_copy_item_raw(struct copy_item_args *arg)
 625 {
 626         const struct rte_flow_item *item = arg->item;
 627         const struct rte_flow_item_raw *spec = item->spec;
 628         const struct rte_flow_item_raw *mask = item->mask;
 629         const uint8_t lvl = arg->header_level;
 630         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 631         struct fm_header_set *fm_data, *fm_mask;
 632
 633         ENICPMD_FUNC_TRACE();
 634         /* Cannot be used for inner packet */
 635         if (lvl > 0)
 636                 return -EINVAL;
 637         /* Need both spec and mask */
 638         if (!spec || !mask)
 639                 return -EINVAL;
 640         /* Only supports relative with offset 0 */
 641         if (!spec->relative || spec->offset != 0 || spec->search ||
 642             spec->limit)
 643                 return -EINVAL;
 644         /* Need non-null pattern that fits within the NIC's filter pattern */
 645         if (spec->length == 0 ||
 646             spec->length + sizeof(struct rte_udp_hdr) > FM_LAYER_SIZE ||
 647             !spec->pattern || !mask->pattern)
 648                 return -EINVAL;
 649         /*
 650          * Mask fields, including length, are often set to zero. Assume that
 651          * means "same as spec" to avoid breaking existing apps. If length
 652          * is not zero, then it should be >= spec length.
 653          *
 654          * No more pattern follows this, so append to the L4 layer instead of
 655          * L5 to work with both recent and older VICs.
 656          */
 657         if (mask->length != 0 && mask->length < spec->length)
 658                 return -EINVAL;
 659
 660         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 661         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 662         fm_data->fk_header_select |= FKH_L4RAW;
 663         fm_mask->fk_header_select |= FKH_L4RAW;
 664         fm_data->fk_header_select &= ~FKH_UDP;
 665         fm_mask->fk_header_select &= ~FKH_UDP;
 666         memcpy(fm_data->l4.rawdata + sizeof(struct rte_udp_hdr),
 667                spec->pattern, spec->length);
 668         memcpy(fm_mask->l4.rawdata + sizeof(struct rte_udp_hdr),
 669                mask->pattern, spec->length);
 670         return 0;
 671 }
 672
 673 static int
 674 flowman_cmd(struct enic_flowman *fm, uint64_t *args, int nargs)
 675 {
 676         return vnic_dev_flowman_cmd(fm->owner_enic->vdev, args, nargs);
 677 }
 678
 679 static int
 680 enic_fet_alloc(struct enic_flowman *fm, uint8_t ingress,
 681                struct fm_key_template *key, int entries,
 682                struct enic_fm_fet **fet_out)
 683 {
 684         struct fm_exact_match_table *cmd;
 685         struct fm_header_set *hdr;
 686         struct enic_fm_fet *fet;
 687         uint64_t args[3];
 688         int ret;
 689
 690         ENICPMD_FUNC_TRACE();
 691         fet = calloc(1, sizeof(struct enic_fm_fet));
 692         if (fet == NULL)
 693                 return -ENOMEM;
 694         cmd = &fm->cmd.va->fm_exact_match_table;
 695         memset(cmd, 0, sizeof(*cmd));
 696         cmd->fet_direction = ingress ? FM_INGRESS : FM_EGRESS;
 697         cmd->fet_stage = FM_STAGE_LAST;
 698         cmd->fet_max_entries = entries ? entries : FM_MAX_EXACT_TABLE_SIZE;
 699         if (key == NULL) {
 700                 hdr = &cmd->fet_key.fk_hdrset[0];
 701                 memset(hdr, 0, sizeof(*hdr));
 702                 hdr->fk_header_select = FKH_IPV4 | FKH_UDP;
 703                 hdr->l3.ip4.fk_saddr = 0xFFFFFFFF;
 704                 hdr->l3.ip4.fk_daddr = 0xFFFFFFFF;
 705                 hdr->l4.udp.fk_source = 0xFFFF;
 706                 hdr->l4.udp.fk_dest = 0xFFFF;
 707                 fet->default_key = 1;
 708         } else {
 709                 memcpy(&cmd->fet_key, key, sizeof(*key));
 710                 memcpy(&fet->key, key, sizeof(*key));
 711                 fet->default_key = 0;
 712         }
 713         cmd->fet_key.fk_packet_tag = 1;
 714
 715         args[0] = FM_EXACT_TABLE_ALLOC;
 716         args[1] = fm->cmd.pa;
 717         ret = flowman_cmd(fm, args, 2);
 718         if (ret) {
 719                 ENICPMD_LOG(ERR, "cannot alloc exact match table: rc=%d", ret);
 720                 free(fet);
 721                 return ret;
 722         }
 723         fet->handle = args[0];
 724         fet->ingress = ingress;
 725         ENICPMD_LOG(DEBUG, "allocated exact match table: handle=0x%" PRIx64,
 726                     fet->handle);
 727         *fet_out = fet;
 728         return 0;
 729 }
 730
 731 static void
 732 enic_fet_free(struct enic_flowman *fm, struct enic_fm_fet *fet)
 733 {
 734         ENICPMD_FUNC_TRACE();
 735         enic_fm_tbl_free(fm, fet->handle);
 736         if (!fet->default_key)
 737                 TAILQ_REMOVE(&fm->fet_list, fet, list);
 738         free(fet);
 739 }
 740
 741 /*
 742  * Get the exact match table for the given combination of
 743  * <group, ingress, key>. Allocate one on the fly as necessary.
 744  */
 745 static int
 746 enic_fet_get(struct enic_flowman *fm,
 747              uint32_t group,
 748              uint8_t ingress,
 749              struct fm_key_template *key,
 750              struct enic_fm_fet **fet_out,
 751              struct rte_flow_error *error)
 752 {
 753         struct enic_fm_fet *fet;
 754
 755         ENICPMD_FUNC_TRACE();
 756         /* See if we already have this table open */
 757         TAILQ_FOREACH(fet, &fm->fet_list, list) {
 758                 if (fet->group == group && fet->ingress == ingress)
 759                         break;
 760         }
 761         if (fet == NULL) {
 762                 /* Jumping to a non-existing group? Use the default table */
 763                 if (key == NULL) {
 764                         fet = ingress ? fm->default_ig_fet : fm->default_eg_fet;
 765                 } else if (enic_fet_alloc(fm, ingress, key, 0, &fet)) {
 766                         return rte_flow_error_set(error, EINVAL,
 767                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 768                                 NULL, "enic: cannot get exact match table");
 769                 }
 770                 fet->group = group;
 771                 /* Default table is never on the open table list */
 772                 if (!fet->default_key)
 773                         TAILQ_INSERT_HEAD(&fm->fet_list, fet, list);
 774         }
 775         fet->ref++;
 776         *fet_out = fet;
 777         ENICPMD_LOG(DEBUG, "fet_get: %s %s group=%u ref=%u",
 778                     fet->default_key ? "default" : "",
 779                     fet->ingress ? "ingress" : "egress",
 780                     fet->group, fet->ref);
 781         return 0;
 782 }
 783
 784 static void
 785 enic_fet_put(struct enic_flowman *fm, struct enic_fm_fet *fet)
 786 {
 787         ENICPMD_FUNC_TRACE();
 788         RTE_ASSERT(fet->ref > 0);
 789         fet->ref--;
 790         ENICPMD_LOG(DEBUG, "fet_put: %s %s group=%u ref=%u",
 791                     fet->default_key ? "default" : "",
 792                     fet->ingress ? "ingress" : "egress",
 793                     fet->group, fet->ref);
 794         if (fet->ref == 0)
 795                 enic_fet_free(fm, fet);
 796 }
 797
 798 /* Return 1 if current item is valid on top of the previous one. */
 799 static int
 800 fm_item_stacking_valid(enum rte_flow_item_type prev_item,
 801                        const struct enic_fm_items *item_info,
 802                        uint8_t is_first_item)
 803 {
 804         enum rte_flow_item_type const *allowed_items = item_info->prev_items;
 805
 806         ENICPMD_FUNC_TRACE();
 807         for (; *allowed_items != RTE_FLOW_ITEM_TYPE_END; allowed_items++) {
 808                 if (prev_item == *allowed_items)
 809                         return 1;
 810         }
 811
 812         /* This is the first item in the stack. Check if that's cool */
 813         if (is_first_item && item_info->valid_start_item)
 814                 return 1;
 815         return 0;
 816 }
 817
 818 /*
 819  * Build the flow manager match entry structure from the provided pattern.
 820  * The pattern is validated as the items are copied.
 821  */
 822 static int
 823 enic_fm_copy_entry(struct enic_flowman *fm,
 824                    const struct rte_flow_item pattern[],
 825                    struct rte_flow_error *error)
 826 {
 827         const struct enic_fm_items *item_info;
 828         enum rte_flow_item_type prev_item;
 829         const struct rte_flow_item *item;
 830         struct copy_item_args args;
 831         uint8_t prev_header_level;
 832         uint8_t is_first_item;
 833         int ret;
 834
 835         ENICPMD_FUNC_TRACE();
 836         item = pattern;
 837         is_first_item = 1;
 838         prev_item = RTE_FLOW_ITEM_TYPE_END;
 839
 840         args.fm_tcam_entry = &fm->tcam_entry;
 841         args.header_level = 0;
 842         prev_header_level = 0;
 843         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 844                 /*
 845                  * Get info about how to validate and copy the item. If NULL
 846                  * is returned the nic does not support the item.
 847                  */
 848                 if (item->type == RTE_FLOW_ITEM_TYPE_VOID)
 849                         continue;
 850
 851                 item_info = &enic_fm_items[item->type];
 852
 853                 if (item->type > FM_MAX_ITEM_TYPE ||
 854                     item_info->copy_item == NULL) {
 855                         return rte_flow_error_set(error, ENOTSUP,
 856                                 RTE_FLOW_ERROR_TYPE_ITEM,
 857                                 NULL, "enic: unsupported item");
 858                 }
 859
 860                 /* check to see if item stacking is valid */
 861                 if (!fm_item_stacking_valid(prev_item, item_info,
 862                                             is_first_item))
 863                         goto stacking_error;
 864
 865                 args.item = item;
 866                 ret = item_info->copy_item(&args);
 867                 if (ret)
 868                         goto item_not_supported;
 869                 /* Going from outer to inner? Treat it as a new packet start */
 870                 if (prev_header_level != args.header_level) {
 871                         prev_item = RTE_FLOW_ITEM_TYPE_END;
 872                         is_first_item = 1;
 873                 } else {
 874                         prev_item = item->type;
 875                         is_first_item = 0;
 876                 }
 877                 prev_header_level = args.header_level;
 878         }
 879         return 0;
 880
 881 item_not_supported:
 882         return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,
 883                                   NULL, "enic: unsupported item type");
 884
 885 stacking_error:
 886         return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
 887                                   item, "enic: unsupported item stack");
 888 }
 889
 890 static void
 891 flow_item_skip_void(const struct rte_flow_item **item)
 892 {
 893         for ( ; ; (*item)++)
 894                 if ((*item)->type != RTE_FLOW_ITEM_TYPE_VOID)
 895                         return;
 896 }
 897
 898 static void
 899 append_template(void **template, uint8_t *off, const void *data, int len)
 900 {
 901         memcpy(*template, data, len);
 902         *template = (char *)*template + len;
 903         *off = *off + len;
 904 }
 905
 906 static int
 907 enic_fm_append_action_op(struct enic_flowman *fm,
 908                          struct fm_action_op *fm_op,
 909                          struct rte_flow_error *error)
 910 {
 911         int count;
 912
 913         count = fm->action_op_count;
 914         ENICPMD_LOG(DEBUG, "append action op: idx=%d op=%u",
 915                     count, fm_op->fa_op);
 916         if (count == FM_ACTION_OP_MAX) {
 917                 return rte_flow_error_set(error, EINVAL,
 918                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 919                         "too many action operations");
 920         }
 921         fm->action.fma_action_ops[count] = *fm_op;
 922         fm->action_op_count = count + 1;
 923         return 0;
 924 }
 925
 926 static struct fm_action_op *
 927 find_prev_action_op(struct enic_flowman *fm, uint32_t opcode)
 928 {
 929         struct fm_action_op *op;
 930         int i;
 931
 932         for (i = 0; i < fm->action_op_count; i++) {
 933                 op = &fm->action.fma_action_ops[i];
 934                 if (op->fa_op == opcode)
 935                         return op;
 936         }
 937         return NULL;
 938 }
 939
 940 /* NIC requires that 1st steer appear before decap.
 941  * Correct example: steer, decap, steer, steer, ...
 942  */
 943 static void
 944 enic_fm_reorder_action_op(struct enic_flowman *fm)
 945 {
 946         struct fm_action_op *op, *steer, *decap;
 947         struct fm_action_op tmp_op;
 948
 949         ENICPMD_FUNC_TRACE();
 950         /* Find 1st steer and decap */
 951         op = fm->action.fma_action_ops;
 952         steer = NULL;
 953         decap = NULL;
 954         while (op->fa_op != FMOP_END) {
 955                 if (!decap && (op->fa_op == FMOP_DECAP_NOSTRIP ||
 956                                op->fa_op == FMOP_DECAP_STRIP))
 957                         decap = op;
 958                 else if (!steer && op->fa_op == FMOP_RQ_STEER)
 959                         steer = op;
 960                 op++;
 961         }
 962         /* If decap is before steer, swap */
 963         if (steer && decap && decap < steer) {
 964                 op = fm->action.fma_action_ops;
 965                 ENICPMD_LOG(DEBUG, "swap decap %ld <-> steer %ld",
 966                             (long)(decap - op), (long)(steer - op));
 967                 tmp_op = *decap;
 968                 *decap = *steer;
 969                 *steer = tmp_op;
 970         }
 971 }
 972
 973 /* VXLAN decap is done via flowman compound action */
 974 static int
 975 enic_fm_copy_vxlan_decap(struct enic_flowman *fm,
 976                          struct fm_tcam_match_entry *fmt,
 977                          const struct rte_flow_action *action,
 978                          struct rte_flow_error *error)
 979 {
 980         struct fm_header_set *fm_data;
 981         struct fm_action_op fm_op;
 982
 983         ENICPMD_FUNC_TRACE();
 984         fm_data = &fmt->ftm_data.fk_hdrset[0];
 985         if (!(fm_data->fk_metadata & FKM_VXLAN)) {
 986                 return rte_flow_error_set(error, EINVAL,
 987                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 988                         "vxlan-decap: vxlan must be in pattern");
 989         }
 990
 991         memset(&fm_op, 0, sizeof(fm_op));
 992         fm_op.fa_op = FMOP_DECAP_NOSTRIP;
 993         return enic_fm_append_action_op(fm, &fm_op, error);
 994 }
 995
 996 /* Generate a reasonable source port number */
 997 static uint16_t
 998 gen_src_port(void)
 999 {
1000         /* Min/max below are the default values in OVS-DPDK and Linux */
1001         uint16_t p = rte_rand();
1002         p = RTE_MAX(p, 32768);
1003         p = RTE_MIN(p, 61000);
1004         return rte_cpu_to_be_16(p);
1005 }
1006
1007 /* VXLAN encap is done via flowman compound action */
1008 static int
1009 enic_fm_copy_vxlan_encap(struct enic_flowman *fm,
1010                          const struct rte_flow_item *item,
1011                          struct rte_flow_error *error)
1012 {
1013         struct fm_action_op fm_op;
1014         struct rte_ether_hdr *eth;
1015         struct rte_udp_hdr *udp;
1016         uint16_t *ethertype;
1017         void *template;
1018         uint8_t off;
1019
1020         ENICPMD_FUNC_TRACE();
1021         memset(&fm_op, 0, sizeof(fm_op));
1022         fm_op.fa_op = FMOP_ENCAP;
1023         template = fm->action.fma_data;
1024         off = 0;
1025         /*
1026          * Copy flow items to the flowman template starting L2.
1027          * L2 must be ethernet.
1028          */
1029         flow_item_skip_void(&item);
1030         if (item->type != RTE_FLOW_ITEM_TYPE_ETH)
1031                 return rte_flow_error_set(error, EINVAL,
1032                         RTE_FLOW_ERROR_TYPE_ITEM, item,
1033                         "vxlan-encap: first item should be ethernet");
1034         eth = (struct rte_ether_hdr *)template;
1035         ethertype = &eth->ether_type;
1036         append_template(&template, &off, item->spec,
1037                         sizeof(struct rte_flow_item_eth));
1038         item++;
1039         flow_item_skip_void(&item);
1040         /* Optional VLAN */
1041         if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
1042                 const struct rte_flow_item_vlan *spec;
1043
1044                 ENICPMD_LOG(DEBUG, "vxlan-encap: vlan");
1045                 spec = item->spec;
1046                 fm_op.encap.outer_vlan = rte_be_to_cpu_16(spec->tci);
1047                 item++;
1048                 flow_item_skip_void(&item);
1049         }
1050         /* L3 must be IPv4, IPv6 */
1051         switch (item->type) {
1052         case RTE_FLOW_ITEM_TYPE_IPV4:
1053         {
1054                 struct rte_ipv4_hdr *ip4;
1055
1056                 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv4");
1057                 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
1058                 ip4 = (struct rte_ipv4_hdr *)template;
1059                 /*
1060                  * Offset of IPv4 length field and its initial value
1061                  * (IP + UDP + VXLAN) are specified in the action. The NIC
1062                  * will add inner packet length.
1063                  */
1064                 fm_op.encap.len1_offset = off +
1065                         offsetof(struct rte_ipv4_hdr, total_length);
1066                 fm_op.encap.len1_delta = sizeof(struct rte_ipv4_hdr) +
1067                         sizeof(struct rte_udp_hdr) +
1068                         sizeof(struct rte_vxlan_hdr);
1069                 append_template(&template, &off, item->spec,
1070                                 sizeof(struct rte_ipv4_hdr));
1071                 ip4->version_ihl = RTE_IPV4_VHL_DEF;
1072                 if (ip4->time_to_live == 0)
1073                         ip4->time_to_live = IP_DEFTTL;
1074                 ip4->next_proto_id = IPPROTO_UDP;
1075                 break;
1076         }
1077         case RTE_FLOW_ITEM_TYPE_IPV6:
1078         {
1079                 struct rte_ipv6_hdr *ip6;
1080
1081                 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv6");
1082                 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
1083                 ip6 = (struct rte_ipv6_hdr *)template;
1084                 fm_op.encap.len1_offset = off +
1085                         offsetof(struct rte_ipv6_hdr, payload_len);
1086                 fm_op.encap.len1_delta = sizeof(struct rte_udp_hdr) +
1087                         sizeof(struct rte_vxlan_hdr);
1088                 append_template(&template, &off, item->spec,
1089                                 sizeof(struct rte_ipv6_hdr));
1090                 ip6->vtc_flow |= rte_cpu_to_be_32(IP6_VTC_FLOW);
1091                 if (ip6->hop_limits == 0)
1092                         ip6->hop_limits = IP_DEFTTL;
1093                 ip6->proto = IPPROTO_UDP;
1094                 break;
1095         }
1096         default:
1097                 return rte_flow_error_set(error,
1098                         EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1099                         "vxlan-encap: L3 must be IPv4/IPv6");
1100         }
1101         item++;
1102         flow_item_skip_void(&item);
1103
1104         /* L4 is UDP */
1105         if (item->type != RTE_FLOW_ITEM_TYPE_UDP)
1106                 return rte_flow_error_set(error, EINVAL,
1107                         RTE_FLOW_ERROR_TYPE_ITEM, item,
1108                         "vxlan-encap: UDP must follow IPv4/IPv6");
1109         /* UDP length = UDP + VXLAN. NIC will add inner packet length. */
1110         fm_op.encap.len2_offset =
1111                 off + offsetof(struct rte_udp_hdr, dgram_len);
1112         fm_op.encap.len2_delta =
1113                 sizeof(struct rte_udp_hdr) + sizeof(struct rte_vxlan_hdr);
1114         udp = (struct rte_udp_hdr *)template;
1115         append_template(&template, &off, item->spec,
1116                         sizeof(struct rte_udp_hdr));
1117         /*
1118          * Firmware does not hash/fill source port yet. Generate a
1119          * random port, as there is *usually* one rte_flow for the
1120          * given inner packet stream (i.e. a single stream has one
1121          * random port).
1122          */
1123         if (udp->src_port == 0)
1124                 udp->src_port = gen_src_port();
1125         item++;
1126         flow_item_skip_void(&item);
1127
1128         /* Finally VXLAN */
1129         if (item->type != RTE_FLOW_ITEM_TYPE_VXLAN)
1130                 return rte_flow_error_set(error,
1131                         EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1132                         "vxlan-encap: VXLAN must follow UDP");
1133         append_template(&template, &off, item->spec,
1134                         sizeof(struct rte_flow_item_vxlan));
1135
1136         /*
1137          * Fill in the rest of the action structure.
1138          * Indicate that we want to encap with vxlan at packet start.
1139          */
1140         fm_op.encap.template_offset = 0;
1141         fm_op.encap.template_len = off;
1142         return enic_fm_append_action_op(fm, &fm_op, error);
1143 }
1144
1145 static int
1146 enic_fm_find_vnic(struct enic *enic, const struct rte_pci_addr *addr,
1147                   uint64_t *handle)
1148 {
1149         uint32_t bdf;
1150         uint64_t args[2];
1151         int rc;
1152
1153         ENICPMD_FUNC_TRACE();
1154         ENICPMD_LOG(DEBUG, "bdf=%x:%x:%x", addr->bus, addr->devid,
1155                     addr->function);
1156         bdf = addr->bus << 8 | addr->devid << 3 | addr->function;
1157         args[0] = FM_VNIC_FIND;
1158         args[1] = bdf;
1159         rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
1160         if (rc != 0) {
1161                 /* Expected to fail if BDF is not on the adapter */
1162                 ENICPMD_LOG(DEBUG, "cannot find vnic handle: rc=%d", rc);
1163                 return rc;
1164         }
1165         *handle = args[0];
1166         ENICPMD_LOG(DEBUG, "found vnic: handle=0x%" PRIx64, *handle);
1167         return 0;
1168 }
1169
1170 /*
1171  * Egress: target port should be either PF uplink or VF.
1172  * Supported cases
1173  * 1. VF egress -> PF uplink
1174  *   PF may be this VF's PF, or another PF, as long as they are on the same VIC.
1175  * 2. VF egress -> VF
1176  *
1177  * Unsupported cases
1178  * 1. PF egress -> VF
1179  *   App should be using representor to pass packets to VF
1180  */
1181 static int
1182 vf_egress_port_id_action(struct enic_flowman *fm,
1183                          struct rte_eth_dev *dst_dev,
1184                          uint64_t dst_vnic_h,
1185                          struct fm_action_op *fm_op,
1186                          struct rte_flow_error *error)
1187 {
1188         struct enic *src_enic, *dst_enic;
1189         struct enic_vf_representor *vf;
1190         uint8_t uif;
1191         int ret;
1192
1193         ENICPMD_FUNC_TRACE();
1194         src_enic = fm->user_enic;
1195         dst_enic = pmd_priv(dst_dev);
1196         if (!(src_enic->rte_dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)) {
1197                 return rte_flow_error_set(error, EINVAL,
1198                         RTE_FLOW_ERROR_TYPE_ACTION,
1199                         NULL, "source port is not VF representor");
1200         }
1201
1202         /* VF -> PF uplink. dst is not VF representor */
1203         if (!(dst_dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)) {
1204                 /* PF is the VF's PF? Then nothing to do */
1205                 vf = VF_ENIC_TO_VF_REP(src_enic);
1206                 if (vf->pf == dst_enic) {
1207                         ENICPMD_LOG(DEBUG, "destination port is VF's PF");
1208                         return 0;
1209                 }
1210                 /* If not, steer to the remote PF's uplink */
1211                 uif = dst_enic->fm_vnic_uif;
1212                 ENICPMD_LOG(DEBUG, "steer to uplink %u", uif);
1213                 memset(fm_op, 0, sizeof(*fm_op));
1214                 fm_op->fa_op = FMOP_SET_EGPORT;
1215                 fm_op->set_egport.egport = uif;
1216                 ret = enic_fm_append_action_op(fm, fm_op, error);
1217                 return ret;
1218         }
1219
1220         /* VF -> VF loopback. Hairpin and steer to vnic */
1221         memset(fm_op, 0, sizeof(*fm_op));
1222         fm_op->fa_op = FMOP_EG_HAIRPIN;
1223         ret = enic_fm_append_action_op(fm, fm_op, error);
1224         if (ret)
1225                 return ret;
1226         ENICPMD_LOG(DEBUG, "egress hairpin");
1227         fm->hairpin_steer_vnic_h = dst_vnic_h;
1228         fm->need_hairpin_steer = 1;
1229         return 0;
1230 }
1231
1232 /* Translate flow actions to flowman TCAM entry actions */
1233 static int
1234 enic_fm_copy_action(struct enic_flowman *fm,
1235                     const struct rte_flow_action actions[],
1236                     uint8_t ingress,
1237                     struct rte_flow_error *error)
1238 {
1239         enum {
1240                 FATE = 1 << 0,
1241                 DECAP = 1 << 1,
1242                 PASSTHRU = 1 << 2,
1243                 COUNT = 1 << 3,
1244                 ENCAP = 1 << 4,
1245                 PUSH_VLAN = 1 << 5,
1246                 PORT_ID = 1 << 6,
1247         };
1248         struct fm_tcam_match_entry *fmt;
1249         struct fm_action_op fm_op;
1250         bool need_ovlan_action;
1251         struct enic *enic;
1252         uint32_t overlap;
1253         uint64_t vnic_h;
1254         uint16_t ovlan;
1255         bool first_rq;
1256         bool steer;
1257         int ret;
1258
1259         ENICPMD_FUNC_TRACE();
1260         fmt = &fm->tcam_entry;
1261         need_ovlan_action = false;
1262         ovlan = 0;
1263         first_rq = true;
1264         steer = false;
1265         enic = fm->user_enic;
1266         overlap = 0;
1267         vnic_h = enic->fm_vnic_handle;
1268
1269         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1270                 switch (actions->type) {
1271                 case RTE_FLOW_ACTION_TYPE_VOID:
1272                         continue;
1273                 case RTE_FLOW_ACTION_TYPE_PASSTHRU: {
1274                         if (overlap & PASSTHRU)
1275                                 goto unsupported;
1276                         overlap |= PASSTHRU;
1277                         break;
1278                 }
1279                 case RTE_FLOW_ACTION_TYPE_JUMP: {
1280                         const struct rte_flow_action_jump *jump =
1281                                 actions->conf;
1282                         struct enic_fm_fet *fet;
1283
1284                         if (overlap & FATE)
1285                                 goto unsupported;
1286                         ret = enic_fet_get(fm, jump->group, ingress, NULL,
1287                                            &fet, error);
1288                         if (ret)
1289                                 return ret;
1290                         overlap |= FATE;
1291                         memset(&fm_op, 0, sizeof(fm_op));
1292                         fm_op.fa_op = FMOP_EXACT_MATCH;
1293                         fm_op.exact.handle = fet->handle;
1294                         fm->fet = fet;
1295                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1296                         if (ret)
1297                                 return ret;
1298                         break;
1299                 }
1300                 case RTE_FLOW_ACTION_TYPE_MARK: {
1301                         const struct rte_flow_action_mark *mark =
1302                                 actions->conf;
1303
1304                         if (mark->id >= ENIC_MAGIC_FILTER_ID - 1)
1305                                 return rte_flow_error_set(error, EINVAL,
1306                                         RTE_FLOW_ERROR_TYPE_ACTION,
1307                                         NULL, "invalid mark id");
1308                         memset(&fm_op, 0, sizeof(fm_op));
1309                         fm_op.fa_op = FMOP_MARK;
1310                         fm_op.mark.mark = mark->id + 1;
1311                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1312                         if (ret)
1313                                 return ret;
1314                         break;
1315                 }
1316                 case RTE_FLOW_ACTION_TYPE_FLAG: {
1317                         /* ENIC_MAGIC_FILTER_ID is reserved for flagging */
1318                         memset(&fm_op, 0, sizeof(fm_op));
1319                         fm_op.fa_op = FMOP_MARK;
1320                         fm_op.mark.mark = ENIC_MAGIC_FILTER_ID;
1321                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1322                         if (ret)
1323                                 return ret;
1324                         break;
1325                 }
1326                 case RTE_FLOW_ACTION_TYPE_QUEUE: {
1327                         const struct rte_flow_action_queue *queue =
1328                                 actions->conf;
1329
1330                         /*
1331                          * If fate other than QUEUE or RSS, fail. Multiple
1332                          * rss and queue actions are ok.
1333                          */
1334                         if ((overlap & FATE) && first_rq)
1335                                 goto unsupported;
1336                         first_rq = false;
1337                         overlap |= FATE;
1338                         memset(&fm_op, 0, sizeof(fm_op));
1339                         fm_op.fa_op = FMOP_RQ_STEER;
1340                         fm_op.rq_steer.rq_index =
1341                                 enic_rte_rq_idx_to_sop_idx(queue->index);
1342                         fm_op.rq_steer.rq_count = 1;
1343                         fm_op.rq_steer.vnic_handle = vnic_h;
1344                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1345                         if (ret)
1346                                 return ret;
1347                         ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1348                                     fm_op.rq_steer.rq_index);
1349                         steer = true;
1350                         break;
1351                 }
1352                 case RTE_FLOW_ACTION_TYPE_DROP: {
1353                         if (overlap & FATE)
1354                                 goto unsupported;
1355                         overlap |= FATE;
1356                         memset(&fm_op, 0, sizeof(fm_op));
1357                         fm_op.fa_op = FMOP_DROP;
1358                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1359                         if (ret)
1360                                 return ret;
1361                         ENICPMD_LOG(DEBUG, "create DROP action");
1362                         break;
1363                 }
1364                 case RTE_FLOW_ACTION_TYPE_COUNT: {
1365                         if (overlap & COUNT)
1366                                 goto unsupported;
1367                         overlap |= COUNT;
1368                         /* Count is associated with entry not action on VIC. */
1369                         fmt->ftm_flags |= FMEF_COUNTER;
1370                         break;
1371                 }
1372                 case RTE_FLOW_ACTION_TYPE_RSS: {
1373                         const struct rte_flow_action_rss *rss = actions->conf;
1374                         bool allow;
1375                         uint16_t i;
1376
1377                         /*
1378                          * If fate other than QUEUE or RSS, fail. Multiple
1379                          * rss and queue actions are ok.
1380                          */
1381                         if ((overlap & FATE) && first_rq)
1382                                 goto unsupported;
1383                         first_rq = false;
1384                         overlap |= FATE;
1385
1386                         /*
1387                          * Hardware only supports RSS actions on outer level
1388                          * with default type and function. Queues must be
1389                          * sequential.
1390                          */
1391                         allow = rss->func == RTE_ETH_HASH_FUNCTION_DEFAULT &&
1392                                 rss->level == 0 && (rss->types == 0 ||
1393                                 rss->types == enic->rss_hf) &&
1394                                 rss->queue_num <= enic->rq_count &&
1395                                 rss->queue[rss->queue_num - 1] < enic->rq_count;
1396
1397
1398                         /* Identity queue map needs to be sequential */
1399                         for (i = 1; i < rss->queue_num; i++)
1400                                 allow = allow && (rss->queue[i] ==
1401                                         rss->queue[i - 1] + 1);
1402                         if (!allow)
1403                                 goto unsupported;
1404
1405                         memset(&fm_op, 0, sizeof(fm_op));
1406                         fm_op.fa_op = FMOP_RQ_STEER;
1407                         fm_op.rq_steer.rq_index =
1408                                 enic_rte_rq_idx_to_sop_idx(rss->queue[0]);
1409                         fm_op.rq_steer.rq_count = rss->queue_num;
1410                         fm_op.rq_steer.vnic_handle = vnic_h;
1411                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1412                         if (ret)
1413                                 return ret;
1414                         ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1415                                     fm_op.rq_steer.rq_index);
1416                         steer = true;
1417                         break;
1418                 }
1419                 case RTE_FLOW_ACTION_TYPE_PORT_ID: {
1420                         const struct rte_flow_action_port_id *port;
1421                         struct rte_eth_dev *dev;
1422
1423                         if (!ingress && (overlap & PORT_ID)) {
1424                                 ENICPMD_LOG(DEBUG, "cannot have multiple egress PORT_ID actions");
1425                                 goto unsupported;
1426                         }
1427                         port = actions->conf;
1428                         if (port->original) {
1429                                 vnic_h = enic->fm_vnic_handle; /* This port */
1430                                 break;
1431                         }
1432                         ENICPMD_LOG(DEBUG, "port id %u", port->id);
1433                         if (!rte_eth_dev_is_valid_port(port->id)) {
1434                                 return rte_flow_error_set(error, EINVAL,
1435                                         RTE_FLOW_ERROR_TYPE_ACTION,
1436                                         NULL, "invalid port_id");
1437                         }
1438                         dev = &rte_eth_devices[port->id];
1439                         if (!dev_is_enic(dev)) {
1440                                 return rte_flow_error_set(error, EINVAL,
1441                                         RTE_FLOW_ERROR_TYPE_ACTION,
1442                                         NULL, "port_id is not enic");
1443                         }
1444                         if (enic->switch_domain_id !=
1445                             pmd_priv(dev)->switch_domain_id) {
1446                                 return rte_flow_error_set(error, EINVAL,
1447                                         RTE_FLOW_ERROR_TYPE_ACTION,
1448                                         NULL, "destination and source ports are not in the same switch domain");
1449                         }
1450                         vnic_h = pmd_priv(dev)->fm_vnic_handle;
1451                         overlap |= PORT_ID;
1452                         /*
1453                          * Ingress. Nothing more to do. We add an implicit
1454                          * steer at the end if needed.
1455                          */
1456                         if (ingress)
1457                                 break;
1458                         /* Egress */
1459                         ret = vf_egress_port_id_action(fm, dev, vnic_h, &fm_op,
1460                                 error);
1461                         if (ret)
1462                                 return ret;
1463                         break;
1464                 }
1465                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: {
1466                         if (overlap & DECAP)
1467                                 goto unsupported;
1468                         overlap |= DECAP;
1469
1470                         ret = enic_fm_copy_vxlan_decap(fm, fmt, actions,
1471                                 error);
1472                         if (ret != 0)
1473                                 return ret;
1474                         break;
1475                 }
1476                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: {
1477                         const struct rte_flow_action_vxlan_encap *encap;
1478
1479                         encap = actions->conf;
1480                         if (overlap & ENCAP)
1481                                 goto unsupported;
1482                         overlap |= ENCAP;
1483                         ret = enic_fm_copy_vxlan_encap(fm, encap->definition,
1484                                 error);
1485                         if (ret != 0)
1486                                 return ret;
1487                         break;
1488                 }
1489                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: {
1490                         struct fm_action_op *decap;
1491
1492                         /*
1493                          * If decap-nostrip appears before pop vlan, this pop
1494                          * applies to the inner packet vlan. Turn it into
1495                          * decap-strip.
1496                          */
1497                         decap = find_prev_action_op(fm, FMOP_DECAP_NOSTRIP);
1498                         if (decap) {
1499                                 ENICPMD_LOG(DEBUG, "pop-vlan inner: decap-nostrip => decap-strip");
1500                                 decap->fa_op = FMOP_DECAP_STRIP;
1501                                 break;
1502                         }
1503                         memset(&fm_op, 0, sizeof(fm_op));
1504                         fm_op.fa_op = FMOP_POP_VLAN;
1505                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1506                         if (ret)
1507                                 return ret;
1508                         break;
1509                 }
1510                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: {
1511                         const struct rte_flow_action_of_push_vlan *vlan;
1512
1513                         if (overlap & PASSTHRU)
1514                                 goto unsupported;
1515                         vlan = actions->conf;
1516                         if (vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN)) {
1517                                 return rte_flow_error_set(error, EINVAL,
1518                                         RTE_FLOW_ERROR_TYPE_ACTION,
1519                                         NULL, "unexpected push_vlan ethertype");
1520                         }
1521                         overlap |= PUSH_VLAN;
1522                         need_ovlan_action = true;
1523                         break;
1524                 }
1525                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: {
1526                         const struct rte_flow_action_of_set_vlan_pcp *pcp;
1527
1528                         pcp = actions->conf;
1529                         if (pcp->vlan_pcp > 7) {
1530                                 return rte_flow_error_set(error, EINVAL,
1531                                         RTE_FLOW_ERROR_TYPE_ACTION,
1532                                         NULL, "invalid vlan_pcp");
1533                         }
1534                         need_ovlan_action = true;
1535                         ovlan |= ((uint16_t)pcp->vlan_pcp) << 13;
1536                         break;
1537                 }
1538                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: {
1539                         const struct rte_flow_action_of_set_vlan_vid *vid;
1540
1541                         vid = actions->conf;
1542                         need_ovlan_action = true;
1543                         ovlan |= rte_be_to_cpu_16(vid->vlan_vid);
1544                         break;
1545                 }
1546                 default:
1547                         goto unsupported;
1548                 }
1549         }
1550
1551         if (!(overlap & (FATE | PASSTHRU | COUNT | PORT_ID)))
1552                 goto unsupported;
1553         /* Egress from VF: need implicit WQ match */
1554         if (enic_is_vf_rep(enic) && !ingress) {
1555                 fmt->ftm_data.fk_wq_id = 0;
1556                 fmt->ftm_mask.fk_wq_id = 0xffff;
1557                 fmt->ftm_data.fk_wq_vnic = enic->fm_vnic_handle;
1558                 ENICPMD_LOG(DEBUG, "add implicit wq id match for vf %d",
1559                             VF_ENIC_TO_VF_REP(enic)->vf_id);
1560         }
1561         if (need_ovlan_action) {
1562                 memset(&fm_op, 0, sizeof(fm_op));
1563                 fm_op.fa_op = FMOP_SET_OVLAN;
1564                 fm_op.ovlan.vlan = ovlan;
1565                 ret = enic_fm_append_action_op(fm, &fm_op, error);
1566                 if (ret)
1567                         return ret;
1568         }
1569         /* Add steer op for PORT_ID without QUEUE */
1570         if ((overlap & PORT_ID) && !steer && ingress) {
1571                 memset(&fm_op, 0, sizeof(fm_op));
1572                 /* Always to queue 0 for now as generic RSS is not available */
1573                 fm_op.fa_op = FMOP_RQ_STEER;
1574                 fm_op.rq_steer.rq_index = 0;
1575                 fm_op.rq_steer.vnic_handle = vnic_h;
1576                 ret = enic_fm_append_action_op(fm, &fm_op, error);
1577                 if (ret)
1578                         return ret;
1579                 ENICPMD_LOG(DEBUG, "add implicit steer op");
1580         }
1581         /* Add required END */
1582         memset(&fm_op, 0, sizeof(fm_op));
1583         fm_op.fa_op = FMOP_END;
1584         ret = enic_fm_append_action_op(fm, &fm_op, error);
1585         if (ret)
1586                 return ret;
1587         enic_fm_reorder_action_op(fm);
1588         return 0;
1589
1590 unsupported:
1591         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
1592                                   NULL, "enic: unsupported action");
1593 }
1594
1595 /** Check if the action is supported */
1596 static int
1597 enic_fm_match_action(const struct rte_flow_action *action,
1598                      const enum rte_flow_action_type *supported_actions)
1599 {
1600         for (; *supported_actions != RTE_FLOW_ACTION_TYPE_END;
1601              supported_actions++) {
1602                 if (action->type == *supported_actions)
1603                         return 1;
1604         }
1605         return 0;
1606 }
1607
1608 /* Debug function to dump internal NIC action structure. */
1609 static void
1610 enic_fm_dump_tcam_actions(const struct fm_action *fm_action)
1611 {
1612         /* Manually keep in sync with FMOP commands */
1613         const char *fmop_str[FMOP_OP_MAX] = {
1614                 [FMOP_END] = "end",
1615                 [FMOP_DROP] = "drop",
1616                 [FMOP_RQ_STEER] = "steer",
1617                 [FMOP_EXACT_MATCH] = "exmatch",
1618                 [FMOP_MARK] = "mark",
1619                 [FMOP_EXT_MARK] = "ext_mark",
1620                 [FMOP_TAG] = "tag",
1621                 [FMOP_EG_HAIRPIN] = "eg_hairpin",
1622                 [FMOP_IG_HAIRPIN] = "ig_hairpin",
1623                 [FMOP_ENCAP_IVLAN] = "encap_ivlan",
1624                 [FMOP_ENCAP_NOIVLAN] = "encap_noivlan",
1625                 [FMOP_ENCAP] = "encap",
1626                 [FMOP_SET_OVLAN] = "set_ovlan",
1627                 [FMOP_DECAP_NOSTRIP] = "decap_nostrip",
1628                 [FMOP_DECAP_STRIP] = "decap_strip",
1629                 [FMOP_POP_VLAN] = "pop_vlan",
1630                 [FMOP_SET_EGPORT] = "set_egport",
1631                 [FMOP_RQ_STEER_ONLY] = "rq_steer_only",
1632                 [FMOP_SET_ENCAP_VLAN] = "set_encap_vlan",
1633                 [FMOP_EMIT] = "emit",
1634                 [FMOP_MODIFY] = "modify",
1635         };
1636         const struct fm_action_op *op = &fm_action->fma_action_ops[0];
1637         char buf[128], *bp = buf;
1638         const char *op_str;
1639         int i, n, buf_len;
1640
1641         buf[0] = '\0';
1642         buf_len = sizeof(buf);
1643         for (i = 0; i < FM_ACTION_OP_MAX; i++) {
1644                 if (op->fa_op == FMOP_END)
1645                         break;
1646                 if (op->fa_op >= FMOP_OP_MAX)
1647                         op_str = "unknown";
1648                 else
1649                         op_str = fmop_str[op->fa_op];
1650                 n = snprintf(bp, buf_len, "%s,", op_str);
1651                 if (n > 0 && n < buf_len) {
1652                         bp += n;
1653                         buf_len -= n;
1654                 }
1655                 op++;
1656         }
1657         /* Remove trailing comma */
1658         if (buf[0])
1659                 *(bp - 1) = '\0';
1660         ENICPMD_LOG(DEBUG, "       Acions: %s", buf);
1661 }
1662
1663 static int
1664 bits_to_str(uint32_t bits, const char *strings[], int max,
1665             char *buf, int buf_len)
1666 {
1667         int i, n = 0, len = 0;
1668
1669         for (i = 0; i < max; i++) {
1670                 if (bits & (1 << i)) {
1671                         n = snprintf(buf, buf_len, "%s,", strings[i]);
1672                         if (n > 0 && n < buf_len) {
1673                                 buf += n;
1674                                 buf_len -= n;
1675                                 len += n;
1676                         }
1677                 }
1678         }
1679         /* Remove trailing comma */
1680         if (len) {
1681                 *(buf - 1) = '\0';
1682                 len--;
1683         }
1684         return len;
1685 }
1686
1687 /* Debug function to dump internal NIC filter structure. */
1688 static void
1689 __enic_fm_dump_tcam_match(const struct fm_header_set *fk_hdrset, char *buf,
1690                           int buf_len)
1691 {
1692         /* Manually keep in sync with FKM_BITS */
1693         const char *fm_fkm_str[FKM_BIT_COUNT] = {
1694                 [FKM_QTAG_BIT] = "qtag",
1695                 [FKM_CMD_BIT] = "cmd",
1696                 [FKM_IPV4_BIT] = "ip4",
1697                 [FKM_IPV6_BIT] = "ip6",
1698                 [FKM_ROCE_BIT] = "roce",
1699                 [FKM_UDP_BIT] = "udp",
1700                 [FKM_TCP_BIT] = "tcp",
1701                 [FKM_TCPORUDP_BIT] = "tcpportudp",
1702                 [FKM_IPFRAG_BIT] = "ipfrag",
1703                 [FKM_NVGRE_BIT] = "nvgre",
1704                 [FKM_VXLAN_BIT] = "vxlan",
1705                 [FKM_GENEVE_BIT] = "geneve",
1706                 [FKM_NSH_BIT] = "nsh",
1707                 [FKM_ROCEV2_BIT] = "rocev2",
1708                 [FKM_VLAN_PRES_BIT] = "vlan_pres",
1709                 [FKM_IPOK_BIT] = "ipok",
1710                 [FKM_L4OK_BIT] = "l4ok",
1711                 [FKM_ROCEOK_BIT] = "roceok",
1712                 [FKM_FCSOK_BIT] = "fcsok",
1713                 [FKM_EG_SPAN_BIT] = "eg_span",
1714                 [FKM_IG_SPAN_BIT] = "ig_span",
1715                 [FKM_EG_HAIRPINNED_BIT] = "eg_hairpinned",
1716         };
1717         /* Manually keep in sync with FKH_BITS */
1718         const char *fm_fkh_str[FKH_BIT_COUNT] = {
1719                 [FKH_ETHER_BIT] = "eth",
1720                 [FKH_QTAG_BIT] = "qtag",
1721                 [FKH_L2RAW_BIT] = "l2raw",
1722                 [FKH_IPV4_BIT] = "ip4",
1723                 [FKH_IPV6_BIT] = "ip6",
1724                 [FKH_L3RAW_BIT] = "l3raw",
1725                 [FKH_UDP_BIT] = "udp",
1726                 [FKH_TCP_BIT] = "tcp",
1727                 [FKH_ICMP_BIT] = "icmp",
1728                 [FKH_VXLAN_BIT] = "vxlan",
1729                 [FKH_L4RAW_BIT] = "l4raw",
1730         };
1731         uint32_t fkh_bits = fk_hdrset->fk_header_select;
1732         uint32_t fkm_bits = fk_hdrset->fk_metadata;
1733         int n;
1734
1735         if (!fkm_bits && !fkh_bits)
1736                 return;
1737         n = snprintf(buf, buf_len, "metadata(");
1738         if (n > 0 && n < buf_len) {
1739                 buf += n;
1740                 buf_len -= n;
1741         }
1742         n = bits_to_str(fkm_bits, fm_fkm_str, FKM_BIT_COUNT, buf, buf_len);
1743         if (n > 0 && n < buf_len) {
1744                 buf += n;
1745                 buf_len -= n;
1746         }
1747         n = snprintf(buf, buf_len, ") valid hdr fields(");
1748         if (n > 0 && n < buf_len) {
1749                 buf += n;
1750                 buf_len -= n;
1751         }
1752         n = bits_to_str(fkh_bits, fm_fkh_str, FKH_BIT_COUNT, buf, buf_len);
1753         if (n > 0 && n < buf_len) {
1754                 buf += n;
1755                 buf_len -= n;
1756         }
1757         snprintf(buf, buf_len, ")");
1758 }
1759
1760 static void
1761 enic_fm_dump_tcam_match(const struct fm_tcam_match_entry *match,
1762                         uint8_t ingress)
1763 {
1764         char buf[256];
1765
1766         memset(buf, 0, sizeof(buf));
1767         __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[0],
1768                                   buf, sizeof(buf));
1769         ENICPMD_LOG(DEBUG, " TCAM %s Outer: %s %scounter position %u",
1770                     (ingress) ? "IG" : "EG", buf,
1771                     (match->ftm_flags & FMEF_COUNTER) ? "" : "no ",
1772                     match->ftm_position);
1773         memset(buf, 0, sizeof(buf));
1774         __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[1],
1775                                   buf, sizeof(buf));
1776         if (buf[0])
1777                 ENICPMD_LOG(DEBUG, "         Inner: %s", buf);
1778 }
1779
1780 /* Debug function to dump internal NIC flow structures. */
1781 static void
1782 enic_fm_dump_tcam_entry(const struct fm_tcam_match_entry *fm_match,
1783                         const struct fm_action *fm_action,
1784                         uint8_t ingress)
1785 {
1786         if (!rte_log_can_log(enic_pmd_logtype, RTE_LOG_DEBUG))
1787                 return;
1788         enic_fm_dump_tcam_match(fm_match, ingress);
1789         enic_fm_dump_tcam_actions(fm_action);
1790 }
1791
1792 static int
1793 enic_fm_flow_parse(struct enic_flowman *fm,
1794                    const struct rte_flow_attr *attrs,
1795                    const struct rte_flow_item pattern[],
1796                    const struct rte_flow_action actions[],
1797                    struct rte_flow_error *error)
1798 {
1799         const struct rte_flow_action *action;
1800         unsigned int ret;
1801         static const enum rte_flow_action_type *sa;
1802
1803         ENICPMD_FUNC_TRACE();
1804         ret = 0;
1805         if (!pattern) {
1806                 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM,
1807                                    NULL, "no pattern specified");
1808                 return -rte_errno;
1809         }
1810
1811         if (!actions) {
1812                 rte_flow_error_set(error, EINVAL,
1813                                    RTE_FLOW_ERROR_TYPE_ACTION_NUM,
1814                                    NULL, "no action specified");
1815                 return -rte_errno;
1816         }
1817
1818         if (attrs) {
1819                 if (attrs->group != FM_TCAM_RTE_GROUP && attrs->priority) {
1820                         rte_flow_error_set(error, ENOTSUP,
1821                                            RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1822                                            NULL,
1823                                            "priorities are not supported for non-default (0) groups");
1824                         return -rte_errno;
1825                 } else if (!fm->owner_enic->switchdev_mode && attrs->transfer) {
1826                         rte_flow_error_set(error, ENOTSUP,
1827                                            RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1828                                            NULL,
1829                                            "transfer is not supported");
1830                         return -rte_errno;
1831                 } else if (attrs->ingress && attrs->egress) {
1832                         rte_flow_error_set(error, ENOTSUP,
1833                                            RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1834                                            NULL,
1835                                            "bidirectional rules not supported");
1836                         return -rte_errno;
1837                 }
1838
1839         } else {
1840                 rte_flow_error_set(error, EINVAL,
1841                                    RTE_FLOW_ERROR_TYPE_ATTR,
1842                                    NULL, "no attribute specified");
1843                 return -rte_errno;
1844         }
1845
1846         /* Verify Actions. */
1847         sa = (attrs->ingress) ? enic_fm_supported_ig_actions :
1848              enic_fm_supported_eg_actions;
1849         for (action = &actions[0]; action->type != RTE_FLOW_ACTION_TYPE_END;
1850              action++) {
1851                 if (action->type == RTE_FLOW_ACTION_TYPE_VOID)
1852                         continue;
1853                 else if (!enic_fm_match_action(action, sa))
1854                         break;
1855         }
1856         if (action->type != RTE_FLOW_ACTION_TYPE_END) {
1857                 rte_flow_error_set(error, EPERM, RTE_FLOW_ERROR_TYPE_ACTION,
1858                                    action, "invalid action");
1859                 return -rte_errno;
1860         }
1861         ret = enic_fm_copy_entry(fm, pattern, error);
1862         if (ret)
1863                 return ret;
1864         ret = enic_fm_copy_action(fm, actions, attrs->ingress, error);
1865         return ret;
1866 }
1867
1868 static void
1869 enic_fm_counter_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
1870 {
1871         if (!fm_flow->counter_valid)
1872                 return;
1873         SLIST_INSERT_HEAD(&fm->counters, fm_flow->counter, next);
1874         fm_flow->counter_valid = false;
1875 }
1876
1877 static int
1878 enic_fm_more_counters(struct enic_flowman *fm)
1879 {
1880         struct enic_fm_counter *new_stack;
1881         struct enic_fm_counter *ctrs;
1882         int i, rc;
1883         uint64_t args[2];
1884
1885         ENICPMD_FUNC_TRACE();
1886         new_stack = rte_realloc(fm->counter_stack, (fm->counters_alloced +
1887                                 FM_COUNTERS_EXPAND) *
1888                                 sizeof(struct enic_fm_counter), 0);
1889         if (new_stack == NULL) {
1890                 ENICPMD_LOG(ERR, "cannot alloc counter memory");
1891                 return -ENOMEM;
1892         }
1893         fm->counter_stack = new_stack;
1894
1895         args[0] = FM_COUNTER_BRK;
1896         args[1] = fm->counters_alloced + FM_COUNTERS_EXPAND;
1897         rc = flowman_cmd(fm, args, 2);
1898         if (rc != 0) {
1899                 ENICPMD_LOG(ERR, "cannot alloc counters rc=%d", rc);
1900                 return rc;
1901         }
1902         ctrs = (struct enic_fm_counter *)fm->counter_stack +
1903                 fm->counters_alloced;
1904         for (i = 0; i < FM_COUNTERS_EXPAND; i++, ctrs++) {
1905                 ctrs->handle = fm->counters_alloced + i;
1906                 SLIST_INSERT_HEAD(&fm->counters, ctrs, next);
1907         }
1908         fm->counters_alloced += FM_COUNTERS_EXPAND;
1909         ENICPMD_LOG(DEBUG, "%u counters allocated, total: %u",
1910                     FM_COUNTERS_EXPAND, fm->counters_alloced);
1911         return 0;
1912 }
1913
1914 static int
1915 enic_fm_counter_zero(struct enic_flowman *fm, struct enic_fm_counter *c)
1916 {
1917         uint64_t args[3];
1918         int ret;
1919
1920         ENICPMD_FUNC_TRACE();
1921         args[0] = FM_COUNTER_QUERY;
1922         args[1] = c->handle;
1923         args[2] = 1; /* clear */
1924         ret = flowman_cmd(fm, args, 3);
1925         if (ret) {
1926                 ENICPMD_LOG(ERR, "counter init: rc=%d handle=0x%x",
1927                             ret, c->handle);
1928                 return ret;
1929         }
1930         return 0;
1931 }
1932
1933 static int
1934 enic_fm_counter_alloc(struct enic_flowman *fm, struct rte_flow_error *error,
1935                       struct enic_fm_counter **ctr)
1936 {
1937         struct enic_fm_counter *c;
1938         int ret;
1939
1940         ENICPMD_FUNC_TRACE();
1941         *ctr = NULL;
1942         if (SLIST_EMPTY(&fm->counters)) {
1943                 ret = enic_fm_more_counters(fm);
1944                 if (ret)
1945                         return rte_flow_error_set(error, -ret,
1946                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1947                                 NULL, "enic: out of counters");
1948         }
1949         c = SLIST_FIRST(&fm->counters);
1950         SLIST_REMOVE_HEAD(&fm->counters, next);
1951         *ctr = c;
1952         return 0;
1953 }
1954
1955 static int
1956 enic_fm_action_free(struct enic_flowman *fm, uint64_t handle)
1957 {
1958         uint64_t args[2];
1959         int rc;
1960
1961         ENICPMD_FUNC_TRACE();
1962         args[0] = FM_ACTION_FREE;
1963         args[1] = handle;
1964         rc = flowman_cmd(fm, args, 2);
1965         if (rc)
1966                 ENICPMD_LOG(ERR, "cannot free action: rc=%d handle=0x%" PRIx64,
1967                             rc, handle);
1968         return rc;
1969 }
1970
1971 static int
1972 enic_fm_entry_free(struct enic_flowman *fm, uint64_t handle)
1973 {
1974         uint64_t args[2];
1975         int rc;
1976
1977         ENICPMD_FUNC_TRACE();
1978         args[0] = FM_MATCH_ENTRY_REMOVE;
1979         args[1] = handle;
1980         rc = flowman_cmd(fm, args, 2);
1981         if (rc)
1982                 ENICPMD_LOG(ERR, "cannot free match entry: rc=%d"
1983                             " handle=0x%" PRIx64, rc, handle);
1984         return rc;
1985 }
1986
1987 static struct enic_fm_jump_flow *
1988 find_jump_flow(struct enic_flowman *fm, uint32_t group)
1989 {
1990         struct enic_fm_jump_flow *j;
1991
1992         ENICPMD_FUNC_TRACE();
1993         TAILQ_FOREACH(j, &fm->jump_list, list) {
1994                 if (j->group == group)
1995                         return j;
1996         }
1997         return NULL;
1998 }
1999
2000 static void
2001 remove_jump_flow(struct enic_flowman *fm, struct rte_flow *flow)
2002 {
2003         struct enic_fm_jump_flow *j;
2004
2005         ENICPMD_FUNC_TRACE();
2006         TAILQ_FOREACH(j, &fm->jump_list, list) {
2007                 if (j->flow == flow) {
2008                         TAILQ_REMOVE(&fm->jump_list, j, list);
2009                         free(j);
2010                         return;
2011                 }
2012         }
2013 }
2014
2015 static int
2016 save_jump_flow(struct enic_flowman *fm,
2017                struct rte_flow *flow,
2018                uint32_t group,
2019                struct fm_tcam_match_entry *match,
2020                struct fm_action *action)
2021 {
2022         struct enic_fm_jump_flow *j;
2023
2024         ENICPMD_FUNC_TRACE();
2025         j = calloc(1, sizeof(struct enic_fm_jump_flow));
2026         if (j == NULL)
2027                 return -ENOMEM;
2028         j->flow = flow;
2029         j->group = group;
2030         j->match = *match;
2031         j->action = *action;
2032         TAILQ_INSERT_HEAD(&fm->jump_list, j, list);
2033         ENICPMD_LOG(DEBUG, "saved jump flow: flow=%p group=%u", flow, group);
2034         return 0;
2035 }
2036
2037 static void
2038 __enic_fm_flow_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
2039 {
2040         if (fm_flow->entry_handle != FM_INVALID_HANDLE) {
2041                 enic_fm_entry_free(fm, fm_flow->entry_handle);
2042                 fm_flow->entry_handle = FM_INVALID_HANDLE;
2043         }
2044         if (fm_flow->action_handle != FM_INVALID_HANDLE) {
2045                 enic_fm_action_free(fm, fm_flow->action_handle);
2046                 fm_flow->action_handle = FM_INVALID_HANDLE;
2047         }
2048         enic_fm_counter_free(fm, fm_flow);
2049         if (fm_flow->fet) {
2050                 enic_fet_put(fm, fm_flow->fet);
2051                 fm_flow->fet = NULL;
2052         }
2053 }
2054
2055 static void
2056 enic_fm_flow_free(struct enic_flowman *fm, struct rte_flow *flow)
2057 {
2058         struct enic_fm_flow *steer = flow->fm->hairpin_steer_flow;
2059
2060         if (flow->fm->fet && flow->fm->fet->default_key)
2061                 remove_jump_flow(fm, flow);
2062         __enic_fm_flow_free(fm, flow->fm);
2063         if (steer) {
2064                 __enic_fm_flow_free(fm, steer);
2065                 free(steer);
2066         }
2067         free(flow->fm);
2068         free(flow);
2069 }
2070
2071 static int
2072 enic_fm_add_tcam_entry(struct enic_flowman *fm,
2073                        struct fm_tcam_match_entry *match_in,
2074                        uint64_t *entry_handle,
2075                        uint8_t ingress,
2076                        struct rte_flow_error *error)
2077 {
2078         struct fm_tcam_match_entry *ftm;
2079         uint64_t args[3];
2080         int ret;
2081
2082         ENICPMD_FUNC_TRACE();
2083         /* Copy entry to the command buffer */
2084         ftm = &fm->cmd.va->fm_tcam_match_entry;
2085         memcpy(ftm, match_in, sizeof(*ftm));
2086         /* Add TCAM entry */
2087         args[0] = FM_TCAM_ENTRY_INSTALL;
2088         args[1] = ingress ? fm->ig_tcam_hndl : fm->eg_tcam_hndl;
2089         args[2] = fm->cmd.pa;
2090         ret = flowman_cmd(fm, args, 3);
2091         if (ret != 0) {
2092                 ENICPMD_LOG(ERR, "cannot add %s TCAM entry: rc=%d",
2093                             ingress ? "ingress" : "egress", ret);
2094                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2095                         NULL, "enic: devcmd(tcam-entry-install)");
2096                 return ret;
2097         }
2098         ENICPMD_LOG(DEBUG, "installed %s TCAM entry: handle=0x%" PRIx64,
2099                     ingress ? "ingress" : "egress", (uint64_t)args[0]);
2100         *entry_handle = args[0];
2101         return 0;
2102 }
2103
2104 static int
2105 enic_fm_add_exact_entry(struct enic_flowman *fm,
2106                         struct fm_tcam_match_entry *match_in,
2107                         uint64_t *entry_handle,
2108                         struct enic_fm_fet *fet,
2109                         struct rte_flow_error *error)
2110 {
2111         struct fm_exact_match_entry *fem;
2112         uint64_t args[3];
2113         int ret;
2114
2115         ENICPMD_FUNC_TRACE();
2116         /* The new entry must have the table's key */
2117         if (memcmp(fet->key.fk_hdrset, match_in->ftm_mask.fk_hdrset,
2118                    sizeof(struct fm_header_set) * FM_HDRSET_MAX)) {
2119                 return rte_flow_error_set(error, EINVAL,
2120                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2121                         "enic: key does not match group's key");
2122         }
2123
2124         /* Copy entry to the command buffer */
2125         fem = &fm->cmd.va->fm_exact_match_entry;
2126         /*
2127          * Translate TCAM entry to exact entry. As is only need to drop
2128          * position and mask. The mask is part of the exact match table.
2129          * Position (aka priority) is not supported in the exact match table.
2130          */
2131         fem->fem_data = match_in->ftm_data;
2132         fem->fem_flags = match_in->ftm_flags;
2133         fem->fem_action = match_in->ftm_action;
2134         fem->fem_counter = match_in->ftm_counter;
2135
2136         /* Add exact entry */
2137         args[0] = FM_EXACT_ENTRY_INSTALL;
2138         args[1] = fet->handle;
2139         args[2] = fm->cmd.pa;
2140         ret = flowman_cmd(fm, args, 3);
2141         if (ret != 0) {
2142                 ENICPMD_LOG(ERR, "cannot add %s exact entry: group=%u",
2143                             fet->ingress ? "ingress" : "egress", fet->group);
2144                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2145                         NULL, "enic: devcmd(exact-entry-install)");
2146                 return ret;
2147         }
2148         ENICPMD_LOG(DEBUG, "installed %s exact entry: group=%u"
2149                     " handle=0x%" PRIx64,
2150                     fet->ingress ? "ingress" : "egress", fet->group,
2151                     (uint64_t)args[0]);
2152         *entry_handle = args[0];
2153         return 0;
2154 }
2155
2156 /* Push match-action to the NIC. */
2157 static int
2158 __enic_fm_flow_add_entry(struct enic_flowman *fm,
2159                          struct enic_fm_flow *fm_flow,
2160                          struct fm_tcam_match_entry *match_in,
2161                          struct fm_action *action_in,
2162                          uint32_t group,
2163                          uint8_t ingress,
2164                          struct rte_flow_error *error)
2165 {
2166         struct enic_fm_counter *ctr;
2167         struct fm_action *fma;
2168         uint64_t action_h;
2169         uint64_t entry_h;
2170         uint64_t args[3];
2171         int ret;
2172
2173         ENICPMD_FUNC_TRACE();
2174         /* Allocate action. */
2175         fma = &fm->cmd.va->fm_action;
2176         memcpy(fma, action_in, sizeof(*fma));
2177         args[0] = FM_ACTION_ALLOC;
2178         args[1] = fm->cmd.pa;
2179         ret = flowman_cmd(fm, args, 2);
2180         if (ret != 0) {
2181                 ENICPMD_LOG(ERR, "allocating TCAM table action rc=%d", ret);
2182                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2183                         NULL, "enic: devcmd(action-alloc)");
2184                 return ret;
2185         }
2186         action_h = args[0];
2187         fm_flow->action_handle = action_h;
2188         match_in->ftm_action = action_h;
2189         ENICPMD_LOG(DEBUG, "action allocated: handle=0x%" PRIx64, action_h);
2190
2191         /* Allocate counter if requested. */
2192         if (match_in->ftm_flags & FMEF_COUNTER) {
2193                 ret = enic_fm_counter_alloc(fm, error, &ctr);
2194                 if (ret) /* error has been filled in */
2195                         return ret;
2196                 fm_flow->counter_valid = true;
2197                 fm_flow->counter = ctr;
2198                 match_in->ftm_counter = ctr->handle;
2199         }
2200
2201         /*
2202          * Get the group's table (either TCAM or exact match table) and
2203          * add entry to it. If we use the exact match table, the handler
2204          * will translate the TCAM entry (match_in) to the appropriate
2205          * exact match entry and use that instead.
2206          */
2207         entry_h = FM_INVALID_HANDLE;
2208         if (group == FM_TCAM_RTE_GROUP) {
2209                 ret = enic_fm_add_tcam_entry(fm, match_in, &entry_h, ingress,
2210                                              error);
2211                 if (ret)
2212                         return ret;
2213                 /* Jump action might have a ref to fet */
2214                 fm_flow->fet = fm->fet;
2215                 fm->fet = NULL;
2216         } else {
2217                 struct enic_fm_fet *fet = NULL;
2218
2219                 ret = enic_fet_get(fm, group, ingress,
2220                                    &match_in->ftm_mask, &fet, error);
2221                 if (ret)
2222                         return ret;
2223                 fm_flow->fet = fet;
2224                 ret = enic_fm_add_exact_entry(fm, match_in, &entry_h, fet,
2225                                               error);
2226                 if (ret)
2227                         return ret;
2228         }
2229         /* Clear counter after adding entry, as it requires in-use counter */
2230         if (fm_flow->counter_valid) {
2231                 ret = enic_fm_counter_zero(fm, fm_flow->counter);
2232                 if (ret)
2233                         return ret;
2234         }
2235         fm_flow->entry_handle = entry_h;
2236         return 0;
2237 }
2238
2239 /* Push match-action to the NIC. */
2240 static struct rte_flow *
2241 enic_fm_flow_add_entry(struct enic_flowman *fm,
2242                        struct fm_tcam_match_entry *match_in,
2243                        struct fm_action *action_in,
2244                        const struct rte_flow_attr *attrs,
2245                        struct rte_flow_error *error)
2246 {
2247         struct enic_fm_flow *fm_flow;
2248         struct rte_flow *flow;
2249
2250         ENICPMD_FUNC_TRACE();
2251         match_in->ftm_position = attrs->priority;
2252         enic_fm_dump_tcam_entry(match_in, action_in, attrs->ingress);
2253         flow = calloc(1, sizeof(*flow));
2254         fm_flow = calloc(1, sizeof(*fm_flow));
2255         if (flow == NULL || fm_flow == NULL) {
2256                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
2257                         NULL, "enic: cannot allocate rte_flow");
2258                 free(flow);
2259                 free(fm_flow);
2260                 return NULL;
2261         }
2262         flow->fm = fm_flow;
2263         fm_flow->action_handle = FM_INVALID_HANDLE;
2264         fm_flow->entry_handle = FM_INVALID_HANDLE;
2265         if (__enic_fm_flow_add_entry(fm, fm_flow, match_in, action_in,
2266                                      attrs->group, attrs->ingress, error)) {
2267                 enic_fm_flow_free(fm, flow);
2268                 return NULL;
2269         }
2270         return flow;
2271 }
2272
2273 static void
2274 convert_jump_flows(struct enic_flowman *fm, struct enic_fm_fet *fet,
2275                    struct rte_flow_error *error)
2276 {
2277         struct enic_fm_flow *fm_flow;
2278         struct enic_fm_jump_flow *j;
2279         struct fm_action *fma;
2280         uint32_t group;
2281
2282         ENICPMD_FUNC_TRACE();
2283         /*
2284          * Find the saved flows that should jump to the new table (fet).
2285          * Then delete the old TCAM entry that jumps to the default table,
2286          * and add a new one that jumps to the new table.
2287          */
2288         group = fet->group;
2289         j = find_jump_flow(fm, group);
2290         while (j) {
2291                 ENICPMD_LOG(DEBUG, "convert jump flow: flow=%p group=%u",
2292                             j->flow, group);
2293                 /* Delete old entry */
2294                 fm_flow = j->flow->fm;
2295                 __enic_fm_flow_free(fm, fm_flow);
2296
2297                 /* Add new entry */
2298                 fma = &j->action;
2299                 fma->fma_action_ops[0].exact.handle = fet->handle;
2300                 if (__enic_fm_flow_add_entry(fm, fm_flow, &j->match, fma,
2301                         FM_TCAM_RTE_GROUP, fet->ingress, error)) {
2302                         /* Cannot roll back changes at the moment */
2303                         ENICPMD_LOG(ERR, "cannot convert jump flow: flow=%p",
2304                                     j->flow);
2305                 } else {
2306                         fm_flow->fet = fet;
2307                         fet->ref++;
2308                         ENICPMD_LOG(DEBUG, "convert ok: group=%u ref=%u",
2309                                     fet->group, fet->ref);
2310                 }
2311
2312                 TAILQ_REMOVE(&fm->jump_list, j, list);
2313                 free(j);
2314                 j = find_jump_flow(fm, group);
2315         }
2316 }
2317
2318 static int
2319 add_hairpin_steer(struct enic_flowman *fm, struct rte_flow *flow,
2320                   struct rte_flow_error *error)
2321 {
2322         struct fm_tcam_match_entry *fm_tcam_entry;
2323         struct enic_fm_flow *fm_flow;
2324         struct fm_action *fm_action;
2325         struct fm_action_op fm_op;
2326         int ret;
2327
2328         ENICPMD_FUNC_TRACE();
2329         fm_flow = calloc(1, sizeof(*fm_flow));
2330         if (fm_flow == NULL) {
2331                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
2332                         NULL, "enic: cannot allocate rte_flow");
2333                 return -ENOMEM;
2334         }
2335         /* Original egress hairpin flow */
2336         fm_tcam_entry = &fm->tcam_entry;
2337         fm_action = &fm->action;
2338         /* Use the match pattern of the egress flow as is, without counters */
2339         fm_tcam_entry->ftm_flags &= ~FMEF_COUNTER;
2340         /* The only action is steer to vnic */
2341         fm->action_op_count = 0;
2342         memset(fm_action, 0, sizeof(*fm_action));
2343         memset(&fm_op, 0, sizeof(fm_op));
2344         /* Always to queue 0 for now */
2345         fm_op.fa_op = FMOP_RQ_STEER;
2346         fm_op.rq_steer.rq_index = 0;
2347         fm_op.rq_steer.vnic_handle = fm->hairpin_steer_vnic_h;
2348         ret = enic_fm_append_action_op(fm, &fm_op, error);
2349         if (ret)
2350                 goto error_with_flow;
2351         ENICPMD_LOG(DEBUG, "add steer op");
2352         /* Add required END */
2353         memset(&fm_op, 0, sizeof(fm_op));
2354         fm_op.fa_op = FMOP_END;
2355         ret = enic_fm_append_action_op(fm, &fm_op, error);
2356         if (ret)
2357                 goto error_with_flow;
2358         /* Add the ingress flow */
2359         fm_flow->action_handle = FM_INVALID_HANDLE;
2360         fm_flow->entry_handle = FM_INVALID_HANDLE;
2361         ret = __enic_fm_flow_add_entry(fm, fm_flow, fm_tcam_entry, fm_action,
2362                                        FM_TCAM_RTE_GROUP, 1 /* ingress */, error);
2363         if (ret) {
2364                 ENICPMD_LOG(ERR, "cannot add hairpin-steer flow");
2365                 goto error_with_flow;
2366         }
2367         /* The new flow is now the egress flow's paired flow */
2368         flow->fm->hairpin_steer_flow = fm_flow;
2369         return 0;
2370
2371 error_with_flow:
2372         free(fm_flow);
2373         return ret;
2374 }
2375
2376 static void
2377 enic_fm_open_scratch(struct enic_flowman *fm)
2378 {
2379         fm->action_op_count = 0;
2380         fm->fet = NULL;
2381         fm->need_hairpin_steer = 0;
2382         fm->hairpin_steer_vnic_h = 0;
2383         memset(&fm->tcam_entry, 0, sizeof(fm->tcam_entry));
2384         memset(&fm->action, 0, sizeof(fm->action));
2385 }
2386
2387 static void
2388 enic_fm_close_scratch(struct enic_flowman *fm)
2389 {
2390         if (fm->fet) {
2391                 enic_fet_put(fm, fm->fet);
2392                 fm->fet = NULL;
2393         }
2394         fm->action_op_count = 0;
2395 }
2396
2397 static int
2398 enic_fm_flow_validate(struct rte_eth_dev *dev,
2399                       const struct rte_flow_attr *attrs,
2400                       const struct rte_flow_item pattern[],
2401                       const struct rte_flow_action actions[],
2402                       struct rte_flow_error *error)
2403 {
2404         struct fm_tcam_match_entry *fm_tcam_entry;
2405         struct fm_action *fm_action;
2406         struct enic_flowman *fm;
2407         int ret;
2408
2409         ENICPMD_FUNC_TRACE();
2410         fm = begin_fm(pmd_priv(dev));
2411         if (fm == NULL)
2412                 return -ENOTSUP;
2413         enic_fm_open_scratch(fm);
2414         ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2415         if (!ret) {
2416                 fm_tcam_entry = &fm->tcam_entry;
2417                 fm_action = &fm->action;
2418                 enic_fm_dump_tcam_entry(fm_tcam_entry, fm_action,
2419                                         attrs->ingress);
2420         }
2421         enic_fm_close_scratch(fm);
2422         end_fm(fm);
2423         return ret;
2424 }
2425
2426 static int
2427 enic_fm_flow_query_count(struct rte_eth_dev *dev,
2428                          struct rte_flow *flow, void *data,
2429                          struct rte_flow_error *error)
2430 {
2431         struct rte_flow_query_count *query;
2432         struct enic_fm_flow *fm_flow;
2433         struct enic_flowman *fm;
2434         uint64_t args[3];
2435         int rc;
2436
2437         ENICPMD_FUNC_TRACE();
2438         fm = begin_fm(pmd_priv(dev));
2439         query = data;
2440         fm_flow = flow->fm;
2441         if (!fm_flow->counter_valid) {
2442                 rc = rte_flow_error_set(error, ENOTSUP,
2443                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2444                         "enic: flow does not have counter");
2445                 goto exit;
2446         }
2447
2448         args[0] = FM_COUNTER_QUERY;
2449         args[1] = fm_flow->counter->handle;
2450         args[2] = query->reset;
2451         rc = flowman_cmd(fm, args, 3);
2452         if (rc) {
2453                 ENICPMD_LOG(ERR, "cannot query counter: rc=%d handle=0x%x",
2454                             rc, fm_flow->counter->handle);
2455                 goto exit;
2456         }
2457         query->hits_set = 1;
2458         query->hits = args[0];
2459         query->bytes_set = 1;
2460         query->bytes = args[1];
2461         rc = 0;
2462 exit:
2463         end_fm(fm);
2464         return rc;
2465 }
2466
2467 static int
2468 enic_fm_flow_query(struct rte_eth_dev *dev,
2469                    struct rte_flow *flow,
2470                    const struct rte_flow_action *actions,
2471                    void *data,
2472                    struct rte_flow_error *error)
2473 {
2474         int ret = 0;
2475
2476         ENICPMD_FUNC_TRACE();
2477         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2478                 switch (actions->type) {
2479                 case RTE_FLOW_ACTION_TYPE_VOID:
2480                         break;
2481                 case RTE_FLOW_ACTION_TYPE_COUNT:
2482                         ret = enic_fm_flow_query_count(dev, flow, data, error);
2483                         break;
2484                 default:
2485                         return rte_flow_error_set(error, ENOTSUP,
2486                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2487                                                   actions,
2488                                                   "action not supported");
2489                 }
2490                 if (ret < 0)
2491                         return ret;
2492         }
2493         return 0;
2494 }
2495
2496 static struct rte_flow *
2497 enic_fm_flow_create(struct rte_eth_dev *dev,
2498                     const struct rte_flow_attr *attrs,
2499                     const struct rte_flow_item pattern[],
2500                     const struct rte_flow_action actions[],
2501                     struct rte_flow_error *error)
2502 {
2503         struct fm_tcam_match_entry *fm_tcam_entry;
2504         struct fm_action *fm_action;
2505         struct enic_flowman *fm;
2506         struct enic_fm_fet *fet;
2507         struct rte_flow *flow;
2508         struct enic *enic;
2509         int ret;
2510
2511         ENICPMD_FUNC_TRACE();
2512         enic = pmd_priv(dev);
2513         fm = begin_fm(enic);
2514         if (fm == NULL) {
2515                 rte_flow_error_set(error, ENOTSUP,
2516                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2517                         "flowman is not initialized");
2518                 return NULL;
2519         }
2520         enic_fm_open_scratch(fm);
2521         flow = NULL;
2522         ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2523         if (ret < 0)
2524                 goto error_with_scratch;
2525         fm_tcam_entry = &fm->tcam_entry;
2526         fm_action = &fm->action;
2527         flow = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2528                                       attrs, error);
2529         if (flow) {
2530                 /* Add ingress rule that pairs with hairpin rule */
2531                 if (fm->need_hairpin_steer) {
2532                         ret = add_hairpin_steer(fm, flow, error);
2533                         if (ret) {
2534                                 enic_fm_flow_free(fm, flow);
2535                                 flow = NULL;
2536                                 goto error_with_scratch;
2537                         }
2538                 }
2539                 LIST_INSERT_HEAD(&enic->flows, flow, next);
2540                 fet = flow->fm->fet;
2541                 if (fet && fet->default_key) {
2542                         /*
2543                          * Jump to non-existent group? Save the relevant info
2544                          * so we can convert this flow when that group
2545                          * materializes.
2546                          */
2547                         save_jump_flow(fm, flow, fet->group,
2548                                        fm_tcam_entry, fm_action);
2549                 } else if (fet && fet->ref == 1) {
2550                         /*
2551                          * A new table is created. Convert the saved flows
2552                          * that should jump to this group.
2553                          */
2554                         convert_jump_flows(fm, fet, error);
2555                 }
2556         }
2557
2558 error_with_scratch:
2559         enic_fm_close_scratch(fm);
2560         end_fm(fm);
2561         return flow;
2562 }
2563
2564 static int
2565 enic_fm_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
2566                      __rte_unused struct rte_flow_error *error)
2567 {
2568         struct enic *enic = pmd_priv(dev);
2569         struct enic_flowman *fm;
2570
2571         ENICPMD_FUNC_TRACE();
2572         fm = begin_fm(enic);
2573         if (fm == NULL)
2574                 return 0;
2575         LIST_REMOVE(flow, next);
2576         enic_fm_flow_free(fm, flow);
2577         end_fm(fm);
2578         return 0;
2579 }
2580
2581 static int
2582 enic_fm_flow_flush(struct rte_eth_dev *dev,
2583                    __rte_unused struct rte_flow_error *error)
2584 {
2585         LIST_HEAD(enic_flows, rte_flow) internal;
2586         struct enic_fm_flow *fm_flow;
2587         struct enic_flowman *fm;
2588         struct rte_flow *flow;
2589         struct enic *enic = pmd_priv(dev);
2590
2591         ENICPMD_FUNC_TRACE();
2592
2593         fm = begin_fm(enic);
2594         if (fm == NULL)
2595                 return 0;
2596         /* Destroy all non-internal flows */
2597         LIST_INIT(&internal);
2598         while (!LIST_EMPTY(&enic->flows)) {
2599                 flow = LIST_FIRST(&enic->flows);
2600                 fm_flow = flow->fm;
2601                 LIST_REMOVE(flow, next);
2602                 if (flow->internal) {
2603                         LIST_INSERT_HEAD(&internal, flow, next);
2604                         continue;
2605                 }
2606                 /*
2607                  * If tables are null, then vNIC is closing, and the firmware
2608                  * has already cleaned up flowman state. So do not try to free
2609                  * resources, as it only causes errors.
2610                  */
2611                 if (fm->ig_tcam_hndl == FM_INVALID_HANDLE) {
2612                         fm_flow->entry_handle = FM_INVALID_HANDLE;
2613                         fm_flow->action_handle = FM_INVALID_HANDLE;
2614                         fm_flow->fet = NULL;
2615                 }
2616                 enic_fm_flow_free(fm, flow);
2617         }
2618         while (!LIST_EMPTY(&internal)) {
2619                 flow = LIST_FIRST(&internal);
2620                 LIST_REMOVE(flow, next);
2621                 LIST_INSERT_HEAD(&enic->flows, flow, next);
2622         }
2623         end_fm(fm);
2624         return 0;
2625 }
2626
2627 static int
2628 enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle)
2629 {
2630         uint64_t args[2];
2631         int rc;
2632
2633         args[0] = FM_MATCH_TABLE_FREE;
2634         args[1] = handle;
2635         rc = flowman_cmd(fm, args, 2);
2636         if (rc)
2637                 ENICPMD_LOG(ERR, "cannot free table: rc=%d handle=0x%" PRIx64,
2638                             rc, handle);
2639         return rc;
2640 }
2641
2642 static int
2643 enic_fm_tcam_tbl_alloc(struct enic_flowman *fm, uint32_t direction,
2644                         uint32_t max_entries, uint64_t *handle)
2645 {
2646         struct fm_tcam_match_table *tcam_tbl;
2647         uint64_t args[2];
2648         int rc;
2649
2650         ENICPMD_FUNC_TRACE();
2651         tcam_tbl = &fm->cmd.va->fm_tcam_match_table;
2652         tcam_tbl->ftt_direction = direction;
2653         tcam_tbl->ftt_stage = FM_STAGE_LAST;
2654         tcam_tbl->ftt_max_entries = max_entries;
2655         args[0] = FM_TCAM_TABLE_ALLOC;
2656         args[1] = fm->cmd.pa;
2657         rc = flowman_cmd(fm, args, 2);
2658         if (rc) {
2659                 ENICPMD_LOG(ERR, "cannot alloc %s TCAM table: rc=%d",
2660                             (direction == FM_INGRESS) ? "IG" : "EG", rc);
2661                 return rc;
2662         }
2663         *handle = args[0];
2664         ENICPMD_LOG(DEBUG, "%s TCAM table allocated, handle=0x%" PRIx64,
2665                     (direction == FM_INGRESS) ? "IG" : "EG", *handle);
2666         return 0;
2667 }
2668
2669 static int
2670 enic_fm_init_counters(struct enic_flowman *fm)
2671 {
2672         ENICPMD_FUNC_TRACE();
2673         SLIST_INIT(&fm->counters);
2674         return enic_fm_more_counters(fm);
2675 }
2676
2677 static void
2678 enic_fm_free_all_counters(struct enic_flowman *fm)
2679 {
2680         uint64_t args[2];
2681         int rc;
2682
2683         args[0] = FM_COUNTER_BRK;
2684         args[1] = 0;
2685         rc = flowman_cmd(fm, args, 2);
2686         if (rc != 0)
2687                 ENICPMD_LOG(ERR, "cannot free counters: rc=%d", rc);
2688         rte_free(fm->counter_stack);
2689 }
2690
2691 static int
2692 enic_fm_alloc_tcam_tables(struct enic_flowman *fm)
2693 {
2694         int rc;
2695
2696         ENICPMD_FUNC_TRACE();
2697         rc = enic_fm_tcam_tbl_alloc(fm, FM_INGRESS, FM_MAX_TCAM_TABLE_SIZE,
2698                                     &fm->ig_tcam_hndl);
2699         if (rc)
2700                 return rc;
2701         rc = enic_fm_tcam_tbl_alloc(fm, FM_EGRESS, FM_MAX_TCAM_TABLE_SIZE,
2702                                     &fm->eg_tcam_hndl);
2703         return rc;
2704 }
2705
2706 static void
2707 enic_fm_free_tcam_tables(struct enic_flowman *fm)
2708 {
2709         ENICPMD_FUNC_TRACE();
2710         if (fm->ig_tcam_hndl) {
2711                 ENICPMD_LOG(DEBUG, "free IG TCAM table handle=0x%" PRIx64,
2712                             fm->ig_tcam_hndl);
2713                 enic_fm_tbl_free(fm, fm->ig_tcam_hndl);
2714                 fm->ig_tcam_hndl = FM_INVALID_HANDLE;
2715         }
2716         if (fm->eg_tcam_hndl) {
2717                 ENICPMD_LOG(DEBUG, "free EG TCAM table handle=0x%" PRIx64,
2718                             fm->eg_tcam_hndl);
2719                 enic_fm_tbl_free(fm, fm->eg_tcam_hndl);
2720                 fm->eg_tcam_hndl = FM_INVALID_HANDLE;
2721         }
2722 }
2723
2724 int
2725 enic_fm_init(struct enic *enic)
2726 {
2727         const struct rte_pci_addr *addr;
2728         struct enic_flowman *fm;
2729         uint8_t name[RTE_MEMZONE_NAMESIZE];
2730         int rc;
2731
2732         if (enic->flow_filter_mode != FILTER_FLOWMAN)
2733                 return 0;
2734         ENICPMD_FUNC_TRACE();
2735         /* Get vnic handle and save for port-id action */
2736         if (enic_is_vf_rep(enic))
2737                 addr = &VF_ENIC_TO_VF_REP(enic)->bdf;
2738         else
2739                 addr = &RTE_ETH_DEV_TO_PCI(enic->rte_dev)->addr;
2740         rc = enic_fm_find_vnic(enic, addr, &enic->fm_vnic_handle);
2741         if (rc) {
2742                 ENICPMD_LOG(ERR, "cannot find vnic handle for %x:%x:%x",
2743                             addr->bus, addr->devid, addr->function);
2744                 return rc;
2745         }
2746         /* Save UIF for egport action */
2747         enic->fm_vnic_uif = vnic_dev_uif(enic->vdev);
2748         ENICPMD_LOG(DEBUG, "uif %u", enic->fm_vnic_uif);
2749         /* Nothing else to do for representor. It will share the PF flowman */
2750         if (enic_is_vf_rep(enic))
2751                 return 0;
2752         fm = calloc(1, sizeof(*fm));
2753         if (fm == NULL) {
2754                 ENICPMD_LOG(ERR, "cannot alloc flowman struct");
2755                 return -ENOMEM;
2756         }
2757         fm->owner_enic = enic;
2758         rte_spinlock_init(&fm->lock);
2759         TAILQ_INIT(&fm->fet_list);
2760         TAILQ_INIT(&fm->jump_list);
2761         /* Allocate host memory for flowman commands */
2762         snprintf((char *)name, sizeof(name), "fm-cmd-%s", enic->bdf_name);
2763         fm->cmd.va = enic_alloc_consistent(enic,
2764                 sizeof(union enic_flowman_cmd_mem), &fm->cmd.pa, name);
2765         if (!fm->cmd.va) {
2766                 ENICPMD_LOG(ERR, "cannot allocate flowman command memory");
2767                 rc = -ENOMEM;
2768                 goto error_fm;
2769         }
2770         /* Allocate TCAM tables upfront as they are the main tables */
2771         rc = enic_fm_alloc_tcam_tables(fm);
2772         if (rc) {
2773                 ENICPMD_LOG(ERR, "cannot alloc TCAM tables");
2774                 goto error_cmd;
2775         }
2776         /* Then a number of counters */
2777         rc = enic_fm_init_counters(fm);
2778         if (rc) {
2779                 ENICPMD_LOG(ERR, "cannot alloc counters");
2780                 goto error_tables;
2781         }
2782         /*
2783          * One default exact match table for each direction. We hold onto
2784          * it until close.
2785          */
2786         rc = enic_fet_alloc(fm, 1, NULL, 128, &fm->default_ig_fet);
2787         if (rc) {
2788                 ENICPMD_LOG(ERR, "cannot alloc default IG exact match table");
2789                 goto error_counters;
2790         }
2791         fm->default_ig_fet->ref = 1;
2792         rc = enic_fet_alloc(fm, 0, NULL, 128, &fm->default_eg_fet);
2793         if (rc) {
2794                 ENICPMD_LOG(ERR, "cannot alloc default EG exact match table");
2795                 goto error_ig_fet;
2796         }
2797         fm->default_eg_fet->ref = 1;
2798         fm->vf_rep_tag = FM_VF_REP_TAG;
2799         enic->fm = fm;
2800         return 0;
2801
2802 error_ig_fet:
2803         enic_fet_free(fm, fm->default_ig_fet);
2804 error_counters:
2805         enic_fm_free_all_counters(fm);
2806 error_tables:
2807         enic_fm_free_tcam_tables(fm);
2808 error_cmd:
2809         enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
2810                 fm->cmd.va, fm->cmd.pa);
2811 error_fm:
2812         free(fm);
2813         return rc;
2814 }
2815
2816 void
2817 enic_fm_destroy(struct enic *enic)
2818 {
2819         struct enic_flowman *fm;
2820         struct enic_fm_fet *fet;
2821
2822         ENICPMD_FUNC_TRACE();
2823         if (enic_is_vf_rep(enic)) {
2824                 delete_rep_flows(enic);
2825                 return;
2826         }
2827         if (enic->fm == NULL)
2828                 return;
2829         fm = enic->fm;
2830         enic_fet_free(fm, fm->default_eg_fet);
2831         enic_fet_free(fm, fm->default_ig_fet);
2832         /* Free all exact match tables still open */
2833         while (!TAILQ_EMPTY(&fm->fet_list)) {
2834                 fet = TAILQ_FIRST(&fm->fet_list);
2835                 enic_fet_free(fm, fet);
2836         }
2837         enic_fm_free_tcam_tables(fm);
2838         enic_fm_free_all_counters(fm);
2839         enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
2840                 fm->cmd.va, fm->cmd.pa);
2841         fm->cmd.va = NULL;
2842         free(fm);
2843         enic->fm = NULL;
2844 }
2845
2846 int
2847 enic_fm_allocate_switch_domain(struct enic *pf)
2848 {
2849         const struct rte_pci_addr *cur_a, *prev_a;
2850         struct rte_eth_dev *dev;
2851         struct enic *cur, *prev;
2852         uint16_t domain_id;
2853         uint64_t vnic_h;
2854         uint16_t pid;
2855         int ret;
2856
2857         ENICPMD_FUNC_TRACE();
2858         if (enic_is_vf_rep(pf))
2859                 return -EINVAL;
2860         cur = pf;
2861         cur_a = &RTE_ETH_DEV_TO_PCI(cur->rte_dev)->addr;
2862         /* Go through ports and find another PF that is on the same adapter */
2863         RTE_ETH_FOREACH_DEV(pid) {
2864                 dev = &rte_eth_devices[pid];
2865                 if (!dev_is_enic(dev))
2866                         continue;
2867                 if (dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)
2868                         continue;
2869                 if (dev == cur->rte_dev)
2870                         continue;
2871                 /* dev is another PF. Is it on the same adapter? */
2872                 prev = pmd_priv(dev);
2873                 prev_a = &RTE_ETH_DEV_TO_PCI(dev)->addr;
2874                 if (!enic_fm_find_vnic(cur, prev_a, &vnic_h)) {
2875                         ENICPMD_LOG(DEBUG, "Port %u (PF BDF %x:%x:%x) and port %u (PF BDF %x:%x:%x domain %u) are on the same VIC",
2876                                 cur->rte_dev->data->port_id,
2877                                 cur_a->bus, cur_a->devid, cur_a->function,
2878                                 dev->data->port_id,
2879                                 prev_a->bus, prev_a->devid, prev_a->function,
2880                                 prev->switch_domain_id);
2881                         cur->switch_domain_id = prev->switch_domain_id;
2882                         return 0;
2883                 }
2884         }
2885         ret = rte_eth_switch_domain_alloc(&domain_id);
2886         if (ret) {
2887                 ENICPMD_LOG(WARNING, "failed to allocate switch domain for device %d",
2888                             ret);
2889         }
2890         cur->switch_domain_id = domain_id;
2891         ENICPMD_LOG(DEBUG, "Port %u (PF BDF %x:%x:%x) is the 1st PF on the VIC. Allocated switch domain id %u",
2892                     cur->rte_dev->data->port_id,
2893                     cur_a->bus, cur_a->devid, cur_a->function,
2894                     domain_id);
2895         return ret;
2896 }
2897
2898 const struct rte_flow_ops enic_fm_flow_ops = {
2899         .validate = enic_fm_flow_validate,
2900         .create = enic_fm_flow_create,
2901         .destroy = enic_fm_flow_destroy,
2902         .flush = enic_fm_flow_flush,
2903         .query = enic_fm_flow_query,
2904 };
2905
2906 /* Add a high priority flow that loops representor packets to VF */
2907 int
2908 enic_fm_add_rep2vf_flow(struct enic_vf_representor *vf)
2909 {
2910         struct fm_tcam_match_entry *fm_tcam_entry;
2911         struct rte_flow *flow0, *flow1;
2912         struct fm_action *fm_action;
2913         struct rte_flow_error error;
2914         struct rte_flow_attr attrs;
2915         struct fm_action_op fm_op;
2916         struct enic_flowman *fm;
2917         struct enic *pf;
2918         uint8_t tag;
2919
2920         pf = vf->pf;
2921         fm = pf->fm;
2922         tag = fm->vf_rep_tag;
2923         enic_fm_open_scratch(fm);
2924         fm_tcam_entry = &fm->tcam_entry;
2925         fm_action = &fm->action;
2926         /* Egress rule: match WQ ID and tag+hairpin */
2927         fm_tcam_entry->ftm_data.fk_wq_id = vf->pf_wq_idx;
2928         fm_tcam_entry->ftm_mask.fk_wq_id = 0xffff;
2929         fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
2930         memset(&fm_op, 0, sizeof(fm_op));
2931         fm_op.fa_op = FMOP_TAG;
2932         fm_op.tag.tag = tag;
2933         enic_fm_append_action_op(fm, &fm_op, &error);
2934         memset(&fm_op, 0, sizeof(fm_op));
2935         fm_op.fa_op = FMOP_EG_HAIRPIN;
2936         enic_fm_append_action_op(fm, &fm_op, &error);
2937         memset(&fm_op, 0, sizeof(fm_op));
2938         fm_op.fa_op = FMOP_END;
2939         enic_fm_append_action_op(fm, &fm_op, &error);
2940         attrs.group = 0;
2941         attrs.ingress = 0;
2942         attrs.egress = 1;
2943         attrs.priority = FM_HIGHEST_PRIORITY;
2944         flow0 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2945                                        &attrs, &error);
2946         enic_fm_close_scratch(fm);
2947         if (flow0 == NULL) {
2948                 ENICPMD_LOG(ERR, "Cannot create flow 0 for representor->VF");
2949                 return -EINVAL;
2950         }
2951         LIST_INSERT_HEAD(&pf->flows, flow0, next);
2952         /* Make this flow internal, so the user app cannot delete it */
2953         flow0->internal = 1;
2954         ENICPMD_LOG(DEBUG, "representor->VF %d flow created: wq %d -> tag %d hairpin",
2955                     vf->vf_id, vf->pf_wq_idx, tag);
2956
2957         /* Ingress: steer hairpinned to VF RQ 0 */
2958         enic_fm_open_scratch(fm);
2959         fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
2960         fm_tcam_entry->ftm_data.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
2961         fm_tcam_entry->ftm_mask.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
2962         fm_tcam_entry->ftm_data.fk_packet_tag = tag;
2963         fm_tcam_entry->ftm_mask.fk_packet_tag = 0xff;
2964         memset(&fm_op, 0, sizeof(fm_op));
2965         fm_op.fa_op = FMOP_RQ_STEER;
2966         fm_op.rq_steer.rq_index = 0;
2967         fm_op.rq_steer.vnic_handle = vf->enic.fm_vnic_handle;
2968         enic_fm_append_action_op(fm, &fm_op, &error);
2969         memset(&fm_op, 0, sizeof(fm_op));
2970         fm_op.fa_op = FMOP_END;
2971         enic_fm_append_action_op(fm, &fm_op, &error);
2972         attrs.group = 0;
2973         attrs.ingress = 1;
2974         attrs.egress = 0;
2975         attrs.priority = FM_HIGHEST_PRIORITY;
2976         flow1 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2977                                        &attrs, &error);
2978         enic_fm_close_scratch(fm);
2979         if (flow1 == NULL) {
2980                 ENICPMD_LOG(ERR, "Cannot create flow 1 for representor->VF");
2981                 enic_fm_flow_destroy(pf->rte_dev, flow0, &error);
2982                 return -EINVAL;
2983         }
2984         LIST_INSERT_HEAD(&pf->flows, flow1, next);
2985         flow1->internal = 1;
2986         ENICPMD_LOG(DEBUG, "representor->VF %d flow created: tag %d hairpinned -> VF RQ %d",
2987                     vf->vf_id, tag, fm_op.rq_steer.rq_index);
2988         vf->rep2vf_flow[0] = flow0;
2989         vf->rep2vf_flow[1] = flow1;
2990         /* Done with this tag, use a different one next time */
2991         fm->vf_rep_tag++;
2992         return 0;
2993 }
2994
2995 /*
2996  * Add a low priority flow that matches all packets from VF and loops them
2997  * back to the representor.
2998  */
2999 int
3000 enic_fm_add_vf2rep_flow(struct enic_vf_representor *vf)
3001 {
3002         struct fm_tcam_match_entry *fm_tcam_entry;
3003         struct rte_flow *flow0, *flow1;
3004         struct fm_action *fm_action;
3005         struct rte_flow_error error;
3006         struct rte_flow_attr attrs;
3007         struct fm_action_op fm_op;
3008         struct enic_flowman *fm;
3009         struct enic *pf;
3010         uint8_t tag;
3011
3012         pf = vf->pf;
3013         fm = pf->fm;
3014         tag = fm->vf_rep_tag;
3015         enic_fm_open_scratch(fm);
3016         fm_tcam_entry = &fm->tcam_entry;
3017         fm_action = &fm->action;
3018         /* Egress rule: match-any and tag+hairpin */
3019         fm_tcam_entry->ftm_data.fk_wq_id = 0;
3020         fm_tcam_entry->ftm_mask.fk_wq_id = 0xffff;
3021         fm_tcam_entry->ftm_data.fk_wq_vnic = vf->enic.fm_vnic_handle;
3022         fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3023         memset(&fm_op, 0, sizeof(fm_op));
3024         fm_op.fa_op = FMOP_TAG;
3025         fm_op.tag.tag = tag;
3026         enic_fm_append_action_op(fm, &fm_op, &error);
3027         memset(&fm_op, 0, sizeof(fm_op));
3028         fm_op.fa_op = FMOP_EG_HAIRPIN;
3029         enic_fm_append_action_op(fm, &fm_op, &error);
3030         memset(&fm_op, 0, sizeof(fm_op));
3031         fm_op.fa_op = FMOP_END;
3032         enic_fm_append_action_op(fm, &fm_op, &error);
3033         attrs.group = 0;
3034         attrs.ingress = 0;
3035         attrs.egress = 1;
3036         attrs.priority = FM_LOWEST_PRIORITY;
3037         flow0 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3038                                        &attrs, &error);
3039         enic_fm_close_scratch(fm);
3040         if (flow0 == NULL) {
3041                 ENICPMD_LOG(ERR, "Cannot create flow 0 for VF->representor");
3042                 return -EINVAL;
3043         }
3044         LIST_INSERT_HEAD(&pf->flows, flow0, next);
3045         /* Make this flow internal, so the user app cannot delete it */
3046         flow0->internal = 1;
3047         ENICPMD_LOG(DEBUG, "VF %d->representor flow created: wq %d (low prio) -> tag %d hairpin",
3048                     vf->vf_id, fm_tcam_entry->ftm_data.fk_wq_id, tag);
3049
3050         /* Ingress: steer hairpinned to VF rep RQ */
3051         enic_fm_open_scratch(fm);
3052         fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3053         fm_tcam_entry->ftm_data.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3054         fm_tcam_entry->ftm_mask.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3055         fm_tcam_entry->ftm_data.fk_packet_tag = tag;
3056         fm_tcam_entry->ftm_mask.fk_packet_tag = 0xff;
3057         memset(&fm_op, 0, sizeof(fm_op));
3058         fm_op.fa_op = FMOP_RQ_STEER;
3059         fm_op.rq_steer.rq_index = vf->pf_rq_sop_idx;
3060         fm_op.rq_steer.vnic_handle = pf->fm_vnic_handle;
3061         enic_fm_append_action_op(fm, &fm_op, &error);
3062         memset(&fm_op, 0, sizeof(fm_op));
3063         fm_op.fa_op = FMOP_END;
3064         enic_fm_append_action_op(fm, &fm_op, &error);
3065         attrs.group = 0;
3066         attrs.ingress = 1;
3067         attrs.egress = 0;
3068         attrs.priority = FM_HIGHEST_PRIORITY;
3069         flow1 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3070                                        &attrs, &error);
3071         enic_fm_close_scratch(fm);
3072         if (flow1 == NULL) {
3073                 ENICPMD_LOG(ERR, "Cannot create flow 1 for VF->representor");
3074                 enic_fm_flow_destroy(pf->rte_dev, flow0, &error);
3075                 return -EINVAL;
3076         }
3077         LIST_INSERT_HEAD(&pf->flows, flow1, next);
3078         flow1->internal = 1;
3079         ENICPMD_LOG(DEBUG, "VF %d->representor flow created: tag %d hairpinned -> PF RQ %d",
3080                     vf->vf_id, tag, vf->pf_rq_sop_idx);
3081         vf->vf2rep_flow[0] = flow0;
3082         vf->vf2rep_flow[1] = flow1;
3083         /* Done with this tag, use a different one next time */
3084         fm->vf_rep_tag++;
3085         return 0;
3086 }
3087
3088 /* Destroy representor flows created by enic_fm_add_{rep2vf,vf2rep}_flow */
3089 static void
3090 delete_rep_flows(struct enic *enic)
3091 {
3092         struct enic_vf_representor *vf;
3093         struct rte_flow_error error;
3094         struct rte_eth_dev *dev;
3095         uint32_t i;
3096
3097         RTE_ASSERT(enic_is_vf_rep(enic));
3098         vf = VF_ENIC_TO_VF_REP(enic);
3099         dev = vf->pf->rte_dev;
3100         for (i = 0; i < ARRAY_SIZE(vf->vf2rep_flow); i++) {
3101                 if (vf->vf2rep_flow[i])
3102                         enic_fm_flow_destroy(dev, vf->vf2rep_flow[i], &error);
3103         }
3104         for (i = 0; i < ARRAY_SIZE(vf->rep2vf_flow); i++) {
3105                 if (vf->rep2vf_flow[i])
3106                         enic_fm_flow_destroy(dev, vf->rep2vf_flow[i], &error);
3107         }
3108 }
3109
3110 static struct enic_flowman *
3111 begin_fm(struct enic *enic)
3112 {
3113         struct enic_vf_representor *vf;
3114         struct enic_flowman *fm;
3115
3116         /* Representor uses PF flowman */
3117         if (enic_is_vf_rep(enic)) {
3118                 vf = VF_ENIC_TO_VF_REP(enic);
3119                 fm = vf->pf->fm;
3120         } else {
3121                 fm = enic->fm;
3122         }
3123         /* Save the API caller and lock if representors exist */
3124         if (fm) {
3125                 if (fm->owner_enic->switchdev_mode)
3126                         rte_spinlock_lock(&fm->lock);
3127                 fm->user_enic = enic;
3128         }
3129         return fm;
3130 }
3131
3132 static void
3133 end_fm(struct enic_flowman *fm)
3134 {
3135         fm->user_enic = NULL;
3136         if (fm->owner_enic->switchdev_mode)
3137                 rte_spinlock_unlock(&fm->lock);
3138 }