drivers/net/enic/enic_fm_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2008-2019 Cisco Systems, Inc.  All rights reserved.
   3  */
   4
   5 #include <errno.h>
   6 #include <stdint.h>
   7 #include <rte_log.h>
   8 #include <rte_ethdev_driver.h>
   9 #include <rte_flow_driver.h>
  10 #include <rte_ether.h>
  11 #include <rte_ip.h>
  12 #include <rte_udp.h>
  13 #include <rte_memzone.h>
  14
  15 #include "enic_compat.h"
  16 #include "enic.h"
  17 #include "vnic_dev.h"
  18 #include "vnic_nic.h"
  19
  20 #define IP_DEFTTL  64   /* from RFC 1340. */
  21 #define IP6_VTC_FLOW 0x60000000
  22
  23 /* Highest Item type supported by Flowman */
  24 #define FM_MAX_ITEM_TYPE RTE_FLOW_ITEM_TYPE_VXLAN
  25
  26 /* Up to 1024 TCAM entries */
  27 #define FM_MAX_TCAM_TABLE_SIZE 1024
  28
  29 /* Up to 4096 entries per exact match table */
  30 #define FM_MAX_EXACT_TABLE_SIZE 4096
  31
  32 /* Number of counters to increase on for each increment */
  33 #define FM_COUNTERS_EXPAND  100
  34
  35 #define FM_INVALID_HANDLE 0
  36
  37 /* Low priority used for implicit VF -> representor flow */
  38 #define FM_LOWEST_PRIORITY 100000
  39
  40 /* High priority used for implicit representor -> VF flow */
  41 #define FM_HIGHEST_PRIORITY 0
  42
  43 /* Tag used for implicit VF <-> representor flows */
  44 #define FM_VF_REP_TAG 1
  45
  46 /*
  47  * Flow exact match tables (FET) in the VIC and rte_flow groups.
  48  * Use a simple scheme to map groups to tables.
  49  * Group 0 uses the single TCAM tables, one for each direction.
  50  * Group 1, 2, ... uses its own exact match table.
  51  *
  52  * The TCAM tables are allocated upfront during init.
  53  *
  54  * Exact match tables are allocated on demand. 3 paths that lead allocations.
  55  *
  56  * 1. Add a flow that jumps from group 0 to group N.
  57  *
  58  * If N does not exist, we allocate an exact match table for it, using
  59  * a dummy key. A key is required for the table.
  60  *
  61  * 2. Add a flow that uses group N.
  62  *
  63  * If N does not exist, we allocate an exact match table for it, using
  64  * the flow's key. Subsequent flows to the same group all should have
  65  * the same key.
  66  *
  67  * Without a jump flow to N, N is not reachable in hardware. No packets
  68  * reach N and match.
  69  *
  70  * 3. Add a flow to an empty group N.
  71  *
  72  * N has been created via (1) and the dummy key. We free that table, allocate
  73  * a new table using the new flow's key. Also re-do the existing jump flow to
  74  * point to the new table.
  75  */
  76 #define FM_TCAM_RTE_GROUP 0
  77
  78 struct enic_fm_fet {
  79         TAILQ_ENTRY(enic_fm_fet) list;
  80         uint32_t group; /* rte_flow group ID */
  81         uint64_t handle; /* Exact match table handle from flowman */
  82         uint8_t ingress;
  83         uint8_t default_key;
  84         int ref; /* Reference count via get/put */
  85         struct fm_key_template key; /* Key associated with the table */
  86 };
  87
  88 struct enic_fm_counter {
  89         SLIST_ENTRY(enic_fm_counter) next;
  90         uint32_t handle;
  91 };
  92
  93 /* rte_flow.fm */
  94 struct enic_fm_flow {
  95         bool counter_valid;
  96         uint64_t entry_handle;
  97         uint64_t action_handle;
  98         struct enic_fm_counter *counter;
  99         struct enic_fm_fet *fet;
 100         /* Auto-added steer action for hairpin flows (e.g. vnic->vnic) */
 101         struct enic_fm_flow *hairpin_steer_flow;
 102 };
 103
 104 struct enic_fm_jump_flow {
 105         TAILQ_ENTRY(enic_fm_jump_flow) list;
 106         struct rte_flow *flow;
 107         uint32_t group;
 108         struct fm_tcam_match_entry match;
 109         struct fm_action action;
 110 };
 111
 112 /*
 113  * Flowman uses host memory for commands. This structure is allocated
 114  * in DMA-able memory.
 115  */
 116 union enic_flowman_cmd_mem {
 117         struct fm_tcam_match_table fm_tcam_match_table;
 118         struct fm_exact_match_table fm_exact_match_table;
 119         struct fm_tcam_match_entry fm_tcam_match_entry;
 120         struct fm_exact_match_entry fm_exact_match_entry;
 121         struct fm_action fm_action;
 122 };
 123
 124 /*
 125  * PF has a flowman instance, and VF representors share it with PF.
 126  * PF allocates this structure and owns it. VF representors borrow
 127  * the PF's structure during API calls (e.g. create, query).
 128  */
 129 struct enic_flowman {
 130         struct enic *owner_enic; /* PF */
 131         struct enic *user_enic;  /* API caller (PF or representor) */
 132         /*
 133          * Representors and PF share the same underlying flowman.
 134          * Lock API calls to serialize accesses from them. Only used
 135          * when VF representors are present.
 136          */
 137         rte_spinlock_t lock;
 138         /* Command buffer */
 139         struct {
 140                 union enic_flowman_cmd_mem *va;
 141                 dma_addr_t pa;
 142         } cmd;
 143         /* TCAM tables allocated upfront, used for group 0 */
 144         uint64_t ig_tcam_hndl;
 145         uint64_t eg_tcam_hndl;
 146         /* Counters */
 147         SLIST_HEAD(enic_free_counters, enic_fm_counter) counters;
 148         void *counter_stack;
 149         uint32_t counters_alloced;
 150         /* Exact match tables for groups != 0, dynamically allocated */
 151         TAILQ_HEAD(fet_list, enic_fm_fet) fet_list;
 152         /*
 153          * Default exact match tables used for jump actions to
 154          * non-existent groups.
 155          */
 156         struct enic_fm_fet *default_eg_fet;
 157         struct enic_fm_fet *default_ig_fet;
 158         /* Flows that jump to the default table above */
 159         TAILQ_HEAD(jump_flow_list, enic_fm_jump_flow) jump_list;
 160         /*
 161          * Scratch data used during each invocation of flow_create
 162          * and flow_validate.
 163          */
 164         struct enic_fm_fet *fet;
 165         struct fm_tcam_match_entry tcam_entry;
 166         struct fm_action action;
 167         struct fm_action action_tmp; /* enic_fm_reorder_action_op */
 168         int action_op_count;
 169         /* Tags used for representor flows */
 170         uint8_t vf_rep_tag;
 171         /* For auto-added steer action for hairpin */
 172         int need_hairpin_steer;
 173         uint64_t hairpin_steer_vnic_h;
 174 };
 175
 176 static int enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle);
 177 /*
 178  * API functions (create, destroy, validate, flush) call begin_fm()
 179  * upon entering to save the caller enic (PF or VF representor) and
 180  * lock. Upon exit, they call end_fm() to unlock.
 181  */
 182 static struct enic_flowman *begin_fm(struct enic *enic);
 183 static void end_fm(struct enic_flowman *fm);
 184 /* Delete internal flows created for representor paths */
 185 static void delete_rep_flows(struct enic *enic);
 186
 187 /*
 188  * Common arguments passed to copy_item functions. Use this structure
 189  * so we can easily add new arguments.
 190  * item: Item specification.
 191  * fm_tcam_entry: Flowman TCAM match entry.
 192  * header_level: 0 for outer header, 1 for inner header.
 193  */
 194 struct copy_item_args {
 195         const struct rte_flow_item *item;
 196         struct fm_tcam_match_entry *fm_tcam_entry;
 197         uint8_t header_level;
 198 };
 199
 200 /* functions for copying items into flowman match */
 201 typedef int (enic_copy_item_fn)(struct copy_item_args *arg);
 202
 203 /* Info about how to copy items into flowman match */
 204 struct enic_fm_items {
 205         /* Function for copying and validating an item. */
 206         enic_copy_item_fn * const copy_item;
 207         /* List of valid previous items. */
 208         const enum rte_flow_item_type * const prev_items;
 209         /*
 210          * True if it's OK for this item to be the first item. For some NIC
 211          * versions, it's invalid to start the stack above layer 3.
 212          */
 213         const uint8_t valid_start_item;
 214 };
 215
 216 static enic_copy_item_fn enic_fm_copy_item_eth;
 217 static enic_copy_item_fn enic_fm_copy_item_ipv4;
 218 static enic_copy_item_fn enic_fm_copy_item_ipv6;
 219 static enic_copy_item_fn enic_fm_copy_item_raw;
 220 static enic_copy_item_fn enic_fm_copy_item_sctp;
 221 static enic_copy_item_fn enic_fm_copy_item_tcp;
 222 static enic_copy_item_fn enic_fm_copy_item_udp;
 223 static enic_copy_item_fn enic_fm_copy_item_vlan;
 224 static enic_copy_item_fn enic_fm_copy_item_vxlan;
 225
 226 /* Ingress actions */
 227 static const enum rte_flow_action_type enic_fm_supported_ig_actions[] = {
 228         RTE_FLOW_ACTION_TYPE_COUNT,
 229         RTE_FLOW_ACTION_TYPE_DROP,
 230         RTE_FLOW_ACTION_TYPE_FLAG,
 231         RTE_FLOW_ACTION_TYPE_JUMP,
 232         RTE_FLOW_ACTION_TYPE_MARK,
 233         RTE_FLOW_ACTION_TYPE_OF_POP_VLAN,
 234         RTE_FLOW_ACTION_TYPE_PORT_ID,
 235         RTE_FLOW_ACTION_TYPE_PASSTHRU,
 236         RTE_FLOW_ACTION_TYPE_QUEUE,
 237         RTE_FLOW_ACTION_TYPE_RSS,
 238         RTE_FLOW_ACTION_TYPE_VOID,
 239         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
 240         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP,
 241         RTE_FLOW_ACTION_TYPE_END, /* END must be the last entry */
 242 };
 243
 244 /* Egress actions */
 245 static const enum rte_flow_action_type enic_fm_supported_eg_actions[] = {
 246         RTE_FLOW_ACTION_TYPE_COUNT,
 247         RTE_FLOW_ACTION_TYPE_DROP,
 248         RTE_FLOW_ACTION_TYPE_JUMP,
 249         RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN,
 250         RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP,
 251         RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID,
 252         RTE_FLOW_ACTION_TYPE_PORT_ID,
 253         RTE_FLOW_ACTION_TYPE_PASSTHRU,
 254         RTE_FLOW_ACTION_TYPE_VOID,
 255         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
 256         RTE_FLOW_ACTION_TYPE_END,
 257 };
 258
 259 static const struct enic_fm_items enic_fm_items[] = {
 260         [RTE_FLOW_ITEM_TYPE_RAW] = {
 261                 .copy_item = enic_fm_copy_item_raw,
 262                 .valid_start_item = 0,
 263                 .prev_items = (const enum rte_flow_item_type[]) {
 264                                RTE_FLOW_ITEM_TYPE_UDP,
 265                                RTE_FLOW_ITEM_TYPE_END,
 266                 },
 267         },
 268         [RTE_FLOW_ITEM_TYPE_ETH] = {
 269                 .copy_item = enic_fm_copy_item_eth,
 270                 .valid_start_item = 1,
 271                 .prev_items = (const enum rte_flow_item_type[]) {
 272                                RTE_FLOW_ITEM_TYPE_END,
 273                 },
 274         },
 275         [RTE_FLOW_ITEM_TYPE_VLAN] = {
 276                 .copy_item = enic_fm_copy_item_vlan,
 277                 .valid_start_item = 1,
 278                 .prev_items = (const enum rte_flow_item_type[]) {
 279                                RTE_FLOW_ITEM_TYPE_ETH,
 280                                RTE_FLOW_ITEM_TYPE_END,
 281                 },
 282         },
 283         [RTE_FLOW_ITEM_TYPE_IPV4] = {
 284                 .copy_item = enic_fm_copy_item_ipv4,
 285                 .valid_start_item = 1,
 286                 .prev_items = (const enum rte_flow_item_type[]) {
 287                                RTE_FLOW_ITEM_TYPE_ETH,
 288                                RTE_FLOW_ITEM_TYPE_VLAN,
 289                                RTE_FLOW_ITEM_TYPE_END,
 290                 },
 291         },
 292         [RTE_FLOW_ITEM_TYPE_IPV6] = {
 293                 .copy_item = enic_fm_copy_item_ipv6,
 294                 .valid_start_item = 1,
 295                 .prev_items = (const enum rte_flow_item_type[]) {
 296                                RTE_FLOW_ITEM_TYPE_ETH,
 297                                RTE_FLOW_ITEM_TYPE_VLAN,
 298                                RTE_FLOW_ITEM_TYPE_END,
 299                 },
 300         },
 301         [RTE_FLOW_ITEM_TYPE_UDP] = {
 302                 .copy_item = enic_fm_copy_item_udp,
 303                 .valid_start_item = 1,
 304                 .prev_items = (const enum rte_flow_item_type[]) {
 305                                RTE_FLOW_ITEM_TYPE_IPV4,
 306                                RTE_FLOW_ITEM_TYPE_IPV6,
 307                                RTE_FLOW_ITEM_TYPE_END,
 308                 },
 309         },
 310         [RTE_FLOW_ITEM_TYPE_TCP] = {
 311                 .copy_item = enic_fm_copy_item_tcp,
 312                 .valid_start_item = 1,
 313                 .prev_items = (const enum rte_flow_item_type[]) {
 314                                RTE_FLOW_ITEM_TYPE_IPV4,
 315                                RTE_FLOW_ITEM_TYPE_IPV6,
 316                                RTE_FLOW_ITEM_TYPE_END,
 317                 },
 318         },
 319         [RTE_FLOW_ITEM_TYPE_SCTP] = {
 320                 .copy_item = enic_fm_copy_item_sctp,
 321                 .valid_start_item = 0,
 322                 .prev_items = (const enum rte_flow_item_type[]) {
 323                                RTE_FLOW_ITEM_TYPE_IPV4,
 324                                RTE_FLOW_ITEM_TYPE_IPV6,
 325                                RTE_FLOW_ITEM_TYPE_END,
 326                 },
 327         },
 328         [RTE_FLOW_ITEM_TYPE_VXLAN] = {
 329                 .copy_item = enic_fm_copy_item_vxlan,
 330                 .valid_start_item = 1,
 331                 .prev_items = (const enum rte_flow_item_type[]) {
 332                                RTE_FLOW_ITEM_TYPE_UDP,
 333                                RTE_FLOW_ITEM_TYPE_END,
 334                 },
 335         },
 336 };
 337
 338 static int
 339 enic_fm_copy_item_eth(struct copy_item_args *arg)
 340 {
 341         const struct rte_flow_item *item = arg->item;
 342         const struct rte_flow_item_eth *spec = item->spec;
 343         const struct rte_flow_item_eth *mask = item->mask;
 344         const uint8_t lvl = arg->header_level;
 345         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 346         struct fm_header_set *fm_data, *fm_mask;
 347
 348         ENICPMD_FUNC_TRACE();
 349         /* Match all if no spec */
 350         if (!spec)
 351                 return 0;
 352         if (!mask)
 353                 mask = &rte_flow_item_eth_mask;
 354         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 355         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 356         fm_data->fk_header_select |= FKH_ETHER;
 357         fm_mask->fk_header_select |= FKH_ETHER;
 358         memcpy(&fm_data->l2.eth, spec, sizeof(*spec));
 359         memcpy(&fm_mask->l2.eth, mask, sizeof(*mask));
 360         return 0;
 361 }
 362
 363 static int
 364 enic_fm_copy_item_vlan(struct copy_item_args *arg)
 365 {
 366         const struct rte_flow_item *item = arg->item;
 367         const struct rte_flow_item_vlan *spec = item->spec;
 368         const struct rte_flow_item_vlan *mask = item->mask;
 369         const uint8_t lvl = arg->header_level;
 370         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 371         struct fm_header_set *fm_data, *fm_mask;
 372         struct rte_ether_hdr *eth_mask;
 373         struct rte_ether_hdr *eth_val;
 374         uint32_t meta;
 375
 376         ENICPMD_FUNC_TRACE();
 377         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 378         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 379         /* Outer and inner packet vlans need different flags */
 380         meta = FKM_VLAN_PRES;
 381         if (lvl > 0)
 382                 meta = FKM_QTAG;
 383         fm_data->fk_metadata |= meta;
 384         fm_mask->fk_metadata |= meta;
 385
 386         /* Match all if no spec */
 387         if (!spec)
 388                 return 0;
 389         if (!mask)
 390                 mask = &rte_flow_item_vlan_mask;
 391
 392         eth_mask = (void *)&fm_mask->l2.eth;
 393         eth_val = (void *)&fm_data->l2.eth;
 394
 395         /* Outer TPID cannot be matched */
 396         if (eth_mask->ether_type)
 397                 return -ENOTSUP;
 398
 399         /*
 400          * When packet matching, the VIC always compares vlan-stripped
 401          * L2, regardless of vlan stripping settings. So, the inner type
 402          * from vlan becomes the ether type of the eth header.
 403          */
 404         eth_mask->ether_type = mask->inner_type;
 405         eth_val->ether_type = spec->inner_type;
 406         fm_data->fk_header_select |= FKH_ETHER | FKH_QTAG;
 407         fm_mask->fk_header_select |= FKH_ETHER | FKH_QTAG;
 408         fm_data->fk_vlan = rte_be_to_cpu_16(spec->tci);
 409         fm_mask->fk_vlan = rte_be_to_cpu_16(mask->tci);
 410         return 0;
 411 }
 412
 413 static int
 414 enic_fm_copy_item_ipv4(struct copy_item_args *arg)
 415 {
 416         const struct rte_flow_item *item = arg->item;
 417         const struct rte_flow_item_ipv4 *spec = item->spec;
 418         const struct rte_flow_item_ipv4 *mask = item->mask;
 419         const uint8_t lvl = arg->header_level;
 420         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 421         struct fm_header_set *fm_data, *fm_mask;
 422
 423         ENICPMD_FUNC_TRACE();
 424         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 425         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 426         fm_data->fk_metadata |= FKM_IPV4;
 427         fm_mask->fk_metadata |= FKM_IPV4;
 428
 429         if (!spec)
 430                 return 0;
 431         if (!mask)
 432                 mask = &rte_flow_item_ipv4_mask;
 433
 434         fm_data->fk_header_select |= FKH_IPV4;
 435         fm_mask->fk_header_select |= FKH_IPV4;
 436         memcpy(&fm_data->l3.ip4, spec, sizeof(*spec));
 437         memcpy(&fm_mask->l3.ip4, mask, sizeof(*mask));
 438         return 0;
 439 }
 440
 441 static int
 442 enic_fm_copy_item_ipv6(struct copy_item_args *arg)
 443 {
 444         const struct rte_flow_item *item = arg->item;
 445         const struct rte_flow_item_ipv6 *spec = item->spec;
 446         const struct rte_flow_item_ipv6 *mask = item->mask;
 447         const uint8_t lvl = arg->header_level;
 448         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 449         struct fm_header_set *fm_data, *fm_mask;
 450
 451         ENICPMD_FUNC_TRACE();
 452         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 453         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 454         fm_data->fk_metadata |= FKM_IPV6;
 455         fm_mask->fk_metadata |= FKM_IPV6;
 456
 457         if (!spec)
 458                 return 0;
 459         if (!mask)
 460                 mask = &rte_flow_item_ipv6_mask;
 461
 462         fm_data->fk_header_select |= FKH_IPV6;
 463         fm_mask->fk_header_select |= FKH_IPV6;
 464         memcpy(&fm_data->l3.ip6, spec, sizeof(*spec));
 465         memcpy(&fm_mask->l3.ip6, mask, sizeof(*mask));
 466         return 0;
 467 }
 468
 469 static int
 470 enic_fm_copy_item_udp(struct copy_item_args *arg)
 471 {
 472         const struct rte_flow_item *item = arg->item;
 473         const struct rte_flow_item_udp *spec = item->spec;
 474         const struct rte_flow_item_udp *mask = item->mask;
 475         const uint8_t lvl = arg->header_level;
 476         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 477         struct fm_header_set *fm_data, *fm_mask;
 478
 479         ENICPMD_FUNC_TRACE();
 480         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 481         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 482         fm_data->fk_metadata |= FKM_UDP;
 483         fm_mask->fk_metadata |= FKM_UDP;
 484
 485         if (!spec)
 486                 return 0;
 487         if (!mask)
 488                 mask = &rte_flow_item_udp_mask;
 489
 490         fm_data->fk_header_select |= FKH_UDP;
 491         fm_mask->fk_header_select |= FKH_UDP;
 492         memcpy(&fm_data->l4.udp, spec, sizeof(*spec));
 493         memcpy(&fm_mask->l4.udp, mask, sizeof(*mask));
 494         return 0;
 495 }
 496
 497 static int
 498 enic_fm_copy_item_tcp(struct copy_item_args *arg)
 499 {
 500         const struct rte_flow_item *item = arg->item;
 501         const struct rte_flow_item_tcp *spec = item->spec;
 502         const struct rte_flow_item_tcp *mask = item->mask;
 503         const uint8_t lvl = arg->header_level;
 504         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 505         struct fm_header_set *fm_data, *fm_mask;
 506
 507         ENICPMD_FUNC_TRACE();
 508         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 509         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 510         fm_data->fk_metadata |= FKM_TCP;
 511         fm_mask->fk_metadata |= FKM_TCP;
 512
 513         if (!spec)
 514                 return 0;
 515         if (!mask)
 516                 mask = &rte_flow_item_tcp_mask;
 517
 518         fm_data->fk_header_select |= FKH_TCP;
 519         fm_mask->fk_header_select |= FKH_TCP;
 520         memcpy(&fm_data->l4.tcp, spec, sizeof(*spec));
 521         memcpy(&fm_mask->l4.tcp, mask, sizeof(*mask));
 522         return 0;
 523 }
 524
 525 static int
 526 enic_fm_copy_item_sctp(struct copy_item_args *arg)
 527 {
 528         const struct rte_flow_item *item = arg->item;
 529         const struct rte_flow_item_sctp *spec = item->spec;
 530         const struct rte_flow_item_sctp *mask = item->mask;
 531         const uint8_t lvl = arg->header_level;
 532         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 533         struct fm_header_set *fm_data, *fm_mask;
 534         uint8_t *ip_proto_mask = NULL;
 535         uint8_t *ip_proto = NULL;
 536         uint32_t l3_fkh;
 537
 538         ENICPMD_FUNC_TRACE();
 539         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 540         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 541         /*
 542          * The NIC filter API has no flags for "match sctp", so explicitly
 543          * set the protocol number in the IP pattern.
 544          */
 545         if (fm_data->fk_metadata & FKM_IPV4) {
 546                 struct rte_ipv4_hdr *ip;
 547                 ip = (struct rte_ipv4_hdr *)&fm_mask->l3.ip4;
 548                 ip_proto_mask = &ip->next_proto_id;
 549                 ip = (struct rte_ipv4_hdr *)&fm_data->l3.ip4;
 550                 ip_proto = &ip->next_proto_id;
 551                 l3_fkh = FKH_IPV4;
 552         } else if (fm_data->fk_metadata & FKM_IPV6) {
 553                 struct rte_ipv6_hdr *ip;
 554                 ip = (struct rte_ipv6_hdr *)&fm_mask->l3.ip6;
 555                 ip_proto_mask = &ip->proto;
 556                 ip = (struct rte_ipv6_hdr *)&fm_data->l3.ip6;
 557                 ip_proto = &ip->proto;
 558                 l3_fkh = FKH_IPV6;
 559         } else {
 560                 /* Need IPv4/IPv6 pattern first */
 561                 return -EINVAL;
 562         }
 563         *ip_proto = IPPROTO_SCTP;
 564         *ip_proto_mask = 0xff;
 565         fm_data->fk_header_select |= l3_fkh;
 566         fm_mask->fk_header_select |= l3_fkh;
 567
 568         if (!spec)
 569                 return 0;
 570         if (!mask)
 571                 mask = &rte_flow_item_sctp_mask;
 572
 573         fm_data->fk_header_select |= FKH_L4RAW;
 574         fm_mask->fk_header_select |= FKH_L4RAW;
 575         memcpy(fm_data->l4.rawdata, spec, sizeof(*spec));
 576         memcpy(fm_mask->l4.rawdata, mask, sizeof(*mask));
 577         return 0;
 578 }
 579
 580 static int
 581 enic_fm_copy_item_vxlan(struct copy_item_args *arg)
 582 {
 583         const struct rte_flow_item *item = arg->item;
 584         const struct rte_flow_item_vxlan *spec = item->spec;
 585         const struct rte_flow_item_vxlan *mask = item->mask;
 586         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 587         struct fm_header_set *fm_data, *fm_mask;
 588
 589         ENICPMD_FUNC_TRACE();
 590         /* Only 2 header levels (outer and inner) allowed */
 591         if (arg->header_level > 0)
 592                 return -EINVAL;
 593
 594         fm_data = &entry->ftm_data.fk_hdrset[0];
 595         fm_mask = &entry->ftm_mask.fk_hdrset[0];
 596         fm_data->fk_metadata |= FKM_VXLAN;
 597         fm_mask->fk_metadata |= FKM_VXLAN;
 598         /* items from here on out are inner header items */
 599         arg->header_level = 1;
 600
 601         /* Match all if no spec */
 602         if (!spec)
 603                 return 0;
 604         if (!mask)
 605                 mask = &rte_flow_item_vxlan_mask;
 606
 607         fm_data->fk_header_select |= FKH_VXLAN;
 608         fm_mask->fk_header_select |= FKH_VXLAN;
 609         memcpy(&fm_data->vxlan, spec, sizeof(*spec));
 610         memcpy(&fm_mask->vxlan, mask, sizeof(*mask));
 611         return 0;
 612 }
 613
 614 /*
 615  * Currently, raw pattern match is very limited. It is intended for matching
 616  * UDP tunnel header (e.g. vxlan or geneve).
 617  */
 618 static int
 619 enic_fm_copy_item_raw(struct copy_item_args *arg)
 620 {
 621         const struct rte_flow_item *item = arg->item;
 622         const struct rte_flow_item_raw *spec = item->spec;
 623         const struct rte_flow_item_raw *mask = item->mask;
 624         const uint8_t lvl = arg->header_level;
 625         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 626         struct fm_header_set *fm_data, *fm_mask;
 627
 628         ENICPMD_FUNC_TRACE();
 629         /* Cannot be used for inner packet */
 630         if (lvl > 0)
 631                 return -EINVAL;
 632         /* Need both spec and mask */
 633         if (!spec || !mask)
 634                 return -EINVAL;
 635         /* Only supports relative with offset 0 */
 636         if (!spec->relative || spec->offset != 0 || spec->search ||
 637             spec->limit)
 638                 return -EINVAL;
 639         /* Need non-null pattern that fits within the NIC's filter pattern */
 640         if (spec->length == 0 ||
 641             spec->length + sizeof(struct rte_udp_hdr) > FM_LAYER_SIZE ||
 642             !spec->pattern || !mask->pattern)
 643                 return -EINVAL;
 644         /*
 645          * Mask fields, including length, are often set to zero. Assume that
 646          * means "same as spec" to avoid breaking existing apps. If length
 647          * is not zero, then it should be >= spec length.
 648          *
 649          * No more pattern follows this, so append to the L4 layer instead of
 650          * L5 to work with both recent and older VICs.
 651          */
 652         if (mask->length != 0 && mask->length < spec->length)
 653                 return -EINVAL;
 654
 655         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 656         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 657         fm_data->fk_header_select |= FKH_L4RAW;
 658         fm_mask->fk_header_select |= FKH_L4RAW;
 659         fm_data->fk_header_select &= ~FKH_UDP;
 660         fm_mask->fk_header_select &= ~FKH_UDP;
 661         memcpy(fm_data->l4.rawdata + sizeof(struct rte_udp_hdr),
 662                spec->pattern, spec->length);
 663         memcpy(fm_mask->l4.rawdata + sizeof(struct rte_udp_hdr),
 664                mask->pattern, spec->length);
 665         return 0;
 666 }
 667
 668 static int
 669 flowman_cmd(struct enic_flowman *fm, uint64_t *args, int nargs)
 670 {
 671         return vnic_dev_flowman_cmd(fm->owner_enic->vdev, args, nargs);
 672 }
 673
 674 static int
 675 enic_fet_alloc(struct enic_flowman *fm, uint8_t ingress,
 676                struct fm_key_template *key, int entries,
 677                struct enic_fm_fet **fet_out)
 678 {
 679         struct fm_exact_match_table *cmd;
 680         struct fm_header_set *hdr;
 681         struct enic_fm_fet *fet;
 682         uint64_t args[3];
 683         int ret;
 684
 685         ENICPMD_FUNC_TRACE();
 686         fet = calloc(1, sizeof(struct enic_fm_fet));
 687         if (fet == NULL)
 688                 return -ENOMEM;
 689         cmd = &fm->cmd.va->fm_exact_match_table;
 690         memset(cmd, 0, sizeof(*cmd));
 691         cmd->fet_direction = ingress ? FM_INGRESS : FM_EGRESS;
 692         cmd->fet_stage = FM_STAGE_LAST;
 693         cmd->fet_max_entries = entries ? entries : FM_MAX_EXACT_TABLE_SIZE;
 694         if (key == NULL) {
 695                 hdr = &cmd->fet_key.fk_hdrset[0];
 696                 memset(hdr, 0, sizeof(*hdr));
 697                 hdr->fk_header_select = FKH_IPV4 | FKH_UDP;
 698                 hdr->l3.ip4.fk_saddr = 0xFFFFFFFF;
 699                 hdr->l3.ip4.fk_daddr = 0xFFFFFFFF;
 700                 hdr->l4.udp.fk_source = 0xFFFF;
 701                 hdr->l4.udp.fk_dest = 0xFFFF;
 702                 fet->default_key = 1;
 703         } else {
 704                 memcpy(&cmd->fet_key, key, sizeof(*key));
 705                 memcpy(&fet->key, key, sizeof(*key));
 706                 fet->default_key = 0;
 707         }
 708         cmd->fet_key.fk_packet_tag = 1;
 709
 710         args[0] = FM_EXACT_TABLE_ALLOC;
 711         args[1] = fm->cmd.pa;
 712         ret = flowman_cmd(fm, args, 2);
 713         if (ret) {
 714                 ENICPMD_LOG(ERR, "cannot alloc exact match table: rc=%d", ret);
 715                 free(fet);
 716                 return ret;
 717         }
 718         fet->handle = args[0];
 719         fet->ingress = ingress;
 720         ENICPMD_LOG(DEBUG, "allocated exact match table: handle=0x%" PRIx64,
 721                     fet->handle);
 722         *fet_out = fet;
 723         return 0;
 724 }
 725
 726 static void
 727 enic_fet_free(struct enic_flowman *fm, struct enic_fm_fet *fet)
 728 {
 729         ENICPMD_FUNC_TRACE();
 730         enic_fm_tbl_free(fm, fet->handle);
 731         if (!fet->default_key)
 732                 TAILQ_REMOVE(&fm->fet_list, fet, list);
 733         free(fet);
 734 }
 735
 736 /*
 737  * Get the exact match table for the given combination of
 738  * <group, ingress, key>. Allocate one on the fly as necessary.
 739  */
 740 static int
 741 enic_fet_get(struct enic_flowman *fm,
 742              uint32_t group,
 743              uint8_t ingress,
 744              struct fm_key_template *key,
 745              struct enic_fm_fet **fet_out,
 746              struct rte_flow_error *error)
 747 {
 748         struct enic_fm_fet *fet;
 749
 750         ENICPMD_FUNC_TRACE();
 751         /* See if we already have this table open */
 752         TAILQ_FOREACH(fet, &fm->fet_list, list) {
 753                 if (fet->group == group && fet->ingress == ingress)
 754                         break;
 755         }
 756         if (fet == NULL) {
 757                 /* Jumping to a non-existing group? Use the default table */
 758                 if (key == NULL) {
 759                         fet = ingress ? fm->default_ig_fet : fm->default_eg_fet;
 760                 } else if (enic_fet_alloc(fm, ingress, key, 0, &fet)) {
 761                         return rte_flow_error_set(error, EINVAL,
 762                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 763                                 NULL, "enic: cannot get exact match table");
 764                 }
 765                 fet->group = group;
 766                 /* Default table is never on the open table list */
 767                 if (!fet->default_key)
 768                         TAILQ_INSERT_HEAD(&fm->fet_list, fet, list);
 769         }
 770         fet->ref++;
 771         *fet_out = fet;
 772         ENICPMD_LOG(DEBUG, "fet_get: %s %s group=%u ref=%u",
 773                     fet->default_key ? "default" : "",
 774                     fet->ingress ? "ingress" : "egress",
 775                     fet->group, fet->ref);
 776         return 0;
 777 }
 778
 779 static void
 780 enic_fet_put(struct enic_flowman *fm, struct enic_fm_fet *fet)
 781 {
 782         ENICPMD_FUNC_TRACE();
 783         RTE_ASSERT(fet->ref > 0);
 784         fet->ref--;
 785         ENICPMD_LOG(DEBUG, "fet_put: %s %s group=%u ref=%u",
 786                     fet->default_key ? "default" : "",
 787                     fet->ingress ? "ingress" : "egress",
 788                     fet->group, fet->ref);
 789         if (fet->ref == 0)
 790                 enic_fet_free(fm, fet);
 791 }
 792
 793 /* Return 1 if current item is valid on top of the previous one. */
 794 static int
 795 fm_item_stacking_valid(enum rte_flow_item_type prev_item,
 796                        const struct enic_fm_items *item_info,
 797                        uint8_t is_first_item)
 798 {
 799         enum rte_flow_item_type const *allowed_items = item_info->prev_items;
 800
 801         ENICPMD_FUNC_TRACE();
 802         for (; *allowed_items != RTE_FLOW_ITEM_TYPE_END; allowed_items++) {
 803                 if (prev_item == *allowed_items)
 804                         return 1;
 805         }
 806
 807         /* This is the first item in the stack. Check if that's cool */
 808         if (is_first_item && item_info->valid_start_item)
 809                 return 1;
 810         return 0;
 811 }
 812
 813 /*
 814  * Build the flow manager match entry structure from the provided pattern.
 815  * The pattern is validated as the items are copied.
 816  */
 817 static int
 818 enic_fm_copy_entry(struct enic_flowman *fm,
 819                    const struct rte_flow_item pattern[],
 820                    struct rte_flow_error *error)
 821 {
 822         const struct enic_fm_items *item_info;
 823         enum rte_flow_item_type prev_item;
 824         const struct rte_flow_item *item;
 825         struct copy_item_args args;
 826         uint8_t prev_header_level;
 827         uint8_t is_first_item;
 828         int ret;
 829
 830         ENICPMD_FUNC_TRACE();
 831         item = pattern;
 832         is_first_item = 1;
 833         prev_item = RTE_FLOW_ITEM_TYPE_END;
 834
 835         args.fm_tcam_entry = &fm->tcam_entry;
 836         args.header_level = 0;
 837         prev_header_level = 0;
 838         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 839                 /*
 840                  * Get info about how to validate and copy the item. If NULL
 841                  * is returned the nic does not support the item.
 842                  */
 843                 if (item->type == RTE_FLOW_ITEM_TYPE_VOID)
 844                         continue;
 845
 846                 item_info = &enic_fm_items[item->type];
 847
 848                 if (item->type > FM_MAX_ITEM_TYPE ||
 849                     item_info->copy_item == NULL) {
 850                         return rte_flow_error_set(error, ENOTSUP,
 851                                 RTE_FLOW_ERROR_TYPE_ITEM,
 852                                 NULL, "enic: unsupported item");
 853                 }
 854
 855                 /* check to see if item stacking is valid */
 856                 if (!fm_item_stacking_valid(prev_item, item_info,
 857                                             is_first_item))
 858                         goto stacking_error;
 859
 860                 args.item = item;
 861                 ret = item_info->copy_item(&args);
 862                 if (ret)
 863                         goto item_not_supported;
 864                 /* Going from outer to inner? Treat it as a new packet start */
 865                 if (prev_header_level != args.header_level) {
 866                         prev_item = RTE_FLOW_ITEM_TYPE_END;
 867                         is_first_item = 1;
 868                 } else {
 869                         prev_item = item->type;
 870                         is_first_item = 0;
 871                 }
 872                 prev_header_level = args.header_level;
 873         }
 874         return 0;
 875
 876 item_not_supported:
 877         return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,
 878                                   NULL, "enic: unsupported item type");
 879
 880 stacking_error:
 881         return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
 882                                   item, "enic: unsupported item stack");
 883 }
 884
 885 static void
 886 flow_item_skip_void(const struct rte_flow_item **item)
 887 {
 888         for ( ; ; (*item)++)
 889                 if ((*item)->type != RTE_FLOW_ITEM_TYPE_VOID)
 890                         return;
 891 }
 892
 893 static void
 894 append_template(void **template, uint8_t *off, const void *data, int len)
 895 {
 896         memcpy(*template, data, len);
 897         *template = (char *)*template + len;
 898         *off = *off + len;
 899 }
 900
 901 static int
 902 enic_fm_append_action_op(struct enic_flowman *fm,
 903                          struct fm_action_op *fm_op,
 904                          struct rte_flow_error *error)
 905 {
 906         int count;
 907
 908         count = fm->action_op_count;
 909         ENICPMD_LOG(DEBUG, "append action op: idx=%d op=%u",
 910                     count, fm_op->fa_op);
 911         if (count == FM_ACTION_OP_MAX) {
 912                 return rte_flow_error_set(error, EINVAL,
 913                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 914                         "too many action operations");
 915         }
 916         fm->action.fma_action_ops[count] = *fm_op;
 917         fm->action_op_count = count + 1;
 918         return 0;
 919 }
 920
 921 /* NIC requires that 1st steer appear before decap.
 922  * Correct example: steer, decap, steer, steer, ...
 923  */
 924 static void
 925 enic_fm_reorder_action_op(struct enic_flowman *fm)
 926 {
 927         struct fm_action_op *op, *steer, *decap;
 928         struct fm_action_op tmp_op;
 929
 930         ENICPMD_FUNC_TRACE();
 931         /* Find 1st steer and decap */
 932         op = fm->action.fma_action_ops;
 933         steer = NULL;
 934         decap = NULL;
 935         while (op->fa_op != FMOP_END) {
 936                 if (!decap && op->fa_op == FMOP_DECAP_NOSTRIP)
 937                         decap = op;
 938                 else if (!steer && op->fa_op == FMOP_RQ_STEER)
 939                         steer = op;
 940                 op++;
 941         }
 942         /* If decap is before steer, swap */
 943         if (steer && decap && decap < steer) {
 944                 op = fm->action.fma_action_ops;
 945                 ENICPMD_LOG(DEBUG, "swap decap %ld <-> steer %ld",
 946                             (long)(decap - op), (long)(steer - op));
 947                 tmp_op = *decap;
 948                 *decap = *steer;
 949                 *steer = tmp_op;
 950         }
 951 }
 952
 953 /* VXLAN decap is done via flowman compound action */
 954 static int
 955 enic_fm_copy_vxlan_decap(struct enic_flowman *fm,
 956                          struct fm_tcam_match_entry *fmt,
 957                          const struct rte_flow_action *action,
 958                          struct rte_flow_error *error)
 959 {
 960         struct fm_header_set *fm_data;
 961         struct fm_action_op fm_op;
 962
 963         ENICPMD_FUNC_TRACE();
 964         fm_data = &fmt->ftm_data.fk_hdrset[0];
 965         if (!(fm_data->fk_metadata & FKM_VXLAN)) {
 966                 return rte_flow_error_set(error, EINVAL,
 967                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 968                         "vxlan-decap: vxlan must be in pattern");
 969         }
 970
 971         memset(&fm_op, 0, sizeof(fm_op));
 972         fm_op.fa_op = FMOP_DECAP_NOSTRIP;
 973         return enic_fm_append_action_op(fm, &fm_op, error);
 974 }
 975
 976 /* VXLAN encap is done via flowman compound action */
 977 static int
 978 enic_fm_copy_vxlan_encap(struct enic_flowman *fm,
 979                          const struct rte_flow_item *item,
 980                          struct rte_flow_error *error)
 981 {
 982         struct fm_action_op fm_op;
 983         struct rte_ether_hdr *eth;
 984         uint16_t *ethertype;
 985         void *template;
 986         uint8_t off;
 987
 988         ENICPMD_FUNC_TRACE();
 989         memset(&fm_op, 0, sizeof(fm_op));
 990         fm_op.fa_op = FMOP_ENCAP;
 991         template = fm->action.fma_data;
 992         off = 0;
 993         /*
 994          * Copy flow items to the flowman template starting L2.
 995          * L2 must be ethernet.
 996          */
 997         flow_item_skip_void(&item);
 998         if (item->type != RTE_FLOW_ITEM_TYPE_ETH)
 999                 return rte_flow_error_set(error, EINVAL,
1000                         RTE_FLOW_ERROR_TYPE_ITEM, item,
1001                         "vxlan-encap: first item should be ethernet");
1002         eth = (struct rte_ether_hdr *)template;
1003         ethertype = &eth->ether_type;
1004         append_template(&template, &off, item->spec,
1005                         sizeof(struct rte_flow_item_eth));
1006         item++;
1007         flow_item_skip_void(&item);
1008         /* Optional VLAN */
1009         if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
1010                 const struct rte_flow_item_vlan *spec;
1011
1012                 ENICPMD_LOG(DEBUG, "vxlan-encap: vlan");
1013                 spec = item->spec;
1014                 fm_op.encap.outer_vlan = rte_be_to_cpu_16(spec->tci);
1015                 item++;
1016                 flow_item_skip_void(&item);
1017         }
1018         /* L3 must be IPv4, IPv6 */
1019         switch (item->type) {
1020         case RTE_FLOW_ITEM_TYPE_IPV4:
1021         {
1022                 struct rte_ipv4_hdr *ip4;
1023
1024                 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv4");
1025                 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
1026                 ip4 = (struct rte_ipv4_hdr *)template;
1027                 /*
1028                  * Offset of IPv4 length field and its initial value
1029                  * (IP + UDP + VXLAN) are specified in the action. The NIC
1030                  * will add inner packet length.
1031                  */
1032                 fm_op.encap.len1_offset = off +
1033                         offsetof(struct rte_ipv4_hdr, total_length);
1034                 fm_op.encap.len1_delta = sizeof(struct rte_ipv4_hdr) +
1035                         sizeof(struct rte_udp_hdr) +
1036                         sizeof(struct rte_vxlan_hdr);
1037                 append_template(&template, &off, item->spec,
1038                                 sizeof(struct rte_ipv4_hdr));
1039                 ip4->version_ihl = RTE_IPV4_VHL_DEF;
1040                 if (ip4->time_to_live == 0)
1041                         ip4->time_to_live = IP_DEFTTL;
1042                 ip4->next_proto_id = IPPROTO_UDP;
1043                 break;
1044         }
1045         case RTE_FLOW_ITEM_TYPE_IPV6:
1046         {
1047                 struct rte_ipv6_hdr *ip6;
1048
1049                 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv6");
1050                 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
1051                 ip6 = (struct rte_ipv6_hdr *)template;
1052                 fm_op.encap.len1_offset = off +
1053                         offsetof(struct rte_ipv6_hdr, payload_len);
1054                 fm_op.encap.len1_delta = sizeof(struct rte_udp_hdr) +
1055                         sizeof(struct rte_vxlan_hdr);
1056                 append_template(&template, &off, item->spec,
1057                                 sizeof(struct rte_ipv6_hdr));
1058                 ip6->vtc_flow |= rte_cpu_to_be_32(IP6_VTC_FLOW);
1059                 if (ip6->hop_limits == 0)
1060                         ip6->hop_limits = IP_DEFTTL;
1061                 ip6->proto = IPPROTO_UDP;
1062                 break;
1063         }
1064         default:
1065                 return rte_flow_error_set(error,
1066                         EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1067                         "vxlan-encap: L3 must be IPv4/IPv6");
1068         }
1069         item++;
1070         flow_item_skip_void(&item);
1071
1072         /* L4 is UDP */
1073         if (item->type != RTE_FLOW_ITEM_TYPE_UDP)
1074                 return rte_flow_error_set(error, EINVAL,
1075                         RTE_FLOW_ERROR_TYPE_ITEM, item,
1076                         "vxlan-encap: UDP must follow IPv4/IPv6");
1077         /* UDP length = UDP + VXLAN. NIC will add inner packet length. */
1078         fm_op.encap.len2_offset =
1079                 off + offsetof(struct rte_udp_hdr, dgram_len);
1080         fm_op.encap.len2_delta =
1081                 sizeof(struct rte_udp_hdr) + sizeof(struct rte_vxlan_hdr);
1082         append_template(&template, &off, item->spec,
1083                         sizeof(struct rte_udp_hdr));
1084         item++;
1085         flow_item_skip_void(&item);
1086
1087         /* Finally VXLAN */
1088         if (item->type != RTE_FLOW_ITEM_TYPE_VXLAN)
1089                 return rte_flow_error_set(error,
1090                         EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1091                         "vxlan-encap: VXLAN must follow UDP");
1092         append_template(&template, &off, item->spec,
1093                         sizeof(struct rte_flow_item_vxlan));
1094
1095         /*
1096          * Fill in the rest of the action structure.
1097          * Indicate that we want to encap with vxlan at packet start.
1098          */
1099         fm_op.encap.template_offset = 0;
1100         fm_op.encap.template_len = off;
1101         return enic_fm_append_action_op(fm, &fm_op, error);
1102 }
1103
1104 static int
1105 enic_fm_find_vnic(struct enic *enic, const struct rte_pci_addr *addr,
1106                   uint64_t *handle)
1107 {
1108         uint32_t bdf;
1109         uint64_t args[2];
1110         int rc;
1111
1112         ENICPMD_FUNC_TRACE();
1113         ENICPMD_LOG(DEBUG, "bdf=%x:%x:%x", addr->bus, addr->devid,
1114                     addr->function);
1115         bdf = addr->bus << 8 | addr->devid << 3 | addr->function;
1116         args[0] = FM_VNIC_FIND;
1117         args[1] = bdf;
1118         rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
1119         if (rc != 0) {
1120                 /* Expected to fail if BDF is not on the adapter */
1121                 ENICPMD_LOG(DEBUG, "cannot find vnic handle: rc=%d", rc);
1122                 return rc;
1123         }
1124         *handle = args[0];
1125         ENICPMD_LOG(DEBUG, "found vnic: handle=0x%" PRIx64, *handle);
1126         return 0;
1127 }
1128
1129 /*
1130  * Egress: target port should be either PF uplink or VF.
1131  * Supported cases
1132  * 1. VF egress -> PF uplink
1133  *   PF may be this VF's PF, or another PF, as long as they are on the same VIC.
1134  * 2. VF egress -> VF
1135  *
1136  * Unsupported cases
1137  * 1. PF egress -> VF
1138  *   App should be using representor to pass packets to VF
1139  */
1140 static int
1141 vf_egress_port_id_action(struct enic_flowman *fm,
1142                          struct rte_eth_dev *dst_dev,
1143                          uint64_t dst_vnic_h,
1144                          struct fm_action_op *fm_op,
1145                          struct rte_flow_error *error)
1146 {
1147         struct enic *src_enic, *dst_enic;
1148         struct enic_vf_representor *vf;
1149         uint8_t uif;
1150         int ret;
1151
1152         ENICPMD_FUNC_TRACE();
1153         src_enic = fm->user_enic;
1154         dst_enic = pmd_priv(dst_dev);
1155         if (!(src_enic->rte_dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)) {
1156                 return rte_flow_error_set(error, EINVAL,
1157                         RTE_FLOW_ERROR_TYPE_ACTION,
1158                         NULL, "source port is not VF representor");
1159         }
1160
1161         /* VF -> PF uplink. dst is not VF representor */
1162         if (!(dst_dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)) {
1163                 /* PF is the VF's PF? Then nothing to do */
1164                 vf = VF_ENIC_TO_VF_REP(src_enic);
1165                 if (vf->pf == dst_enic) {
1166                         ENICPMD_LOG(DEBUG, "destination port is VF's PF");
1167                         return 0;
1168                 }
1169                 /* If not, steer to the remote PF's uplink */
1170                 uif = dst_enic->fm_vnic_uif;
1171                 ENICPMD_LOG(DEBUG, "steer to uplink %u", uif);
1172                 memset(fm_op, 0, sizeof(*fm_op));
1173                 fm_op->fa_op = FMOP_SET_EGPORT;
1174                 fm_op->set_egport.egport = uif;
1175                 ret = enic_fm_append_action_op(fm, fm_op, error);
1176                 return ret;
1177         }
1178
1179         /* VF -> VF loopback. Hairpin and steer to vnic */
1180         memset(fm_op, 0, sizeof(*fm_op));
1181         fm_op->fa_op = FMOP_EG_HAIRPIN;
1182         ret = enic_fm_append_action_op(fm, fm_op, error);
1183         if (ret)
1184                 return ret;
1185         ENICPMD_LOG(DEBUG, "egress hairpin");
1186         fm->hairpin_steer_vnic_h = dst_vnic_h;
1187         fm->need_hairpin_steer = 1;
1188         return 0;
1189 }
1190
1191 /* Translate flow actions to flowman TCAM entry actions */
1192 static int
1193 enic_fm_copy_action(struct enic_flowman *fm,
1194                     const struct rte_flow_action actions[],
1195                     uint8_t ingress,
1196                     struct rte_flow_error *error)
1197 {
1198         enum {
1199                 FATE = 1 << 0,
1200                 DECAP = 1 << 1,
1201                 PASSTHRU = 1 << 2,
1202                 COUNT = 1 << 3,
1203                 ENCAP = 1 << 4,
1204                 PUSH_VLAN = 1 << 5,
1205                 PORT_ID = 1 << 6,
1206         };
1207         struct fm_tcam_match_entry *fmt;
1208         struct fm_action_op fm_op;
1209         bool need_ovlan_action;
1210         struct enic *enic;
1211         uint32_t overlap;
1212         uint64_t vnic_h;
1213         uint16_t ovlan;
1214         bool first_rq;
1215         bool steer;
1216         int ret;
1217
1218         ENICPMD_FUNC_TRACE();
1219         fmt = &fm->tcam_entry;
1220         need_ovlan_action = false;
1221         ovlan = 0;
1222         first_rq = true;
1223         steer = false;
1224         enic = fm->user_enic;
1225         overlap = 0;
1226         vnic_h = enic->fm_vnic_handle;
1227
1228         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1229                 switch (actions->type) {
1230                 case RTE_FLOW_ACTION_TYPE_VOID:
1231                         continue;
1232                 case RTE_FLOW_ACTION_TYPE_PASSTHRU: {
1233                         if (overlap & PASSTHRU)
1234                                 goto unsupported;
1235                         overlap |= PASSTHRU;
1236                         break;
1237                 }
1238                 case RTE_FLOW_ACTION_TYPE_JUMP: {
1239                         const struct rte_flow_action_jump *jump =
1240                                 actions->conf;
1241                         struct enic_fm_fet *fet;
1242
1243                         if (overlap & FATE)
1244                                 goto unsupported;
1245                         ret = enic_fet_get(fm, jump->group, ingress, NULL,
1246                                            &fet, error);
1247                         if (ret)
1248                                 return ret;
1249                         overlap |= FATE;
1250                         memset(&fm_op, 0, sizeof(fm_op));
1251                         fm_op.fa_op = FMOP_EXACT_MATCH;
1252                         fm_op.exact.handle = fet->handle;
1253                         fm->fet = fet;
1254                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1255                         if (ret)
1256                                 return ret;
1257                         break;
1258                 }
1259                 case RTE_FLOW_ACTION_TYPE_MARK: {
1260                         const struct rte_flow_action_mark *mark =
1261                                 actions->conf;
1262
1263                         if (mark->id >= ENIC_MAGIC_FILTER_ID - 1)
1264                                 return rte_flow_error_set(error, EINVAL,
1265                                         RTE_FLOW_ERROR_TYPE_ACTION,
1266                                         NULL, "invalid mark id");
1267                         memset(&fm_op, 0, sizeof(fm_op));
1268                         fm_op.fa_op = FMOP_MARK;
1269                         fm_op.mark.mark = mark->id + 1;
1270                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1271                         if (ret)
1272                                 return ret;
1273                         break;
1274                 }
1275                 case RTE_FLOW_ACTION_TYPE_FLAG: {
1276                         /* ENIC_MAGIC_FILTER_ID is reserved for flagging */
1277                         memset(&fm_op, 0, sizeof(fm_op));
1278                         fm_op.fa_op = FMOP_MARK;
1279                         fm_op.mark.mark = ENIC_MAGIC_FILTER_ID;
1280                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1281                         if (ret)
1282                                 return ret;
1283                         break;
1284                 }
1285                 case RTE_FLOW_ACTION_TYPE_QUEUE: {
1286                         const struct rte_flow_action_queue *queue =
1287                                 actions->conf;
1288
1289                         /*
1290                          * If fate other than QUEUE or RSS, fail. Multiple
1291                          * rss and queue actions are ok.
1292                          */
1293                         if ((overlap & FATE) && first_rq)
1294                                 goto unsupported;
1295                         first_rq = false;
1296                         overlap |= FATE;
1297                         memset(&fm_op, 0, sizeof(fm_op));
1298                         fm_op.fa_op = FMOP_RQ_STEER;
1299                         fm_op.rq_steer.rq_index =
1300                                 enic_rte_rq_idx_to_sop_idx(queue->index);
1301                         fm_op.rq_steer.rq_count = 1;
1302                         fm_op.rq_steer.vnic_handle = vnic_h;
1303                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1304                         if (ret)
1305                                 return ret;
1306                         ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1307                                     fm_op.rq_steer.rq_index);
1308                         steer = true;
1309                         break;
1310                 }
1311                 case RTE_FLOW_ACTION_TYPE_DROP: {
1312                         if (overlap & FATE)
1313                                 goto unsupported;
1314                         overlap |= FATE;
1315                         memset(&fm_op, 0, sizeof(fm_op));
1316                         fm_op.fa_op = FMOP_DROP;
1317                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1318                         if (ret)
1319                                 return ret;
1320                         ENICPMD_LOG(DEBUG, "create DROP action");
1321                         break;
1322                 }
1323                 case RTE_FLOW_ACTION_TYPE_COUNT: {
1324                         if (overlap & COUNT)
1325                                 goto unsupported;
1326                         overlap |= COUNT;
1327                         /* Count is associated with entry not action on VIC. */
1328                         fmt->ftm_flags |= FMEF_COUNTER;
1329                         break;
1330                 }
1331                 case RTE_FLOW_ACTION_TYPE_RSS: {
1332                         const struct rte_flow_action_rss *rss = actions->conf;
1333                         bool allow;
1334                         uint16_t i;
1335
1336                         /*
1337                          * If fate other than QUEUE or RSS, fail. Multiple
1338                          * rss and queue actions are ok.
1339                          */
1340                         if ((overlap & FATE) && first_rq)
1341                                 goto unsupported;
1342                         first_rq = false;
1343                         overlap |= FATE;
1344
1345                         /*
1346                          * Hardware only supports RSS actions on outer level
1347                          * with default type and function. Queues must be
1348                          * sequential.
1349                          */
1350                         allow = rss->func == RTE_ETH_HASH_FUNCTION_DEFAULT &&
1351                                 rss->level == 0 && (rss->types == 0 ||
1352                                 rss->types == enic->rss_hf) &&
1353                                 rss->queue_num <= enic->rq_count &&
1354                                 rss->queue[rss->queue_num - 1] < enic->rq_count;
1355
1356
1357                         /* Identity queue map needs to be sequential */
1358                         for (i = 1; i < rss->queue_num; i++)
1359                                 allow = allow && (rss->queue[i] ==
1360                                         rss->queue[i - 1] + 1);
1361                         if (!allow)
1362                                 goto unsupported;
1363
1364                         memset(&fm_op, 0, sizeof(fm_op));
1365                         fm_op.fa_op = FMOP_RQ_STEER;
1366                         fm_op.rq_steer.rq_index =
1367                                 enic_rte_rq_idx_to_sop_idx(rss->queue[0]);
1368                         fm_op.rq_steer.rq_count = rss->queue_num;
1369                         fm_op.rq_steer.vnic_handle = vnic_h;
1370                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1371                         if (ret)
1372                                 return ret;
1373                         ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1374                                     fm_op.rq_steer.rq_index);
1375                         steer = true;
1376                         break;
1377                 }
1378                 case RTE_FLOW_ACTION_TYPE_PORT_ID: {
1379                         const struct rte_flow_action_port_id *port;
1380                         struct rte_eth_dev *dev;
1381
1382                         if (!ingress && (overlap & PORT_ID)) {
1383                                 ENICPMD_LOG(DEBUG, "cannot have multiple egress PORT_ID actions");
1384                                 goto unsupported;
1385                         }
1386                         port = actions->conf;
1387                         if (port->original) {
1388                                 vnic_h = enic->fm_vnic_handle; /* This port */
1389                                 break;
1390                         }
1391                         ENICPMD_LOG(DEBUG, "port id %u", port->id);
1392                         if (!rte_eth_dev_is_valid_port(port->id)) {
1393                                 return rte_flow_error_set(error, EINVAL,
1394                                         RTE_FLOW_ERROR_TYPE_ACTION,
1395                                         NULL, "invalid port_id");
1396                         }
1397                         dev = &rte_eth_devices[port->id];
1398                         if (!dev_is_enic(dev)) {
1399                                 return rte_flow_error_set(error, EINVAL,
1400                                         RTE_FLOW_ERROR_TYPE_ACTION,
1401                                         NULL, "port_id is not enic");
1402                         }
1403                         if (enic->switch_domain_id !=
1404                             pmd_priv(dev)->switch_domain_id) {
1405                                 return rte_flow_error_set(error, EINVAL,
1406                                         RTE_FLOW_ERROR_TYPE_ACTION,
1407                                         NULL, "destination and source ports are not in the same switch domain");
1408                         }
1409                         vnic_h = pmd_priv(dev)->fm_vnic_handle;
1410                         overlap |= PORT_ID;
1411                         /*
1412                          * Ingress. Nothing more to do. We add an implicit
1413                          * steer at the end if needed.
1414                          */
1415                         if (ingress)
1416                                 break;
1417                         /* Egress */
1418                         ret = vf_egress_port_id_action(fm, dev, vnic_h, &fm_op,
1419                                 error);
1420                         if (ret)
1421                                 return ret;
1422                         break;
1423                 }
1424                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: {
1425                         if (overlap & DECAP)
1426                                 goto unsupported;
1427                         overlap |= DECAP;
1428
1429                         ret = enic_fm_copy_vxlan_decap(fm, fmt, actions,
1430                                 error);
1431                         if (ret != 0)
1432                                 return ret;
1433                         break;
1434                 }
1435                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: {
1436                         const struct rte_flow_action_vxlan_encap *encap;
1437
1438                         encap = actions->conf;
1439                         if (overlap & ENCAP)
1440                                 goto unsupported;
1441                         overlap |= ENCAP;
1442                         ret = enic_fm_copy_vxlan_encap(fm, encap->definition,
1443                                 error);
1444                         if (ret != 0)
1445                                 return ret;
1446                         break;
1447                 }
1448                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: {
1449                         memset(&fm_op, 0, sizeof(fm_op));
1450                         fm_op.fa_op = FMOP_POP_VLAN;
1451                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1452                         if (ret)
1453                                 return ret;
1454                         break;
1455                 }
1456                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: {
1457                         const struct rte_flow_action_of_push_vlan *vlan;
1458
1459                         if (overlap & PASSTHRU)
1460                                 goto unsupported;
1461                         vlan = actions->conf;
1462                         if (vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN)) {
1463                                 return rte_flow_error_set(error, EINVAL,
1464                                         RTE_FLOW_ERROR_TYPE_ACTION,
1465                                         NULL, "unexpected push_vlan ethertype");
1466                         }
1467                         overlap |= PUSH_VLAN;
1468                         need_ovlan_action = true;
1469                         break;
1470                 }
1471                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: {
1472                         const struct rte_flow_action_of_set_vlan_pcp *pcp;
1473
1474                         pcp = actions->conf;
1475                         if (pcp->vlan_pcp > 7) {
1476                                 return rte_flow_error_set(error, EINVAL,
1477                                         RTE_FLOW_ERROR_TYPE_ACTION,
1478                                         NULL, "invalid vlan_pcp");
1479                         }
1480                         need_ovlan_action = true;
1481                         ovlan |= ((uint16_t)pcp->vlan_pcp) << 13;
1482                         break;
1483                 }
1484                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: {
1485                         const struct rte_flow_action_of_set_vlan_vid *vid;
1486
1487                         vid = actions->conf;
1488                         need_ovlan_action = true;
1489                         ovlan |= rte_be_to_cpu_16(vid->vlan_vid);
1490                         break;
1491                 }
1492                 default:
1493                         goto unsupported;
1494                 }
1495         }
1496
1497         if (!(overlap & (FATE | PASSTHRU | COUNT | PORT_ID)))
1498                 goto unsupported;
1499         /* Egress from VF: need implicit WQ match */
1500         if (enic_is_vf_rep(enic) && !ingress) {
1501                 fmt->ftm_data.fk_wq_id = 0;
1502                 fmt->ftm_mask.fk_wq_id = 0xffff;
1503                 fmt->ftm_data.fk_wq_vnic = enic->fm_vnic_handle;
1504                 ENICPMD_LOG(DEBUG, "add implicit wq id match for vf %d",
1505                             VF_ENIC_TO_VF_REP(enic)->vf_id);
1506         }
1507         if (need_ovlan_action) {
1508                 memset(&fm_op, 0, sizeof(fm_op));
1509                 fm_op.fa_op = FMOP_SET_OVLAN;
1510                 fm_op.ovlan.vlan = ovlan;
1511                 ret = enic_fm_append_action_op(fm, &fm_op, error);
1512                 if (ret)
1513                         return ret;
1514         }
1515         /* Add steer op for PORT_ID without QUEUE */
1516         if ((overlap & PORT_ID) && !steer && ingress) {
1517                 memset(&fm_op, 0, sizeof(fm_op));
1518                 /* Always to queue 0 for now as generic RSS is not available */
1519                 fm_op.fa_op = FMOP_RQ_STEER;
1520                 fm_op.rq_steer.rq_index = 0;
1521                 fm_op.rq_steer.vnic_handle = vnic_h;
1522                 ret = enic_fm_append_action_op(fm, &fm_op, error);
1523                 if (ret)
1524                         return ret;
1525                 ENICPMD_LOG(DEBUG, "add implicit steer op");
1526         }
1527         /* Add required END */
1528         memset(&fm_op, 0, sizeof(fm_op));
1529         fm_op.fa_op = FMOP_END;
1530         ret = enic_fm_append_action_op(fm, &fm_op, error);
1531         if (ret)
1532                 return ret;
1533         enic_fm_reorder_action_op(fm);
1534         return 0;
1535
1536 unsupported:
1537         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
1538                                   NULL, "enic: unsupported action");
1539 }
1540
1541 /** Check if the action is supported */
1542 static int
1543 enic_fm_match_action(const struct rte_flow_action *action,
1544                      const enum rte_flow_action_type *supported_actions)
1545 {
1546         for (; *supported_actions != RTE_FLOW_ACTION_TYPE_END;
1547              supported_actions++) {
1548                 if (action->type == *supported_actions)
1549                         return 1;
1550         }
1551         return 0;
1552 }
1553
1554 /* Debug function to dump internal NIC action structure. */
1555 static void
1556 enic_fm_dump_tcam_actions(const struct fm_action *fm_action)
1557 {
1558         /* Manually keep in sync with FMOP commands */
1559         const char *fmop_str[FMOP_OP_MAX] = {
1560                 [FMOP_END] = "end",
1561                 [FMOP_DROP] = "drop",
1562                 [FMOP_RQ_STEER] = "steer",
1563                 [FMOP_EXACT_MATCH] = "exmatch",
1564                 [FMOP_MARK] = "mark",
1565                 [FMOP_EXT_MARK] = "ext_mark",
1566                 [FMOP_TAG] = "tag",
1567                 [FMOP_EG_HAIRPIN] = "eg_hairpin",
1568                 [FMOP_IG_HAIRPIN] = "ig_hairpin",
1569                 [FMOP_ENCAP_IVLAN] = "encap_ivlan",
1570                 [FMOP_ENCAP_NOIVLAN] = "encap_noivlan",
1571                 [FMOP_ENCAP] = "encap",
1572                 [FMOP_SET_OVLAN] = "set_ovlan",
1573                 [FMOP_DECAP_NOSTRIP] = "decap_nostrip",
1574                 [FMOP_DECAP_STRIP] = "decap_strip",
1575                 [FMOP_POP_VLAN] = "pop_vlan",
1576                 [FMOP_SET_EGPORT] = "set_egport",
1577                 [FMOP_RQ_STEER_ONLY] = "rq_steer_only",
1578                 [FMOP_SET_ENCAP_VLAN] = "set_encap_vlan",
1579                 [FMOP_EMIT] = "emit",
1580                 [FMOP_MODIFY] = "modify",
1581         };
1582         const struct fm_action_op *op = &fm_action->fma_action_ops[0];
1583         char buf[128], *bp = buf;
1584         const char *op_str;
1585         int i, n, buf_len;
1586
1587         buf[0] = '\0';
1588         buf_len = sizeof(buf);
1589         for (i = 0; i < FM_ACTION_OP_MAX; i++) {
1590                 if (op->fa_op == FMOP_END)
1591                         break;
1592                 if (op->fa_op >= FMOP_OP_MAX)
1593                         op_str = "unknown";
1594                 else
1595                         op_str = fmop_str[op->fa_op];
1596                 n = snprintf(bp, buf_len, "%s,", op_str);
1597                 if (n > 0 && n < buf_len) {
1598                         bp += n;
1599                         buf_len -= n;
1600                 }
1601                 op++;
1602         }
1603         /* Remove trailing comma */
1604         if (buf[0])
1605                 *(bp - 1) = '\0';
1606         ENICPMD_LOG(DEBUG, "       Acions: %s", buf);
1607 }
1608
1609 static int
1610 bits_to_str(uint32_t bits, const char *strings[], int max,
1611             char *buf, int buf_len)
1612 {
1613         int i, n = 0, len = 0;
1614
1615         for (i = 0; i < max; i++) {
1616                 if (bits & (1 << i)) {
1617                         n = snprintf(buf, buf_len, "%s,", strings[i]);
1618                         if (n > 0 && n < buf_len) {
1619                                 buf += n;
1620                                 buf_len -= n;
1621                                 len += n;
1622                         }
1623                 }
1624         }
1625         /* Remove trailing comma */
1626         if (len) {
1627                 *(buf - 1) = '\0';
1628                 len--;
1629         }
1630         return len;
1631 }
1632
1633 /* Debug function to dump internal NIC filter structure. */
1634 static void
1635 __enic_fm_dump_tcam_match(const struct fm_header_set *fk_hdrset, char *buf,
1636                           int buf_len)
1637 {
1638         /* Manually keep in sync with FKM_BITS */
1639         const char *fm_fkm_str[FKM_BIT_COUNT] = {
1640                 [FKM_QTAG_BIT] = "qtag",
1641                 [FKM_CMD_BIT] = "cmd",
1642                 [FKM_IPV4_BIT] = "ip4",
1643                 [FKM_IPV6_BIT] = "ip6",
1644                 [FKM_ROCE_BIT] = "roce",
1645                 [FKM_UDP_BIT] = "udp",
1646                 [FKM_TCP_BIT] = "tcp",
1647                 [FKM_TCPORUDP_BIT] = "tcpportudp",
1648                 [FKM_IPFRAG_BIT] = "ipfrag",
1649                 [FKM_NVGRE_BIT] = "nvgre",
1650                 [FKM_VXLAN_BIT] = "vxlan",
1651                 [FKM_GENEVE_BIT] = "geneve",
1652                 [FKM_NSH_BIT] = "nsh",
1653                 [FKM_ROCEV2_BIT] = "rocev2",
1654                 [FKM_VLAN_PRES_BIT] = "vlan_pres",
1655                 [FKM_IPOK_BIT] = "ipok",
1656                 [FKM_L4OK_BIT] = "l4ok",
1657                 [FKM_ROCEOK_BIT] = "roceok",
1658                 [FKM_FCSOK_BIT] = "fcsok",
1659                 [FKM_EG_SPAN_BIT] = "eg_span",
1660                 [FKM_IG_SPAN_BIT] = "ig_span",
1661                 [FKM_EG_HAIRPINNED_BIT] = "eg_hairpinned",
1662         };
1663         /* Manually keep in sync with FKH_BITS */
1664         const char *fm_fkh_str[FKH_BIT_COUNT] = {
1665                 [FKH_ETHER_BIT] = "eth",
1666                 [FKH_QTAG_BIT] = "qtag",
1667                 [FKH_L2RAW_BIT] = "l2raw",
1668                 [FKH_IPV4_BIT] = "ip4",
1669                 [FKH_IPV6_BIT] = "ip6",
1670                 [FKH_L3RAW_BIT] = "l3raw",
1671                 [FKH_UDP_BIT] = "udp",
1672                 [FKH_TCP_BIT] = "tcp",
1673                 [FKH_ICMP_BIT] = "icmp",
1674                 [FKH_VXLAN_BIT] = "vxlan",
1675                 [FKH_L4RAW_BIT] = "l4raw",
1676         };
1677         uint32_t fkh_bits = fk_hdrset->fk_header_select;
1678         uint32_t fkm_bits = fk_hdrset->fk_metadata;
1679         int n;
1680
1681         if (!fkm_bits && !fkh_bits)
1682                 return;
1683         n = snprintf(buf, buf_len, "metadata(");
1684         if (n > 0 && n < buf_len) {
1685                 buf += n;
1686                 buf_len -= n;
1687         }
1688         n = bits_to_str(fkm_bits, fm_fkm_str, FKM_BIT_COUNT, buf, buf_len);
1689         if (n > 0 && n < buf_len) {
1690                 buf += n;
1691                 buf_len -= n;
1692         }
1693         n = snprintf(buf, buf_len, ") valid hdr fields(");
1694         if (n > 0 && n < buf_len) {
1695                 buf += n;
1696                 buf_len -= n;
1697         }
1698         n = bits_to_str(fkh_bits, fm_fkh_str, FKH_BIT_COUNT, buf, buf_len);
1699         if (n > 0 && n < buf_len) {
1700                 buf += n;
1701                 buf_len -= n;
1702         }
1703         snprintf(buf, buf_len, ")");
1704 }
1705
1706 static void
1707 enic_fm_dump_tcam_match(const struct fm_tcam_match_entry *match,
1708                         uint8_t ingress)
1709 {
1710         char buf[256];
1711
1712         memset(buf, 0, sizeof(buf));
1713         __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[0],
1714                                   buf, sizeof(buf));
1715         ENICPMD_LOG(DEBUG, " TCAM %s Outer: %s %scounter position %u",
1716                     (ingress) ? "IG" : "EG", buf,
1717                     (match->ftm_flags & FMEF_COUNTER) ? "" : "no ",
1718                     match->ftm_position);
1719         memset(buf, 0, sizeof(buf));
1720         __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[1],
1721                                   buf, sizeof(buf));
1722         if (buf[0])
1723                 ENICPMD_LOG(DEBUG, "         Inner: %s", buf);
1724 }
1725
1726 /* Debug function to dump internal NIC flow structures. */
1727 static void
1728 enic_fm_dump_tcam_entry(const struct fm_tcam_match_entry *fm_match,
1729                         const struct fm_action *fm_action,
1730                         uint8_t ingress)
1731 {
1732         if (!rte_log_can_log(enic_pmd_logtype, RTE_LOG_DEBUG))
1733                 return;
1734         enic_fm_dump_tcam_match(fm_match, ingress);
1735         enic_fm_dump_tcam_actions(fm_action);
1736 }
1737
1738 static int
1739 enic_fm_flow_parse(struct enic_flowman *fm,
1740                    const struct rte_flow_attr *attrs,
1741                    const struct rte_flow_item pattern[],
1742                    const struct rte_flow_action actions[],
1743                    struct rte_flow_error *error)
1744 {
1745         const struct rte_flow_action *action;
1746         unsigned int ret;
1747         static const enum rte_flow_action_type *sa;
1748
1749         ENICPMD_FUNC_TRACE();
1750         ret = 0;
1751         if (!pattern) {
1752                 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM,
1753                                    NULL, "no pattern specified");
1754                 return -rte_errno;
1755         }
1756
1757         if (!actions) {
1758                 rte_flow_error_set(error, EINVAL,
1759                                    RTE_FLOW_ERROR_TYPE_ACTION_NUM,
1760                                    NULL, "no action specified");
1761                 return -rte_errno;
1762         }
1763
1764         if (attrs) {
1765                 if (attrs->group != FM_TCAM_RTE_GROUP && attrs->priority) {
1766                         rte_flow_error_set(error, ENOTSUP,
1767                                            RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1768                                            NULL,
1769                                            "priorities are not supported for non-default (0) groups");
1770                         return -rte_errno;
1771                 } else if (!fm->owner_enic->switchdev_mode && attrs->transfer) {
1772                         rte_flow_error_set(error, ENOTSUP,
1773                                            RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1774                                            NULL,
1775                                            "transfer is not supported");
1776                         return -rte_errno;
1777                 } else if (attrs->ingress && attrs->egress) {
1778                         rte_flow_error_set(error, ENOTSUP,
1779                                            RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1780                                            NULL,
1781                                            "bidirectional rules not supported");
1782                         return -rte_errno;
1783                 }
1784
1785         } else {
1786                 rte_flow_error_set(error, EINVAL,
1787                                    RTE_FLOW_ERROR_TYPE_ATTR,
1788                                    NULL, "no attribute specified");
1789                 return -rte_errno;
1790         }
1791
1792         /* Verify Actions. */
1793         sa = (attrs->ingress) ? enic_fm_supported_ig_actions :
1794              enic_fm_supported_eg_actions;
1795         for (action = &actions[0]; action->type != RTE_FLOW_ACTION_TYPE_END;
1796              action++) {
1797                 if (action->type == RTE_FLOW_ACTION_TYPE_VOID)
1798                         continue;
1799                 else if (!enic_fm_match_action(action, sa))
1800                         break;
1801         }
1802         if (action->type != RTE_FLOW_ACTION_TYPE_END) {
1803                 rte_flow_error_set(error, EPERM, RTE_FLOW_ERROR_TYPE_ACTION,
1804                                    action, "invalid action");
1805                 return -rte_errno;
1806         }
1807         ret = enic_fm_copy_entry(fm, pattern, error);
1808         if (ret)
1809                 return ret;
1810         ret = enic_fm_copy_action(fm, actions, attrs->ingress, error);
1811         return ret;
1812 }
1813
1814 static void
1815 enic_fm_counter_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
1816 {
1817         if (!fm_flow->counter_valid)
1818                 return;
1819         SLIST_INSERT_HEAD(&fm->counters, fm_flow->counter, next);
1820         fm_flow->counter_valid = false;
1821 }
1822
1823 static int
1824 enic_fm_more_counters(struct enic_flowman *fm)
1825 {
1826         struct enic_fm_counter *new_stack;
1827         struct enic_fm_counter *ctrs;
1828         int i, rc;
1829         uint64_t args[2];
1830
1831         ENICPMD_FUNC_TRACE();
1832         new_stack = rte_realloc(fm->counter_stack, (fm->counters_alloced +
1833                                 FM_COUNTERS_EXPAND) *
1834                                 sizeof(struct enic_fm_counter), 0);
1835         if (new_stack == NULL) {
1836                 ENICPMD_LOG(ERR, "cannot alloc counter memory");
1837                 return -ENOMEM;
1838         }
1839         fm->counter_stack = new_stack;
1840
1841         args[0] = FM_COUNTER_BRK;
1842         args[1] = fm->counters_alloced + FM_COUNTERS_EXPAND;
1843         rc = flowman_cmd(fm, args, 2);
1844         if (rc != 0) {
1845                 ENICPMD_LOG(ERR, "cannot alloc counters rc=%d", rc);
1846                 return rc;
1847         }
1848         ctrs = (struct enic_fm_counter *)fm->counter_stack +
1849                 fm->counters_alloced;
1850         for (i = 0; i < FM_COUNTERS_EXPAND; i++, ctrs++) {
1851                 ctrs->handle = fm->counters_alloced + i;
1852                 SLIST_INSERT_HEAD(&fm->counters, ctrs, next);
1853         }
1854         fm->counters_alloced += FM_COUNTERS_EXPAND;
1855         ENICPMD_LOG(DEBUG, "%u counters allocated, total: %u",
1856                     FM_COUNTERS_EXPAND, fm->counters_alloced);
1857         return 0;
1858 }
1859
1860 static int
1861 enic_fm_counter_zero(struct enic_flowman *fm, struct enic_fm_counter *c)
1862 {
1863         uint64_t args[3];
1864         int ret;
1865
1866         ENICPMD_FUNC_TRACE();
1867         args[0] = FM_COUNTER_QUERY;
1868         args[1] = c->handle;
1869         args[2] = 1; /* clear */
1870         ret = flowman_cmd(fm, args, 3);
1871         if (ret) {
1872                 ENICPMD_LOG(ERR, "counter init: rc=%d handle=0x%x",
1873                             ret, c->handle);
1874                 return ret;
1875         }
1876         return 0;
1877 }
1878
1879 static int
1880 enic_fm_counter_alloc(struct enic_flowman *fm, struct rte_flow_error *error,
1881                       struct enic_fm_counter **ctr)
1882 {
1883         struct enic_fm_counter *c;
1884         int ret;
1885
1886         ENICPMD_FUNC_TRACE();
1887         *ctr = NULL;
1888         if (SLIST_EMPTY(&fm->counters)) {
1889                 ret = enic_fm_more_counters(fm);
1890                 if (ret)
1891                         return rte_flow_error_set(error, -ret,
1892                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1893                                 NULL, "enic: out of counters");
1894         }
1895         c = SLIST_FIRST(&fm->counters);
1896         SLIST_REMOVE_HEAD(&fm->counters, next);
1897         *ctr = c;
1898         return 0;
1899 }
1900
1901 static int
1902 enic_fm_action_free(struct enic_flowman *fm, uint64_t handle)
1903 {
1904         uint64_t args[2];
1905         int rc;
1906
1907         ENICPMD_FUNC_TRACE();
1908         args[0] = FM_ACTION_FREE;
1909         args[1] = handle;
1910         rc = flowman_cmd(fm, args, 2);
1911         if (rc)
1912                 ENICPMD_LOG(ERR, "cannot free action: rc=%d handle=0x%" PRIx64,
1913                             rc, handle);
1914         return rc;
1915 }
1916
1917 static int
1918 enic_fm_entry_free(struct enic_flowman *fm, uint64_t handle)
1919 {
1920         uint64_t args[2];
1921         int rc;
1922
1923         ENICPMD_FUNC_TRACE();
1924         args[0] = FM_MATCH_ENTRY_REMOVE;
1925         args[1] = handle;
1926         rc = flowman_cmd(fm, args, 2);
1927         if (rc)
1928                 ENICPMD_LOG(ERR, "cannot free match entry: rc=%d"
1929                             " handle=0x%" PRIx64, rc, handle);
1930         return rc;
1931 }
1932
1933 static struct enic_fm_jump_flow *
1934 find_jump_flow(struct enic_flowman *fm, uint32_t group)
1935 {
1936         struct enic_fm_jump_flow *j;
1937
1938         ENICPMD_FUNC_TRACE();
1939         TAILQ_FOREACH(j, &fm->jump_list, list) {
1940                 if (j->group == group)
1941                         return j;
1942         }
1943         return NULL;
1944 }
1945
1946 static void
1947 remove_jump_flow(struct enic_flowman *fm, struct rte_flow *flow)
1948 {
1949         struct enic_fm_jump_flow *j;
1950
1951         ENICPMD_FUNC_TRACE();
1952         TAILQ_FOREACH(j, &fm->jump_list, list) {
1953                 if (j->flow == flow) {
1954                         TAILQ_REMOVE(&fm->jump_list, j, list);
1955                         free(j);
1956                         return;
1957                 }
1958         }
1959 }
1960
1961 static int
1962 save_jump_flow(struct enic_flowman *fm,
1963                struct rte_flow *flow,
1964                uint32_t group,
1965                struct fm_tcam_match_entry *match,
1966                struct fm_action *action)
1967 {
1968         struct enic_fm_jump_flow *j;
1969
1970         ENICPMD_FUNC_TRACE();
1971         j = calloc(1, sizeof(struct enic_fm_jump_flow));
1972         if (j == NULL)
1973                 return -ENOMEM;
1974         j->flow = flow;
1975         j->group = group;
1976         j->match = *match;
1977         j->action = *action;
1978         TAILQ_INSERT_HEAD(&fm->jump_list, j, list);
1979         ENICPMD_LOG(DEBUG, "saved jump flow: flow=%p group=%u", flow, group);
1980         return 0;
1981 }
1982
1983 static void
1984 __enic_fm_flow_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
1985 {
1986         if (fm_flow->entry_handle != FM_INVALID_HANDLE) {
1987                 enic_fm_entry_free(fm, fm_flow->entry_handle);
1988                 fm_flow->entry_handle = FM_INVALID_HANDLE;
1989         }
1990         if (fm_flow->action_handle != FM_INVALID_HANDLE) {
1991                 enic_fm_action_free(fm, fm_flow->action_handle);
1992                 fm_flow->action_handle = FM_INVALID_HANDLE;
1993         }
1994         enic_fm_counter_free(fm, fm_flow);
1995         if (fm_flow->fet) {
1996                 enic_fet_put(fm, fm_flow->fet);
1997                 fm_flow->fet = NULL;
1998         }
1999 }
2000
2001 static void
2002 enic_fm_flow_free(struct enic_flowman *fm, struct rte_flow *flow)
2003 {
2004         struct enic_fm_flow *steer = flow->fm->hairpin_steer_flow;
2005
2006         if (flow->fm->fet && flow->fm->fet->default_key)
2007                 remove_jump_flow(fm, flow);
2008         __enic_fm_flow_free(fm, flow->fm);
2009         if (steer) {
2010                 __enic_fm_flow_free(fm, steer);
2011                 free(steer);
2012         }
2013         free(flow->fm);
2014         free(flow);
2015 }
2016
2017 static int
2018 enic_fm_add_tcam_entry(struct enic_flowman *fm,
2019                        struct fm_tcam_match_entry *match_in,
2020                        uint64_t *entry_handle,
2021                        uint8_t ingress,
2022                        struct rte_flow_error *error)
2023 {
2024         struct fm_tcam_match_entry *ftm;
2025         uint64_t args[3];
2026         int ret;
2027
2028         ENICPMD_FUNC_TRACE();
2029         /* Copy entry to the command buffer */
2030         ftm = &fm->cmd.va->fm_tcam_match_entry;
2031         memcpy(ftm, match_in, sizeof(*ftm));
2032         /* Add TCAM entry */
2033         args[0] = FM_TCAM_ENTRY_INSTALL;
2034         args[1] = ingress ? fm->ig_tcam_hndl : fm->eg_tcam_hndl;
2035         args[2] = fm->cmd.pa;
2036         ret = flowman_cmd(fm, args, 3);
2037         if (ret != 0) {
2038                 ENICPMD_LOG(ERR, "cannot add %s TCAM entry: rc=%d",
2039                             ingress ? "ingress" : "egress", ret);
2040                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2041                         NULL, "enic: devcmd(tcam-entry-install)");
2042                 return ret;
2043         }
2044         ENICPMD_LOG(DEBUG, "installed %s TCAM entry: handle=0x%" PRIx64,
2045                     ingress ? "ingress" : "egress", (uint64_t)args[0]);
2046         *entry_handle = args[0];
2047         return 0;
2048 }
2049
2050 static int
2051 enic_fm_add_exact_entry(struct enic_flowman *fm,
2052                         struct fm_tcam_match_entry *match_in,
2053                         uint64_t *entry_handle,
2054                         struct enic_fm_fet *fet,
2055                         struct rte_flow_error *error)
2056 {
2057         struct fm_exact_match_entry *fem;
2058         uint64_t args[3];
2059         int ret;
2060
2061         ENICPMD_FUNC_TRACE();
2062         /* The new entry must have the table's key */
2063         if (memcmp(fet->key.fk_hdrset, match_in->ftm_mask.fk_hdrset,
2064                    sizeof(struct fm_header_set) * FM_HDRSET_MAX)) {
2065                 return rte_flow_error_set(error, EINVAL,
2066                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2067                         "enic: key does not match group's key");
2068         }
2069
2070         /* Copy entry to the command buffer */
2071         fem = &fm->cmd.va->fm_exact_match_entry;
2072         /*
2073          * Translate TCAM entry to exact entry. As is only need to drop
2074          * position and mask. The mask is part of the exact match table.
2075          * Position (aka priority) is not supported in the exact match table.
2076          */
2077         fem->fem_data = match_in->ftm_data;
2078         fem->fem_flags = match_in->ftm_flags;
2079         fem->fem_action = match_in->ftm_action;
2080         fem->fem_counter = match_in->ftm_counter;
2081
2082         /* Add exact entry */
2083         args[0] = FM_EXACT_ENTRY_INSTALL;
2084         args[1] = fet->handle;
2085         args[2] = fm->cmd.pa;
2086         ret = flowman_cmd(fm, args, 3);
2087         if (ret != 0) {
2088                 ENICPMD_LOG(ERR, "cannot add %s exact entry: group=%u",
2089                             fet->ingress ? "ingress" : "egress", fet->group);
2090                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2091                         NULL, "enic: devcmd(exact-entry-install)");
2092                 return ret;
2093         }
2094         ENICPMD_LOG(DEBUG, "installed %s exact entry: group=%u"
2095                     " handle=0x%" PRIx64,
2096                     fet->ingress ? "ingress" : "egress", fet->group,
2097                     (uint64_t)args[0]);
2098         *entry_handle = args[0];
2099         return 0;
2100 }
2101
2102 /* Push match-action to the NIC. */
2103 static int
2104 __enic_fm_flow_add_entry(struct enic_flowman *fm,
2105                          struct enic_fm_flow *fm_flow,
2106                          struct fm_tcam_match_entry *match_in,
2107                          struct fm_action *action_in,
2108                          uint32_t group,
2109                          uint8_t ingress,
2110                          struct rte_flow_error *error)
2111 {
2112         struct enic_fm_counter *ctr;
2113         struct fm_action *fma;
2114         uint64_t action_h;
2115         uint64_t entry_h;
2116         uint64_t args[3];
2117         int ret;
2118
2119         ENICPMD_FUNC_TRACE();
2120         /* Allocate action. */
2121         fma = &fm->cmd.va->fm_action;
2122         memcpy(fma, action_in, sizeof(*fma));
2123         args[0] = FM_ACTION_ALLOC;
2124         args[1] = fm->cmd.pa;
2125         ret = flowman_cmd(fm, args, 2);
2126         if (ret != 0) {
2127                 ENICPMD_LOG(ERR, "allocating TCAM table action rc=%d", ret);
2128                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2129                         NULL, "enic: devcmd(action-alloc)");
2130                 return ret;
2131         }
2132         action_h = args[0];
2133         fm_flow->action_handle = action_h;
2134         match_in->ftm_action = action_h;
2135         ENICPMD_LOG(DEBUG, "action allocated: handle=0x%" PRIx64, action_h);
2136
2137         /* Allocate counter if requested. */
2138         if (match_in->ftm_flags & FMEF_COUNTER) {
2139                 ret = enic_fm_counter_alloc(fm, error, &ctr);
2140                 if (ret) /* error has been filled in */
2141                         return ret;
2142                 fm_flow->counter_valid = true;
2143                 fm_flow->counter = ctr;
2144                 match_in->ftm_counter = ctr->handle;
2145         }
2146
2147         /*
2148          * Get the group's table (either TCAM or exact match table) and
2149          * add entry to it. If we use the exact match table, the handler
2150          * will translate the TCAM entry (match_in) to the appropriate
2151          * exact match entry and use that instead.
2152          */
2153         entry_h = FM_INVALID_HANDLE;
2154         if (group == FM_TCAM_RTE_GROUP) {
2155                 ret = enic_fm_add_tcam_entry(fm, match_in, &entry_h, ingress,
2156                                              error);
2157                 if (ret)
2158                         return ret;
2159                 /* Jump action might have a ref to fet */
2160                 fm_flow->fet = fm->fet;
2161                 fm->fet = NULL;
2162         } else {
2163                 struct enic_fm_fet *fet = NULL;
2164
2165                 ret = enic_fet_get(fm, group, ingress,
2166                                    &match_in->ftm_mask, &fet, error);
2167                 if (ret)
2168                         return ret;
2169                 fm_flow->fet = fet;
2170                 ret = enic_fm_add_exact_entry(fm, match_in, &entry_h, fet,
2171                                               error);
2172                 if (ret)
2173                         return ret;
2174         }
2175         /* Clear counter after adding entry, as it requires in-use counter */
2176         if (fm_flow->counter_valid) {
2177                 ret = enic_fm_counter_zero(fm, fm_flow->counter);
2178                 if (ret)
2179                         return ret;
2180         }
2181         fm_flow->entry_handle = entry_h;
2182         return 0;
2183 }
2184
2185 /* Push match-action to the NIC. */
2186 static struct rte_flow *
2187 enic_fm_flow_add_entry(struct enic_flowman *fm,
2188                        struct fm_tcam_match_entry *match_in,
2189                        struct fm_action *action_in,
2190                        const struct rte_flow_attr *attrs,
2191                        struct rte_flow_error *error)
2192 {
2193         struct enic_fm_flow *fm_flow;
2194         struct rte_flow *flow;
2195
2196         ENICPMD_FUNC_TRACE();
2197         match_in->ftm_position = attrs->priority;
2198         enic_fm_dump_tcam_entry(match_in, action_in, attrs->ingress);
2199         flow = calloc(1, sizeof(*flow));
2200         fm_flow = calloc(1, sizeof(*fm_flow));
2201         if (flow == NULL || fm_flow == NULL) {
2202                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
2203                         NULL, "enic: cannot allocate rte_flow");
2204                 free(flow);
2205                 free(fm_flow);
2206                 return NULL;
2207         }
2208         flow->fm = fm_flow;
2209         fm_flow->action_handle = FM_INVALID_HANDLE;
2210         fm_flow->entry_handle = FM_INVALID_HANDLE;
2211         if (__enic_fm_flow_add_entry(fm, fm_flow, match_in, action_in,
2212                                      attrs->group, attrs->ingress, error)) {
2213                 enic_fm_flow_free(fm, flow);
2214                 return NULL;
2215         }
2216         return flow;
2217 }
2218
2219 static void
2220 convert_jump_flows(struct enic_flowman *fm, struct enic_fm_fet *fet,
2221                    struct rte_flow_error *error)
2222 {
2223         struct enic_fm_flow *fm_flow;
2224         struct enic_fm_jump_flow *j;
2225         struct fm_action *fma;
2226         uint32_t group;
2227
2228         ENICPMD_FUNC_TRACE();
2229         /*
2230          * Find the saved flows that should jump to the new table (fet).
2231          * Then delete the old TCAM entry that jumps to the default table,
2232          * and add a new one that jumps to the new table.
2233          */
2234         group = fet->group;
2235         j = find_jump_flow(fm, group);
2236         while (j) {
2237                 ENICPMD_LOG(DEBUG, "convert jump flow: flow=%p group=%u",
2238                             j->flow, group);
2239                 /* Delete old entry */
2240                 fm_flow = j->flow->fm;
2241                 __enic_fm_flow_free(fm, fm_flow);
2242
2243                 /* Add new entry */
2244                 fma = &j->action;
2245                 fma->fma_action_ops[0].exact.handle = fet->handle;
2246                 if (__enic_fm_flow_add_entry(fm, fm_flow, &j->match, fma,
2247                         FM_TCAM_RTE_GROUP, fet->ingress, error)) {
2248                         /* Cannot roll back changes at the moment */
2249                         ENICPMD_LOG(ERR, "cannot convert jump flow: flow=%p",
2250                                     j->flow);
2251                 } else {
2252                         fm_flow->fet = fet;
2253                         fet->ref++;
2254                         ENICPMD_LOG(DEBUG, "convert ok: group=%u ref=%u",
2255                                     fet->group, fet->ref);
2256                 }
2257
2258                 TAILQ_REMOVE(&fm->jump_list, j, list);
2259                 free(j);
2260                 j = find_jump_flow(fm, group);
2261         }
2262 }
2263
2264 static int
2265 add_hairpin_steer(struct enic_flowman *fm, struct rte_flow *flow,
2266                   struct rte_flow_error *error)
2267 {
2268         struct fm_tcam_match_entry *fm_tcam_entry;
2269         struct enic_fm_flow *fm_flow;
2270         struct fm_action *fm_action;
2271         struct fm_action_op fm_op;
2272         int ret;
2273
2274         ENICPMD_FUNC_TRACE();
2275         fm_flow = calloc(1, sizeof(*fm_flow));
2276         if (fm_flow == NULL) {
2277                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
2278                         NULL, "enic: cannot allocate rte_flow");
2279                 return -ENOMEM;
2280         }
2281         /* Original egress hairpin flow */
2282         fm_tcam_entry = &fm->tcam_entry;
2283         fm_action = &fm->action;
2284         /* Use the match pattern of the egress flow as is, without counters */
2285         fm_tcam_entry->ftm_flags &= ~FMEF_COUNTER;
2286         /* The only action is steer to vnic */
2287         fm->action_op_count = 0;
2288         memset(fm_action, 0, sizeof(*fm_action));
2289         memset(&fm_op, 0, sizeof(fm_op));
2290         /* Always to queue 0 for now */
2291         fm_op.fa_op = FMOP_RQ_STEER;
2292         fm_op.rq_steer.rq_index = 0;
2293         fm_op.rq_steer.vnic_handle = fm->hairpin_steer_vnic_h;
2294         ret = enic_fm_append_action_op(fm, &fm_op, error);
2295         if (ret)
2296                 goto error_with_flow;
2297         ENICPMD_LOG(DEBUG, "add steer op");
2298         /* Add required END */
2299         memset(&fm_op, 0, sizeof(fm_op));
2300         fm_op.fa_op = FMOP_END;
2301         ret = enic_fm_append_action_op(fm, &fm_op, error);
2302         if (ret)
2303                 goto error_with_flow;
2304         /* Add the ingress flow */
2305         fm_flow->action_handle = FM_INVALID_HANDLE;
2306         fm_flow->entry_handle = FM_INVALID_HANDLE;
2307         ret = __enic_fm_flow_add_entry(fm, fm_flow, fm_tcam_entry, fm_action,
2308                                        FM_TCAM_RTE_GROUP, 1 /* ingress */, error);
2309         if (ret) {
2310                 ENICPMD_LOG(ERR, "cannot add hairpin-steer flow");
2311                 goto error_with_flow;
2312         }
2313         /* The new flow is now the egress flow's paired flow */
2314         flow->fm->hairpin_steer_flow = fm_flow;
2315         return 0;
2316
2317 error_with_flow:
2318         free(fm_flow);
2319         return ret;
2320 }
2321
2322 static void
2323 enic_fm_open_scratch(struct enic_flowman *fm)
2324 {
2325         fm->action_op_count = 0;
2326         fm->fet = NULL;
2327         fm->need_hairpin_steer = 0;
2328         fm->hairpin_steer_vnic_h = 0;
2329         memset(&fm->tcam_entry, 0, sizeof(fm->tcam_entry));
2330         memset(&fm->action, 0, sizeof(fm->action));
2331 }
2332
2333 static void
2334 enic_fm_close_scratch(struct enic_flowman *fm)
2335 {
2336         if (fm->fet) {
2337                 enic_fet_put(fm, fm->fet);
2338                 fm->fet = NULL;
2339         }
2340         fm->action_op_count = 0;
2341 }
2342
2343 static int
2344 enic_fm_flow_validate(struct rte_eth_dev *dev,
2345                       const struct rte_flow_attr *attrs,
2346                       const struct rte_flow_item pattern[],
2347                       const struct rte_flow_action actions[],
2348                       struct rte_flow_error *error)
2349 {
2350         struct fm_tcam_match_entry *fm_tcam_entry;
2351         struct fm_action *fm_action;
2352         struct enic_flowman *fm;
2353         int ret;
2354
2355         ENICPMD_FUNC_TRACE();
2356         fm = begin_fm(pmd_priv(dev));
2357         if (fm == NULL)
2358                 return -ENOTSUP;
2359         enic_fm_open_scratch(fm);
2360         ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2361         if (!ret) {
2362                 fm_tcam_entry = &fm->tcam_entry;
2363                 fm_action = &fm->action;
2364                 enic_fm_dump_tcam_entry(fm_tcam_entry, fm_action,
2365                                         attrs->ingress);
2366         }
2367         enic_fm_close_scratch(fm);
2368         end_fm(fm);
2369         return ret;
2370 }
2371
2372 static int
2373 enic_fm_flow_query_count(struct rte_eth_dev *dev,
2374                          struct rte_flow *flow, void *data,
2375                          struct rte_flow_error *error)
2376 {
2377         struct rte_flow_query_count *query;
2378         struct enic_fm_flow *fm_flow;
2379         struct enic_flowman *fm;
2380         uint64_t args[3];
2381         int rc;
2382
2383         ENICPMD_FUNC_TRACE();
2384         fm = begin_fm(pmd_priv(dev));
2385         query = data;
2386         fm_flow = flow->fm;
2387         if (!fm_flow->counter_valid) {
2388                 rc = rte_flow_error_set(error, ENOTSUP,
2389                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2390                         "enic: flow does not have counter");
2391                 goto exit;
2392         }
2393
2394         args[0] = FM_COUNTER_QUERY;
2395         args[1] = fm_flow->counter->handle;
2396         args[2] = query->reset;
2397         rc = flowman_cmd(fm, args, 3);
2398         if (rc) {
2399                 ENICPMD_LOG(ERR, "cannot query counter: rc=%d handle=0x%x",
2400                             rc, fm_flow->counter->handle);
2401                 goto exit;
2402         }
2403         query->hits_set = 1;
2404         query->hits = args[0];
2405         query->bytes_set = 1;
2406         query->bytes = args[1];
2407         rc = 0;
2408 exit:
2409         end_fm(fm);
2410         return rc;
2411 }
2412
2413 static int
2414 enic_fm_flow_query(struct rte_eth_dev *dev,
2415                    struct rte_flow *flow,
2416                    const struct rte_flow_action *actions,
2417                    void *data,
2418                    struct rte_flow_error *error)
2419 {
2420         int ret = 0;
2421
2422         ENICPMD_FUNC_TRACE();
2423         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2424                 switch (actions->type) {
2425                 case RTE_FLOW_ACTION_TYPE_VOID:
2426                         break;
2427                 case RTE_FLOW_ACTION_TYPE_COUNT:
2428                         ret = enic_fm_flow_query_count(dev, flow, data, error);
2429                         break;
2430                 default:
2431                         return rte_flow_error_set(error, ENOTSUP,
2432                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2433                                                   actions,
2434                                                   "action not supported");
2435                 }
2436                 if (ret < 0)
2437                         return ret;
2438         }
2439         return 0;
2440 }
2441
2442 static struct rte_flow *
2443 enic_fm_flow_create(struct rte_eth_dev *dev,
2444                     const struct rte_flow_attr *attrs,
2445                     const struct rte_flow_item pattern[],
2446                     const struct rte_flow_action actions[],
2447                     struct rte_flow_error *error)
2448 {
2449         struct fm_tcam_match_entry *fm_tcam_entry;
2450         struct fm_action *fm_action;
2451         struct enic_flowman *fm;
2452         struct enic_fm_fet *fet;
2453         struct rte_flow *flow;
2454         struct enic *enic;
2455         int ret;
2456
2457         ENICPMD_FUNC_TRACE();
2458         enic = pmd_priv(dev);
2459         fm = begin_fm(enic);
2460         if (fm == NULL) {
2461                 rte_flow_error_set(error, ENOTSUP,
2462                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2463                         "flowman is not initialized");
2464                 return NULL;
2465         }
2466         enic_fm_open_scratch(fm);
2467         flow = NULL;
2468         ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2469         if (ret < 0)
2470                 goto error_with_scratch;
2471         fm_tcam_entry = &fm->tcam_entry;
2472         fm_action = &fm->action;
2473         flow = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2474                                       attrs, error);
2475         if (flow) {
2476                 /* Add ingress rule that pairs with hairpin rule */
2477                 if (fm->need_hairpin_steer) {
2478                         ret = add_hairpin_steer(fm, flow, error);
2479                         if (ret) {
2480                                 enic_fm_flow_free(fm, flow);
2481                                 flow = NULL;
2482                                 goto error_with_scratch;
2483                         }
2484                 }
2485                 LIST_INSERT_HEAD(&enic->flows, flow, next);
2486                 fet = flow->fm->fet;
2487                 if (fet && fet->default_key) {
2488                         /*
2489                          * Jump to non-existent group? Save the relevant info
2490                          * so we can convert this flow when that group
2491                          * materializes.
2492                          */
2493                         save_jump_flow(fm, flow, fet->group,
2494                                        fm_tcam_entry, fm_action);
2495                 } else if (fet && fet->ref == 1) {
2496                         /*
2497                          * A new table is created. Convert the saved flows
2498                          * that should jump to this group.
2499                          */
2500                         convert_jump_flows(fm, fet, error);
2501                 }
2502         }
2503
2504 error_with_scratch:
2505         enic_fm_close_scratch(fm);
2506         end_fm(fm);
2507         return flow;
2508 }
2509
2510 static int
2511 enic_fm_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
2512                      __rte_unused struct rte_flow_error *error)
2513 {
2514         struct enic *enic = pmd_priv(dev);
2515         struct enic_flowman *fm;
2516
2517         ENICPMD_FUNC_TRACE();
2518         fm = begin_fm(enic);
2519         if (fm == NULL)
2520                 return 0;
2521         LIST_REMOVE(flow, next);
2522         enic_fm_flow_free(fm, flow);
2523         end_fm(fm);
2524         return 0;
2525 }
2526
2527 static int
2528 enic_fm_flow_flush(struct rte_eth_dev *dev,
2529                    __rte_unused struct rte_flow_error *error)
2530 {
2531         LIST_HEAD(enic_flows, rte_flow) internal;
2532         struct enic_fm_flow *fm_flow;
2533         struct enic_flowman *fm;
2534         struct rte_flow *flow;
2535         struct enic *enic = pmd_priv(dev);
2536
2537         ENICPMD_FUNC_TRACE();
2538
2539         fm = begin_fm(enic);
2540         if (fm == NULL)
2541                 return 0;
2542         /* Destroy all non-internal flows */
2543         LIST_INIT(&internal);
2544         while (!LIST_EMPTY(&enic->flows)) {
2545                 flow = LIST_FIRST(&enic->flows);
2546                 fm_flow = flow->fm;
2547                 LIST_REMOVE(flow, next);
2548                 if (flow->internal) {
2549                         LIST_INSERT_HEAD(&internal, flow, next);
2550                         continue;
2551                 }
2552                 /*
2553                  * If tables are null, then vNIC is closing, and the firmware
2554                  * has already cleaned up flowman state. So do not try to free
2555                  * resources, as it only causes errors.
2556                  */
2557                 if (fm->ig_tcam_hndl == FM_INVALID_HANDLE) {
2558                         fm_flow->entry_handle = FM_INVALID_HANDLE;
2559                         fm_flow->action_handle = FM_INVALID_HANDLE;
2560                         fm_flow->fet = NULL;
2561                 }
2562                 enic_fm_flow_free(fm, flow);
2563         }
2564         while (!LIST_EMPTY(&internal)) {
2565                 flow = LIST_FIRST(&internal);
2566                 LIST_REMOVE(flow, next);
2567                 LIST_INSERT_HEAD(&enic->flows, flow, next);
2568         }
2569         end_fm(fm);
2570         return 0;
2571 }
2572
2573 static int
2574 enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle)
2575 {
2576         uint64_t args[2];
2577         int rc;
2578
2579         args[0] = FM_MATCH_TABLE_FREE;
2580         args[1] = handle;
2581         rc = flowman_cmd(fm, args, 2);
2582         if (rc)
2583                 ENICPMD_LOG(ERR, "cannot free table: rc=%d handle=0x%" PRIx64,
2584                             rc, handle);
2585         return rc;
2586 }
2587
2588 static int
2589 enic_fm_tcam_tbl_alloc(struct enic_flowman *fm, uint32_t direction,
2590                         uint32_t max_entries, uint64_t *handle)
2591 {
2592         struct fm_tcam_match_table *tcam_tbl;
2593         uint64_t args[2];
2594         int rc;
2595
2596         ENICPMD_FUNC_TRACE();
2597         tcam_tbl = &fm->cmd.va->fm_tcam_match_table;
2598         tcam_tbl->ftt_direction = direction;
2599         tcam_tbl->ftt_stage = FM_STAGE_LAST;
2600         tcam_tbl->ftt_max_entries = max_entries;
2601         args[0] = FM_TCAM_TABLE_ALLOC;
2602         args[1] = fm->cmd.pa;
2603         rc = flowman_cmd(fm, args, 2);
2604         if (rc) {
2605                 ENICPMD_LOG(ERR, "cannot alloc %s TCAM table: rc=%d",
2606                             (direction == FM_INGRESS) ? "IG" : "EG", rc);
2607                 return rc;
2608         }
2609         *handle = args[0];
2610         ENICPMD_LOG(DEBUG, "%s TCAM table allocated, handle=0x%" PRIx64,
2611                     (direction == FM_INGRESS) ? "IG" : "EG", *handle);
2612         return 0;
2613 }
2614
2615 static int
2616 enic_fm_init_counters(struct enic_flowman *fm)
2617 {
2618         ENICPMD_FUNC_TRACE();
2619         SLIST_INIT(&fm->counters);
2620         return enic_fm_more_counters(fm);
2621 }
2622
2623 static void
2624 enic_fm_free_all_counters(struct enic_flowman *fm)
2625 {
2626         uint64_t args[2];
2627         int rc;
2628
2629         args[0] = FM_COUNTER_BRK;
2630         args[1] = 0;
2631         rc = flowman_cmd(fm, args, 2);
2632         if (rc != 0)
2633                 ENICPMD_LOG(ERR, "cannot free counters: rc=%d", rc);
2634         rte_free(fm->counter_stack);
2635 }
2636
2637 static int
2638 enic_fm_alloc_tcam_tables(struct enic_flowman *fm)
2639 {
2640         int rc;
2641
2642         ENICPMD_FUNC_TRACE();
2643         rc = enic_fm_tcam_tbl_alloc(fm, FM_INGRESS, FM_MAX_TCAM_TABLE_SIZE,
2644                                     &fm->ig_tcam_hndl);
2645         if (rc)
2646                 return rc;
2647         rc = enic_fm_tcam_tbl_alloc(fm, FM_EGRESS, FM_MAX_TCAM_TABLE_SIZE,
2648                                     &fm->eg_tcam_hndl);
2649         return rc;
2650 }
2651
2652 static void
2653 enic_fm_free_tcam_tables(struct enic_flowman *fm)
2654 {
2655         ENICPMD_FUNC_TRACE();
2656         if (fm->ig_tcam_hndl) {
2657                 ENICPMD_LOG(DEBUG, "free IG TCAM table handle=0x%" PRIx64,
2658                             fm->ig_tcam_hndl);
2659                 enic_fm_tbl_free(fm, fm->ig_tcam_hndl);
2660                 fm->ig_tcam_hndl = FM_INVALID_HANDLE;
2661         }
2662         if (fm->eg_tcam_hndl) {
2663                 ENICPMD_LOG(DEBUG, "free EG TCAM table handle=0x%" PRIx64,
2664                             fm->eg_tcam_hndl);
2665                 enic_fm_tbl_free(fm, fm->eg_tcam_hndl);
2666                 fm->eg_tcam_hndl = FM_INVALID_HANDLE;
2667         }
2668 }
2669
2670 int
2671 enic_fm_init(struct enic *enic)
2672 {
2673         const struct rte_pci_addr *addr;
2674         struct enic_flowman *fm;
2675         uint8_t name[RTE_MEMZONE_NAMESIZE];
2676         int rc;
2677
2678         if (enic->flow_filter_mode != FILTER_FLOWMAN)
2679                 return 0;
2680         ENICPMD_FUNC_TRACE();
2681         /* Get vnic handle and save for port-id action */
2682         if (enic_is_vf_rep(enic))
2683                 addr = &VF_ENIC_TO_VF_REP(enic)->bdf;
2684         else
2685                 addr = &RTE_ETH_DEV_TO_PCI(enic->rte_dev)->addr;
2686         rc = enic_fm_find_vnic(enic, addr, &enic->fm_vnic_handle);
2687         if (rc) {
2688                 ENICPMD_LOG(ERR, "cannot find vnic handle for %x:%x:%x",
2689                             addr->bus, addr->devid, addr->function);
2690                 return rc;
2691         }
2692         /* Save UIF for egport action */
2693         enic->fm_vnic_uif = vnic_dev_uif(enic->vdev);
2694         ENICPMD_LOG(DEBUG, "uif %u", enic->fm_vnic_uif);
2695         /* Nothing else to do for representor. It will share the PF flowman */
2696         if (enic_is_vf_rep(enic))
2697                 return 0;
2698         fm = calloc(1, sizeof(*fm));
2699         if (fm == NULL) {
2700                 ENICPMD_LOG(ERR, "cannot alloc flowman struct");
2701                 return -ENOMEM;
2702         }
2703         fm->owner_enic = enic;
2704         rte_spinlock_init(&fm->lock);
2705         TAILQ_INIT(&fm->fet_list);
2706         TAILQ_INIT(&fm->jump_list);
2707         /* Allocate host memory for flowman commands */
2708         snprintf((char *)name, sizeof(name), "fm-cmd-%s", enic->bdf_name);
2709         fm->cmd.va = enic_alloc_consistent(enic,
2710                 sizeof(union enic_flowman_cmd_mem), &fm->cmd.pa, name);
2711         if (!fm->cmd.va) {
2712                 ENICPMD_LOG(ERR, "cannot allocate flowman command memory");
2713                 rc = -ENOMEM;
2714                 goto error_fm;
2715         }
2716         /* Allocate TCAM tables upfront as they are the main tables */
2717         rc = enic_fm_alloc_tcam_tables(fm);
2718         if (rc) {
2719                 ENICPMD_LOG(ERR, "cannot alloc TCAM tables");
2720                 goto error_cmd;
2721         }
2722         /* Then a number of counters */
2723         rc = enic_fm_init_counters(fm);
2724         if (rc) {
2725                 ENICPMD_LOG(ERR, "cannot alloc counters");
2726                 goto error_tables;
2727         }
2728         /*
2729          * One default exact match table for each direction. We hold onto
2730          * it until close.
2731          */
2732         rc = enic_fet_alloc(fm, 1, NULL, 128, &fm->default_ig_fet);
2733         if (rc) {
2734                 ENICPMD_LOG(ERR, "cannot alloc default IG exact match table");
2735                 goto error_counters;
2736         }
2737         fm->default_ig_fet->ref = 1;
2738         rc = enic_fet_alloc(fm, 0, NULL, 128, &fm->default_eg_fet);
2739         if (rc) {
2740                 ENICPMD_LOG(ERR, "cannot alloc default EG exact match table");
2741                 goto error_ig_fet;
2742         }
2743         fm->default_eg_fet->ref = 1;
2744         fm->vf_rep_tag = FM_VF_REP_TAG;
2745         enic->fm = fm;
2746         return 0;
2747
2748 error_ig_fet:
2749         enic_fet_free(fm, fm->default_ig_fet);
2750 error_counters:
2751         enic_fm_free_all_counters(fm);
2752 error_tables:
2753         enic_fm_free_tcam_tables(fm);
2754 error_cmd:
2755         enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
2756                 fm->cmd.va, fm->cmd.pa);
2757 error_fm:
2758         free(fm);
2759         return rc;
2760 }
2761
2762 void
2763 enic_fm_destroy(struct enic *enic)
2764 {
2765         struct enic_flowman *fm;
2766         struct enic_fm_fet *fet;
2767
2768         ENICPMD_FUNC_TRACE();
2769         if (enic_is_vf_rep(enic)) {
2770                 delete_rep_flows(enic);
2771                 return;
2772         }
2773         if (enic->fm == NULL)
2774                 return;
2775         fm = enic->fm;
2776         enic_fet_free(fm, fm->default_eg_fet);
2777         enic_fet_free(fm, fm->default_ig_fet);
2778         /* Free all exact match tables still open */
2779         while (!TAILQ_EMPTY(&fm->fet_list)) {
2780                 fet = TAILQ_FIRST(&fm->fet_list);
2781                 enic_fet_free(fm, fet);
2782         }
2783         enic_fm_free_tcam_tables(fm);
2784         enic_fm_free_all_counters(fm);
2785         enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
2786                 fm->cmd.va, fm->cmd.pa);
2787         fm->cmd.va = NULL;
2788         free(fm);
2789         enic->fm = NULL;
2790 }
2791
2792 int
2793 enic_fm_allocate_switch_domain(struct enic *pf)
2794 {
2795         const struct rte_pci_addr *cur_a, *prev_a;
2796         struct rte_eth_dev *dev;
2797         struct enic *cur, *prev;
2798         uint16_t domain_id;
2799         uint64_t vnic_h;
2800         uint16_t pid;
2801         int ret;
2802
2803         ENICPMD_FUNC_TRACE();
2804         if (enic_is_vf_rep(pf))
2805                 return -EINVAL;
2806         cur = pf;
2807         cur_a = &RTE_ETH_DEV_TO_PCI(cur->rte_dev)->addr;
2808         /* Go through ports and find another PF that is on the same adapter */
2809         RTE_ETH_FOREACH_DEV(pid) {
2810                 dev = &rte_eth_devices[pid];
2811                 if (!dev_is_enic(dev))
2812                         continue;
2813                 if (dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)
2814                         continue;
2815                 if (dev == cur->rte_dev)
2816                         continue;
2817                 /* dev is another PF. Is it on the same adapter? */
2818                 prev = pmd_priv(dev);
2819                 prev_a = &RTE_ETH_DEV_TO_PCI(dev)->addr;
2820                 if (!enic_fm_find_vnic(cur, prev_a, &vnic_h)) {
2821                         ENICPMD_LOG(DEBUG, "Port %u (PF BDF %x:%x:%x) and port %u (PF BDF %x:%x:%x domain %u) are on the same VIC",
2822                                 cur->rte_dev->data->port_id,
2823                                 cur_a->bus, cur_a->devid, cur_a->function,
2824                                 dev->data->port_id,
2825                                 prev_a->bus, prev_a->devid, prev_a->function,
2826                                 prev->switch_domain_id);
2827                         cur->switch_domain_id = prev->switch_domain_id;
2828                         return 0;
2829                 }
2830         }
2831         ret = rte_eth_switch_domain_alloc(&domain_id);
2832         if (ret) {
2833                 ENICPMD_LOG(WARNING, "failed to allocate switch domain for device %d",
2834                             ret);
2835         }
2836         cur->switch_domain_id = domain_id;
2837         ENICPMD_LOG(DEBUG, "Port %u (PF BDF %x:%x:%x) is the 1st PF on the VIC. Allocated switch domain id %u",
2838                     cur->rte_dev->data->port_id,
2839                     cur_a->bus, cur_a->devid, cur_a->function,
2840                     domain_id);
2841         return ret;
2842 }
2843
2844 const struct rte_flow_ops enic_fm_flow_ops = {
2845         .validate = enic_fm_flow_validate,
2846         .create = enic_fm_flow_create,
2847         .destroy = enic_fm_flow_destroy,
2848         .flush = enic_fm_flow_flush,
2849         .query = enic_fm_flow_query,
2850 };
2851
2852 /* Add a high priority flow that loops representor packets to VF */
2853 int
2854 enic_fm_add_rep2vf_flow(struct enic_vf_representor *vf)
2855 {
2856         struct fm_tcam_match_entry *fm_tcam_entry;
2857         struct rte_flow *flow0, *flow1;
2858         struct fm_action *fm_action;
2859         struct rte_flow_error error;
2860         struct rte_flow_attr attrs;
2861         struct fm_action_op fm_op;
2862         struct enic_flowman *fm;
2863         struct enic *pf;
2864         uint8_t tag;
2865
2866         pf = vf->pf;
2867         fm = pf->fm;
2868         tag = fm->vf_rep_tag;
2869         enic_fm_open_scratch(fm);
2870         fm_tcam_entry = &fm->tcam_entry;
2871         fm_action = &fm->action;
2872         /* Egress rule: match WQ ID and tag+hairpin */
2873         fm_tcam_entry->ftm_data.fk_wq_id = vf->pf_wq_idx;
2874         fm_tcam_entry->ftm_mask.fk_wq_id = 0xffff;
2875         fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
2876         memset(&fm_op, 0, sizeof(fm_op));
2877         fm_op.fa_op = FMOP_TAG;
2878         fm_op.tag.tag = tag;
2879         enic_fm_append_action_op(fm, &fm_op, &error);
2880         memset(&fm_op, 0, sizeof(fm_op));
2881         fm_op.fa_op = FMOP_EG_HAIRPIN;
2882         enic_fm_append_action_op(fm, &fm_op, &error);
2883         memset(&fm_op, 0, sizeof(fm_op));
2884         fm_op.fa_op = FMOP_END;
2885         enic_fm_append_action_op(fm, &fm_op, &error);
2886         attrs.group = 0;
2887         attrs.ingress = 0;
2888         attrs.egress = 1;
2889         attrs.priority = FM_HIGHEST_PRIORITY;
2890         flow0 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2891                                        &attrs, &error);
2892         enic_fm_close_scratch(fm);
2893         if (flow0 == NULL) {
2894                 ENICPMD_LOG(ERR, "Cannot create flow 0 for representor->VF");
2895                 return -EINVAL;
2896         }
2897         LIST_INSERT_HEAD(&pf->flows, flow0, next);
2898         /* Make this flow internal, so the user app cannot delete it */
2899         flow0->internal = 1;
2900         ENICPMD_LOG(DEBUG, "representor->VF %d flow created: wq %d -> tag %d hairpin",
2901                     vf->vf_id, vf->pf_wq_idx, tag);
2902
2903         /* Ingress: steer hairpinned to VF RQ 0 */
2904         enic_fm_open_scratch(fm);
2905         fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
2906         fm_tcam_entry->ftm_data.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
2907         fm_tcam_entry->ftm_mask.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
2908         fm_tcam_entry->ftm_data.fk_packet_tag = tag;
2909         fm_tcam_entry->ftm_mask.fk_packet_tag = 0xff;
2910         memset(&fm_op, 0, sizeof(fm_op));
2911         fm_op.fa_op = FMOP_RQ_STEER;
2912         fm_op.rq_steer.rq_index = 0;
2913         fm_op.rq_steer.vnic_handle = vf->enic.fm_vnic_handle;
2914         enic_fm_append_action_op(fm, &fm_op, &error);
2915         memset(&fm_op, 0, sizeof(fm_op));
2916         fm_op.fa_op = FMOP_END;
2917         enic_fm_append_action_op(fm, &fm_op, &error);
2918         attrs.group = 0;
2919         attrs.ingress = 1;
2920         attrs.egress = 0;
2921         attrs.priority = FM_HIGHEST_PRIORITY;
2922         flow1 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2923                                        &attrs, &error);
2924         enic_fm_close_scratch(fm);
2925         if (flow1 == NULL) {
2926                 ENICPMD_LOG(ERR, "Cannot create flow 1 for representor->VF");
2927                 enic_fm_flow_destroy(pf->rte_dev, flow0, &error);
2928                 return -EINVAL;
2929         }
2930         LIST_INSERT_HEAD(&pf->flows, flow1, next);
2931         flow1->internal = 1;
2932         ENICPMD_LOG(DEBUG, "representor->VF %d flow created: tag %d hairpinned -> VF RQ %d",
2933                     vf->vf_id, tag, fm_op.rq_steer.rq_index);
2934         vf->rep2vf_flow[0] = flow0;
2935         vf->rep2vf_flow[1] = flow1;
2936         /* Done with this tag, use a different one next time */
2937         fm->vf_rep_tag++;
2938         return 0;
2939 }
2940
2941 /*
2942  * Add a low priority flow that matches all packets from VF and loops them
2943  * back to the representor.
2944  */
2945 int
2946 enic_fm_add_vf2rep_flow(struct enic_vf_representor *vf)
2947 {
2948         struct fm_tcam_match_entry *fm_tcam_entry;
2949         struct rte_flow *flow0, *flow1;
2950         struct fm_action *fm_action;
2951         struct rte_flow_error error;
2952         struct rte_flow_attr attrs;
2953         struct fm_action_op fm_op;
2954         struct enic_flowman *fm;
2955         struct enic *pf;
2956         uint8_t tag;
2957
2958         pf = vf->pf;
2959         fm = pf->fm;
2960         tag = fm->vf_rep_tag;
2961         enic_fm_open_scratch(fm);
2962         fm_tcam_entry = &fm->tcam_entry;
2963         fm_action = &fm->action;
2964         /* Egress rule: match-any and tag+hairpin */
2965         fm_tcam_entry->ftm_data.fk_wq_id = 0;
2966         fm_tcam_entry->ftm_mask.fk_wq_id = 0xffff;
2967         fm_tcam_entry->ftm_data.fk_wq_vnic = vf->enic.fm_vnic_handle;
2968         fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
2969         memset(&fm_op, 0, sizeof(fm_op));
2970         fm_op.fa_op = FMOP_TAG;
2971         fm_op.tag.tag = tag;
2972         enic_fm_append_action_op(fm, &fm_op, &error);
2973         memset(&fm_op, 0, sizeof(fm_op));
2974         fm_op.fa_op = FMOP_EG_HAIRPIN;
2975         enic_fm_append_action_op(fm, &fm_op, &error);
2976         memset(&fm_op, 0, sizeof(fm_op));
2977         fm_op.fa_op = FMOP_END;
2978         enic_fm_append_action_op(fm, &fm_op, &error);
2979         attrs.group = 0;
2980         attrs.ingress = 0;
2981         attrs.egress = 1;
2982         attrs.priority = FM_LOWEST_PRIORITY;
2983         flow0 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2984                                        &attrs, &error);
2985         enic_fm_close_scratch(fm);
2986         if (flow0 == NULL) {
2987                 ENICPMD_LOG(ERR, "Cannot create flow 0 for VF->representor");
2988                 return -EINVAL;
2989         }
2990         LIST_INSERT_HEAD(&pf->flows, flow0, next);
2991         /* Make this flow internal, so the user app cannot delete it */
2992         flow0->internal = 1;
2993         ENICPMD_LOG(DEBUG, "VF %d->representor flow created: wq %d (low prio) -> tag %d hairpin",
2994                     vf->vf_id, fm_tcam_entry->ftm_data.fk_wq_id, tag);
2995
2996         /* Ingress: steer hairpinned to VF rep RQ */
2997         enic_fm_open_scratch(fm);
2998         fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
2999         fm_tcam_entry->ftm_data.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3000         fm_tcam_entry->ftm_mask.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3001         fm_tcam_entry->ftm_data.fk_packet_tag = tag;
3002         fm_tcam_entry->ftm_mask.fk_packet_tag = 0xff;
3003         memset(&fm_op, 0, sizeof(fm_op));
3004         fm_op.fa_op = FMOP_RQ_STEER;
3005         fm_op.rq_steer.rq_index = vf->pf_rq_sop_idx;
3006         fm_op.rq_steer.vnic_handle = pf->fm_vnic_handle;
3007         enic_fm_append_action_op(fm, &fm_op, &error);
3008         memset(&fm_op, 0, sizeof(fm_op));
3009         fm_op.fa_op = FMOP_END;
3010         enic_fm_append_action_op(fm, &fm_op, &error);
3011         attrs.group = 0;
3012         attrs.ingress = 1;
3013         attrs.egress = 0;
3014         attrs.priority = FM_HIGHEST_PRIORITY;
3015         flow1 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3016                                        &attrs, &error);
3017         enic_fm_close_scratch(fm);
3018         if (flow1 == NULL) {
3019                 ENICPMD_LOG(ERR, "Cannot create flow 1 for VF->representor");
3020                 enic_fm_flow_destroy(pf->rte_dev, flow0, &error);
3021                 return -EINVAL;
3022         }
3023         LIST_INSERT_HEAD(&pf->flows, flow1, next);
3024         flow1->internal = 1;
3025         ENICPMD_LOG(DEBUG, "VF %d->representor flow created: tag %d hairpinned -> PF RQ %d",
3026                     vf->vf_id, tag, vf->pf_rq_sop_idx);
3027         vf->vf2rep_flow[0] = flow0;
3028         vf->vf2rep_flow[1] = flow1;
3029         /* Done with this tag, use a different one next time */
3030         fm->vf_rep_tag++;
3031         return 0;
3032 }
3033
3034 /* Destroy representor flows created by enic_fm_add_{rep2vf,vf2rep}_flow */
3035 static void
3036 delete_rep_flows(struct enic *enic)
3037 {
3038         struct enic_vf_representor *vf;
3039         struct rte_flow_error error;
3040         struct rte_eth_dev *dev;
3041         uint32_t i;
3042
3043         RTE_ASSERT(enic_is_vf_rep(enic));
3044         vf = VF_ENIC_TO_VF_REP(enic);
3045         dev = vf->pf->rte_dev;
3046         for (i = 0; i < ARRAY_SIZE(vf->vf2rep_flow); i++) {
3047                 if (vf->vf2rep_flow[i])
3048                         enic_fm_flow_destroy(dev, vf->vf2rep_flow[i], &error);
3049         }
3050         for (i = 0; i < ARRAY_SIZE(vf->rep2vf_flow); i++) {
3051                 if (vf->rep2vf_flow[i])
3052                         enic_fm_flow_destroy(dev, vf->rep2vf_flow[i], &error);
3053         }
3054 }
3055
3056 static struct enic_flowman *
3057 begin_fm(struct enic *enic)
3058 {
3059         struct enic_vf_representor *vf;
3060         struct enic_flowman *fm;
3061
3062         /* Representor uses PF flowman */
3063         if (enic_is_vf_rep(enic)) {
3064                 vf = VF_ENIC_TO_VF_REP(enic);
3065                 fm = vf->pf->fm;
3066         } else {
3067                 fm = enic->fm;
3068         }
3069         /* Save the API caller and lock if representors exist */
3070         if (fm) {
3071                 if (fm->owner_enic->switchdev_mode)
3072                         rte_spinlock_lock(&fm->lock);
3073                 fm->user_enic = enic;
3074         }
3075         return fm;
3076 }
3077
3078 static void
3079 end_fm(struct enic_flowman *fm)
3080 {
3081         fm->user_enic = NULL;
3082         if (fm->owner_enic->switchdev_mode)
3083                 rte_spinlock_unlock(&fm->lock);
3084 }