drivers/net/enic/enic_fm_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2008-2019 Cisco Systems, Inc.  All rights reserved.
   3  */
   4
   5 #include <errno.h>
   6 #include <stdint.h>
   7 #include <rte_log.h>
   8 #include <rte_ethdev_driver.h>
   9 #include <rte_flow_driver.h>
  10 #include <rte_ether.h>
  11 #include <rte_ip.h>
  12 #include <rte_udp.h>
  13 #include <rte_memzone.h>
  14
  15 #include "enic_compat.h"
  16 #include "enic.h"
  17 #include "vnic_dev.h"
  18 #include "vnic_nic.h"
  19
  20 #define IP_DEFTTL  64   /* from RFC 1340. */
  21 #define IP6_VTC_FLOW 0x60000000
  22
  23 /* Highest Item type supported by Flowman */
  24 #define FM_MAX_ITEM_TYPE RTE_FLOW_ITEM_TYPE_VXLAN
  25
  26 /* Up to 1024 TCAM entries */
  27 #define FM_MAX_TCAM_TABLE_SIZE 1024
  28
  29 /* Up to 4096 entries per exact match table */
  30 #define FM_MAX_EXACT_TABLE_SIZE 4096
  31
  32 /* Number of counters to increase on for each increment */
  33 #define FM_COUNTERS_EXPAND  100
  34
  35 #define FM_INVALID_HANDLE 0
  36
  37 /* Low priority used for implicit VF -> representor flow */
  38 #define FM_LOWEST_PRIORITY 100000
  39
  40 /* High priority used for implicit representor -> VF flow */
  41 #define FM_HIGHEST_PRIORITY 0
  42
  43 /* Tag used for implicit VF <-> representor flows */
  44 #define FM_VF_REP_TAG 1
  45
  46 /*
  47  * Flow exact match tables (FET) in the VIC and rte_flow groups.
  48  * Use a simple scheme to map groups to tables.
  49  * Group 0 uses the single TCAM tables, one for each direction.
  50  * Group 1, 2, ... uses its own exact match table.
  51  *
  52  * The TCAM tables are allocated upfront during init.
  53  *
  54  * Exact match tables are allocated on demand. 3 paths that lead allocations.
  55  *
  56  * 1. Add a flow that jumps from group 0 to group N.
  57  *
  58  * If N does not exist, we allocate an exact match table for it, using
  59  * a dummy key. A key is required for the table.
  60  *
  61  * 2. Add a flow that uses group N.
  62  *
  63  * If N does not exist, we allocate an exact match table for it, using
  64  * the flow's key. Subsequent flows to the same group all should have
  65  * the same key.
  66  *
  67  * Without a jump flow to N, N is not reachable in hardware. No packets
  68  * reach N and match.
  69  *
  70  * 3. Add a flow to an empty group N.
  71  *
  72  * N has been created via (1) and the dummy key. We free that table, allocate
  73  * a new table using the new flow's key. Also re-do the existing jump flow to
  74  * point to the new table.
  75  */
  76 #define FM_TCAM_RTE_GROUP 0
  77
  78 struct enic_fm_fet {
  79         TAILQ_ENTRY(enic_fm_fet) list;
  80         uint32_t group; /* rte_flow group ID */
  81         uint64_t handle; /* Exact match table handle from flowman */
  82         uint8_t ingress;
  83         uint8_t default_key;
  84         int ref; /* Reference count via get/put */
  85         struct fm_key_template key; /* Key associated with the table */
  86 };
  87
  88 struct enic_fm_counter {
  89         SLIST_ENTRY(enic_fm_counter) next;
  90         uint32_t handle;
  91 };
  92
  93 /* rte_flow.fm */
  94 struct enic_fm_flow {
  95         bool counter_valid;
  96         uint64_t entry_handle;
  97         uint64_t action_handle;
  98         struct enic_fm_counter *counter;
  99         struct enic_fm_fet *fet;
 100 };
 101
 102 struct enic_fm_jump_flow {
 103         TAILQ_ENTRY(enic_fm_jump_flow) list;
 104         struct rte_flow *flow;
 105         uint32_t group;
 106         struct fm_tcam_match_entry match;
 107         struct fm_action action;
 108 };
 109
 110 /*
 111  * Flowman uses host memory for commands. This structure is allocated
 112  * in DMA-able memory.
 113  */
 114 union enic_flowman_cmd_mem {
 115         struct fm_tcam_match_table fm_tcam_match_table;
 116         struct fm_exact_match_table fm_exact_match_table;
 117         struct fm_tcam_match_entry fm_tcam_match_entry;
 118         struct fm_exact_match_entry fm_exact_match_entry;
 119         struct fm_action fm_action;
 120 };
 121
 122 /*
 123  * PF has a flowman instance, and VF representors share it with PF.
 124  * PF allocates this structure and owns it. VF representors borrow
 125  * the PF's structure during API calls (e.g. create, query).
 126  */
 127 struct enic_flowman {
 128         struct enic *owner_enic; /* PF */
 129         struct enic *user_enic;  /* API caller (PF or representor) */
 130         /*
 131          * Representors and PF share the same underlying flowman.
 132          * Lock API calls to serialize accesses from them. Only used
 133          * when VF representors are present.
 134          */
 135         rte_spinlock_t lock;
 136         /* Command buffer */
 137         struct {
 138                 union enic_flowman_cmd_mem *va;
 139                 dma_addr_t pa;
 140         } cmd;
 141         /* TCAM tables allocated upfront, used for group 0 */
 142         uint64_t ig_tcam_hndl;
 143         uint64_t eg_tcam_hndl;
 144         /* Counters */
 145         SLIST_HEAD(enic_free_counters, enic_fm_counter) counters;
 146         void *counter_stack;
 147         uint32_t counters_alloced;
 148         /* Exact match tables for groups != 0, dynamically allocated */
 149         TAILQ_HEAD(fet_list, enic_fm_fet) fet_list;
 150         /*
 151          * Default exact match tables used for jump actions to
 152          * non-existent groups.
 153          */
 154         struct enic_fm_fet *default_eg_fet;
 155         struct enic_fm_fet *default_ig_fet;
 156         /* Flows that jump to the default table above */
 157         TAILQ_HEAD(jump_flow_list, enic_fm_jump_flow) jump_list;
 158         /*
 159          * Scratch data used during each invocation of flow_create
 160          * and flow_validate.
 161          */
 162         struct enic_fm_fet *fet;
 163         struct fm_tcam_match_entry tcam_entry;
 164         struct fm_action action;
 165         struct fm_action action_tmp; /* enic_fm_reorder_action_op */
 166         int action_op_count;
 167         /* Tags used for representor flows */
 168         uint8_t vf_rep_tag;
 169 };
 170
 171 static int enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle);
 172 /*
 173  * API functions (create, destroy, validate, flush) call begin_fm()
 174  * upon entering to save the caller enic (PF or VF representor) and
 175  * lock. Upon exit, they call end_fm() to unlock.
 176  */
 177 static struct enic_flowman *begin_fm(struct enic *enic);
 178 static void end_fm(struct enic_flowman *fm);
 179 /* Delete internal flows created for representor paths */
 180 static void delete_rep_flows(struct enic *enic);
 181
 182 /*
 183  * Common arguments passed to copy_item functions. Use this structure
 184  * so we can easily add new arguments.
 185  * item: Item specification.
 186  * fm_tcam_entry: Flowman TCAM match entry.
 187  * header_level: 0 for outer header, 1 for inner header.
 188  */
 189 struct copy_item_args {
 190         const struct rte_flow_item *item;
 191         struct fm_tcam_match_entry *fm_tcam_entry;
 192         uint8_t header_level;
 193 };
 194
 195 /* functions for copying items into flowman match */
 196 typedef int (enic_copy_item_fn)(struct copy_item_args *arg);
 197
 198 /* Info about how to copy items into flowman match */
 199 struct enic_fm_items {
 200         /* Function for copying and validating an item. */
 201         enic_copy_item_fn * const copy_item;
 202         /* List of valid previous items. */
 203         const enum rte_flow_item_type * const prev_items;
 204         /*
 205          * True if it's OK for this item to be the first item. For some NIC
 206          * versions, it's invalid to start the stack above layer 3.
 207          */
 208         const uint8_t valid_start_item;
 209 };
 210
 211 static enic_copy_item_fn enic_fm_copy_item_eth;
 212 static enic_copy_item_fn enic_fm_copy_item_ipv4;
 213 static enic_copy_item_fn enic_fm_copy_item_ipv6;
 214 static enic_copy_item_fn enic_fm_copy_item_raw;
 215 static enic_copy_item_fn enic_fm_copy_item_sctp;
 216 static enic_copy_item_fn enic_fm_copy_item_tcp;
 217 static enic_copy_item_fn enic_fm_copy_item_udp;
 218 static enic_copy_item_fn enic_fm_copy_item_vlan;
 219 static enic_copy_item_fn enic_fm_copy_item_vxlan;
 220
 221 /* Ingress actions */
 222 static const enum rte_flow_action_type enic_fm_supported_ig_actions[] = {
 223         RTE_FLOW_ACTION_TYPE_COUNT,
 224         RTE_FLOW_ACTION_TYPE_DROP,
 225         RTE_FLOW_ACTION_TYPE_FLAG,
 226         RTE_FLOW_ACTION_TYPE_JUMP,
 227         RTE_FLOW_ACTION_TYPE_MARK,
 228         RTE_FLOW_ACTION_TYPE_OF_POP_VLAN,
 229         RTE_FLOW_ACTION_TYPE_PORT_ID,
 230         RTE_FLOW_ACTION_TYPE_PASSTHRU,
 231         RTE_FLOW_ACTION_TYPE_QUEUE,
 232         RTE_FLOW_ACTION_TYPE_RSS,
 233         RTE_FLOW_ACTION_TYPE_VOID,
 234         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
 235         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP,
 236         RTE_FLOW_ACTION_TYPE_END, /* END must be the last entry */
 237 };
 238
 239 /* Egress actions */
 240 static const enum rte_flow_action_type enic_fm_supported_eg_actions[] = {
 241         RTE_FLOW_ACTION_TYPE_COUNT,
 242         RTE_FLOW_ACTION_TYPE_DROP,
 243         RTE_FLOW_ACTION_TYPE_JUMP,
 244         RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN,
 245         RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP,
 246         RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID,
 247         RTE_FLOW_ACTION_TYPE_PASSTHRU,
 248         RTE_FLOW_ACTION_TYPE_VOID,
 249         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
 250         RTE_FLOW_ACTION_TYPE_END,
 251 };
 252
 253 static const struct enic_fm_items enic_fm_items[] = {
 254         [RTE_FLOW_ITEM_TYPE_RAW] = {
 255                 .copy_item = enic_fm_copy_item_raw,
 256                 .valid_start_item = 0,
 257                 .prev_items = (const enum rte_flow_item_type[]) {
 258                                RTE_FLOW_ITEM_TYPE_UDP,
 259                                RTE_FLOW_ITEM_TYPE_END,
 260                 },
 261         },
 262         [RTE_FLOW_ITEM_TYPE_ETH] = {
 263                 .copy_item = enic_fm_copy_item_eth,
 264                 .valid_start_item = 1,
 265                 .prev_items = (const enum rte_flow_item_type[]) {
 266                                RTE_FLOW_ITEM_TYPE_END,
 267                 },
 268         },
 269         [RTE_FLOW_ITEM_TYPE_VLAN] = {
 270                 .copy_item = enic_fm_copy_item_vlan,
 271                 .valid_start_item = 1,
 272                 .prev_items = (const enum rte_flow_item_type[]) {
 273                                RTE_FLOW_ITEM_TYPE_ETH,
 274                                RTE_FLOW_ITEM_TYPE_END,
 275                 },
 276         },
 277         [RTE_FLOW_ITEM_TYPE_IPV4] = {
 278                 .copy_item = enic_fm_copy_item_ipv4,
 279                 .valid_start_item = 1,
 280                 .prev_items = (const enum rte_flow_item_type[]) {
 281                                RTE_FLOW_ITEM_TYPE_ETH,
 282                                RTE_FLOW_ITEM_TYPE_VLAN,
 283                                RTE_FLOW_ITEM_TYPE_END,
 284                 },
 285         },
 286         [RTE_FLOW_ITEM_TYPE_IPV6] = {
 287                 .copy_item = enic_fm_copy_item_ipv6,
 288                 .valid_start_item = 1,
 289                 .prev_items = (const enum rte_flow_item_type[]) {
 290                                RTE_FLOW_ITEM_TYPE_ETH,
 291                                RTE_FLOW_ITEM_TYPE_VLAN,
 292                                RTE_FLOW_ITEM_TYPE_END,
 293                 },
 294         },
 295         [RTE_FLOW_ITEM_TYPE_UDP] = {
 296                 .copy_item = enic_fm_copy_item_udp,
 297                 .valid_start_item = 1,
 298                 .prev_items = (const enum rte_flow_item_type[]) {
 299                                RTE_FLOW_ITEM_TYPE_IPV4,
 300                                RTE_FLOW_ITEM_TYPE_IPV6,
 301                                RTE_FLOW_ITEM_TYPE_END,
 302                 },
 303         },
 304         [RTE_FLOW_ITEM_TYPE_TCP] = {
 305                 .copy_item = enic_fm_copy_item_tcp,
 306                 .valid_start_item = 1,
 307                 .prev_items = (const enum rte_flow_item_type[]) {
 308                                RTE_FLOW_ITEM_TYPE_IPV4,
 309                                RTE_FLOW_ITEM_TYPE_IPV6,
 310                                RTE_FLOW_ITEM_TYPE_END,
 311                 },
 312         },
 313         [RTE_FLOW_ITEM_TYPE_SCTP] = {
 314                 .copy_item = enic_fm_copy_item_sctp,
 315                 .valid_start_item = 0,
 316                 .prev_items = (const enum rte_flow_item_type[]) {
 317                                RTE_FLOW_ITEM_TYPE_IPV4,
 318                                RTE_FLOW_ITEM_TYPE_IPV6,
 319                                RTE_FLOW_ITEM_TYPE_END,
 320                 },
 321         },
 322         [RTE_FLOW_ITEM_TYPE_VXLAN] = {
 323                 .copy_item = enic_fm_copy_item_vxlan,
 324                 .valid_start_item = 1,
 325                 .prev_items = (const enum rte_flow_item_type[]) {
 326                                RTE_FLOW_ITEM_TYPE_UDP,
 327                                RTE_FLOW_ITEM_TYPE_END,
 328                 },
 329         },
 330 };
 331
 332 static int
 333 enic_fm_copy_item_eth(struct copy_item_args *arg)
 334 {
 335         const struct rte_flow_item *item = arg->item;
 336         const struct rte_flow_item_eth *spec = item->spec;
 337         const struct rte_flow_item_eth *mask = item->mask;
 338         const uint8_t lvl = arg->header_level;
 339         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 340         struct fm_header_set *fm_data, *fm_mask;
 341
 342         ENICPMD_FUNC_TRACE();
 343         /* Match all if no spec */
 344         if (!spec)
 345                 return 0;
 346         if (!mask)
 347                 mask = &rte_flow_item_eth_mask;
 348         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 349         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 350         fm_data->fk_header_select |= FKH_ETHER;
 351         fm_mask->fk_header_select |= FKH_ETHER;
 352         memcpy(&fm_data->l2.eth, spec, sizeof(*spec));
 353         memcpy(&fm_mask->l2.eth, mask, sizeof(*mask));
 354         return 0;
 355 }
 356
 357 static int
 358 enic_fm_copy_item_vlan(struct copy_item_args *arg)
 359 {
 360         const struct rte_flow_item *item = arg->item;
 361         const struct rte_flow_item_vlan *spec = item->spec;
 362         const struct rte_flow_item_vlan *mask = item->mask;
 363         const uint8_t lvl = arg->header_level;
 364         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 365         struct fm_header_set *fm_data, *fm_mask;
 366         struct rte_ether_hdr *eth_mask;
 367         struct rte_ether_hdr *eth_val;
 368         uint32_t meta;
 369
 370         ENICPMD_FUNC_TRACE();
 371         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 372         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 373         /* Outer and inner packet vlans need different flags */
 374         meta = FKM_VLAN_PRES;
 375         if (lvl > 0)
 376                 meta = FKM_QTAG;
 377         fm_data->fk_metadata |= meta;
 378         fm_mask->fk_metadata |= meta;
 379
 380         /* Match all if no spec */
 381         if (!spec)
 382                 return 0;
 383         if (!mask)
 384                 mask = &rte_flow_item_vlan_mask;
 385
 386         eth_mask = (void *)&fm_mask->l2.eth;
 387         eth_val = (void *)&fm_data->l2.eth;
 388
 389         /* Outer TPID cannot be matched */
 390         if (eth_mask->ether_type)
 391                 return -ENOTSUP;
 392
 393         /*
 394          * When packet matching, the VIC always compares vlan-stripped
 395          * L2, regardless of vlan stripping settings. So, the inner type
 396          * from vlan becomes the ether type of the eth header.
 397          */
 398         eth_mask->ether_type = mask->inner_type;
 399         eth_val->ether_type = spec->inner_type;
 400         fm_data->fk_header_select |= FKH_ETHER | FKH_QTAG;
 401         fm_mask->fk_header_select |= FKH_ETHER | FKH_QTAG;
 402         fm_data->fk_vlan = rte_be_to_cpu_16(spec->tci);
 403         fm_mask->fk_vlan = rte_be_to_cpu_16(mask->tci);
 404         return 0;
 405 }
 406
 407 static int
 408 enic_fm_copy_item_ipv4(struct copy_item_args *arg)
 409 {
 410         const struct rte_flow_item *item = arg->item;
 411         const struct rte_flow_item_ipv4 *spec = item->spec;
 412         const struct rte_flow_item_ipv4 *mask = item->mask;
 413         const uint8_t lvl = arg->header_level;
 414         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 415         struct fm_header_set *fm_data, *fm_mask;
 416
 417         ENICPMD_FUNC_TRACE();
 418         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 419         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 420         fm_data->fk_metadata |= FKM_IPV4;
 421         fm_mask->fk_metadata |= FKM_IPV4;
 422
 423         if (!spec)
 424                 return 0;
 425         if (!mask)
 426                 mask = &rte_flow_item_ipv4_mask;
 427
 428         fm_data->fk_header_select |= FKH_IPV4;
 429         fm_mask->fk_header_select |= FKH_IPV4;
 430         memcpy(&fm_data->l3.ip4, spec, sizeof(*spec));
 431         memcpy(&fm_mask->l3.ip4, mask, sizeof(*mask));
 432         return 0;
 433 }
 434
 435 static int
 436 enic_fm_copy_item_ipv6(struct copy_item_args *arg)
 437 {
 438         const struct rte_flow_item *item = arg->item;
 439         const struct rte_flow_item_ipv6 *spec = item->spec;
 440         const struct rte_flow_item_ipv6 *mask = item->mask;
 441         const uint8_t lvl = arg->header_level;
 442         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 443         struct fm_header_set *fm_data, *fm_mask;
 444
 445         ENICPMD_FUNC_TRACE();
 446         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 447         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 448         fm_data->fk_metadata |= FKM_IPV6;
 449         fm_mask->fk_metadata |= FKM_IPV6;
 450
 451         if (!spec)
 452                 return 0;
 453         if (!mask)
 454                 mask = &rte_flow_item_ipv6_mask;
 455
 456         fm_data->fk_header_select |= FKH_IPV6;
 457         fm_mask->fk_header_select |= FKH_IPV6;
 458         memcpy(&fm_data->l3.ip6, spec, sizeof(*spec));
 459         memcpy(&fm_mask->l3.ip6, mask, sizeof(*mask));
 460         return 0;
 461 }
 462
 463 static int
 464 enic_fm_copy_item_udp(struct copy_item_args *arg)
 465 {
 466         const struct rte_flow_item *item = arg->item;
 467         const struct rte_flow_item_udp *spec = item->spec;
 468         const struct rte_flow_item_udp *mask = item->mask;
 469         const uint8_t lvl = arg->header_level;
 470         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 471         struct fm_header_set *fm_data, *fm_mask;
 472
 473         ENICPMD_FUNC_TRACE();
 474         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 475         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 476         fm_data->fk_metadata |= FKM_UDP;
 477         fm_mask->fk_metadata |= FKM_UDP;
 478
 479         if (!spec)
 480                 return 0;
 481         if (!mask)
 482                 mask = &rte_flow_item_udp_mask;
 483
 484         fm_data->fk_header_select |= FKH_UDP;
 485         fm_mask->fk_header_select |= FKH_UDP;
 486         memcpy(&fm_data->l4.udp, spec, sizeof(*spec));
 487         memcpy(&fm_mask->l4.udp, mask, sizeof(*mask));
 488         return 0;
 489 }
 490
 491 static int
 492 enic_fm_copy_item_tcp(struct copy_item_args *arg)
 493 {
 494         const struct rte_flow_item *item = arg->item;
 495         const struct rte_flow_item_tcp *spec = item->spec;
 496         const struct rte_flow_item_tcp *mask = item->mask;
 497         const uint8_t lvl = arg->header_level;
 498         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 499         struct fm_header_set *fm_data, *fm_mask;
 500
 501         ENICPMD_FUNC_TRACE();
 502         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 503         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 504         fm_data->fk_metadata |= FKM_TCP;
 505         fm_mask->fk_metadata |= FKM_TCP;
 506
 507         if (!spec)
 508                 return 0;
 509         if (!mask)
 510                 mask = &rte_flow_item_tcp_mask;
 511
 512         fm_data->fk_header_select |= FKH_TCP;
 513         fm_mask->fk_header_select |= FKH_TCP;
 514         memcpy(&fm_data->l4.tcp, spec, sizeof(*spec));
 515         memcpy(&fm_mask->l4.tcp, mask, sizeof(*mask));
 516         return 0;
 517 }
 518
 519 static int
 520 enic_fm_copy_item_sctp(struct copy_item_args *arg)
 521 {
 522         const struct rte_flow_item *item = arg->item;
 523         const struct rte_flow_item_sctp *spec = item->spec;
 524         const struct rte_flow_item_sctp *mask = item->mask;
 525         const uint8_t lvl = arg->header_level;
 526         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 527         struct fm_header_set *fm_data, *fm_mask;
 528         uint8_t *ip_proto_mask = NULL;
 529         uint8_t *ip_proto = NULL;
 530         uint32_t l3_fkh;
 531
 532         ENICPMD_FUNC_TRACE();
 533         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 534         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 535         /*
 536          * The NIC filter API has no flags for "match sctp", so explicitly
 537          * set the protocol number in the IP pattern.
 538          */
 539         if (fm_data->fk_metadata & FKM_IPV4) {
 540                 struct rte_ipv4_hdr *ip;
 541                 ip = (struct rte_ipv4_hdr *)&fm_mask->l3.ip4;
 542                 ip_proto_mask = &ip->next_proto_id;
 543                 ip = (struct rte_ipv4_hdr *)&fm_data->l3.ip4;
 544                 ip_proto = &ip->next_proto_id;
 545                 l3_fkh = FKH_IPV4;
 546         } else if (fm_data->fk_metadata & FKM_IPV6) {
 547                 struct rte_ipv6_hdr *ip;
 548                 ip = (struct rte_ipv6_hdr *)&fm_mask->l3.ip6;
 549                 ip_proto_mask = &ip->proto;
 550                 ip = (struct rte_ipv6_hdr *)&fm_data->l3.ip6;
 551                 ip_proto = &ip->proto;
 552                 l3_fkh = FKH_IPV6;
 553         } else {
 554                 /* Need IPv4/IPv6 pattern first */
 555                 return -EINVAL;
 556         }
 557         *ip_proto = IPPROTO_SCTP;
 558         *ip_proto_mask = 0xff;
 559         fm_data->fk_header_select |= l3_fkh;
 560         fm_mask->fk_header_select |= l3_fkh;
 561
 562         if (!spec)
 563                 return 0;
 564         if (!mask)
 565                 mask = &rte_flow_item_sctp_mask;
 566
 567         fm_data->fk_header_select |= FKH_L4RAW;
 568         fm_mask->fk_header_select |= FKH_L4RAW;
 569         memcpy(fm_data->l4.rawdata, spec, sizeof(*spec));
 570         memcpy(fm_mask->l4.rawdata, mask, sizeof(*mask));
 571         return 0;
 572 }
 573
 574 static int
 575 enic_fm_copy_item_vxlan(struct copy_item_args *arg)
 576 {
 577         const struct rte_flow_item *item = arg->item;
 578         const struct rte_flow_item_vxlan *spec = item->spec;
 579         const struct rte_flow_item_vxlan *mask = item->mask;
 580         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 581         struct fm_header_set *fm_data, *fm_mask;
 582
 583         ENICPMD_FUNC_TRACE();
 584         /* Only 2 header levels (outer and inner) allowed */
 585         if (arg->header_level > 0)
 586                 return -EINVAL;
 587
 588         fm_data = &entry->ftm_data.fk_hdrset[0];
 589         fm_mask = &entry->ftm_mask.fk_hdrset[0];
 590         fm_data->fk_metadata |= FKM_VXLAN;
 591         fm_mask->fk_metadata |= FKM_VXLAN;
 592         /* items from here on out are inner header items */
 593         arg->header_level = 1;
 594
 595         /* Match all if no spec */
 596         if (!spec)
 597                 return 0;
 598         if (!mask)
 599                 mask = &rte_flow_item_vxlan_mask;
 600
 601         fm_data->fk_header_select |= FKH_VXLAN;
 602         fm_mask->fk_header_select |= FKH_VXLAN;
 603         memcpy(&fm_data->vxlan, spec, sizeof(*spec));
 604         memcpy(&fm_mask->vxlan, mask, sizeof(*mask));
 605         return 0;
 606 }
 607
 608 /*
 609  * Currently, raw pattern match is very limited. It is intended for matching
 610  * UDP tunnel header (e.g. vxlan or geneve).
 611  */
 612 static int
 613 enic_fm_copy_item_raw(struct copy_item_args *arg)
 614 {
 615         const struct rte_flow_item *item = arg->item;
 616         const struct rte_flow_item_raw *spec = item->spec;
 617         const struct rte_flow_item_raw *mask = item->mask;
 618         const uint8_t lvl = arg->header_level;
 619         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 620         struct fm_header_set *fm_data, *fm_mask;
 621
 622         ENICPMD_FUNC_TRACE();
 623         /* Cannot be used for inner packet */
 624         if (lvl > 0)
 625                 return -EINVAL;
 626         /* Need both spec and mask */
 627         if (!spec || !mask)
 628                 return -EINVAL;
 629         /* Only supports relative with offset 0 */
 630         if (!spec->relative || spec->offset != 0 || spec->search ||
 631             spec->limit)
 632                 return -EINVAL;
 633         /* Need non-null pattern that fits within the NIC's filter pattern */
 634         if (spec->length == 0 ||
 635             spec->length + sizeof(struct rte_udp_hdr) > FM_LAYER_SIZE ||
 636             !spec->pattern || !mask->pattern)
 637                 return -EINVAL;
 638         /*
 639          * Mask fields, including length, are often set to zero. Assume that
 640          * means "same as spec" to avoid breaking existing apps. If length
 641          * is not zero, then it should be >= spec length.
 642          *
 643          * No more pattern follows this, so append to the L4 layer instead of
 644          * L5 to work with both recent and older VICs.
 645          */
 646         if (mask->length != 0 && mask->length < spec->length)
 647                 return -EINVAL;
 648
 649         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 650         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 651         fm_data->fk_header_select |= FKH_L4RAW;
 652         fm_mask->fk_header_select |= FKH_L4RAW;
 653         fm_data->fk_header_select &= ~FKH_UDP;
 654         fm_mask->fk_header_select &= ~FKH_UDP;
 655         memcpy(fm_data->l4.rawdata + sizeof(struct rte_udp_hdr),
 656                spec->pattern, spec->length);
 657         memcpy(fm_mask->l4.rawdata + sizeof(struct rte_udp_hdr),
 658                mask->pattern, spec->length);
 659         return 0;
 660 }
 661
 662 static int
 663 flowman_cmd(struct enic_flowman *fm, uint64_t *args, int nargs)
 664 {
 665         return vnic_dev_flowman_cmd(fm->owner_enic->vdev, args, nargs);
 666 }
 667
 668 static int
 669 enic_fet_alloc(struct enic_flowman *fm, uint8_t ingress,
 670                struct fm_key_template *key, int entries,
 671                struct enic_fm_fet **fet_out)
 672 {
 673         struct fm_exact_match_table *cmd;
 674         struct fm_header_set *hdr;
 675         struct enic_fm_fet *fet;
 676         uint64_t args[3];
 677         int ret;
 678
 679         ENICPMD_FUNC_TRACE();
 680         fet = calloc(1, sizeof(struct enic_fm_fet));
 681         if (fet == NULL)
 682                 return -ENOMEM;
 683         cmd = &fm->cmd.va->fm_exact_match_table;
 684         memset(cmd, 0, sizeof(*cmd));
 685         cmd->fet_direction = ingress ? FM_INGRESS : FM_EGRESS;
 686         cmd->fet_stage = FM_STAGE_LAST;
 687         cmd->fet_max_entries = entries ? entries : FM_MAX_EXACT_TABLE_SIZE;
 688         if (key == NULL) {
 689                 hdr = &cmd->fet_key.fk_hdrset[0];
 690                 memset(hdr, 0, sizeof(*hdr));
 691                 hdr->fk_header_select = FKH_IPV4 | FKH_UDP;
 692                 hdr->l3.ip4.fk_saddr = 0xFFFFFFFF;
 693                 hdr->l3.ip4.fk_daddr = 0xFFFFFFFF;
 694                 hdr->l4.udp.fk_source = 0xFFFF;
 695                 hdr->l4.udp.fk_dest = 0xFFFF;
 696                 fet->default_key = 1;
 697         } else {
 698                 memcpy(&cmd->fet_key, key, sizeof(*key));
 699                 memcpy(&fet->key, key, sizeof(*key));
 700                 fet->default_key = 0;
 701         }
 702         cmd->fet_key.fk_packet_tag = 1;
 703
 704         args[0] = FM_EXACT_TABLE_ALLOC;
 705         args[1] = fm->cmd.pa;
 706         ret = flowman_cmd(fm, args, 2);
 707         if (ret) {
 708                 ENICPMD_LOG(ERR, "cannot alloc exact match table: rc=%d", ret);
 709                 free(fet);
 710                 return ret;
 711         }
 712         fet->handle = args[0];
 713         fet->ingress = ingress;
 714         ENICPMD_LOG(DEBUG, "allocated exact match table: handle=0x%" PRIx64,
 715                     fet->handle);
 716         *fet_out = fet;
 717         return 0;
 718 }
 719
 720 static void
 721 enic_fet_free(struct enic_flowman *fm, struct enic_fm_fet *fet)
 722 {
 723         ENICPMD_FUNC_TRACE();
 724         enic_fm_tbl_free(fm, fet->handle);
 725         if (!fet->default_key)
 726                 TAILQ_REMOVE(&fm->fet_list, fet, list);
 727         free(fet);
 728 }
 729
 730 /*
 731  * Get the exact match table for the given combination of
 732  * <group, ingress, key>. Allocate one on the fly as necessary.
 733  */
 734 static int
 735 enic_fet_get(struct enic_flowman *fm,
 736              uint32_t group,
 737              uint8_t ingress,
 738              struct fm_key_template *key,
 739              struct enic_fm_fet **fet_out,
 740              struct rte_flow_error *error)
 741 {
 742         struct enic_fm_fet *fet;
 743
 744         ENICPMD_FUNC_TRACE();
 745         /* See if we already have this table open */
 746         TAILQ_FOREACH(fet, &fm->fet_list, list) {
 747                 if (fet->group == group && fet->ingress == ingress)
 748                         break;
 749         }
 750         if (fet == NULL) {
 751                 /* Jumping to a non-existing group? Use the default table */
 752                 if (key == NULL) {
 753                         fet = ingress ? fm->default_ig_fet : fm->default_eg_fet;
 754                 } else if (enic_fet_alloc(fm, ingress, key, 0, &fet)) {
 755                         return rte_flow_error_set(error, EINVAL,
 756                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 757                                 NULL, "enic: cannot get exact match table");
 758                 }
 759                 fet->group = group;
 760                 /* Default table is never on the open table list */
 761                 if (!fet->default_key)
 762                         TAILQ_INSERT_HEAD(&fm->fet_list, fet, list);
 763         }
 764         fet->ref++;
 765         *fet_out = fet;
 766         ENICPMD_LOG(DEBUG, "fet_get: %s %s group=%u ref=%u",
 767                     fet->default_key ? "default" : "",
 768                     fet->ingress ? "ingress" : "egress",
 769                     fet->group, fet->ref);
 770         return 0;
 771 }
 772
 773 static void
 774 enic_fet_put(struct enic_flowman *fm, struct enic_fm_fet *fet)
 775 {
 776         ENICPMD_FUNC_TRACE();
 777         RTE_ASSERT(fet->ref > 0);
 778         fet->ref--;
 779         ENICPMD_LOG(DEBUG, "fet_put: %s %s group=%u ref=%u",
 780                     fet->default_key ? "default" : "",
 781                     fet->ingress ? "ingress" : "egress",
 782                     fet->group, fet->ref);
 783         if (fet->ref == 0)
 784                 enic_fet_free(fm, fet);
 785 }
 786
 787 /* Return 1 if current item is valid on top of the previous one. */
 788 static int
 789 fm_item_stacking_valid(enum rte_flow_item_type prev_item,
 790                        const struct enic_fm_items *item_info,
 791                        uint8_t is_first_item)
 792 {
 793         enum rte_flow_item_type const *allowed_items = item_info->prev_items;
 794
 795         ENICPMD_FUNC_TRACE();
 796         for (; *allowed_items != RTE_FLOW_ITEM_TYPE_END; allowed_items++) {
 797                 if (prev_item == *allowed_items)
 798                         return 1;
 799         }
 800
 801         /* This is the first item in the stack. Check if that's cool */
 802         if (is_first_item && item_info->valid_start_item)
 803                 return 1;
 804         return 0;
 805 }
 806
 807 /*
 808  * Build the flow manager match entry structure from the provided pattern.
 809  * The pattern is validated as the items are copied.
 810  */
 811 static int
 812 enic_fm_copy_entry(struct enic_flowman *fm,
 813                    const struct rte_flow_item pattern[],
 814                    struct rte_flow_error *error)
 815 {
 816         const struct enic_fm_items *item_info;
 817         enum rte_flow_item_type prev_item;
 818         const struct rte_flow_item *item;
 819         struct copy_item_args args;
 820         uint8_t prev_header_level;
 821         uint8_t is_first_item;
 822         int ret;
 823
 824         ENICPMD_FUNC_TRACE();
 825         item = pattern;
 826         is_first_item = 1;
 827         prev_item = RTE_FLOW_ITEM_TYPE_END;
 828
 829         args.fm_tcam_entry = &fm->tcam_entry;
 830         args.header_level = 0;
 831         prev_header_level = 0;
 832         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 833                 /*
 834                  * Get info about how to validate and copy the item. If NULL
 835                  * is returned the nic does not support the item.
 836                  */
 837                 if (item->type == RTE_FLOW_ITEM_TYPE_VOID)
 838                         continue;
 839
 840                 item_info = &enic_fm_items[item->type];
 841
 842                 if (item->type > FM_MAX_ITEM_TYPE ||
 843                     item_info->copy_item == NULL) {
 844                         return rte_flow_error_set(error, ENOTSUP,
 845                                 RTE_FLOW_ERROR_TYPE_ITEM,
 846                                 NULL, "enic: unsupported item");
 847                 }
 848
 849                 /* check to see if item stacking is valid */
 850                 if (!fm_item_stacking_valid(prev_item, item_info,
 851                                             is_first_item))
 852                         goto stacking_error;
 853
 854                 args.item = item;
 855                 ret = item_info->copy_item(&args);
 856                 if (ret)
 857                         goto item_not_supported;
 858                 /* Going from outer to inner? Treat it as a new packet start */
 859                 if (prev_header_level != args.header_level) {
 860                         prev_item = RTE_FLOW_ITEM_TYPE_END;
 861                         is_first_item = 1;
 862                 } else {
 863                         prev_item = item->type;
 864                         is_first_item = 0;
 865                 }
 866                 prev_header_level = args.header_level;
 867         }
 868         return 0;
 869
 870 item_not_supported:
 871         return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,
 872                                   NULL, "enic: unsupported item type");
 873
 874 stacking_error:
 875         return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
 876                                   item, "enic: unsupported item stack");
 877 }
 878
 879 static void
 880 flow_item_skip_void(const struct rte_flow_item **item)
 881 {
 882         for ( ; ; (*item)++)
 883                 if ((*item)->type != RTE_FLOW_ITEM_TYPE_VOID)
 884                         return;
 885 }
 886
 887 static void
 888 append_template(void **template, uint8_t *off, const void *data, int len)
 889 {
 890         memcpy(*template, data, len);
 891         *template = (char *)*template + len;
 892         *off = *off + len;
 893 }
 894
 895 static int
 896 enic_fm_append_action_op(struct enic_flowman *fm,
 897                          struct fm_action_op *fm_op,
 898                          struct rte_flow_error *error)
 899 {
 900         int count;
 901
 902         count = fm->action_op_count;
 903         ENICPMD_LOG(DEBUG, "append action op: idx=%d op=%u",
 904                     count, fm_op->fa_op);
 905         if (count == FM_ACTION_OP_MAX) {
 906                 return rte_flow_error_set(error, EINVAL,
 907                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 908                         "too many action operations");
 909         }
 910         fm->action.fma_action_ops[count] = *fm_op;
 911         fm->action_op_count = count + 1;
 912         return 0;
 913 }
 914
 915 /* NIC requires that 1st steer appear before decap.
 916  * Correct example: steer, decap, steer, steer, ...
 917  */
 918 static void
 919 enic_fm_reorder_action_op(struct enic_flowman *fm)
 920 {
 921         struct fm_action_op *op, *steer, *decap;
 922         struct fm_action_op tmp_op;
 923
 924         ENICPMD_FUNC_TRACE();
 925         /* Find 1st steer and decap */
 926         op = fm->action.fma_action_ops;
 927         steer = NULL;
 928         decap = NULL;
 929         while (op->fa_op != FMOP_END) {
 930                 if (!decap && op->fa_op == FMOP_DECAP_NOSTRIP)
 931                         decap = op;
 932                 else if (!steer && op->fa_op == FMOP_RQ_STEER)
 933                         steer = op;
 934                 op++;
 935         }
 936         /* If decap is before steer, swap */
 937         if (steer && decap && decap < steer) {
 938                 op = fm->action.fma_action_ops;
 939                 ENICPMD_LOG(DEBUG, "swap decap %ld <-> steer %ld",
 940                             (long)(decap - op), (long)(steer - op));
 941                 tmp_op = *decap;
 942                 *decap = *steer;
 943                 *steer = tmp_op;
 944         }
 945 }
 946
 947 /* VXLAN decap is done via flowman compound action */
 948 static int
 949 enic_fm_copy_vxlan_decap(struct enic_flowman *fm,
 950                          struct fm_tcam_match_entry *fmt,
 951                          const struct rte_flow_action *action,
 952                          struct rte_flow_error *error)
 953 {
 954         struct fm_header_set *fm_data;
 955         struct fm_action_op fm_op;
 956
 957         ENICPMD_FUNC_TRACE();
 958         fm_data = &fmt->ftm_data.fk_hdrset[0];
 959         if (!(fm_data->fk_metadata & FKM_VXLAN)) {
 960                 return rte_flow_error_set(error, EINVAL,
 961                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 962                         "vxlan-decap: vxlan must be in pattern");
 963         }
 964
 965         memset(&fm_op, 0, sizeof(fm_op));
 966         fm_op.fa_op = FMOP_DECAP_NOSTRIP;
 967         return enic_fm_append_action_op(fm, &fm_op, error);
 968 }
 969
 970 /* VXLAN encap is done via flowman compound action */
 971 static int
 972 enic_fm_copy_vxlan_encap(struct enic_flowman *fm,
 973                          const struct rte_flow_item *item,
 974                          struct rte_flow_error *error)
 975 {
 976         struct fm_action_op fm_op;
 977         struct rte_ether_hdr *eth;
 978         uint16_t *ethertype;
 979         void *template;
 980         uint8_t off;
 981
 982         ENICPMD_FUNC_TRACE();
 983         memset(&fm_op, 0, sizeof(fm_op));
 984         fm_op.fa_op = FMOP_ENCAP;
 985         template = fm->action.fma_data;
 986         off = 0;
 987         /*
 988          * Copy flow items to the flowman template starting L2.
 989          * L2 must be ethernet.
 990          */
 991         flow_item_skip_void(&item);
 992         if (item->type != RTE_FLOW_ITEM_TYPE_ETH)
 993                 return rte_flow_error_set(error, EINVAL,
 994                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 995                         "vxlan-encap: first item should be ethernet");
 996         eth = (struct rte_ether_hdr *)template;
 997         ethertype = &eth->ether_type;
 998         append_template(&template, &off, item->spec,
 999                         sizeof(struct rte_flow_item_eth));
1000         item++;
1001         flow_item_skip_void(&item);
1002         /* Optional VLAN */
1003         if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
1004                 const struct rte_flow_item_vlan *spec;
1005
1006                 ENICPMD_LOG(DEBUG, "vxlan-encap: vlan");
1007                 spec = item->spec;
1008                 fm_op.encap.outer_vlan = rte_be_to_cpu_16(spec->tci);
1009                 item++;
1010                 flow_item_skip_void(&item);
1011         }
1012         /* L3 must be IPv4, IPv6 */
1013         switch (item->type) {
1014         case RTE_FLOW_ITEM_TYPE_IPV4:
1015         {
1016                 struct rte_ipv4_hdr *ip4;
1017
1018                 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv4");
1019                 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
1020                 ip4 = (struct rte_ipv4_hdr *)template;
1021                 /*
1022                  * Offset of IPv4 length field and its initial value
1023                  * (IP + UDP + VXLAN) are specified in the action. The NIC
1024                  * will add inner packet length.
1025                  */
1026                 fm_op.encap.len1_offset = off +
1027                         offsetof(struct rte_ipv4_hdr, total_length);
1028                 fm_op.encap.len1_delta = sizeof(struct rte_ipv4_hdr) +
1029                         sizeof(struct rte_udp_hdr) +
1030                         sizeof(struct rte_vxlan_hdr);
1031                 append_template(&template, &off, item->spec,
1032                                 sizeof(struct rte_ipv4_hdr));
1033                 ip4->version_ihl = RTE_IPV4_VHL_DEF;
1034                 if (ip4->time_to_live == 0)
1035                         ip4->time_to_live = IP_DEFTTL;
1036                 ip4->next_proto_id = IPPROTO_UDP;
1037                 break;
1038         }
1039         case RTE_FLOW_ITEM_TYPE_IPV6:
1040         {
1041                 struct rte_ipv6_hdr *ip6;
1042
1043                 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv6");
1044                 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
1045                 ip6 = (struct rte_ipv6_hdr *)template;
1046                 fm_op.encap.len1_offset = off +
1047                         offsetof(struct rte_ipv6_hdr, payload_len);
1048                 fm_op.encap.len1_delta = sizeof(struct rte_udp_hdr) +
1049                         sizeof(struct rte_vxlan_hdr);
1050                 append_template(&template, &off, item->spec,
1051                                 sizeof(struct rte_ipv6_hdr));
1052                 ip6->vtc_flow |= rte_cpu_to_be_32(IP6_VTC_FLOW);
1053                 if (ip6->hop_limits == 0)
1054                         ip6->hop_limits = IP_DEFTTL;
1055                 ip6->proto = IPPROTO_UDP;
1056                 break;
1057         }
1058         default:
1059                 return rte_flow_error_set(error,
1060                         EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1061                         "vxlan-encap: L3 must be IPv4/IPv6");
1062         }
1063         item++;
1064         flow_item_skip_void(&item);
1065
1066         /* L4 is UDP */
1067         if (item->type != RTE_FLOW_ITEM_TYPE_UDP)
1068                 return rte_flow_error_set(error, EINVAL,
1069                         RTE_FLOW_ERROR_TYPE_ITEM, item,
1070                         "vxlan-encap: UDP must follow IPv4/IPv6");
1071         /* UDP length = UDP + VXLAN. NIC will add inner packet length. */
1072         fm_op.encap.len2_offset =
1073                 off + offsetof(struct rte_udp_hdr, dgram_len);
1074         fm_op.encap.len2_delta =
1075                 sizeof(struct rte_udp_hdr) + sizeof(struct rte_vxlan_hdr);
1076         append_template(&template, &off, item->spec,
1077                         sizeof(struct rte_udp_hdr));
1078         item++;
1079         flow_item_skip_void(&item);
1080
1081         /* Finally VXLAN */
1082         if (item->type != RTE_FLOW_ITEM_TYPE_VXLAN)
1083                 return rte_flow_error_set(error,
1084                         EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1085                         "vxlan-encap: VXLAN must follow UDP");
1086         append_template(&template, &off, item->spec,
1087                         sizeof(struct rte_flow_item_vxlan));
1088
1089         /*
1090          * Fill in the rest of the action structure.
1091          * Indicate that we want to encap with vxlan at packet start.
1092          */
1093         fm_op.encap.template_offset = 0;
1094         fm_op.encap.template_len = off;
1095         return enic_fm_append_action_op(fm, &fm_op, error);
1096 }
1097
1098 static int
1099 enic_fm_find_vnic(struct enic *enic, const struct rte_pci_addr *addr,
1100                   uint64_t *handle)
1101 {
1102         uint32_t bdf;
1103         uint64_t args[2];
1104         int rc;
1105
1106         ENICPMD_FUNC_TRACE();
1107         ENICPMD_LOG(DEBUG, "bdf=%x:%x:%x", addr->bus, addr->devid,
1108                     addr->function);
1109         bdf = addr->bus << 8 | addr->devid << 3 | addr->function;
1110         args[0] = FM_VNIC_FIND;
1111         args[1] = bdf;
1112         rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
1113         if (rc != 0) {
1114                 /* Expected to fail if BDF is not on the adapter */
1115                 ENICPMD_LOG(DEBUG, "cannot find vnic handle: rc=%d", rc);
1116                 return rc;
1117         }
1118         *handle = args[0];
1119         ENICPMD_LOG(DEBUG, "found vnic: handle=0x%" PRIx64, *handle);
1120         return 0;
1121 }
1122
1123 /* Translate flow actions to flowman TCAM entry actions */
1124 static int
1125 enic_fm_copy_action(struct enic_flowman *fm,
1126                     const struct rte_flow_action actions[],
1127                     uint8_t ingress,
1128                     struct rte_flow_error *error)
1129 {
1130         enum {
1131                 FATE = 1 << 0,
1132                 DECAP = 1 << 1,
1133                 PASSTHRU = 1 << 2,
1134                 COUNT = 1 << 3,
1135                 ENCAP = 1 << 4,
1136                 PUSH_VLAN = 1 << 5,
1137                 PORT_ID = 1 << 6,
1138         };
1139         struct fm_tcam_match_entry *fmt;
1140         struct fm_action_op fm_op;
1141         bool need_ovlan_action;
1142         struct enic *enic;
1143         uint32_t overlap;
1144         uint64_t vnic_h;
1145         uint16_t ovlan;
1146         bool first_rq;
1147         bool steer;
1148         int ret;
1149
1150         ENICPMD_FUNC_TRACE();
1151         fmt = &fm->tcam_entry;
1152         need_ovlan_action = false;
1153         ovlan = 0;
1154         first_rq = true;
1155         steer = false;
1156         enic = fm->user_enic;
1157         overlap = 0;
1158         vnic_h = enic->fm_vnic_handle;
1159
1160         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1161                 switch (actions->type) {
1162                 case RTE_FLOW_ACTION_TYPE_VOID:
1163                         continue;
1164                 case RTE_FLOW_ACTION_TYPE_PASSTHRU: {
1165                         if (overlap & PASSTHRU)
1166                                 goto unsupported;
1167                         overlap |= PASSTHRU;
1168                         break;
1169                 }
1170                 case RTE_FLOW_ACTION_TYPE_JUMP: {
1171                         const struct rte_flow_action_jump *jump =
1172                                 actions->conf;
1173                         struct enic_fm_fet *fet;
1174
1175                         if (overlap & FATE)
1176                                 goto unsupported;
1177                         ret = enic_fet_get(fm, jump->group, ingress, NULL,
1178                                            &fet, error);
1179                         if (ret)
1180                                 return ret;
1181                         overlap |= FATE;
1182                         memset(&fm_op, 0, sizeof(fm_op));
1183                         fm_op.fa_op = FMOP_EXACT_MATCH;
1184                         fm_op.exact.handle = fet->handle;
1185                         fm->fet = fet;
1186                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1187                         if (ret)
1188                                 return ret;
1189                         break;
1190                 }
1191                 case RTE_FLOW_ACTION_TYPE_MARK: {
1192                         const struct rte_flow_action_mark *mark =
1193                                 actions->conf;
1194
1195                         if (mark->id >= ENIC_MAGIC_FILTER_ID - 1)
1196                                 return rte_flow_error_set(error, EINVAL,
1197                                         RTE_FLOW_ERROR_TYPE_ACTION,
1198                                         NULL, "invalid mark id");
1199                         memset(&fm_op, 0, sizeof(fm_op));
1200                         fm_op.fa_op = FMOP_MARK;
1201                         fm_op.mark.mark = mark->id + 1;
1202                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1203                         if (ret)
1204                                 return ret;
1205                         break;
1206                 }
1207                 case RTE_FLOW_ACTION_TYPE_FLAG: {
1208                         /* ENIC_MAGIC_FILTER_ID is reserved for flagging */
1209                         memset(&fm_op, 0, sizeof(fm_op));
1210                         fm_op.fa_op = FMOP_MARK;
1211                         fm_op.mark.mark = ENIC_MAGIC_FILTER_ID;
1212                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1213                         if (ret)
1214                                 return ret;
1215                         break;
1216                 }
1217                 case RTE_FLOW_ACTION_TYPE_QUEUE: {
1218                         const struct rte_flow_action_queue *queue =
1219                                 actions->conf;
1220
1221                         /*
1222                          * If fate other than QUEUE or RSS, fail. Multiple
1223                          * rss and queue actions are ok.
1224                          */
1225                         if ((overlap & FATE) && first_rq)
1226                                 goto unsupported;
1227                         first_rq = false;
1228                         overlap |= FATE;
1229                         memset(&fm_op, 0, sizeof(fm_op));
1230                         fm_op.fa_op = FMOP_RQ_STEER;
1231                         fm_op.rq_steer.rq_index =
1232                                 enic_rte_rq_idx_to_sop_idx(queue->index);
1233                         fm_op.rq_steer.rq_count = 1;
1234                         fm_op.rq_steer.vnic_handle = vnic_h;
1235                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1236                         if (ret)
1237                                 return ret;
1238                         ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1239                                     fm_op.rq_steer.rq_index);
1240                         steer = true;
1241                         break;
1242                 }
1243                 case RTE_FLOW_ACTION_TYPE_DROP: {
1244                         if (overlap & FATE)
1245                                 goto unsupported;
1246                         overlap |= FATE;
1247                         memset(&fm_op, 0, sizeof(fm_op));
1248                         fm_op.fa_op = FMOP_DROP;
1249                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1250                         if (ret)
1251                                 return ret;
1252                         ENICPMD_LOG(DEBUG, "create DROP action");
1253                         break;
1254                 }
1255                 case RTE_FLOW_ACTION_TYPE_COUNT: {
1256                         if (overlap & COUNT)
1257                                 goto unsupported;
1258                         overlap |= COUNT;
1259                         /* Count is associated with entry not action on VIC. */
1260                         fmt->ftm_flags |= FMEF_COUNTER;
1261                         break;
1262                 }
1263                 case RTE_FLOW_ACTION_TYPE_RSS: {
1264                         const struct rte_flow_action_rss *rss = actions->conf;
1265                         bool allow;
1266                         uint16_t i;
1267
1268                         /*
1269                          * If fate other than QUEUE or RSS, fail. Multiple
1270                          * rss and queue actions are ok.
1271                          */
1272                         if ((overlap & FATE) && first_rq)
1273                                 goto unsupported;
1274                         first_rq = false;
1275                         overlap |= FATE;
1276
1277                         /*
1278                          * Hardware only supports RSS actions on outer level
1279                          * with default type and function. Queues must be
1280                          * sequential.
1281                          */
1282                         allow = rss->func == RTE_ETH_HASH_FUNCTION_DEFAULT &&
1283                                 rss->level == 0 && (rss->types == 0 ||
1284                                 rss->types == enic->rss_hf) &&
1285                                 rss->queue_num <= enic->rq_count &&
1286                                 rss->queue[rss->queue_num - 1] < enic->rq_count;
1287
1288
1289                         /* Identity queue map needs to be sequential */
1290                         for (i = 1; i < rss->queue_num; i++)
1291                                 allow = allow && (rss->queue[i] ==
1292                                         rss->queue[i - 1] + 1);
1293                         if (!allow)
1294                                 goto unsupported;
1295
1296                         memset(&fm_op, 0, sizeof(fm_op));
1297                         fm_op.fa_op = FMOP_RQ_STEER;
1298                         fm_op.rq_steer.rq_index =
1299                                 enic_rte_rq_idx_to_sop_idx(rss->queue[0]);
1300                         fm_op.rq_steer.rq_count = rss->queue_num;
1301                         fm_op.rq_steer.vnic_handle = vnic_h;
1302                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1303                         if (ret)
1304                                 return ret;
1305                         ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1306                                     fm_op.rq_steer.rq_index);
1307                         steer = true;
1308                         break;
1309                 }
1310                 case RTE_FLOW_ACTION_TYPE_PORT_ID: {
1311                         const struct rte_flow_action_port_id *port;
1312                         struct rte_eth_dev *dev;
1313
1314                         port = actions->conf;
1315                         if (port->original) {
1316                                 vnic_h = enic->fm_vnic_handle; /* This port */
1317                                 break;
1318                         }
1319                         ENICPMD_LOG(DEBUG, "port id %u", port->id);
1320                         if (!rte_eth_dev_is_valid_port(port->id)) {
1321                                 return rte_flow_error_set(error, EINVAL,
1322                                         RTE_FLOW_ERROR_TYPE_ACTION,
1323                                         NULL, "invalid port_id");
1324                         }
1325                         dev = &rte_eth_devices[port->id];
1326                         if (!dev_is_enic(dev)) {
1327                                 return rte_flow_error_set(error, EINVAL,
1328                                         RTE_FLOW_ERROR_TYPE_ACTION,
1329                                         NULL, "port_id is not enic");
1330                         }
1331                         if (enic->switch_domain_id !=
1332                             pmd_priv(dev)->switch_domain_id) {
1333                                 return rte_flow_error_set(error, EINVAL,
1334                                         RTE_FLOW_ERROR_TYPE_ACTION,
1335                                         NULL, "destination and source ports are not in the same switch domain");
1336                         }
1337                         vnic_h = pmd_priv(dev)->fm_vnic_handle;
1338                         overlap |= PORT_ID;
1339                         /*
1340                          * Ingress. Nothing more to do. We add an implicit
1341                          * steer at the end if needed.
1342                          */
1343                         break;
1344                 }
1345                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: {
1346                         if (overlap & DECAP)
1347                                 goto unsupported;
1348                         overlap |= DECAP;
1349
1350                         ret = enic_fm_copy_vxlan_decap(fm, fmt, actions,
1351                                 error);
1352                         if (ret != 0)
1353                                 return ret;
1354                         break;
1355                 }
1356                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: {
1357                         const struct rte_flow_action_vxlan_encap *encap;
1358
1359                         encap = actions->conf;
1360                         if (overlap & ENCAP)
1361                                 goto unsupported;
1362                         overlap |= ENCAP;
1363                         ret = enic_fm_copy_vxlan_encap(fm, encap->definition,
1364                                 error);
1365                         if (ret != 0)
1366                                 return ret;
1367                         break;
1368                 }
1369                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: {
1370                         memset(&fm_op, 0, sizeof(fm_op));
1371                         fm_op.fa_op = FMOP_POP_VLAN;
1372                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1373                         if (ret)
1374                                 return ret;
1375                         break;
1376                 }
1377                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: {
1378                         const struct rte_flow_action_of_push_vlan *vlan;
1379
1380                         if (overlap & PASSTHRU)
1381                                 goto unsupported;
1382                         vlan = actions->conf;
1383                         if (vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN)) {
1384                                 return rte_flow_error_set(error, EINVAL,
1385                                         RTE_FLOW_ERROR_TYPE_ACTION,
1386                                         NULL, "unexpected push_vlan ethertype");
1387                         }
1388                         overlap |= PUSH_VLAN;
1389                         need_ovlan_action = true;
1390                         break;
1391                 }
1392                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: {
1393                         const struct rte_flow_action_of_set_vlan_pcp *pcp;
1394
1395                         pcp = actions->conf;
1396                         if (pcp->vlan_pcp > 7) {
1397                                 return rte_flow_error_set(error, EINVAL,
1398                                         RTE_FLOW_ERROR_TYPE_ACTION,
1399                                         NULL, "invalid vlan_pcp");
1400                         }
1401                         need_ovlan_action = true;
1402                         ovlan |= ((uint16_t)pcp->vlan_pcp) << 13;
1403                         break;
1404                 }
1405                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: {
1406                         const struct rte_flow_action_of_set_vlan_vid *vid;
1407
1408                         vid = actions->conf;
1409                         need_ovlan_action = true;
1410                         ovlan |= rte_be_to_cpu_16(vid->vlan_vid);
1411                         break;
1412                 }
1413                 default:
1414                         goto unsupported;
1415                 }
1416         }
1417
1418         if (!(overlap & (FATE | PASSTHRU | COUNT | PORT_ID)))
1419                 goto unsupported;
1420         /* Egress from VF: need implicit WQ match */
1421         if (enic_is_vf_rep(enic) && !ingress) {
1422                 fmt->ftm_data.fk_wq_id = 0;
1423                 fmt->ftm_mask.fk_wq_id = 0xffff;
1424                 fmt->ftm_data.fk_wq_vnic = enic->fm_vnic_handle;
1425                 ENICPMD_LOG(DEBUG, "add implicit wq id match for vf %d",
1426                             VF_ENIC_TO_VF_REP(enic)->vf_id);
1427         }
1428         if (need_ovlan_action) {
1429                 memset(&fm_op, 0, sizeof(fm_op));
1430                 fm_op.fa_op = FMOP_SET_OVLAN;
1431                 fm_op.ovlan.vlan = ovlan;
1432                 ret = enic_fm_append_action_op(fm, &fm_op, error);
1433                 if (ret)
1434                         return ret;
1435         }
1436         /* Add steer op for PORT_ID without QUEUE */
1437         if ((overlap & PORT_ID) && !steer && ingress) {
1438                 memset(&fm_op, 0, sizeof(fm_op));
1439                 /* Always to queue 0 for now as generic RSS is not available */
1440                 fm_op.fa_op = FMOP_RQ_STEER;
1441                 fm_op.rq_steer.rq_index = 0;
1442                 fm_op.rq_steer.vnic_handle = vnic_h;
1443                 ret = enic_fm_append_action_op(fm, &fm_op, error);
1444                 if (ret)
1445                         return ret;
1446                 ENICPMD_LOG(DEBUG, "add implicit steer op");
1447         }
1448         /* Add required END */
1449         memset(&fm_op, 0, sizeof(fm_op));
1450         fm_op.fa_op = FMOP_END;
1451         ret = enic_fm_append_action_op(fm, &fm_op, error);
1452         if (ret)
1453                 return ret;
1454         enic_fm_reorder_action_op(fm);
1455         return 0;
1456
1457 unsupported:
1458         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
1459                                   NULL, "enic: unsupported action");
1460 }
1461
1462 /** Check if the action is supported */
1463 static int
1464 enic_fm_match_action(const struct rte_flow_action *action,
1465                      const enum rte_flow_action_type *supported_actions)
1466 {
1467         for (; *supported_actions != RTE_FLOW_ACTION_TYPE_END;
1468              supported_actions++) {
1469                 if (action->type == *supported_actions)
1470                         return 1;
1471         }
1472         return 0;
1473 }
1474
1475 /* Debug function to dump internal NIC action structure. */
1476 static void
1477 enic_fm_dump_tcam_actions(const struct fm_action *fm_action)
1478 {
1479         /* Manually keep in sync with FMOP commands */
1480         const char *fmop_str[FMOP_OP_MAX] = {
1481                 [FMOP_END] = "end",
1482                 [FMOP_DROP] = "drop",
1483                 [FMOP_RQ_STEER] = "steer",
1484                 [FMOP_EXACT_MATCH] = "exmatch",
1485                 [FMOP_MARK] = "mark",
1486                 [FMOP_EXT_MARK] = "ext_mark",
1487                 [FMOP_TAG] = "tag",
1488                 [FMOP_EG_HAIRPIN] = "eg_hairpin",
1489                 [FMOP_IG_HAIRPIN] = "ig_hairpin",
1490                 [FMOP_ENCAP_IVLAN] = "encap_ivlan",
1491                 [FMOP_ENCAP_NOIVLAN] = "encap_noivlan",
1492                 [FMOP_ENCAP] = "encap",
1493                 [FMOP_SET_OVLAN] = "set_ovlan",
1494                 [FMOP_DECAP_NOSTRIP] = "decap_nostrip",
1495                 [FMOP_DECAP_STRIP] = "decap_strip",
1496                 [FMOP_POP_VLAN] = "pop_vlan",
1497                 [FMOP_SET_EGPORT] = "set_egport",
1498                 [FMOP_RQ_STEER_ONLY] = "rq_steer_only",
1499                 [FMOP_SET_ENCAP_VLAN] = "set_encap_vlan",
1500                 [FMOP_EMIT] = "emit",
1501                 [FMOP_MODIFY] = "modify",
1502         };
1503         const struct fm_action_op *op = &fm_action->fma_action_ops[0];
1504         char buf[128], *bp = buf;
1505         const char *op_str;
1506         int i, n, buf_len;
1507
1508         buf[0] = '\0';
1509         buf_len = sizeof(buf);
1510         for (i = 0; i < FM_ACTION_OP_MAX; i++) {
1511                 if (op->fa_op == FMOP_END)
1512                         break;
1513                 if (op->fa_op >= FMOP_OP_MAX)
1514                         op_str = "unknown";
1515                 else
1516                         op_str = fmop_str[op->fa_op];
1517                 n = snprintf(bp, buf_len, "%s,", op_str);
1518                 if (n > 0 && n < buf_len) {
1519                         bp += n;
1520                         buf_len -= n;
1521                 }
1522                 op++;
1523         }
1524         /* Remove trailing comma */
1525         if (buf[0])
1526                 *(bp - 1) = '\0';
1527         ENICPMD_LOG(DEBUG, "       Acions: %s", buf);
1528 }
1529
1530 static int
1531 bits_to_str(uint32_t bits, const char *strings[], int max,
1532             char *buf, int buf_len)
1533 {
1534         int i, n = 0, len = 0;
1535
1536         for (i = 0; i < max; i++) {
1537                 if (bits & (1 << i)) {
1538                         n = snprintf(buf, buf_len, "%s,", strings[i]);
1539                         if (n > 0 && n < buf_len) {
1540                                 buf += n;
1541                                 buf_len -= n;
1542                                 len += n;
1543                         }
1544                 }
1545         }
1546         /* Remove trailing comma */
1547         if (len) {
1548                 *(buf - 1) = '\0';
1549                 len--;
1550         }
1551         return len;
1552 }
1553
1554 /* Debug function to dump internal NIC filter structure. */
1555 static void
1556 __enic_fm_dump_tcam_match(const struct fm_header_set *fk_hdrset, char *buf,
1557                           int buf_len)
1558 {
1559         /* Manually keep in sync with FKM_BITS */
1560         const char *fm_fkm_str[FKM_BIT_COUNT] = {
1561                 [FKM_QTAG_BIT] = "qtag",
1562                 [FKM_CMD_BIT] = "cmd",
1563                 [FKM_IPV4_BIT] = "ip4",
1564                 [FKM_IPV6_BIT] = "ip6",
1565                 [FKM_ROCE_BIT] = "roce",
1566                 [FKM_UDP_BIT] = "udp",
1567                 [FKM_TCP_BIT] = "tcp",
1568                 [FKM_TCPORUDP_BIT] = "tcpportudp",
1569                 [FKM_IPFRAG_BIT] = "ipfrag",
1570                 [FKM_NVGRE_BIT] = "nvgre",
1571                 [FKM_VXLAN_BIT] = "vxlan",
1572                 [FKM_GENEVE_BIT] = "geneve",
1573                 [FKM_NSH_BIT] = "nsh",
1574                 [FKM_ROCEV2_BIT] = "rocev2",
1575                 [FKM_VLAN_PRES_BIT] = "vlan_pres",
1576                 [FKM_IPOK_BIT] = "ipok",
1577                 [FKM_L4OK_BIT] = "l4ok",
1578                 [FKM_ROCEOK_BIT] = "roceok",
1579                 [FKM_FCSOK_BIT] = "fcsok",
1580                 [FKM_EG_SPAN_BIT] = "eg_span",
1581                 [FKM_IG_SPAN_BIT] = "ig_span",
1582                 [FKM_EG_HAIRPINNED_BIT] = "eg_hairpinned",
1583         };
1584         /* Manually keep in sync with FKH_BITS */
1585         const char *fm_fkh_str[FKH_BIT_COUNT] = {
1586                 [FKH_ETHER_BIT] = "eth",
1587                 [FKH_QTAG_BIT] = "qtag",
1588                 [FKH_L2RAW_BIT] = "l2raw",
1589                 [FKH_IPV4_BIT] = "ip4",
1590                 [FKH_IPV6_BIT] = "ip6",
1591                 [FKH_L3RAW_BIT] = "l3raw",
1592                 [FKH_UDP_BIT] = "udp",
1593                 [FKH_TCP_BIT] = "tcp",
1594                 [FKH_ICMP_BIT] = "icmp",
1595                 [FKH_VXLAN_BIT] = "vxlan",
1596                 [FKH_L4RAW_BIT] = "l4raw",
1597         };
1598         uint32_t fkh_bits = fk_hdrset->fk_header_select;
1599         uint32_t fkm_bits = fk_hdrset->fk_metadata;
1600         int n;
1601
1602         if (!fkm_bits && !fkh_bits)
1603                 return;
1604         n = snprintf(buf, buf_len, "metadata(");
1605         if (n > 0 && n < buf_len) {
1606                 buf += n;
1607                 buf_len -= n;
1608         }
1609         n = bits_to_str(fkm_bits, fm_fkm_str, FKM_BIT_COUNT, buf, buf_len);
1610         if (n > 0 && n < buf_len) {
1611                 buf += n;
1612                 buf_len -= n;
1613         }
1614         n = snprintf(buf, buf_len, ") valid hdr fields(");
1615         if (n > 0 && n < buf_len) {
1616                 buf += n;
1617                 buf_len -= n;
1618         }
1619         n = bits_to_str(fkh_bits, fm_fkh_str, FKH_BIT_COUNT, buf, buf_len);
1620         if (n > 0 && n < buf_len) {
1621                 buf += n;
1622                 buf_len -= n;
1623         }
1624         snprintf(buf, buf_len, ")");
1625 }
1626
1627 static void
1628 enic_fm_dump_tcam_match(const struct fm_tcam_match_entry *match,
1629                         uint8_t ingress)
1630 {
1631         char buf[256];
1632
1633         memset(buf, 0, sizeof(buf));
1634         __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[0],
1635                                   buf, sizeof(buf));
1636         ENICPMD_LOG(DEBUG, " TCAM %s Outer: %s %scounter",
1637                     (ingress) ? "IG" : "EG", buf,
1638                     (match->ftm_flags & FMEF_COUNTER) ? "" : "no ");
1639         memset(buf, 0, sizeof(buf));
1640         __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[1],
1641                                   buf, sizeof(buf));
1642         if (buf[0])
1643                 ENICPMD_LOG(DEBUG, "         Inner: %s", buf);
1644 }
1645
1646 /* Debug function to dump internal NIC flow structures. */
1647 static void
1648 enic_fm_dump_tcam_entry(const struct fm_tcam_match_entry *fm_match,
1649                         const struct fm_action *fm_action,
1650                         uint8_t ingress)
1651 {
1652         if (!rte_log_can_log(enic_pmd_logtype, RTE_LOG_DEBUG))
1653                 return;
1654         enic_fm_dump_tcam_match(fm_match, ingress);
1655         enic_fm_dump_tcam_actions(fm_action);
1656 }
1657
1658 static int
1659 enic_fm_flow_parse(struct enic_flowman *fm,
1660                    const struct rte_flow_attr *attrs,
1661                    const struct rte_flow_item pattern[],
1662                    const struct rte_flow_action actions[],
1663                    struct rte_flow_error *error)
1664 {
1665         const struct rte_flow_action *action;
1666         unsigned int ret;
1667         static const enum rte_flow_action_type *sa;
1668
1669         ENICPMD_FUNC_TRACE();
1670         ret = 0;
1671         if (!pattern) {
1672                 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM,
1673                                    NULL, "no pattern specified");
1674                 return -rte_errno;
1675         }
1676
1677         if (!actions) {
1678                 rte_flow_error_set(error, EINVAL,
1679                                    RTE_FLOW_ERROR_TYPE_ACTION_NUM,
1680                                    NULL, "no action specified");
1681                 return -rte_errno;
1682         }
1683
1684         if (attrs) {
1685                 if (attrs->priority) {
1686                         rte_flow_error_set(error, ENOTSUP,
1687                                            RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1688                                            NULL,
1689                                            "priorities are not supported");
1690                         return -rte_errno;
1691                 } else if (!fm->owner_enic->switchdev_mode && attrs->transfer) {
1692                         rte_flow_error_set(error, ENOTSUP,
1693                                            RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1694                                            NULL,
1695                                            "transfer is not supported");
1696                         return -rte_errno;
1697                 } else if (attrs->ingress && attrs->egress) {
1698                         rte_flow_error_set(error, ENOTSUP,
1699                                            RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1700                                            NULL,
1701                                            "bidirectional rules not supported");
1702                         return -rte_errno;
1703                 }
1704
1705         } else {
1706                 rte_flow_error_set(error, EINVAL,
1707                                    RTE_FLOW_ERROR_TYPE_ATTR,
1708                                    NULL, "no attribute specified");
1709                 return -rte_errno;
1710         }
1711
1712         /* Verify Actions. */
1713         sa = (attrs->ingress) ? enic_fm_supported_ig_actions :
1714              enic_fm_supported_eg_actions;
1715         for (action = &actions[0]; action->type != RTE_FLOW_ACTION_TYPE_END;
1716              action++) {
1717                 if (action->type == RTE_FLOW_ACTION_TYPE_VOID)
1718                         continue;
1719                 else if (!enic_fm_match_action(action, sa))
1720                         break;
1721         }
1722         if (action->type != RTE_FLOW_ACTION_TYPE_END) {
1723                 rte_flow_error_set(error, EPERM, RTE_FLOW_ERROR_TYPE_ACTION,
1724                                    action, "invalid action");
1725                 return -rte_errno;
1726         }
1727         ret = enic_fm_copy_entry(fm, pattern, error);
1728         if (ret)
1729                 return ret;
1730         ret = enic_fm_copy_action(fm, actions, attrs->ingress, error);
1731         return ret;
1732 }
1733
1734 static void
1735 enic_fm_counter_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
1736 {
1737         if (!fm_flow->counter_valid)
1738                 return;
1739         SLIST_INSERT_HEAD(&fm->counters, fm_flow->counter, next);
1740         fm_flow->counter_valid = false;
1741 }
1742
1743 static int
1744 enic_fm_more_counters(struct enic_flowman *fm)
1745 {
1746         struct enic_fm_counter *new_stack;
1747         struct enic_fm_counter *ctrs;
1748         int i, rc;
1749         uint64_t args[2];
1750
1751         ENICPMD_FUNC_TRACE();
1752         new_stack = rte_realloc(fm->counter_stack, (fm->counters_alloced +
1753                                 FM_COUNTERS_EXPAND) *
1754                                 sizeof(struct enic_fm_counter), 0);
1755         if (new_stack == NULL) {
1756                 ENICPMD_LOG(ERR, "cannot alloc counter memory");
1757                 return -ENOMEM;
1758         }
1759         fm->counter_stack = new_stack;
1760
1761         args[0] = FM_COUNTER_BRK;
1762         args[1] = fm->counters_alloced + FM_COUNTERS_EXPAND;
1763         rc = flowman_cmd(fm, args, 2);
1764         if (rc != 0) {
1765                 ENICPMD_LOG(ERR, "cannot alloc counters rc=%d", rc);
1766                 return rc;
1767         }
1768         ctrs = (struct enic_fm_counter *)fm->counter_stack +
1769                 fm->counters_alloced;
1770         for (i = 0; i < FM_COUNTERS_EXPAND; i++, ctrs++) {
1771                 ctrs->handle = fm->counters_alloced + i;
1772                 SLIST_INSERT_HEAD(&fm->counters, ctrs, next);
1773         }
1774         fm->counters_alloced += FM_COUNTERS_EXPAND;
1775         ENICPMD_LOG(DEBUG, "%u counters allocated, total: %u",
1776                     FM_COUNTERS_EXPAND, fm->counters_alloced);
1777         return 0;
1778 }
1779
1780 static int
1781 enic_fm_counter_zero(struct enic_flowman *fm, struct enic_fm_counter *c)
1782 {
1783         uint64_t args[3];
1784         int ret;
1785
1786         ENICPMD_FUNC_TRACE();
1787         args[0] = FM_COUNTER_QUERY;
1788         args[1] = c->handle;
1789         args[2] = 1; /* clear */
1790         ret = flowman_cmd(fm, args, 3);
1791         if (ret) {
1792                 ENICPMD_LOG(ERR, "counter init: rc=%d handle=0x%x",
1793                             ret, c->handle);
1794                 return ret;
1795         }
1796         return 0;
1797 }
1798
1799 static int
1800 enic_fm_counter_alloc(struct enic_flowman *fm, struct rte_flow_error *error,
1801                       struct enic_fm_counter **ctr)
1802 {
1803         struct enic_fm_counter *c;
1804         int ret;
1805
1806         ENICPMD_FUNC_TRACE();
1807         *ctr = NULL;
1808         if (SLIST_EMPTY(&fm->counters)) {
1809                 ret = enic_fm_more_counters(fm);
1810                 if (ret)
1811                         return rte_flow_error_set(error, -ret,
1812                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1813                                 NULL, "enic: out of counters");
1814         }
1815         c = SLIST_FIRST(&fm->counters);
1816         SLIST_REMOVE_HEAD(&fm->counters, next);
1817         *ctr = c;
1818         return 0;
1819 }
1820
1821 static int
1822 enic_fm_action_free(struct enic_flowman *fm, uint64_t handle)
1823 {
1824         uint64_t args[2];
1825         int rc;
1826
1827         ENICPMD_FUNC_TRACE();
1828         args[0] = FM_ACTION_FREE;
1829         args[1] = handle;
1830         rc = flowman_cmd(fm, args, 2);
1831         if (rc)
1832                 ENICPMD_LOG(ERR, "cannot free action: rc=%d handle=0x%" PRIx64,
1833                             rc, handle);
1834         return rc;
1835 }
1836
1837 static int
1838 enic_fm_entry_free(struct enic_flowman *fm, uint64_t handle)
1839 {
1840         uint64_t args[2];
1841         int rc;
1842
1843         ENICPMD_FUNC_TRACE();
1844         args[0] = FM_MATCH_ENTRY_REMOVE;
1845         args[1] = handle;
1846         rc = flowman_cmd(fm, args, 2);
1847         if (rc)
1848                 ENICPMD_LOG(ERR, "cannot free match entry: rc=%d"
1849                             " handle=0x%" PRIx64, rc, handle);
1850         return rc;
1851 }
1852
1853 static struct enic_fm_jump_flow *
1854 find_jump_flow(struct enic_flowman *fm, uint32_t group)
1855 {
1856         struct enic_fm_jump_flow *j;
1857
1858         ENICPMD_FUNC_TRACE();
1859         TAILQ_FOREACH(j, &fm->jump_list, list) {
1860                 if (j->group == group)
1861                         return j;
1862         }
1863         return NULL;
1864 }
1865
1866 static void
1867 remove_jump_flow(struct enic_flowman *fm, struct rte_flow *flow)
1868 {
1869         struct enic_fm_jump_flow *j;
1870
1871         ENICPMD_FUNC_TRACE();
1872         TAILQ_FOREACH(j, &fm->jump_list, list) {
1873                 if (j->flow == flow) {
1874                         TAILQ_REMOVE(&fm->jump_list, j, list);
1875                         free(j);
1876                         return;
1877                 }
1878         }
1879 }
1880
1881 static int
1882 save_jump_flow(struct enic_flowman *fm,
1883                struct rte_flow *flow,
1884                uint32_t group,
1885                struct fm_tcam_match_entry *match,
1886                struct fm_action *action)
1887 {
1888         struct enic_fm_jump_flow *j;
1889
1890         ENICPMD_FUNC_TRACE();
1891         j = calloc(1, sizeof(struct enic_fm_jump_flow));
1892         if (j == NULL)
1893                 return -ENOMEM;
1894         j->flow = flow;
1895         j->group = group;
1896         j->match = *match;
1897         j->action = *action;
1898         TAILQ_INSERT_HEAD(&fm->jump_list, j, list);
1899         ENICPMD_LOG(DEBUG, "saved jump flow: flow=%p group=%u", flow, group);
1900         return 0;
1901 }
1902
1903 static void
1904 __enic_fm_flow_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
1905 {
1906         if (fm_flow->entry_handle != FM_INVALID_HANDLE) {
1907                 enic_fm_entry_free(fm, fm_flow->entry_handle);
1908                 fm_flow->entry_handle = FM_INVALID_HANDLE;
1909         }
1910         if (fm_flow->action_handle != FM_INVALID_HANDLE) {
1911                 enic_fm_action_free(fm, fm_flow->action_handle);
1912                 fm_flow->action_handle = FM_INVALID_HANDLE;
1913         }
1914         enic_fm_counter_free(fm, fm_flow);
1915         if (fm_flow->fet) {
1916                 enic_fet_put(fm, fm_flow->fet);
1917                 fm_flow->fet = NULL;
1918         }
1919 }
1920
1921 static void
1922 enic_fm_flow_free(struct enic_flowman *fm, struct rte_flow *flow)
1923 {
1924         if (flow->fm->fet && flow->fm->fet->default_key)
1925                 remove_jump_flow(fm, flow);
1926         __enic_fm_flow_free(fm, flow->fm);
1927         free(flow->fm);
1928         free(flow);
1929 }
1930
1931 static int
1932 enic_fm_add_tcam_entry(struct enic_flowman *fm,
1933                        struct fm_tcam_match_entry *match_in,
1934                        uint64_t *entry_handle,
1935                        uint8_t ingress,
1936                        struct rte_flow_error *error)
1937 {
1938         struct fm_tcam_match_entry *ftm;
1939         uint64_t args[3];
1940         int ret;
1941
1942         ENICPMD_FUNC_TRACE();
1943         /* Copy entry to the command buffer */
1944         ftm = &fm->cmd.va->fm_tcam_match_entry;
1945         memcpy(ftm, match_in, sizeof(*ftm));
1946         /* Add TCAM entry */
1947         args[0] = FM_TCAM_ENTRY_INSTALL;
1948         args[1] = ingress ? fm->ig_tcam_hndl : fm->eg_tcam_hndl;
1949         args[2] = fm->cmd.pa;
1950         ret = flowman_cmd(fm, args, 3);
1951         if (ret != 0) {
1952                 ENICPMD_LOG(ERR, "cannot add %s TCAM entry: rc=%d",
1953                             ingress ? "ingress" : "egress", ret);
1954                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1955                         NULL, "enic: devcmd(tcam-entry-install)");
1956                 return ret;
1957         }
1958         ENICPMD_LOG(DEBUG, "installed %s TCAM entry: handle=0x%" PRIx64,
1959                     ingress ? "ingress" : "egress", (uint64_t)args[0]);
1960         *entry_handle = args[0];
1961         return 0;
1962 }
1963
1964 static int
1965 enic_fm_add_exact_entry(struct enic_flowman *fm,
1966                         struct fm_tcam_match_entry *match_in,
1967                         uint64_t *entry_handle,
1968                         struct enic_fm_fet *fet,
1969                         struct rte_flow_error *error)
1970 {
1971         struct fm_exact_match_entry *fem;
1972         uint64_t args[3];
1973         int ret;
1974
1975         ENICPMD_FUNC_TRACE();
1976         /* The new entry must have the table's key */
1977         if (memcmp(fet->key.fk_hdrset, match_in->ftm_mask.fk_hdrset,
1978                    sizeof(struct fm_header_set) * FM_HDRSET_MAX)) {
1979                 return rte_flow_error_set(error, EINVAL,
1980                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1981                         "enic: key does not match group's key");
1982         }
1983
1984         /* Copy entry to the command buffer */
1985         fem = &fm->cmd.va->fm_exact_match_entry;
1986         /*
1987          * Translate TCAM entry to exact entry. As is only need to drop
1988          * position and mask. The mask is part of the exact match table.
1989          * Position (aka priority) is not supported in the exact match table.
1990          */
1991         fem->fem_data = match_in->ftm_data;
1992         fem->fem_flags = match_in->ftm_flags;
1993         fem->fem_action = match_in->ftm_action;
1994         fem->fem_counter = match_in->ftm_counter;
1995
1996         /* Add exact entry */
1997         args[0] = FM_EXACT_ENTRY_INSTALL;
1998         args[1] = fet->handle;
1999         args[2] = fm->cmd.pa;
2000         ret = flowman_cmd(fm, args, 3);
2001         if (ret != 0) {
2002                 ENICPMD_LOG(ERR, "cannot add %s exact entry: group=%u",
2003                             fet->ingress ? "ingress" : "egress", fet->group);
2004                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2005                         NULL, "enic: devcmd(exact-entry-install)");
2006                 return ret;
2007         }
2008         ENICPMD_LOG(DEBUG, "installed %s exact entry: group=%u"
2009                     " handle=0x%" PRIx64,
2010                     fet->ingress ? "ingress" : "egress", fet->group,
2011                     (uint64_t)args[0]);
2012         *entry_handle = args[0];
2013         return 0;
2014 }
2015
2016 /* Push match-action to the NIC. */
2017 static int
2018 __enic_fm_flow_add_entry(struct enic_flowman *fm,
2019                          struct enic_fm_flow *fm_flow,
2020                          struct fm_tcam_match_entry *match_in,
2021                          struct fm_action *action_in,
2022                          uint32_t group,
2023                          uint8_t ingress,
2024                          struct rte_flow_error *error)
2025 {
2026         struct enic_fm_counter *ctr;
2027         struct fm_action *fma;
2028         uint64_t action_h;
2029         uint64_t entry_h;
2030         uint64_t args[3];
2031         int ret;
2032
2033         ENICPMD_FUNC_TRACE();
2034         /* Allocate action. */
2035         fma = &fm->cmd.va->fm_action;
2036         memcpy(fma, action_in, sizeof(*fma));
2037         args[0] = FM_ACTION_ALLOC;
2038         args[1] = fm->cmd.pa;
2039         ret = flowman_cmd(fm, args, 2);
2040         if (ret != 0) {
2041                 ENICPMD_LOG(ERR, "allocating TCAM table action rc=%d", ret);
2042                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2043                         NULL, "enic: devcmd(action-alloc)");
2044                 return ret;
2045         }
2046         action_h = args[0];
2047         fm_flow->action_handle = action_h;
2048         match_in->ftm_action = action_h;
2049         ENICPMD_LOG(DEBUG, "action allocated: handle=0x%" PRIx64, action_h);
2050
2051         /* Allocate counter if requested. */
2052         if (match_in->ftm_flags & FMEF_COUNTER) {
2053                 ret = enic_fm_counter_alloc(fm, error, &ctr);
2054                 if (ret) /* error has been filled in */
2055                         return ret;
2056                 fm_flow->counter_valid = true;
2057                 fm_flow->counter = ctr;
2058                 match_in->ftm_counter = ctr->handle;
2059         }
2060
2061         /*
2062          * Get the group's table (either TCAM or exact match table) and
2063          * add entry to it. If we use the exact match table, the handler
2064          * will translate the TCAM entry (match_in) to the appropriate
2065          * exact match entry and use that instead.
2066          */
2067         entry_h = FM_INVALID_HANDLE;
2068         if (group == FM_TCAM_RTE_GROUP) {
2069                 ret = enic_fm_add_tcam_entry(fm, match_in, &entry_h, ingress,
2070                                              error);
2071                 if (ret)
2072                         return ret;
2073                 /* Jump action might have a ref to fet */
2074                 fm_flow->fet = fm->fet;
2075                 fm->fet = NULL;
2076         } else {
2077                 struct enic_fm_fet *fet = NULL;
2078
2079                 ret = enic_fet_get(fm, group, ingress,
2080                                    &match_in->ftm_mask, &fet, error);
2081                 if (ret)
2082                         return ret;
2083                 fm_flow->fet = fet;
2084                 ret = enic_fm_add_exact_entry(fm, match_in, &entry_h, fet,
2085                                               error);
2086                 if (ret)
2087                         return ret;
2088         }
2089         /* Clear counter after adding entry, as it requires in-use counter */
2090         if (fm_flow->counter_valid) {
2091                 ret = enic_fm_counter_zero(fm, fm_flow->counter);
2092                 if (ret)
2093                         return ret;
2094         }
2095         fm_flow->entry_handle = entry_h;
2096         return 0;
2097 }
2098
2099 /* Push match-action to the NIC. */
2100 static struct rte_flow *
2101 enic_fm_flow_add_entry(struct enic_flowman *fm,
2102                        struct fm_tcam_match_entry *match_in,
2103                        struct fm_action *action_in,
2104                        const struct rte_flow_attr *attrs,
2105                        struct rte_flow_error *error)
2106 {
2107         struct enic_fm_flow *fm_flow;
2108         struct rte_flow *flow;
2109
2110         ENICPMD_FUNC_TRACE();
2111         enic_fm_dump_tcam_entry(match_in, action_in, attrs->ingress);
2112         flow = calloc(1, sizeof(*flow));
2113         fm_flow = calloc(1, sizeof(*fm_flow));
2114         if (flow == NULL || fm_flow == NULL) {
2115                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
2116                         NULL, "enic: cannot allocate rte_flow");
2117                 free(flow);
2118                 free(fm_flow);
2119                 return NULL;
2120         }
2121         flow->fm = fm_flow;
2122         fm_flow->action_handle = FM_INVALID_HANDLE;
2123         fm_flow->entry_handle = FM_INVALID_HANDLE;
2124         if (__enic_fm_flow_add_entry(fm, fm_flow, match_in, action_in,
2125                                      attrs->group, attrs->ingress, error)) {
2126                 enic_fm_flow_free(fm, flow);
2127                 return NULL;
2128         }
2129         return flow;
2130 }
2131
2132 static void
2133 convert_jump_flows(struct enic_flowman *fm, struct enic_fm_fet *fet,
2134                    struct rte_flow_error *error)
2135 {
2136         struct enic_fm_flow *fm_flow;
2137         struct enic_fm_jump_flow *j;
2138         struct fm_action *fma;
2139         uint32_t group;
2140
2141         ENICPMD_FUNC_TRACE();
2142         /*
2143          * Find the saved flows that should jump to the new table (fet).
2144          * Then delete the old TCAM entry that jumps to the default table,
2145          * and add a new one that jumps to the new table.
2146          */
2147         group = fet->group;
2148         j = find_jump_flow(fm, group);
2149         while (j) {
2150                 ENICPMD_LOG(DEBUG, "convert jump flow: flow=%p group=%u",
2151                             j->flow, group);
2152                 /* Delete old entry */
2153                 fm_flow = j->flow->fm;
2154                 __enic_fm_flow_free(fm, fm_flow);
2155
2156                 /* Add new entry */
2157                 fma = &j->action;
2158                 fma->fma_action_ops[0].exact.handle = fet->handle;
2159                 if (__enic_fm_flow_add_entry(fm, fm_flow, &j->match, fma,
2160                         FM_TCAM_RTE_GROUP, fet->ingress, error)) {
2161                         /* Cannot roll back changes at the moment */
2162                         ENICPMD_LOG(ERR, "cannot convert jump flow: flow=%p",
2163                                     j->flow);
2164                 } else {
2165                         fm_flow->fet = fet;
2166                         fet->ref++;
2167                         ENICPMD_LOG(DEBUG, "convert ok: group=%u ref=%u",
2168                                     fet->group, fet->ref);
2169                 }
2170
2171                 TAILQ_REMOVE(&fm->jump_list, j, list);
2172                 free(j);
2173                 j = find_jump_flow(fm, group);
2174         }
2175 }
2176
2177 static void
2178 enic_fm_open_scratch(struct enic_flowman *fm)
2179 {
2180         fm->action_op_count = 0;
2181         fm->fet = NULL;
2182         memset(&fm->tcam_entry, 0, sizeof(fm->tcam_entry));
2183         memset(&fm->action, 0, sizeof(fm->action));
2184 }
2185
2186 static void
2187 enic_fm_close_scratch(struct enic_flowman *fm)
2188 {
2189         if (fm->fet) {
2190                 enic_fet_put(fm, fm->fet);
2191                 fm->fet = NULL;
2192         }
2193         fm->action_op_count = 0;
2194 }
2195
2196 static int
2197 enic_fm_flow_validate(struct rte_eth_dev *dev,
2198                       const struct rte_flow_attr *attrs,
2199                       const struct rte_flow_item pattern[],
2200                       const struct rte_flow_action actions[],
2201                       struct rte_flow_error *error)
2202 {
2203         struct fm_tcam_match_entry *fm_tcam_entry;
2204         struct fm_action *fm_action;
2205         struct enic_flowman *fm;
2206         int ret;
2207
2208         ENICPMD_FUNC_TRACE();
2209         fm = begin_fm(pmd_priv(dev));
2210         if (fm == NULL)
2211                 return -ENOTSUP;
2212         enic_fm_open_scratch(fm);
2213         ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2214         if (!ret) {
2215                 fm_tcam_entry = &fm->tcam_entry;
2216                 fm_action = &fm->action;
2217                 enic_fm_dump_tcam_entry(fm_tcam_entry, fm_action,
2218                                         attrs->ingress);
2219         }
2220         enic_fm_close_scratch(fm);
2221         end_fm(fm);
2222         return ret;
2223 }
2224
2225 static int
2226 enic_fm_flow_query_count(struct rte_eth_dev *dev,
2227                          struct rte_flow *flow, void *data,
2228                          struct rte_flow_error *error)
2229 {
2230         struct rte_flow_query_count *query;
2231         struct enic_fm_flow *fm_flow;
2232         struct enic_flowman *fm;
2233         uint64_t args[3];
2234         int rc;
2235
2236         ENICPMD_FUNC_TRACE();
2237         fm = begin_fm(pmd_priv(dev));
2238         query = data;
2239         fm_flow = flow->fm;
2240         if (!fm_flow->counter_valid) {
2241                 rc = rte_flow_error_set(error, ENOTSUP,
2242                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2243                         "enic: flow does not have counter");
2244                 goto exit;
2245         }
2246
2247         args[0] = FM_COUNTER_QUERY;
2248         args[1] = fm_flow->counter->handle;
2249         args[2] = query->reset;
2250         rc = flowman_cmd(fm, args, 3);
2251         if (rc) {
2252                 ENICPMD_LOG(ERR, "cannot query counter: rc=%d handle=0x%x",
2253                             rc, fm_flow->counter->handle);
2254                 goto exit;
2255         }
2256         query->hits_set = 1;
2257         query->hits = args[0];
2258         query->bytes_set = 1;
2259         query->bytes = args[1];
2260         rc = 0;
2261 exit:
2262         end_fm(fm);
2263         return rc;
2264 }
2265
2266 static int
2267 enic_fm_flow_query(struct rte_eth_dev *dev,
2268                    struct rte_flow *flow,
2269                    const struct rte_flow_action *actions,
2270                    void *data,
2271                    struct rte_flow_error *error)
2272 {
2273         int ret = 0;
2274
2275         ENICPMD_FUNC_TRACE();
2276         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2277                 switch (actions->type) {
2278                 case RTE_FLOW_ACTION_TYPE_VOID:
2279                         break;
2280                 case RTE_FLOW_ACTION_TYPE_COUNT:
2281                         ret = enic_fm_flow_query_count(dev, flow, data, error);
2282                         break;
2283                 default:
2284                         return rte_flow_error_set(error, ENOTSUP,
2285                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2286                                                   actions,
2287                                                   "action not supported");
2288                 }
2289                 if (ret < 0)
2290                         return ret;
2291         }
2292         return 0;
2293 }
2294
2295 static struct rte_flow *
2296 enic_fm_flow_create(struct rte_eth_dev *dev,
2297                     const struct rte_flow_attr *attrs,
2298                     const struct rte_flow_item pattern[],
2299                     const struct rte_flow_action actions[],
2300                     struct rte_flow_error *error)
2301 {
2302         struct fm_tcam_match_entry *fm_tcam_entry;
2303         struct fm_action *fm_action;
2304         struct enic_flowman *fm;
2305         struct enic_fm_fet *fet;
2306         struct rte_flow *flow;
2307         struct enic *enic;
2308         int ret;
2309
2310         ENICPMD_FUNC_TRACE();
2311         enic = pmd_priv(dev);
2312         fm = begin_fm(enic);
2313         if (fm == NULL) {
2314                 rte_flow_error_set(error, ENOTSUP,
2315                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2316                         "flowman is not initialized");
2317                 return NULL;
2318         }
2319         enic_fm_open_scratch(fm);
2320         flow = NULL;
2321         ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2322         if (ret < 0)
2323                 goto error_with_scratch;
2324         fm_tcam_entry = &fm->tcam_entry;
2325         fm_action = &fm->action;
2326         flow = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2327                                       attrs, error);
2328         if (flow) {
2329                 LIST_INSERT_HEAD(&enic->flows, flow, next);
2330                 fet = flow->fm->fet;
2331                 if (fet && fet->default_key) {
2332                         /*
2333                          * Jump to non-existent group? Save the relevant info
2334                          * so we can convert this flow when that group
2335                          * materializes.
2336                          */
2337                         save_jump_flow(fm, flow, fet->group,
2338                                        fm_tcam_entry, fm_action);
2339                 } else if (fet && fet->ref == 1) {
2340                         /*
2341                          * A new table is created. Convert the saved flows
2342                          * that should jump to this group.
2343                          */
2344                         convert_jump_flows(fm, fet, error);
2345                 }
2346         }
2347
2348 error_with_scratch:
2349         enic_fm_close_scratch(fm);
2350         end_fm(fm);
2351         return flow;
2352 }
2353
2354 static int
2355 enic_fm_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
2356                      __rte_unused struct rte_flow_error *error)
2357 {
2358         struct enic *enic = pmd_priv(dev);
2359         struct enic_flowman *fm;
2360
2361         ENICPMD_FUNC_TRACE();
2362         fm = begin_fm(enic);
2363         if (fm == NULL)
2364                 return 0;
2365         LIST_REMOVE(flow, next);
2366         enic_fm_flow_free(fm, flow);
2367         end_fm(fm);
2368         return 0;
2369 }
2370
2371 static int
2372 enic_fm_flow_flush(struct rte_eth_dev *dev,
2373                    __rte_unused struct rte_flow_error *error)
2374 {
2375         LIST_HEAD(enic_flows, rte_flow) internal;
2376         struct enic_fm_flow *fm_flow;
2377         struct enic_flowman *fm;
2378         struct rte_flow *flow;
2379         struct enic *enic = pmd_priv(dev);
2380
2381         ENICPMD_FUNC_TRACE();
2382
2383         fm = begin_fm(enic);
2384         if (fm == NULL)
2385                 return 0;
2386         /* Destroy all non-internal flows */
2387         LIST_INIT(&internal);
2388         while (!LIST_EMPTY(&enic->flows)) {
2389                 flow = LIST_FIRST(&enic->flows);
2390                 fm_flow = flow->fm;
2391                 LIST_REMOVE(flow, next);
2392                 if (flow->internal) {
2393                         LIST_INSERT_HEAD(&internal, flow, next);
2394                         continue;
2395                 }
2396                 /*
2397                  * If tables are null, then vNIC is closing, and the firmware
2398                  * has already cleaned up flowman state. So do not try to free
2399                  * resources, as it only causes errors.
2400                  */
2401                 if (fm->ig_tcam_hndl == FM_INVALID_HANDLE) {
2402                         fm_flow->entry_handle = FM_INVALID_HANDLE;
2403                         fm_flow->action_handle = FM_INVALID_HANDLE;
2404                         fm_flow->fet = NULL;
2405                 }
2406                 enic_fm_flow_free(fm, flow);
2407         }
2408         while (!LIST_EMPTY(&internal)) {
2409                 flow = LIST_FIRST(&internal);
2410                 LIST_REMOVE(flow, next);
2411                 LIST_INSERT_HEAD(&enic->flows, flow, next);
2412         }
2413         end_fm(fm);
2414         return 0;
2415 }
2416
2417 static int
2418 enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle)
2419 {
2420         uint64_t args[2];
2421         int rc;
2422
2423         args[0] = FM_MATCH_TABLE_FREE;
2424         args[1] = handle;
2425         rc = flowman_cmd(fm, args, 2);
2426         if (rc)
2427                 ENICPMD_LOG(ERR, "cannot free table: rc=%d handle=0x%" PRIx64,
2428                             rc, handle);
2429         return rc;
2430 }
2431
2432 static int
2433 enic_fm_tcam_tbl_alloc(struct enic_flowman *fm, uint32_t direction,
2434                         uint32_t max_entries, uint64_t *handle)
2435 {
2436         struct fm_tcam_match_table *tcam_tbl;
2437         uint64_t args[2];
2438         int rc;
2439
2440         ENICPMD_FUNC_TRACE();
2441         tcam_tbl = &fm->cmd.va->fm_tcam_match_table;
2442         tcam_tbl->ftt_direction = direction;
2443         tcam_tbl->ftt_stage = FM_STAGE_LAST;
2444         tcam_tbl->ftt_max_entries = max_entries;
2445         args[0] = FM_TCAM_TABLE_ALLOC;
2446         args[1] = fm->cmd.pa;
2447         rc = flowman_cmd(fm, args, 2);
2448         if (rc) {
2449                 ENICPMD_LOG(ERR, "cannot alloc %s TCAM table: rc=%d",
2450                             (direction == FM_INGRESS) ? "IG" : "EG", rc);
2451                 return rc;
2452         }
2453         *handle = args[0];
2454         ENICPMD_LOG(DEBUG, "%s TCAM table allocated, handle=0x%" PRIx64,
2455                     (direction == FM_INGRESS) ? "IG" : "EG", *handle);
2456         return 0;
2457 }
2458
2459 static int
2460 enic_fm_init_counters(struct enic_flowman *fm)
2461 {
2462         ENICPMD_FUNC_TRACE();
2463         SLIST_INIT(&fm->counters);
2464         return enic_fm_more_counters(fm);
2465 }
2466
2467 static void
2468 enic_fm_free_all_counters(struct enic_flowman *fm)
2469 {
2470         uint64_t args[2];
2471         int rc;
2472
2473         args[0] = FM_COUNTER_BRK;
2474         args[1] = 0;
2475         rc = flowman_cmd(fm, args, 2);
2476         if (rc != 0)
2477                 ENICPMD_LOG(ERR, "cannot free counters: rc=%d", rc);
2478         rte_free(fm->counter_stack);
2479 }
2480
2481 static int
2482 enic_fm_alloc_tcam_tables(struct enic_flowman *fm)
2483 {
2484         int rc;
2485
2486         ENICPMD_FUNC_TRACE();
2487         rc = enic_fm_tcam_tbl_alloc(fm, FM_INGRESS, FM_MAX_TCAM_TABLE_SIZE,
2488                                     &fm->ig_tcam_hndl);
2489         if (rc)
2490                 return rc;
2491         rc = enic_fm_tcam_tbl_alloc(fm, FM_EGRESS, FM_MAX_TCAM_TABLE_SIZE,
2492                                     &fm->eg_tcam_hndl);
2493         return rc;
2494 }
2495
2496 static void
2497 enic_fm_free_tcam_tables(struct enic_flowman *fm)
2498 {
2499         ENICPMD_FUNC_TRACE();
2500         if (fm->ig_tcam_hndl) {
2501                 ENICPMD_LOG(DEBUG, "free IG TCAM table handle=0x%" PRIx64,
2502                             fm->ig_tcam_hndl);
2503                 enic_fm_tbl_free(fm, fm->ig_tcam_hndl);
2504                 fm->ig_tcam_hndl = FM_INVALID_HANDLE;
2505         }
2506         if (fm->eg_tcam_hndl) {
2507                 ENICPMD_LOG(DEBUG, "free EG TCAM table handle=0x%" PRIx64,
2508                             fm->eg_tcam_hndl);
2509                 enic_fm_tbl_free(fm, fm->eg_tcam_hndl);
2510                 fm->eg_tcam_hndl = FM_INVALID_HANDLE;
2511         }
2512 }
2513
2514 int
2515 enic_fm_init(struct enic *enic)
2516 {
2517         const struct rte_pci_addr *addr;
2518         struct enic_flowman *fm;
2519         uint8_t name[RTE_MEMZONE_NAMESIZE];
2520         int rc;
2521
2522         if (enic->flow_filter_mode != FILTER_FLOWMAN)
2523                 return 0;
2524         ENICPMD_FUNC_TRACE();
2525         /* Get vnic handle and save for port-id action */
2526         if (enic_is_vf_rep(enic))
2527                 addr = &VF_ENIC_TO_VF_REP(enic)->bdf;
2528         else
2529                 addr = &RTE_ETH_DEV_TO_PCI(enic->rte_dev)->addr;
2530         rc = enic_fm_find_vnic(enic, addr, &enic->fm_vnic_handle);
2531         if (rc) {
2532                 ENICPMD_LOG(ERR, "cannot find vnic handle for %x:%x:%x",
2533                             addr->bus, addr->devid, addr->function);
2534                 return rc;
2535         }
2536         /* Save UIF for egport action */
2537         enic->fm_vnic_uif = vnic_dev_uif(enic->vdev);
2538         ENICPMD_LOG(DEBUG, "uif %u", enic->fm_vnic_uif);
2539         /* Nothing else to do for representor. It will share the PF flowman */
2540         if (enic_is_vf_rep(enic))
2541                 return 0;
2542         fm = calloc(1, sizeof(*fm));
2543         if (fm == NULL) {
2544                 ENICPMD_LOG(ERR, "cannot alloc flowman struct");
2545                 return -ENOMEM;
2546         }
2547         fm->owner_enic = enic;
2548         rte_spinlock_init(&fm->lock);
2549         TAILQ_INIT(&fm->fet_list);
2550         TAILQ_INIT(&fm->jump_list);
2551         /* Allocate host memory for flowman commands */
2552         snprintf((char *)name, sizeof(name), "fm-cmd-%s", enic->bdf_name);
2553         fm->cmd.va = enic_alloc_consistent(enic,
2554                 sizeof(union enic_flowman_cmd_mem), &fm->cmd.pa, name);
2555         if (!fm->cmd.va) {
2556                 ENICPMD_LOG(ERR, "cannot allocate flowman command memory");
2557                 rc = -ENOMEM;
2558                 goto error_fm;
2559         }
2560         /* Allocate TCAM tables upfront as they are the main tables */
2561         rc = enic_fm_alloc_tcam_tables(fm);
2562         if (rc) {
2563                 ENICPMD_LOG(ERR, "cannot alloc TCAM tables");
2564                 goto error_cmd;
2565         }
2566         /* Then a number of counters */
2567         rc = enic_fm_init_counters(fm);
2568         if (rc) {
2569                 ENICPMD_LOG(ERR, "cannot alloc counters");
2570                 goto error_tables;
2571         }
2572         /*
2573          * One default exact match table for each direction. We hold onto
2574          * it until close.
2575          */
2576         rc = enic_fet_alloc(fm, 1, NULL, 128, &fm->default_ig_fet);
2577         if (rc) {
2578                 ENICPMD_LOG(ERR, "cannot alloc default IG exact match table");
2579                 goto error_counters;
2580         }
2581         fm->default_ig_fet->ref = 1;
2582         rc = enic_fet_alloc(fm, 0, NULL, 128, &fm->default_eg_fet);
2583         if (rc) {
2584                 ENICPMD_LOG(ERR, "cannot alloc default EG exact match table");
2585                 goto error_ig_fet;
2586         }
2587         fm->default_eg_fet->ref = 1;
2588         fm->vf_rep_tag = FM_VF_REP_TAG;
2589         enic->fm = fm;
2590         return 0;
2591
2592 error_ig_fet:
2593         enic_fet_free(fm, fm->default_ig_fet);
2594 error_counters:
2595         enic_fm_free_all_counters(fm);
2596 error_tables:
2597         enic_fm_free_tcam_tables(fm);
2598 error_cmd:
2599         enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
2600                 fm->cmd.va, fm->cmd.pa);
2601 error_fm:
2602         free(fm);
2603         return rc;
2604 }
2605
2606 void
2607 enic_fm_destroy(struct enic *enic)
2608 {
2609         struct enic_flowman *fm;
2610         struct enic_fm_fet *fet;
2611
2612         ENICPMD_FUNC_TRACE();
2613         if (enic_is_vf_rep(enic)) {
2614                 delete_rep_flows(enic);
2615                 return;
2616         }
2617         if (enic->fm == NULL)
2618                 return;
2619         fm = enic->fm;
2620         enic_fet_free(fm, fm->default_eg_fet);
2621         enic_fet_free(fm, fm->default_ig_fet);
2622         /* Free all exact match tables still open */
2623         while (!TAILQ_EMPTY(&fm->fet_list)) {
2624                 fet = TAILQ_FIRST(&fm->fet_list);
2625                 enic_fet_free(fm, fet);
2626         }
2627         enic_fm_free_tcam_tables(fm);
2628         enic_fm_free_all_counters(fm);
2629         enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
2630                 fm->cmd.va, fm->cmd.pa);
2631         fm->cmd.va = NULL;
2632         free(fm);
2633         enic->fm = NULL;
2634 }
2635
2636 int
2637 enic_fm_allocate_switch_domain(struct enic *pf)
2638 {
2639         const struct rte_pci_addr *cur_a, *prev_a;
2640         struct rte_eth_dev *dev;
2641         struct enic *cur, *prev;
2642         uint16_t domain_id;
2643         uint64_t vnic_h;
2644         uint16_t pid;
2645         int ret;
2646
2647         ENICPMD_FUNC_TRACE();
2648         if (enic_is_vf_rep(pf))
2649                 return -EINVAL;
2650         cur = pf;
2651         cur_a = &RTE_ETH_DEV_TO_PCI(cur->rte_dev)->addr;
2652         /* Go through ports and find another PF that is on the same adapter */
2653         RTE_ETH_FOREACH_DEV(pid) {
2654                 dev = &rte_eth_devices[pid];
2655                 if (!dev_is_enic(dev))
2656                         continue;
2657                 if (dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)
2658                         continue;
2659                 if (dev == cur->rte_dev)
2660                         continue;
2661                 /* dev is another PF. Is it on the same adapter? */
2662                 prev = pmd_priv(dev);
2663                 prev_a = &RTE_ETH_DEV_TO_PCI(dev)->addr;
2664                 if (!enic_fm_find_vnic(cur, prev_a, &vnic_h)) {
2665                         ENICPMD_LOG(DEBUG, "Port %u (PF BDF %x:%x:%x) and port %u (PF BDF %x:%x:%x domain %u) are on the same VIC",
2666                                 cur->rte_dev->data->port_id,
2667                                 cur_a->bus, cur_a->devid, cur_a->function,
2668                                 dev->data->port_id,
2669                                 prev_a->bus, prev_a->devid, prev_a->function,
2670                                 prev->switch_domain_id);
2671                         cur->switch_domain_id = prev->switch_domain_id;
2672                         return 0;
2673                 }
2674         }
2675         ret = rte_eth_switch_domain_alloc(&domain_id);
2676         if (ret) {
2677                 ENICPMD_LOG(WARNING, "failed to allocate switch domain for device %d",
2678                             ret);
2679         }
2680         cur->switch_domain_id = domain_id;
2681         ENICPMD_LOG(DEBUG, "Port %u (PF BDF %x:%x:%x) is the 1st PF on the VIC. Allocated switch domain id %u",
2682                     cur->rte_dev->data->port_id,
2683                     cur_a->bus, cur_a->devid, cur_a->function,
2684                     domain_id);
2685         return ret;
2686 }
2687
2688 const struct rte_flow_ops enic_fm_flow_ops = {
2689         .validate = enic_fm_flow_validate,
2690         .create = enic_fm_flow_create,
2691         .destroy = enic_fm_flow_destroy,
2692         .flush = enic_fm_flow_flush,
2693         .query = enic_fm_flow_query,
2694 };
2695
2696 /* Add a high priority flow that loops representor packets to VF */
2697 int
2698 enic_fm_add_rep2vf_flow(struct enic_vf_representor *vf)
2699 {
2700         struct fm_tcam_match_entry *fm_tcam_entry;
2701         struct rte_flow *flow0, *flow1;
2702         struct fm_action *fm_action;
2703         struct rte_flow_error error;
2704         struct rte_flow_attr attrs;
2705         struct fm_action_op fm_op;
2706         struct enic_flowman *fm;
2707         struct enic *pf;
2708         uint8_t tag;
2709
2710         pf = vf->pf;
2711         fm = pf->fm;
2712         tag = fm->vf_rep_tag;
2713         enic_fm_open_scratch(fm);
2714         fm_tcam_entry = &fm->tcam_entry;
2715         fm_action = &fm->action;
2716         /* Egress rule: match WQ ID and tag+hairpin */
2717         fm_tcam_entry->ftm_data.fk_wq_id = vf->pf_wq_idx;
2718         fm_tcam_entry->ftm_mask.fk_wq_id = 0xffff;
2719         fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
2720         memset(&fm_op, 0, sizeof(fm_op));
2721         fm_op.fa_op = FMOP_TAG;
2722         fm_op.tag.tag = tag;
2723         enic_fm_append_action_op(fm, &fm_op, &error);
2724         memset(&fm_op, 0, sizeof(fm_op));
2725         fm_op.fa_op = FMOP_EG_HAIRPIN;
2726         enic_fm_append_action_op(fm, &fm_op, &error);
2727         memset(&fm_op, 0, sizeof(fm_op));
2728         fm_op.fa_op = FMOP_END;
2729         enic_fm_append_action_op(fm, &fm_op, &error);
2730         attrs.group = 0;
2731         attrs.ingress = 0;
2732         attrs.egress = 1;
2733         attrs.priority = FM_HIGHEST_PRIORITY;
2734         flow0 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2735                                        &attrs, &error);
2736         enic_fm_close_scratch(fm);
2737         if (flow0 == NULL) {
2738                 ENICPMD_LOG(ERR, "Cannot create flow 0 for representor->VF");
2739                 return -EINVAL;
2740         }
2741         LIST_INSERT_HEAD(&pf->flows, flow0, next);
2742         /* Make this flow internal, so the user app cannot delete it */
2743         flow0->internal = 1;
2744         ENICPMD_LOG(DEBUG, "representor->VF %d flow created: wq %d -> tag %d hairpin",
2745                     vf->vf_id, vf->pf_wq_idx, tag);
2746
2747         /* Ingress: steer hairpinned to VF RQ 0 */
2748         enic_fm_open_scratch(fm);
2749         fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
2750         fm_tcam_entry->ftm_data.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
2751         fm_tcam_entry->ftm_mask.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
2752         fm_tcam_entry->ftm_data.fk_packet_tag = tag;
2753         fm_tcam_entry->ftm_mask.fk_packet_tag = 0xff;
2754         memset(&fm_op, 0, sizeof(fm_op));
2755         fm_op.fa_op = FMOP_RQ_STEER;
2756         fm_op.rq_steer.rq_index = 0;
2757         fm_op.rq_steer.vnic_handle = vf->enic.fm_vnic_handle;
2758         enic_fm_append_action_op(fm, &fm_op, &error);
2759         memset(&fm_op, 0, sizeof(fm_op));
2760         fm_op.fa_op = FMOP_END;
2761         enic_fm_append_action_op(fm, &fm_op, &error);
2762         attrs.group = 0;
2763         attrs.ingress = 1;
2764         attrs.egress = 0;
2765         attrs.priority = FM_HIGHEST_PRIORITY;
2766         flow1 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2767                                        &attrs, &error);
2768         enic_fm_close_scratch(fm);
2769         if (flow1 == NULL) {
2770                 ENICPMD_LOG(ERR, "Cannot create flow 1 for representor->VF");
2771                 enic_fm_flow_destroy(pf->rte_dev, flow0, &error);
2772                 return -EINVAL;
2773         }
2774         LIST_INSERT_HEAD(&pf->flows, flow1, next);
2775         flow1->internal = 1;
2776         ENICPMD_LOG(DEBUG, "representor->VF %d flow created: tag %d hairpinned -> VF RQ %d",
2777                     vf->vf_id, tag, fm_op.rq_steer.rq_index);
2778         vf->rep2vf_flow[0] = flow0;
2779         vf->rep2vf_flow[1] = flow1;
2780         /* Done with this tag, use a different one next time */
2781         fm->vf_rep_tag++;
2782         return 0;
2783 }
2784
2785 /*
2786  * Add a low priority flow that matches all packets from VF and loops them
2787  * back to the representor.
2788  */
2789 int
2790 enic_fm_add_vf2rep_flow(struct enic_vf_representor *vf)
2791 {
2792         struct fm_tcam_match_entry *fm_tcam_entry;
2793         struct rte_flow *flow0, *flow1;
2794         struct fm_action *fm_action;
2795         struct rte_flow_error error;
2796         struct rte_flow_attr attrs;
2797         struct fm_action_op fm_op;
2798         struct enic_flowman *fm;
2799         struct enic *pf;
2800         uint8_t tag;
2801
2802         pf = vf->pf;
2803         fm = pf->fm;
2804         tag = fm->vf_rep_tag;
2805         enic_fm_open_scratch(fm);
2806         fm_tcam_entry = &fm->tcam_entry;
2807         fm_action = &fm->action;
2808         /* Egress rule: match-any and tag+hairpin */
2809         fm_tcam_entry->ftm_data.fk_wq_id = 0;
2810         fm_tcam_entry->ftm_mask.fk_wq_id = 0xffff;
2811         fm_tcam_entry->ftm_data.fk_wq_vnic = vf->enic.fm_vnic_handle;
2812         fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
2813         memset(&fm_op, 0, sizeof(fm_op));
2814         fm_op.fa_op = FMOP_TAG;
2815         fm_op.tag.tag = tag;
2816         enic_fm_append_action_op(fm, &fm_op, &error);
2817         memset(&fm_op, 0, sizeof(fm_op));
2818         fm_op.fa_op = FMOP_EG_HAIRPIN;
2819         enic_fm_append_action_op(fm, &fm_op, &error);
2820         memset(&fm_op, 0, sizeof(fm_op));
2821         fm_op.fa_op = FMOP_END;
2822         enic_fm_append_action_op(fm, &fm_op, &error);
2823         attrs.group = 0;
2824         attrs.ingress = 0;
2825         attrs.egress = 1;
2826         attrs.priority = FM_LOWEST_PRIORITY;
2827         flow0 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2828                                        &attrs, &error);
2829         enic_fm_close_scratch(fm);
2830         if (flow0 == NULL) {
2831                 ENICPMD_LOG(ERR, "Cannot create flow 0 for VF->representor");
2832                 return -EINVAL;
2833         }
2834         LIST_INSERT_HEAD(&pf->flows, flow0, next);
2835         /* Make this flow internal, so the user app cannot delete it */
2836         flow0->internal = 1;
2837         ENICPMD_LOG(DEBUG, "VF %d->representor flow created: wq %d (low prio) -> tag %d hairpin",
2838                     vf->vf_id, fm_tcam_entry->ftm_data.fk_wq_id, tag);
2839
2840         /* Ingress: steer hairpinned to VF rep RQ */
2841         enic_fm_open_scratch(fm);
2842         fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
2843         fm_tcam_entry->ftm_data.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
2844         fm_tcam_entry->ftm_mask.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
2845         fm_tcam_entry->ftm_data.fk_packet_tag = tag;
2846         fm_tcam_entry->ftm_mask.fk_packet_tag = 0xff;
2847         memset(&fm_op, 0, sizeof(fm_op));
2848         fm_op.fa_op = FMOP_RQ_STEER;
2849         fm_op.rq_steer.rq_index = vf->pf_rq_sop_idx;
2850         fm_op.rq_steer.vnic_handle = pf->fm_vnic_handle;
2851         enic_fm_append_action_op(fm, &fm_op, &error);
2852         memset(&fm_op, 0, sizeof(fm_op));
2853         fm_op.fa_op = FMOP_END;
2854         enic_fm_append_action_op(fm, &fm_op, &error);
2855         attrs.group = 0;
2856         attrs.ingress = 1;
2857         attrs.egress = 0;
2858         attrs.priority = FM_HIGHEST_PRIORITY;
2859         flow1 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2860                                        &attrs, &error);
2861         enic_fm_close_scratch(fm);
2862         if (flow1 == NULL) {
2863                 ENICPMD_LOG(ERR, "Cannot create flow 1 for VF->representor");
2864                 enic_fm_flow_destroy(pf->rte_dev, flow0, &error);
2865                 return -EINVAL;
2866         }
2867         LIST_INSERT_HEAD(&pf->flows, flow1, next);
2868         flow1->internal = 1;
2869         ENICPMD_LOG(DEBUG, "VF %d->representor flow created: tag %d hairpinned -> PF RQ %d",
2870                     vf->vf_id, tag, vf->pf_rq_sop_idx);
2871         vf->vf2rep_flow[0] = flow0;
2872         vf->vf2rep_flow[1] = flow1;
2873         /* Done with this tag, use a different one next time */
2874         fm->vf_rep_tag++;
2875         return 0;
2876 }
2877
2878 /* Destroy representor flows created by enic_fm_add_{rep2vf,vf2rep}_flow */
2879 static void
2880 delete_rep_flows(struct enic *enic)
2881 {
2882         struct enic_vf_representor *vf;
2883         struct rte_flow_error error;
2884         struct rte_eth_dev *dev;
2885         uint32_t i;
2886
2887         RTE_ASSERT(enic_is_vf_rep(enic));
2888         vf = VF_ENIC_TO_VF_REP(enic);
2889         dev = vf->pf->rte_dev;
2890         for (i = 0; i < ARRAY_SIZE(vf->vf2rep_flow); i++) {
2891                 if (vf->vf2rep_flow[i])
2892                         enic_fm_flow_destroy(dev, vf->vf2rep_flow[i], &error);
2893         }
2894         for (i = 0; i < ARRAY_SIZE(vf->rep2vf_flow); i++) {
2895                 if (vf->rep2vf_flow[i])
2896                         enic_fm_flow_destroy(dev, vf->rep2vf_flow[i], &error);
2897         }
2898 }
2899
2900 static struct enic_flowman *
2901 begin_fm(struct enic *enic)
2902 {
2903         struct enic_vf_representor *vf;
2904         struct enic_flowman *fm;
2905
2906         /* Representor uses PF flowman */
2907         if (enic_is_vf_rep(enic)) {
2908                 vf = VF_ENIC_TO_VF_REP(enic);
2909                 fm = vf->pf->fm;
2910         } else {
2911                 fm = enic->fm;
2912         }
2913         /* Save the API caller and lock if representors exist */
2914         if (fm) {
2915                 if (fm->owner_enic->switchdev_mode)
2916                         rte_spinlock_lock(&fm->lock);
2917                 fm->user_enic = enic;
2918         }
2919         return fm;
2920 }
2921
2922 static void
2923 end_fm(struct enic_flowman *fm)
2924 {
2925         fm->user_enic = NULL;
2926         if (fm->owner_enic->switchdev_mode)
2927                 rte_spinlock_unlock(&fm->lock);
2928 }