drivers/net/enic/enic_fm_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2008-2019 Cisco Systems, Inc.  All rights reserved.
   3  */
   4
   5 #include <errno.h>
   6 #include <stdint.h>
   7 #include <rte_log.h>
   8 #include <ethdev_driver.h>
   9 #include <rte_flow_driver.h>
  10 #include <rte_ether.h>
  11 #include <rte_hash.h>
  12 #include <rte_jhash.h>
  13 #include <rte_ip.h>
  14 #include <rte_udp.h>
  15 #include <rte_memzone.h>
  16
  17 #include "enic_compat.h"
  18 #include "enic.h"
  19 #include "vnic_dev.h"
  20 #include "vnic_nic.h"
  21
  22 #define IP_DEFTTL  64   /* from RFC 1340. */
  23 #define IP6_VTC_FLOW 0x60000000
  24
  25 /* Highest Item type supported by Flowman */
  26 #define FM_MAX_ITEM_TYPE RTE_FLOW_ITEM_TYPE_VXLAN
  27
  28 /* Up to 1024 TCAM entries */
  29 #define FM_MAX_TCAM_TABLE_SIZE 1024
  30
  31 /* Up to 4096 entries per exact match table */
  32 #define FM_MAX_EXACT_TABLE_SIZE 4096
  33
  34 /* Number of counters to increase on for each increment */
  35 #define FM_COUNTERS_EXPAND  100
  36
  37 #define FM_INVALID_HANDLE 0
  38
  39 /* Low priority used for implicit VF -> representor flow */
  40 #define FM_LOWEST_PRIORITY 100000
  41
  42 /* High priority used for implicit representor -> VF flow */
  43 #define FM_HIGHEST_PRIORITY 0
  44
  45 /* Tag used for implicit VF <-> representor flows */
  46 #define FM_VF_REP_TAG 1
  47
  48 /* Max number of actions supported by VIC is 2K. Make hash table double that. */
  49 #define FM_MAX_ACTION_TABLE_SIZE 4096
  50
  51 /*
  52  * Flow exact match tables (FET) in the VIC and rte_flow groups.
  53  * Use a simple scheme to map groups to tables.
  54  * Group 0 uses the single TCAM tables, one for each direction.
  55  * Group 1, 2, ... uses its own exact match table.
  56  *
  57  * The TCAM tables are allocated upfront during init.
  58  *
  59  * Exact match tables are allocated on demand. 3 paths that lead allocations.
  60  *
  61  * 1. Add a flow that jumps from group 0 to group N.
  62  *
  63  * If N does not exist, we allocate an exact match table for it, using
  64  * a dummy key. A key is required for the table.
  65  *
  66  * 2. Add a flow that uses group N.
  67  *
  68  * If N does not exist, we allocate an exact match table for it, using
  69  * the flow's key. Subsequent flows to the same group all should have
  70  * the same key.
  71  *
  72  * Without a jump flow to N, N is not reachable in hardware. No packets
  73  * reach N and match.
  74  *
  75  * 3. Add a flow to an empty group N.
  76  *
  77  * N has been created via (1) and the dummy key. We free that table, allocate
  78  * a new table using the new flow's key. Also re-do the existing jump flow to
  79  * point to the new table.
  80  */
  81 #define FM_TCAM_RTE_GROUP 0
  82
  83 struct enic_fm_fet {
  84         TAILQ_ENTRY(enic_fm_fet) list;
  85         uint32_t group; /* rte_flow group ID */
  86         uint64_t handle; /* Exact match table handle from flowman */
  87         uint8_t ingress;
  88         uint8_t default_key;
  89         int ref; /* Reference count via get/put */
  90         struct fm_key_template key; /* Key associated with the table */
  91 };
  92
  93 struct enic_fm_counter {
  94         SLIST_ENTRY(enic_fm_counter) next;
  95         uint32_t handle;
  96 };
  97
  98 struct enic_fm_action {
  99         int ref;
 100         uint64_t handle;
 101         struct fm_action key;
 102 };
 103
 104 /* rte_flow.fm */
 105 struct enic_fm_flow {
 106         bool counter_valid;
 107         uint64_t entry_handle;
 108         struct enic_fm_action  *action;
 109         struct enic_fm_counter *counter;
 110         struct enic_fm_fet *fet;
 111         /* Auto-added steer action for hairpin flows (e.g. vnic->vnic) */
 112         struct enic_fm_flow *hairpin_steer_flow;
 113 };
 114
 115 struct enic_fm_jump_flow {
 116         TAILQ_ENTRY(enic_fm_jump_flow) list;
 117         struct rte_flow *flow;
 118         uint32_t group;
 119         struct fm_tcam_match_entry match;
 120         struct fm_action action;
 121 };
 122
 123 /*
 124  * Flowman uses host memory for commands. This structure is allocated
 125  * in DMA-able memory.
 126  */
 127 union enic_flowman_cmd_mem {
 128         struct fm_tcam_match_table fm_tcam_match_table;
 129         struct fm_exact_match_table fm_exact_match_table;
 130         struct fm_tcam_match_entry fm_tcam_match_entry;
 131         struct fm_exact_match_entry fm_exact_match_entry;
 132         struct fm_action fm_action;
 133 };
 134
 135 /*
 136  * PF has a flowman instance, and VF representors share it with PF.
 137  * PF allocates this structure and owns it. VF representors borrow
 138  * the PF's structure during API calls (e.g. create, query).
 139  */
 140 struct enic_flowman {
 141         struct enic *owner_enic; /* PF */
 142         struct enic *user_enic;  /* API caller (PF or representor) */
 143         /*
 144          * Representors and PF share the same underlying flowman.
 145          * Lock API calls to serialize accesses from them. Only used
 146          * when VF representors are present.
 147          */
 148         rte_spinlock_t lock;
 149         /* Command buffer */
 150         struct {
 151                 union enic_flowman_cmd_mem *va;
 152                 dma_addr_t pa;
 153         } cmd;
 154         /* TCAM tables allocated upfront, used for group 0 */
 155         uint64_t ig_tcam_hndl;
 156         uint64_t eg_tcam_hndl;
 157         /* Counters */
 158         SLIST_HEAD(enic_free_counters, enic_fm_counter) counters;
 159         void *counter_stack;
 160         uint32_t counters_alloced;
 161         /* Exact match tables for groups != 0, dynamically allocated */
 162         TAILQ_HEAD(fet_list, enic_fm_fet) fet_list;
 163         /*
 164          * Default exact match tables used for jump actions to
 165          * non-existent groups.
 166          */
 167         struct enic_fm_fet *default_eg_fet;
 168         struct enic_fm_fet *default_ig_fet;
 169         /* hash table for Action reuse */
 170         struct rte_hash *action_hash;
 171         /* Flows that jump to the default table above */
 172         TAILQ_HEAD(jump_flow_list, enic_fm_jump_flow) jump_list;
 173         /*
 174          * Scratch data used during each invocation of flow_create
 175          * and flow_validate.
 176          */
 177         struct enic_fm_fet *fet;
 178         struct fm_tcam_match_entry tcam_entry;
 179         struct fm_action action;
 180         struct fm_action action_tmp; /* enic_fm_reorder_action_op */
 181         int action_op_count;
 182         /* Tags used for representor flows */
 183         uint8_t vf_rep_tag;
 184         /* For auto-added steer action for hairpin */
 185         int need_hairpin_steer;
 186         uint64_t hairpin_steer_vnic_h;
 187 };
 188
 189 static int enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle);
 190 /*
 191  * API functions (create, destroy, validate, flush) call begin_fm()
 192  * upon entering to save the caller enic (PF or VF representor) and
 193  * lock. Upon exit, they call end_fm() to unlock.
 194  */
 195 static struct enic_flowman *begin_fm(struct enic *enic);
 196 static void end_fm(struct enic_flowman *fm);
 197 /* Delete internal flows created for representor paths */
 198 static void delete_rep_flows(struct enic *enic);
 199
 200 /*
 201  * Common arguments passed to copy_item functions. Use this structure
 202  * so we can easily add new arguments.
 203  * item: Item specification.
 204  * fm_tcam_entry: Flowman TCAM match entry.
 205  * header_level: 0 for outer header, 1 for inner header.
 206  */
 207 struct copy_item_args {
 208         const struct rte_flow_item *item;
 209         struct fm_tcam_match_entry *fm_tcam_entry;
 210         uint8_t header_level;
 211 };
 212
 213 /* functions for copying items into flowman match */
 214 typedef int (enic_copy_item_fn)(struct copy_item_args *arg);
 215
 216 /* Info about how to copy items into flowman match */
 217 struct enic_fm_items {
 218         /* Function for copying and validating an item. */
 219         enic_copy_item_fn * const copy_item;
 220         /* List of valid previous items. */
 221         const enum rte_flow_item_type * const prev_items;
 222         /*
 223          * True if it's OK for this item to be the first item. For some NIC
 224          * versions, it's invalid to start the stack above layer 3.
 225          */
 226         const uint8_t valid_start_item;
 227 };
 228
 229 static enic_copy_item_fn enic_fm_copy_item_eth;
 230 static enic_copy_item_fn enic_fm_copy_item_ipv4;
 231 static enic_copy_item_fn enic_fm_copy_item_ipv6;
 232 static enic_copy_item_fn enic_fm_copy_item_raw;
 233 static enic_copy_item_fn enic_fm_copy_item_sctp;
 234 static enic_copy_item_fn enic_fm_copy_item_tcp;
 235 static enic_copy_item_fn enic_fm_copy_item_udp;
 236 static enic_copy_item_fn enic_fm_copy_item_vlan;
 237 static enic_copy_item_fn enic_fm_copy_item_vxlan;
 238
 239 /* Ingress actions */
 240 static const enum rte_flow_action_type enic_fm_supported_ig_actions[] = {
 241         RTE_FLOW_ACTION_TYPE_COUNT,
 242         RTE_FLOW_ACTION_TYPE_DROP,
 243         RTE_FLOW_ACTION_TYPE_FLAG,
 244         RTE_FLOW_ACTION_TYPE_JUMP,
 245         RTE_FLOW_ACTION_TYPE_MARK,
 246         RTE_FLOW_ACTION_TYPE_OF_POP_VLAN,
 247         RTE_FLOW_ACTION_TYPE_PORT_ID,
 248         RTE_FLOW_ACTION_TYPE_PASSTHRU,
 249         RTE_FLOW_ACTION_TYPE_QUEUE,
 250         RTE_FLOW_ACTION_TYPE_RSS,
 251         RTE_FLOW_ACTION_TYPE_VOID,
 252         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
 253         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP,
 254         RTE_FLOW_ACTION_TYPE_END, /* END must be the last entry */
 255 };
 256
 257 /* Egress actions */
 258 static const enum rte_flow_action_type enic_fm_supported_eg_actions[] = {
 259         RTE_FLOW_ACTION_TYPE_COUNT,
 260         RTE_FLOW_ACTION_TYPE_DROP,
 261         RTE_FLOW_ACTION_TYPE_JUMP,
 262         RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN,
 263         RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP,
 264         RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID,
 265         RTE_FLOW_ACTION_TYPE_PORT_ID,
 266         RTE_FLOW_ACTION_TYPE_PASSTHRU,
 267         RTE_FLOW_ACTION_TYPE_VOID,
 268         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
 269         RTE_FLOW_ACTION_TYPE_END,
 270 };
 271
 272 static const struct enic_fm_items enic_fm_items[] = {
 273         [RTE_FLOW_ITEM_TYPE_RAW] = {
 274                 .copy_item = enic_fm_copy_item_raw,
 275                 .valid_start_item = 0,
 276                 .prev_items = (const enum rte_flow_item_type[]) {
 277                                RTE_FLOW_ITEM_TYPE_UDP,
 278                                RTE_FLOW_ITEM_TYPE_END,
 279                 },
 280         },
 281         [RTE_FLOW_ITEM_TYPE_ETH] = {
 282                 .copy_item = enic_fm_copy_item_eth,
 283                 .valid_start_item = 1,
 284                 .prev_items = (const enum rte_flow_item_type[]) {
 285                                RTE_FLOW_ITEM_TYPE_END,
 286                 },
 287         },
 288         [RTE_FLOW_ITEM_TYPE_VLAN] = {
 289                 .copy_item = enic_fm_copy_item_vlan,
 290                 .valid_start_item = 1,
 291                 .prev_items = (const enum rte_flow_item_type[]) {
 292                                RTE_FLOW_ITEM_TYPE_ETH,
 293                                RTE_FLOW_ITEM_TYPE_END,
 294                 },
 295         },
 296         [RTE_FLOW_ITEM_TYPE_IPV4] = {
 297                 .copy_item = enic_fm_copy_item_ipv4,
 298                 .valid_start_item = 1,
 299                 .prev_items = (const enum rte_flow_item_type[]) {
 300                                RTE_FLOW_ITEM_TYPE_ETH,
 301                                RTE_FLOW_ITEM_TYPE_VLAN,
 302                                RTE_FLOW_ITEM_TYPE_END,
 303                 },
 304         },
 305         [RTE_FLOW_ITEM_TYPE_IPV6] = {
 306                 .copy_item = enic_fm_copy_item_ipv6,
 307                 .valid_start_item = 1,
 308                 .prev_items = (const enum rte_flow_item_type[]) {
 309                                RTE_FLOW_ITEM_TYPE_ETH,
 310                                RTE_FLOW_ITEM_TYPE_VLAN,
 311                                RTE_FLOW_ITEM_TYPE_END,
 312                 },
 313         },
 314         [RTE_FLOW_ITEM_TYPE_UDP] = {
 315                 .copy_item = enic_fm_copy_item_udp,
 316                 .valid_start_item = 1,
 317                 .prev_items = (const enum rte_flow_item_type[]) {
 318                                RTE_FLOW_ITEM_TYPE_IPV4,
 319                                RTE_FLOW_ITEM_TYPE_IPV6,
 320                                RTE_FLOW_ITEM_TYPE_END,
 321                 },
 322         },
 323         [RTE_FLOW_ITEM_TYPE_TCP] = {
 324                 .copy_item = enic_fm_copy_item_tcp,
 325                 .valid_start_item = 1,
 326                 .prev_items = (const enum rte_flow_item_type[]) {
 327                                RTE_FLOW_ITEM_TYPE_IPV4,
 328                                RTE_FLOW_ITEM_TYPE_IPV6,
 329                                RTE_FLOW_ITEM_TYPE_END,
 330                 },
 331         },
 332         [RTE_FLOW_ITEM_TYPE_SCTP] = {
 333                 .copy_item = enic_fm_copy_item_sctp,
 334                 .valid_start_item = 0,
 335                 .prev_items = (const enum rte_flow_item_type[]) {
 336                                RTE_FLOW_ITEM_TYPE_IPV4,
 337                                RTE_FLOW_ITEM_TYPE_IPV6,
 338                                RTE_FLOW_ITEM_TYPE_END,
 339                 },
 340         },
 341         [RTE_FLOW_ITEM_TYPE_VXLAN] = {
 342                 .copy_item = enic_fm_copy_item_vxlan,
 343                 .valid_start_item = 1,
 344                 .prev_items = (const enum rte_flow_item_type[]) {
 345                                RTE_FLOW_ITEM_TYPE_UDP,
 346                                RTE_FLOW_ITEM_TYPE_END,
 347                 },
 348         },
 349 };
 350
 351 static int
 352 enic_fm_copy_item_eth(struct copy_item_args *arg)
 353 {
 354         const struct rte_flow_item *item = arg->item;
 355         const struct rte_flow_item_eth *spec = item->spec;
 356         const struct rte_flow_item_eth *mask = item->mask;
 357         const uint8_t lvl = arg->header_level;
 358         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 359         struct fm_header_set *fm_data, *fm_mask;
 360
 361         ENICPMD_FUNC_TRACE();
 362         /* Match all if no spec */
 363         if (!spec)
 364                 return 0;
 365         if (!mask)
 366                 mask = &rte_flow_item_eth_mask;
 367         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 368         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 369         fm_data->fk_header_select |= FKH_ETHER;
 370         fm_mask->fk_header_select |= FKH_ETHER;
 371         memcpy(&fm_data->l2.eth, spec, sizeof(struct rte_ether_hdr));
 372         memcpy(&fm_mask->l2.eth, mask, sizeof(struct rte_ether_hdr));
 373         return 0;
 374 }
 375
 376 static int
 377 enic_fm_copy_item_vlan(struct copy_item_args *arg)
 378 {
 379         const struct rte_flow_item *item = arg->item;
 380         const struct rte_flow_item_vlan *spec = item->spec;
 381         const struct rte_flow_item_vlan *mask = item->mask;
 382         const uint8_t lvl = arg->header_level;
 383         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 384         struct fm_header_set *fm_data, *fm_mask;
 385         struct rte_ether_hdr *eth_mask;
 386         struct rte_ether_hdr *eth_val;
 387         uint32_t meta;
 388
 389         ENICPMD_FUNC_TRACE();
 390         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 391         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 392         /* Outer and inner packet vlans need different flags */
 393         meta = FKM_VLAN_PRES;
 394         if (lvl > 0)
 395                 meta = FKM_QTAG;
 396         fm_data->fk_metadata |= meta;
 397         fm_mask->fk_metadata |= meta;
 398
 399         /* Match all if no spec */
 400         if (!spec)
 401                 return 0;
 402         if (!mask)
 403                 mask = &rte_flow_item_vlan_mask;
 404
 405         eth_mask = (void *)&fm_mask->l2.eth;
 406         eth_val = (void *)&fm_data->l2.eth;
 407
 408         /*
 409          * Outer TPID cannot be matched. If inner_type is 0, use what is
 410          * in the eth header.
 411          */
 412         if (eth_mask->ether_type && mask->inner_type)
 413                 return -ENOTSUP;
 414
 415         /*
 416          * When packet matching, the VIC always compares vlan-stripped
 417          * L2, regardless of vlan stripping settings. So, the inner type
 418          * from vlan becomes the ether type of the eth header.
 419          */
 420         if (mask->inner_type) {
 421                 eth_mask->ether_type = mask->inner_type;
 422                 eth_val->ether_type = spec->inner_type;
 423         }
 424         fm_data->fk_header_select |= FKH_ETHER | FKH_QTAG;
 425         fm_mask->fk_header_select |= FKH_ETHER | FKH_QTAG;
 426         fm_data->fk_vlan = rte_be_to_cpu_16(spec->tci);
 427         fm_mask->fk_vlan = rte_be_to_cpu_16(mask->tci);
 428         return 0;
 429 }
 430
 431 static int
 432 enic_fm_copy_item_ipv4(struct copy_item_args *arg)
 433 {
 434         const struct rte_flow_item *item = arg->item;
 435         const struct rte_flow_item_ipv4 *spec = item->spec;
 436         const struct rte_flow_item_ipv4 *mask = item->mask;
 437         const uint8_t lvl = arg->header_level;
 438         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 439         struct fm_header_set *fm_data, *fm_mask;
 440
 441         ENICPMD_FUNC_TRACE();
 442         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 443         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 444         fm_data->fk_metadata |= FKM_IPV4;
 445         fm_mask->fk_metadata |= FKM_IPV4;
 446
 447         if (!spec)
 448                 return 0;
 449         if (!mask)
 450                 mask = &rte_flow_item_ipv4_mask;
 451
 452         fm_data->fk_header_select |= FKH_IPV4;
 453         fm_mask->fk_header_select |= FKH_IPV4;
 454         memcpy(&fm_data->l3.ip4, spec, sizeof(*spec));
 455         memcpy(&fm_mask->l3.ip4, mask, sizeof(*mask));
 456         return 0;
 457 }
 458
 459 static int
 460 enic_fm_copy_item_ipv6(struct copy_item_args *arg)
 461 {
 462         const struct rte_flow_item *item = arg->item;
 463         const struct rte_flow_item_ipv6 *spec = item->spec;
 464         const struct rte_flow_item_ipv6 *mask = item->mask;
 465         const uint8_t lvl = arg->header_level;
 466         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 467         struct fm_header_set *fm_data, *fm_mask;
 468
 469         ENICPMD_FUNC_TRACE();
 470         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 471         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 472         fm_data->fk_metadata |= FKM_IPV6;
 473         fm_mask->fk_metadata |= FKM_IPV6;
 474
 475         if (!spec)
 476                 return 0;
 477         if (!mask)
 478                 mask = &rte_flow_item_ipv6_mask;
 479
 480         fm_data->fk_header_select |= FKH_IPV6;
 481         fm_mask->fk_header_select |= FKH_IPV6;
 482         memcpy(&fm_data->l3.ip6, spec, sizeof(struct rte_ipv6_hdr));
 483         memcpy(&fm_mask->l3.ip6, mask, sizeof(struct rte_ipv6_hdr));
 484         return 0;
 485 }
 486
 487 static int
 488 enic_fm_copy_item_udp(struct copy_item_args *arg)
 489 {
 490         const struct rte_flow_item *item = arg->item;
 491         const struct rte_flow_item_udp *spec = item->spec;
 492         const struct rte_flow_item_udp *mask = item->mask;
 493         const uint8_t lvl = arg->header_level;
 494         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 495         struct fm_header_set *fm_data, *fm_mask;
 496
 497         ENICPMD_FUNC_TRACE();
 498         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 499         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 500         fm_data->fk_metadata |= FKM_UDP;
 501         fm_mask->fk_metadata |= FKM_UDP;
 502
 503         if (!spec)
 504                 return 0;
 505         if (!mask)
 506                 mask = &rte_flow_item_udp_mask;
 507
 508         fm_data->fk_header_select |= FKH_UDP;
 509         fm_mask->fk_header_select |= FKH_UDP;
 510         memcpy(&fm_data->l4.udp, spec, sizeof(*spec));
 511         memcpy(&fm_mask->l4.udp, mask, sizeof(*mask));
 512         return 0;
 513 }
 514
 515 static int
 516 enic_fm_copy_item_tcp(struct copy_item_args *arg)
 517 {
 518         const struct rte_flow_item *item = arg->item;
 519         const struct rte_flow_item_tcp *spec = item->spec;
 520         const struct rte_flow_item_tcp *mask = item->mask;
 521         const uint8_t lvl = arg->header_level;
 522         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 523         struct fm_header_set *fm_data, *fm_mask;
 524
 525         ENICPMD_FUNC_TRACE();
 526         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 527         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 528         fm_data->fk_metadata |= FKM_TCP;
 529         fm_mask->fk_metadata |= FKM_TCP;
 530
 531         if (!spec)
 532                 return 0;
 533         if (!mask)
 534                 mask = &rte_flow_item_tcp_mask;
 535
 536         fm_data->fk_header_select |= FKH_TCP;
 537         fm_mask->fk_header_select |= FKH_TCP;
 538         memcpy(&fm_data->l4.tcp, spec, sizeof(*spec));
 539         memcpy(&fm_mask->l4.tcp, mask, sizeof(*mask));
 540         return 0;
 541 }
 542
 543 static int
 544 enic_fm_copy_item_sctp(struct copy_item_args *arg)
 545 {
 546         const struct rte_flow_item *item = arg->item;
 547         const struct rte_flow_item_sctp *spec = item->spec;
 548         const struct rte_flow_item_sctp *mask = item->mask;
 549         const uint8_t lvl = arg->header_level;
 550         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 551         struct fm_header_set *fm_data, *fm_mask;
 552         uint8_t *ip_proto_mask = NULL;
 553         uint8_t *ip_proto = NULL;
 554         uint32_t l3_fkh;
 555
 556         ENICPMD_FUNC_TRACE();
 557         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 558         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 559         /*
 560          * The NIC filter API has no flags for "match sctp", so explicitly
 561          * set the protocol number in the IP pattern.
 562          */
 563         if (fm_data->fk_metadata & FKM_IPV4) {
 564                 struct rte_ipv4_hdr *ip;
 565                 ip = (struct rte_ipv4_hdr *)&fm_mask->l3.ip4;
 566                 ip_proto_mask = &ip->next_proto_id;
 567                 ip = (struct rte_ipv4_hdr *)&fm_data->l3.ip4;
 568                 ip_proto = &ip->next_proto_id;
 569                 l3_fkh = FKH_IPV4;
 570         } else if (fm_data->fk_metadata & FKM_IPV6) {
 571                 struct rte_ipv6_hdr *ip;
 572                 ip = (struct rte_ipv6_hdr *)&fm_mask->l3.ip6;
 573                 ip_proto_mask = &ip->proto;
 574                 ip = (struct rte_ipv6_hdr *)&fm_data->l3.ip6;
 575                 ip_proto = &ip->proto;
 576                 l3_fkh = FKH_IPV6;
 577         } else {
 578                 /* Need IPv4/IPv6 pattern first */
 579                 return -EINVAL;
 580         }
 581         *ip_proto = IPPROTO_SCTP;
 582         *ip_proto_mask = 0xff;
 583         fm_data->fk_header_select |= l3_fkh;
 584         fm_mask->fk_header_select |= l3_fkh;
 585
 586         if (!spec)
 587                 return 0;
 588         if (!mask)
 589                 mask = &rte_flow_item_sctp_mask;
 590
 591         fm_data->fk_header_select |= FKH_L4RAW;
 592         fm_mask->fk_header_select |= FKH_L4RAW;
 593         memcpy(fm_data->l4.rawdata, spec, sizeof(*spec));
 594         memcpy(fm_mask->l4.rawdata, mask, sizeof(*mask));
 595         return 0;
 596 }
 597
 598 static int
 599 enic_fm_copy_item_vxlan(struct copy_item_args *arg)
 600 {
 601         const struct rte_flow_item *item = arg->item;
 602         const struct rte_flow_item_vxlan *spec = item->spec;
 603         const struct rte_flow_item_vxlan *mask = item->mask;
 604         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 605         struct fm_header_set *fm_data, *fm_mask;
 606
 607         ENICPMD_FUNC_TRACE();
 608         /* Only 2 header levels (outer and inner) allowed */
 609         if (arg->header_level > 0)
 610                 return -EINVAL;
 611
 612         fm_data = &entry->ftm_data.fk_hdrset[0];
 613         fm_mask = &entry->ftm_mask.fk_hdrset[0];
 614         fm_data->fk_metadata |= FKM_VXLAN;
 615         fm_mask->fk_metadata |= FKM_VXLAN;
 616         /* items from here on out are inner header items */
 617         arg->header_level = 1;
 618
 619         /* Match all if no spec */
 620         if (!spec)
 621                 return 0;
 622         if (!mask)
 623                 mask = &rte_flow_item_vxlan_mask;
 624
 625         fm_data->fk_header_select |= FKH_VXLAN;
 626         fm_mask->fk_header_select |= FKH_VXLAN;
 627         memcpy(&fm_data->vxlan, spec, sizeof(*spec));
 628         memcpy(&fm_mask->vxlan, mask, sizeof(*mask));
 629         return 0;
 630 }
 631
 632 /*
 633  * Currently, raw pattern match is very limited. It is intended for matching
 634  * UDP tunnel header (e.g. vxlan or geneve).
 635  */
 636 static int
 637 enic_fm_copy_item_raw(struct copy_item_args *arg)
 638 {
 639         const struct rte_flow_item *item = arg->item;
 640         const struct rte_flow_item_raw *spec = item->spec;
 641         const struct rte_flow_item_raw *mask = item->mask;
 642         const uint8_t lvl = arg->header_level;
 643         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 644         struct fm_header_set *fm_data, *fm_mask;
 645
 646         ENICPMD_FUNC_TRACE();
 647         /* Cannot be used for inner packet */
 648         if (lvl > 0)
 649                 return -EINVAL;
 650         /* Need both spec and mask */
 651         if (!spec || !mask)
 652                 return -EINVAL;
 653         /* Only supports relative with offset 0 */
 654         if (!spec->relative || spec->offset != 0 || spec->search ||
 655             spec->limit)
 656                 return -EINVAL;
 657         /* Need non-null pattern that fits within the NIC's filter pattern */
 658         if (spec->length == 0 ||
 659             spec->length + sizeof(struct rte_udp_hdr) > FM_LAYER_SIZE ||
 660             !spec->pattern || !mask->pattern)
 661                 return -EINVAL;
 662         /*
 663          * Mask fields, including length, are often set to zero. Assume that
 664          * means "same as spec" to avoid breaking existing apps. If length
 665          * is not zero, then it should be >= spec length.
 666          *
 667          * No more pattern follows this, so append to the L4 layer instead of
 668          * L5 to work with both recent and older VICs.
 669          */
 670         if (mask->length != 0 && mask->length < spec->length)
 671                 return -EINVAL;
 672
 673         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 674         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 675         fm_data->fk_header_select |= FKH_L4RAW;
 676         fm_mask->fk_header_select |= FKH_L4RAW;
 677         fm_data->fk_header_select &= ~FKH_UDP;
 678         fm_mask->fk_header_select &= ~FKH_UDP;
 679         memcpy(fm_data->l4.rawdata + sizeof(struct rte_udp_hdr),
 680                spec->pattern, spec->length);
 681         memcpy(fm_mask->l4.rawdata + sizeof(struct rte_udp_hdr),
 682                mask->pattern, spec->length);
 683         return 0;
 684 }
 685
 686 static int
 687 flowman_cmd(struct enic_flowman *fm, uint64_t *args, int nargs)
 688 {
 689         return vnic_dev_flowman_cmd(fm->owner_enic->vdev, args, nargs);
 690 }
 691
 692 static int
 693 enic_fet_alloc(struct enic_flowman *fm, uint8_t ingress,
 694                struct fm_key_template *key, int entries,
 695                struct enic_fm_fet **fet_out)
 696 {
 697         struct fm_exact_match_table *cmd;
 698         struct fm_header_set *hdr;
 699         struct enic_fm_fet *fet;
 700         uint64_t args[3];
 701         int ret;
 702
 703         ENICPMD_FUNC_TRACE();
 704         fet = calloc(1, sizeof(struct enic_fm_fet));
 705         if (fet == NULL)
 706                 return -ENOMEM;
 707         cmd = &fm->cmd.va->fm_exact_match_table;
 708         memset(cmd, 0, sizeof(*cmd));
 709         cmd->fet_direction = ingress ? FM_INGRESS : FM_EGRESS;
 710         cmd->fet_stage = FM_STAGE_LAST;
 711         cmd->fet_max_entries = entries ? entries : FM_MAX_EXACT_TABLE_SIZE;
 712         if (key == NULL) {
 713                 hdr = &cmd->fet_key.fk_hdrset[0];
 714                 memset(hdr, 0, sizeof(*hdr));
 715                 hdr->fk_header_select = FKH_IPV4 | FKH_UDP;
 716                 hdr->l3.ip4.fk_saddr = 0xFFFFFFFF;
 717                 hdr->l3.ip4.fk_daddr = 0xFFFFFFFF;
 718                 hdr->l4.udp.fk_source = 0xFFFF;
 719                 hdr->l4.udp.fk_dest = 0xFFFF;
 720                 fet->default_key = 1;
 721         } else {
 722                 memcpy(&cmd->fet_key, key, sizeof(*key));
 723                 memcpy(&fet->key, key, sizeof(*key));
 724                 fet->default_key = 0;
 725         }
 726         cmd->fet_key.fk_packet_tag = 1;
 727
 728         args[0] = FM_EXACT_TABLE_ALLOC;
 729         args[1] = fm->cmd.pa;
 730         ret = flowman_cmd(fm, args, 2);
 731         if (ret) {
 732                 ENICPMD_LOG(ERR, "cannot alloc exact match table: rc=%d", ret);
 733                 free(fet);
 734                 return ret;
 735         }
 736         fet->handle = args[0];
 737         fet->ingress = ingress;
 738         ENICPMD_LOG(DEBUG, "allocated exact match table: handle=0x%" PRIx64,
 739                     fet->handle);
 740         *fet_out = fet;
 741         return 0;
 742 }
 743
 744 static void
 745 enic_fet_free(struct enic_flowman *fm, struct enic_fm_fet *fet)
 746 {
 747         ENICPMD_FUNC_TRACE();
 748         enic_fm_tbl_free(fm, fet->handle);
 749         if (!fet->default_key)
 750                 TAILQ_REMOVE(&fm->fet_list, fet, list);
 751         free(fet);
 752 }
 753
 754 /*
 755  * Get the exact match table for the given combination of
 756  * <group, ingress, key>. Allocate one on the fly as necessary.
 757  */
 758 static int
 759 enic_fet_get(struct enic_flowman *fm,
 760              uint32_t group,
 761              uint8_t ingress,
 762              struct fm_key_template *key,
 763              struct enic_fm_fet **fet_out,
 764              struct rte_flow_error *error)
 765 {
 766         struct enic_fm_fet *fet;
 767
 768         ENICPMD_FUNC_TRACE();
 769         /* See if we already have this table open */
 770         TAILQ_FOREACH(fet, &fm->fet_list, list) {
 771                 if (fet->group == group && fet->ingress == ingress)
 772                         break;
 773         }
 774         if (fet == NULL) {
 775                 /* Jumping to a non-existing group? Use the default table */
 776                 if (key == NULL) {
 777                         fet = ingress ? fm->default_ig_fet : fm->default_eg_fet;
 778                 } else if (enic_fet_alloc(fm, ingress, key, 0, &fet)) {
 779                         return rte_flow_error_set(error, EINVAL,
 780                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 781                                 NULL, "enic: cannot get exact match table");
 782                 }
 783                 fet->group = group;
 784                 /* Default table is never on the open table list */
 785                 if (!fet->default_key)
 786                         TAILQ_INSERT_HEAD(&fm->fet_list, fet, list);
 787         }
 788         fet->ref++;
 789         *fet_out = fet;
 790         ENICPMD_LOG(DEBUG, "fet_get: %s %s group=%u ref=%u",
 791                     fet->default_key ? "default" : "",
 792                     fet->ingress ? "ingress" : "egress",
 793                     fet->group, fet->ref);
 794         return 0;
 795 }
 796
 797 static void
 798 enic_fet_put(struct enic_flowman *fm, struct enic_fm_fet *fet)
 799 {
 800         ENICPMD_FUNC_TRACE();
 801         RTE_ASSERT(fet->ref > 0);
 802         fet->ref--;
 803         ENICPMD_LOG(DEBUG, "fet_put: %s %s group=%u ref=%u",
 804                     fet->default_key ? "default" : "",
 805                     fet->ingress ? "ingress" : "egress",
 806                     fet->group, fet->ref);
 807         if (fet->ref == 0)
 808                 enic_fet_free(fm, fet);
 809 }
 810
 811 /* Return 1 if current item is valid on top of the previous one. */
 812 static int
 813 fm_item_stacking_valid(enum rte_flow_item_type prev_item,
 814                        const struct enic_fm_items *item_info,
 815                        uint8_t is_first_item)
 816 {
 817         enum rte_flow_item_type const *allowed_items = item_info->prev_items;
 818
 819         ENICPMD_FUNC_TRACE();
 820         for (; *allowed_items != RTE_FLOW_ITEM_TYPE_END; allowed_items++) {
 821                 if (prev_item == *allowed_items)
 822                         return 1;
 823         }
 824
 825         /* This is the first item in the stack. Check if that's cool */
 826         if (is_first_item && item_info->valid_start_item)
 827                 return 1;
 828         return 0;
 829 }
 830
 831 /*
 832  * Build the flow manager match entry structure from the provided pattern.
 833  * The pattern is validated as the items are copied.
 834  */
 835 static int
 836 enic_fm_copy_entry(struct enic_flowman *fm,
 837                    const struct rte_flow_item pattern[],
 838                    struct rte_flow_error *error)
 839 {
 840         const struct enic_fm_items *item_info;
 841         enum rte_flow_item_type prev_item;
 842         const struct rte_flow_item *item;
 843         struct copy_item_args args;
 844         uint8_t prev_header_level;
 845         uint8_t is_first_item;
 846         int ret;
 847
 848         ENICPMD_FUNC_TRACE();
 849         item = pattern;
 850         is_first_item = 1;
 851         prev_item = RTE_FLOW_ITEM_TYPE_END;
 852
 853         args.fm_tcam_entry = &fm->tcam_entry;
 854         args.header_level = 0;
 855         prev_header_level = 0;
 856         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 857                 /*
 858                  * Get info about how to validate and copy the item. If NULL
 859                  * is returned the nic does not support the item.
 860                  */
 861                 if (item->type == RTE_FLOW_ITEM_TYPE_VOID)
 862                         continue;
 863
 864                 item_info = &enic_fm_items[item->type];
 865
 866                 if (item->type > FM_MAX_ITEM_TYPE ||
 867                     item_info->copy_item == NULL) {
 868                         return rte_flow_error_set(error, ENOTSUP,
 869                                 RTE_FLOW_ERROR_TYPE_ITEM,
 870                                 NULL, "enic: unsupported item");
 871                 }
 872
 873                 /* check to see if item stacking is valid */
 874                 if (!fm_item_stacking_valid(prev_item, item_info,
 875                                             is_first_item))
 876                         goto stacking_error;
 877
 878                 args.item = item;
 879                 ret = item_info->copy_item(&args);
 880                 if (ret)
 881                         goto item_not_supported;
 882                 /* Going from outer to inner? Treat it as a new packet start */
 883                 if (prev_header_level != args.header_level) {
 884                         prev_item = RTE_FLOW_ITEM_TYPE_END;
 885                         is_first_item = 1;
 886                 } else {
 887                         prev_item = item->type;
 888                         is_first_item = 0;
 889                 }
 890                 prev_header_level = args.header_level;
 891         }
 892         return 0;
 893
 894 item_not_supported:
 895         return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,
 896                                   NULL, "enic: unsupported item type");
 897
 898 stacking_error:
 899         return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
 900                                   item, "enic: unsupported item stack");
 901 }
 902
 903 static void
 904 flow_item_skip_void(const struct rte_flow_item **item)
 905 {
 906         for ( ; ; (*item)++)
 907                 if ((*item)->type != RTE_FLOW_ITEM_TYPE_VOID)
 908                         return;
 909 }
 910
 911 static void
 912 append_template(void **template, uint8_t *off, const void *data, int len)
 913 {
 914         memcpy(*template, data, len);
 915         *template = (char *)*template + len;
 916         *off = *off + len;
 917 }
 918
 919 static int
 920 enic_fm_append_action_op(struct enic_flowman *fm,
 921                          struct fm_action_op *fm_op,
 922                          struct rte_flow_error *error)
 923 {
 924         int count;
 925
 926         count = fm->action_op_count;
 927         ENICPMD_LOG(DEBUG, "append action op: idx=%d op=%u",
 928                     count, fm_op->fa_op);
 929         if (count == FM_ACTION_OP_MAX) {
 930                 return rte_flow_error_set(error, EINVAL,
 931                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 932                         "too many action operations");
 933         }
 934         fm->action.fma_action_ops[count] = *fm_op;
 935         fm->action_op_count = count + 1;
 936         return 0;
 937 }
 938
 939 static struct fm_action_op *
 940 find_prev_action_op(struct enic_flowman *fm, uint32_t opcode)
 941 {
 942         struct fm_action_op *op;
 943         int i;
 944
 945         for (i = 0; i < fm->action_op_count; i++) {
 946                 op = &fm->action.fma_action_ops[i];
 947                 if (op->fa_op == opcode)
 948                         return op;
 949         }
 950         return NULL;
 951 }
 952
 953 /* NIC requires that 1st steer appear before decap.
 954  * Correct example: steer, decap, steer, steer, ...
 955  */
 956 static void
 957 enic_fm_reorder_action_op(struct enic_flowman *fm)
 958 {
 959         struct fm_action_op *op, *steer, *decap;
 960         struct fm_action_op tmp_op;
 961
 962         ENICPMD_FUNC_TRACE();
 963         /* Find 1st steer and decap */
 964         op = fm->action.fma_action_ops;
 965         steer = NULL;
 966         decap = NULL;
 967         while (op->fa_op != FMOP_END) {
 968                 if (!decap && (op->fa_op == FMOP_DECAP_NOSTRIP ||
 969                                op->fa_op == FMOP_DECAP_STRIP))
 970                         decap = op;
 971                 else if (!steer && op->fa_op == FMOP_RQ_STEER)
 972                         steer = op;
 973                 op++;
 974         }
 975         /* If decap is before steer, swap */
 976         if (steer && decap && decap < steer) {
 977                 op = fm->action.fma_action_ops;
 978                 ENICPMD_LOG(DEBUG, "swap decap %ld <-> steer %ld",
 979                             (long)(decap - op), (long)(steer - op));
 980                 tmp_op = *decap;
 981                 *decap = *steer;
 982                 *steer = tmp_op;
 983         }
 984 }
 985
 986 /* VXLAN decap is done via flowman compound action */
 987 static int
 988 enic_fm_copy_vxlan_decap(struct enic_flowman *fm,
 989                          struct fm_tcam_match_entry *fmt,
 990                          const struct rte_flow_action *action,
 991                          struct rte_flow_error *error)
 992 {
 993         struct fm_header_set *fm_data;
 994         struct fm_action_op fm_op;
 995
 996         ENICPMD_FUNC_TRACE();
 997         fm_data = &fmt->ftm_data.fk_hdrset[0];
 998         if (!(fm_data->fk_metadata & FKM_VXLAN)) {
 999                 return rte_flow_error_set(error, EINVAL,
1000                         RTE_FLOW_ERROR_TYPE_ACTION, action,
1001                         "vxlan-decap: vxlan must be in pattern");
1002         }
1003
1004         memset(&fm_op, 0, sizeof(fm_op));
1005         fm_op.fa_op = FMOP_DECAP_NOSTRIP;
1006         return enic_fm_append_action_op(fm, &fm_op, error);
1007 }
1008
1009 /* Generate a reasonable source port number */
1010 static uint16_t
1011 gen_src_port(void)
1012 {
1013         /* Min/max below are the default values in OVS-DPDK and Linux */
1014         uint16_t p = rte_rand();
1015         p = RTE_MAX(p, 32768);
1016         p = RTE_MIN(p, 61000);
1017         return rte_cpu_to_be_16(p);
1018 }
1019
1020 /* VXLAN encap is done via flowman compound action */
1021 static int
1022 enic_fm_copy_vxlan_encap(struct enic_flowman *fm,
1023                          const struct rte_flow_item *item,
1024                          struct rte_flow_error *error)
1025 {
1026         struct fm_action_op fm_op;
1027         struct rte_ether_hdr *eth;
1028         struct rte_udp_hdr *udp;
1029         uint16_t *ethertype;
1030         void *template;
1031         uint8_t off;
1032
1033         ENICPMD_FUNC_TRACE();
1034         memset(&fm_op, 0, sizeof(fm_op));
1035         fm_op.fa_op = FMOP_ENCAP;
1036         template = fm->action.fma_data;
1037         off = 0;
1038         /*
1039          * Copy flow items to the flowman template starting L2.
1040          * L2 must be ethernet.
1041          */
1042         flow_item_skip_void(&item);
1043         if (item->type != RTE_FLOW_ITEM_TYPE_ETH)
1044                 return rte_flow_error_set(error, EINVAL,
1045                         RTE_FLOW_ERROR_TYPE_ITEM, item,
1046                         "vxlan-encap: first item should be ethernet");
1047         eth = (struct rte_ether_hdr *)template;
1048         ethertype = &eth->ether_type;
1049         append_template(&template, &off, item->spec,
1050                         sizeof(struct rte_ether_hdr));
1051         item++;
1052         flow_item_skip_void(&item);
1053         /* Optional VLAN */
1054         if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
1055                 const struct rte_flow_item_vlan *spec;
1056
1057                 ENICPMD_LOG(DEBUG, "vxlan-encap: vlan");
1058                 spec = item->spec;
1059                 fm_op.encap.outer_vlan = rte_be_to_cpu_16(spec->tci);
1060                 item++;
1061                 flow_item_skip_void(&item);
1062         }
1063         /* L3 must be IPv4, IPv6 */
1064         switch (item->type) {
1065         case RTE_FLOW_ITEM_TYPE_IPV4:
1066         {
1067                 struct rte_ipv4_hdr *ip4;
1068
1069                 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv4");
1070                 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
1071                 ip4 = (struct rte_ipv4_hdr *)template;
1072                 /*
1073                  * Offset of IPv4 length field and its initial value
1074                  * (IP + UDP + VXLAN) are specified in the action. The NIC
1075                  * will add inner packet length.
1076                  */
1077                 fm_op.encap.len1_offset = off +
1078                         offsetof(struct rte_ipv4_hdr, total_length);
1079                 fm_op.encap.len1_delta = sizeof(struct rte_ipv4_hdr) +
1080                         sizeof(struct rte_udp_hdr) +
1081                         sizeof(struct rte_vxlan_hdr);
1082                 append_template(&template, &off, item->spec,
1083                                 sizeof(struct rte_ipv4_hdr));
1084                 ip4->version_ihl = RTE_IPV4_VHL_DEF;
1085                 if (ip4->time_to_live == 0)
1086                         ip4->time_to_live = IP_DEFTTL;
1087                 ip4->next_proto_id = IPPROTO_UDP;
1088                 break;
1089         }
1090         case RTE_FLOW_ITEM_TYPE_IPV6:
1091         {
1092                 struct rte_ipv6_hdr *ip6;
1093
1094                 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv6");
1095                 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
1096                 ip6 = (struct rte_ipv6_hdr *)template;
1097                 fm_op.encap.len1_offset = off +
1098                         offsetof(struct rte_ipv6_hdr, payload_len);
1099                 fm_op.encap.len1_delta = sizeof(struct rte_udp_hdr) +
1100                         sizeof(struct rte_vxlan_hdr);
1101                 append_template(&template, &off, item->spec,
1102                                 sizeof(struct rte_ipv6_hdr));
1103                 ip6->vtc_flow |= rte_cpu_to_be_32(IP6_VTC_FLOW);
1104                 if (ip6->hop_limits == 0)
1105                         ip6->hop_limits = IP_DEFTTL;
1106                 ip6->proto = IPPROTO_UDP;
1107                 break;
1108         }
1109         default:
1110                 return rte_flow_error_set(error,
1111                         EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1112                         "vxlan-encap: L3 must be IPv4/IPv6");
1113         }
1114         item++;
1115         flow_item_skip_void(&item);
1116
1117         /* L4 is UDP */
1118         if (item->type != RTE_FLOW_ITEM_TYPE_UDP)
1119                 return rte_flow_error_set(error, EINVAL,
1120                         RTE_FLOW_ERROR_TYPE_ITEM, item,
1121                         "vxlan-encap: UDP must follow IPv4/IPv6");
1122         /* UDP length = UDP + VXLAN. NIC will add inner packet length. */
1123         fm_op.encap.len2_offset =
1124                 off + offsetof(struct rte_udp_hdr, dgram_len);
1125         fm_op.encap.len2_delta =
1126                 sizeof(struct rte_udp_hdr) + sizeof(struct rte_vxlan_hdr);
1127         udp = (struct rte_udp_hdr *)template;
1128         append_template(&template, &off, item->spec,
1129                         sizeof(struct rte_udp_hdr));
1130         /*
1131          * Firmware does not hash/fill source port yet. Generate a
1132          * random port, as there is *usually* one rte_flow for the
1133          * given inner packet stream (i.e. a single stream has one
1134          * random port).
1135          */
1136         if (udp->src_port == 0)
1137                 udp->src_port = gen_src_port();
1138         item++;
1139         flow_item_skip_void(&item);
1140
1141         /* Finally VXLAN */
1142         if (item->type != RTE_FLOW_ITEM_TYPE_VXLAN)
1143                 return rte_flow_error_set(error,
1144                         EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1145                         "vxlan-encap: VXLAN must follow UDP");
1146         append_template(&template, &off, item->spec,
1147                         sizeof(struct rte_flow_item_vxlan));
1148
1149         /*
1150          * Fill in the rest of the action structure.
1151          * Indicate that we want to encap with vxlan at packet start.
1152          */
1153         fm_op.encap.template_offset = 0;
1154         fm_op.encap.template_len = off;
1155         return enic_fm_append_action_op(fm, &fm_op, error);
1156 }
1157
1158 static int
1159 enic_fm_find_vnic(struct enic *enic, const struct rte_pci_addr *addr,
1160                   uint64_t *handle)
1161 {
1162         uint32_t bdf;
1163         uint64_t args[2];
1164         int rc;
1165
1166         ENICPMD_FUNC_TRACE();
1167         ENICPMD_LOG(DEBUG, "bdf=%x:%x:%x", addr->bus, addr->devid,
1168                     addr->function);
1169         bdf = addr->bus << 8 | addr->devid << 3 | addr->function;
1170         args[0] = FM_VNIC_FIND;
1171         args[1] = bdf;
1172         rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
1173         if (rc != 0) {
1174                 /* Expected to fail if BDF is not on the adapter */
1175                 ENICPMD_LOG(DEBUG, "cannot find vnic handle: rc=%d", rc);
1176                 return rc;
1177         }
1178         *handle = args[0];
1179         ENICPMD_LOG(DEBUG, "found vnic: handle=0x%" PRIx64, *handle);
1180         return 0;
1181 }
1182
1183 /*
1184  * Egress: target port should be either PF uplink or VF.
1185  * Supported cases
1186  * 1. VF egress -> PF uplink
1187  *   PF may be this VF's PF, or another PF, as long as they are on the same VIC.
1188  * 2. VF egress -> VF
1189  *
1190  * Unsupported cases
1191  * 1. PF egress -> VF
1192  *   App should be using representor to pass packets to VF
1193  */
1194 static int
1195 vf_egress_port_id_action(struct enic_flowman *fm,
1196                          struct rte_eth_dev *dst_dev,
1197                          uint64_t dst_vnic_h,
1198                          struct fm_action_op *fm_op,
1199                          struct rte_flow_error *error)
1200 {
1201         struct enic *src_enic, *dst_enic;
1202         struct enic_vf_representor *vf;
1203         uint8_t uif;
1204         int ret;
1205
1206         ENICPMD_FUNC_TRACE();
1207         src_enic = fm->user_enic;
1208         dst_enic = pmd_priv(dst_dev);
1209         if (!(src_enic->rte_dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)) {
1210                 return rte_flow_error_set(error, EINVAL,
1211                         RTE_FLOW_ERROR_TYPE_ACTION,
1212                         NULL, "source port is not VF representor");
1213         }
1214
1215         /* VF -> PF uplink. dst is not VF representor */
1216         if (!(dst_dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)) {
1217                 /* PF is the VF's PF? Then nothing to do */
1218                 vf = VF_ENIC_TO_VF_REP(src_enic);
1219                 if (vf->pf == dst_enic) {
1220                         ENICPMD_LOG(DEBUG, "destination port is VF's PF");
1221                         return 0;
1222                 }
1223                 /* If not, steer to the remote PF's uplink */
1224                 uif = dst_enic->fm_vnic_uif;
1225                 ENICPMD_LOG(DEBUG, "steer to uplink %u", uif);
1226                 memset(fm_op, 0, sizeof(*fm_op));
1227                 fm_op->fa_op = FMOP_SET_EGPORT;
1228                 fm_op->set_egport.egport = uif;
1229                 ret = enic_fm_append_action_op(fm, fm_op, error);
1230                 return ret;
1231         }
1232
1233         /* VF -> VF loopback. Hairpin and steer to vnic */
1234         memset(fm_op, 0, sizeof(*fm_op));
1235         fm_op->fa_op = FMOP_EG_HAIRPIN;
1236         ret = enic_fm_append_action_op(fm, fm_op, error);
1237         if (ret)
1238                 return ret;
1239         ENICPMD_LOG(DEBUG, "egress hairpin");
1240         fm->hairpin_steer_vnic_h = dst_vnic_h;
1241         fm->need_hairpin_steer = 1;
1242         return 0;
1243 }
1244
1245 static int
1246 enic_fm_check_transfer_dst(struct enic *enic, uint16_t dst_port_id,
1247                            struct rte_eth_dev **dst_dev,
1248                            struct rte_flow_error *error)
1249 {
1250         struct rte_eth_dev *dev;
1251
1252         ENICPMD_LOG(DEBUG, "port id %u", dst_port_id);
1253         if (!rte_eth_dev_is_valid_port(dst_port_id)) {
1254                 return rte_flow_error_set(error, EINVAL,
1255                         RTE_FLOW_ERROR_TYPE_ACTION,
1256                         NULL, "invalid port_id");
1257         }
1258         dev = &rte_eth_devices[dst_port_id];
1259         if (!dev_is_enic(dev)) {
1260                 return rte_flow_error_set(error, EINVAL,
1261                         RTE_FLOW_ERROR_TYPE_ACTION,
1262                         NULL, "port_id is not enic");
1263         }
1264         if (enic->switch_domain_id != pmd_priv(dev)->switch_domain_id) {
1265                 return rte_flow_error_set(error, EINVAL,
1266                         RTE_FLOW_ERROR_TYPE_ACTION,
1267                         NULL, "destination and source ports are not in the same switch domain");
1268         }
1269
1270         *dst_dev = dev;
1271         return 0;
1272 }
1273
1274 /* Translate flow actions to flowman TCAM entry actions */
1275 static int
1276 enic_fm_copy_action(struct enic_flowman *fm,
1277                     const struct rte_flow_action actions[],
1278                     uint8_t ingress,
1279                     struct rte_flow_error *error)
1280 {
1281         enum {
1282                 FATE = 1 << 0,
1283                 DECAP = 1 << 1,
1284                 PASSTHRU = 1 << 2,
1285                 COUNT = 1 << 3,
1286                 ENCAP = 1 << 4,
1287                 PUSH_VLAN = 1 << 5,
1288                 PORT_ID = 1 << 6,
1289         };
1290         struct fm_tcam_match_entry *fmt;
1291         struct fm_action_op fm_op;
1292         bool need_ovlan_action;
1293         struct enic *enic;
1294         uint32_t overlap;
1295         uint64_t vnic_h;
1296         uint16_t ovlan;
1297         bool first_rq;
1298         bool steer;
1299         int ret;
1300
1301         ENICPMD_FUNC_TRACE();
1302         fmt = &fm->tcam_entry;
1303         need_ovlan_action = false;
1304         ovlan = 0;
1305         first_rq = true;
1306         steer = false;
1307         enic = fm->user_enic;
1308         overlap = 0;
1309         vnic_h = enic->fm_vnic_handle;
1310
1311         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1312                 switch (actions->type) {
1313                 case RTE_FLOW_ACTION_TYPE_VOID:
1314                         continue;
1315                 case RTE_FLOW_ACTION_TYPE_PASSTHRU: {
1316                         if (overlap & PASSTHRU)
1317                                 goto unsupported;
1318                         overlap |= PASSTHRU;
1319                         break;
1320                 }
1321                 case RTE_FLOW_ACTION_TYPE_JUMP: {
1322                         const struct rte_flow_action_jump *jump =
1323                                 actions->conf;
1324                         struct enic_fm_fet *fet;
1325
1326                         if (overlap & FATE)
1327                                 goto unsupported;
1328                         ret = enic_fet_get(fm, jump->group, ingress, NULL,
1329                                            &fet, error);
1330                         if (ret)
1331                                 return ret;
1332                         overlap |= FATE;
1333                         memset(&fm_op, 0, sizeof(fm_op));
1334                         fm_op.fa_op = FMOP_EXACT_MATCH;
1335                         fm_op.exact.handle = fet->handle;
1336                         fm->fet = fet;
1337                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1338                         if (ret)
1339                                 return ret;
1340                         break;
1341                 }
1342                 case RTE_FLOW_ACTION_TYPE_MARK: {
1343                         const struct rte_flow_action_mark *mark =
1344                                 actions->conf;
1345
1346                         if (enic->use_noscatter_vec_rx_handler)
1347                                 goto unsupported;
1348                         if (mark->id >= ENIC_MAGIC_FILTER_ID - 1)
1349                                 return rte_flow_error_set(error, EINVAL,
1350                                         RTE_FLOW_ERROR_TYPE_ACTION,
1351                                         NULL, "invalid mark id");
1352                         memset(&fm_op, 0, sizeof(fm_op));
1353                         fm_op.fa_op = FMOP_MARK;
1354                         fm_op.mark.mark = mark->id + 1;
1355                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1356                         if (ret)
1357                                 return ret;
1358                         break;
1359                 }
1360                 case RTE_FLOW_ACTION_TYPE_FLAG: {
1361                         if (enic->use_noscatter_vec_rx_handler)
1362                                 goto unsupported;
1363                         /* ENIC_MAGIC_FILTER_ID is reserved for flagging */
1364                         memset(&fm_op, 0, sizeof(fm_op));
1365                         fm_op.fa_op = FMOP_MARK;
1366                         fm_op.mark.mark = ENIC_MAGIC_FILTER_ID;
1367                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1368                         if (ret)
1369                                 return ret;
1370                         break;
1371                 }
1372                 case RTE_FLOW_ACTION_TYPE_QUEUE: {
1373                         const struct rte_flow_action_queue *queue =
1374                                 actions->conf;
1375
1376                         /*
1377                          * If fate other than QUEUE or RSS, fail. Multiple
1378                          * rss and queue actions are ok.
1379                          */
1380                         if ((overlap & FATE) && first_rq)
1381                                 goto unsupported;
1382                         first_rq = false;
1383                         overlap |= FATE;
1384                         memset(&fm_op, 0, sizeof(fm_op));
1385                         fm_op.fa_op = FMOP_RQ_STEER;
1386                         fm_op.rq_steer.rq_index =
1387                                 enic_rte_rq_idx_to_sop_idx(queue->index);
1388                         fm_op.rq_steer.rq_count = 1;
1389                         fm_op.rq_steer.vnic_handle = vnic_h;
1390                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1391                         if (ret)
1392                                 return ret;
1393                         ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1394                                     fm_op.rq_steer.rq_index);
1395                         steer = true;
1396                         break;
1397                 }
1398                 case RTE_FLOW_ACTION_TYPE_DROP: {
1399                         if (overlap & FATE)
1400                                 goto unsupported;
1401                         overlap |= FATE;
1402                         memset(&fm_op, 0, sizeof(fm_op));
1403                         fm_op.fa_op = FMOP_DROP;
1404                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1405                         if (ret)
1406                                 return ret;
1407                         ENICPMD_LOG(DEBUG, "create DROP action");
1408                         break;
1409                 }
1410                 case RTE_FLOW_ACTION_TYPE_COUNT: {
1411                         if (overlap & COUNT)
1412                                 goto unsupported;
1413                         overlap |= COUNT;
1414                         /* Count is associated with entry not action on VIC. */
1415                         fmt->ftm_flags |= FMEF_COUNTER;
1416                         break;
1417                 }
1418                 case RTE_FLOW_ACTION_TYPE_RSS: {
1419                         const struct rte_flow_action_rss *rss = actions->conf;
1420                         bool allow;
1421                         uint16_t i;
1422
1423                         /*
1424                          * If fate other than QUEUE or RSS, fail. Multiple
1425                          * rss and queue actions are ok.
1426                          */
1427                         if ((overlap & FATE) && first_rq)
1428                                 goto unsupported;
1429                         first_rq = false;
1430                         overlap |= FATE;
1431
1432                         /*
1433                          * Hardware only supports RSS actions on outer level
1434                          * with default type and function. Queues must be
1435                          * sequential.
1436                          */
1437                         allow = rss->func == RTE_ETH_HASH_FUNCTION_DEFAULT &&
1438                                 rss->level == 0 && (rss->types == 0 ||
1439                                 rss->types == enic->rss_hf) &&
1440                                 rss->queue_num <= enic->rq_count &&
1441                                 rss->queue[rss->queue_num - 1] < enic->rq_count;
1442
1443
1444                         /* Identity queue map needs to be sequential */
1445                         for (i = 1; i < rss->queue_num; i++)
1446                                 allow = allow && (rss->queue[i] ==
1447                                         rss->queue[i - 1] + 1);
1448                         if (!allow)
1449                                 goto unsupported;
1450
1451                         memset(&fm_op, 0, sizeof(fm_op));
1452                         fm_op.fa_op = FMOP_RQ_STEER;
1453                         fm_op.rq_steer.rq_index =
1454                                 enic_rte_rq_idx_to_sop_idx(rss->queue[0]);
1455                         fm_op.rq_steer.rq_count = rss->queue_num;
1456                         fm_op.rq_steer.vnic_handle = vnic_h;
1457                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1458                         if (ret)
1459                                 return ret;
1460                         ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1461                                     fm_op.rq_steer.rq_index);
1462                         steer = true;
1463                         break;
1464                 }
1465                 case RTE_FLOW_ACTION_TYPE_PORT_ID: {
1466                         const struct rte_flow_action_port_id *port;
1467                         struct rte_eth_dev *dev = NULL;
1468
1469                         if (!ingress && (overlap & PORT_ID)) {
1470                                 ENICPMD_LOG(DEBUG, "cannot have multiple egress PORT_ID actions");
1471                                 goto unsupported;
1472                         }
1473                         port = actions->conf;
1474                         if (port->original) {
1475                                 vnic_h = enic->fm_vnic_handle; /* This port */
1476                                 break;
1477                         }
1478                         ret = enic_fm_check_transfer_dst(enic, port->id, &dev,
1479                                                          error);
1480                         if (ret)
1481                                 return ret;
1482                         vnic_h = pmd_priv(dev)->fm_vnic_handle;
1483                         overlap |= PORT_ID;
1484                         /*
1485                          * Ingress. Nothing more to do. We add an implicit
1486                          * steer at the end if needed.
1487                          */
1488                         if (ingress)
1489                                 break;
1490                         /* Egress */
1491                         ret = vf_egress_port_id_action(fm, dev, vnic_h, &fm_op,
1492                                 error);
1493                         if (ret)
1494                                 return ret;
1495                         break;
1496                 }
1497                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: {
1498                         if (overlap & DECAP)
1499                                 goto unsupported;
1500                         overlap |= DECAP;
1501
1502                         ret = enic_fm_copy_vxlan_decap(fm, fmt, actions,
1503                                 error);
1504                         if (ret != 0)
1505                                 return ret;
1506                         break;
1507                 }
1508                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: {
1509                         const struct rte_flow_action_vxlan_encap *encap;
1510
1511                         encap = actions->conf;
1512                         if (overlap & ENCAP)
1513                                 goto unsupported;
1514                         overlap |= ENCAP;
1515                         ret = enic_fm_copy_vxlan_encap(fm, encap->definition,
1516                                 error);
1517                         if (ret != 0)
1518                                 return ret;
1519                         break;
1520                 }
1521                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: {
1522                         struct fm_action_op *decap;
1523
1524                         /*
1525                          * If decap-nostrip appears before pop vlan, this pop
1526                          * applies to the inner packet vlan. Turn it into
1527                          * decap-strip.
1528                          */
1529                         decap = find_prev_action_op(fm, FMOP_DECAP_NOSTRIP);
1530                         if (decap) {
1531                                 ENICPMD_LOG(DEBUG, "pop-vlan inner: decap-nostrip => decap-strip");
1532                                 decap->fa_op = FMOP_DECAP_STRIP;
1533                                 break;
1534                         }
1535                         memset(&fm_op, 0, sizeof(fm_op));
1536                         fm_op.fa_op = FMOP_POP_VLAN;
1537                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1538                         if (ret)
1539                                 return ret;
1540                         break;
1541                 }
1542                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: {
1543                         const struct rte_flow_action_of_push_vlan *vlan;
1544
1545                         if (overlap & PASSTHRU)
1546                                 goto unsupported;
1547                         vlan = actions->conf;
1548                         if (vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN)) {
1549                                 return rte_flow_error_set(error, EINVAL,
1550                                         RTE_FLOW_ERROR_TYPE_ACTION,
1551                                         NULL, "unexpected push_vlan ethertype");
1552                         }
1553                         overlap |= PUSH_VLAN;
1554                         need_ovlan_action = true;
1555                         break;
1556                 }
1557                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: {
1558                         const struct rte_flow_action_of_set_vlan_pcp *pcp;
1559
1560                         pcp = actions->conf;
1561                         if (pcp->vlan_pcp > 7) {
1562                                 return rte_flow_error_set(error, EINVAL,
1563                                         RTE_FLOW_ERROR_TYPE_ACTION,
1564                                         NULL, "invalid vlan_pcp");
1565                         }
1566                         need_ovlan_action = true;
1567                         ovlan |= ((uint16_t)pcp->vlan_pcp) << 13;
1568                         break;
1569                 }
1570                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: {
1571                         const struct rte_flow_action_of_set_vlan_vid *vid;
1572
1573                         vid = actions->conf;
1574                         need_ovlan_action = true;
1575                         ovlan |= rte_be_to_cpu_16(vid->vlan_vid);
1576                         break;
1577                 }
1578                 case RTE_FLOW_ACTION_TYPE_PORT_REPRESENTOR: {
1579                         const struct rte_flow_action_ethdev *ethdev;
1580                         struct rte_eth_dev *dev = NULL;
1581
1582                         ethdev = actions->conf;
1583                         ret = enic_fm_check_transfer_dst(enic, ethdev->port_id,
1584                                                          &dev, error);
1585                         if (ret)
1586                                 return ret;
1587                         vnic_h = pmd_priv(dev)->fm_vnic_handle;
1588                         overlap |= PORT_ID;
1589                         /*
1590                          * Action PORT_REPRESENTOR implies ingress destination.
1591                          * Noting to do. We add an implicit stree at the
1592                          * end if needed.
1593                          */
1594                         ingress = 1;
1595                         break;
1596                 }
1597                 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: {
1598                         const struct rte_flow_action_ethdev *ethdev;
1599                         struct rte_eth_dev *dev = NULL;
1600
1601                         if (overlap & PORT_ID) {
1602                                 ENICPMD_LOG(DEBUG, "cannot have multiple egress PORT_ID actions");
1603                                 goto unsupported;
1604                         }
1605                         ethdev = actions->conf;
1606                         ret = enic_fm_check_transfer_dst(enic, ethdev->port_id,
1607                                                          &dev, error);
1608                         if (ret)
1609                                 return ret;
1610                         vnic_h = pmd_priv(dev)->fm_vnic_handle;
1611                         overlap |= PORT_ID;
1612                         /* Action REPRESENTED_PORT: always egress destination */
1613                         ingress = 0;
1614                         ret = vf_egress_port_id_action(fm, dev, vnic_h, &fm_op,
1615                                 error);
1616                         if (ret)
1617                                 return ret;
1618                         break;
1619                 }
1620                 default:
1621                         goto unsupported;
1622                 }
1623         }
1624
1625         if (!(overlap & (FATE | PASSTHRU | COUNT | PORT_ID)))
1626                 goto unsupported;
1627         /* Egress from VF: need implicit WQ match */
1628         if (enic_is_vf_rep(enic) && !ingress) {
1629                 fmt->ftm_data.fk_wq_id = 0;
1630                 fmt->ftm_mask.fk_wq_id = 0xffff;
1631                 fmt->ftm_data.fk_wq_vnic = enic->fm_vnic_handle;
1632                 ENICPMD_LOG(DEBUG, "add implicit wq id match for vf %d",
1633                             VF_ENIC_TO_VF_REP(enic)->vf_id);
1634         }
1635         if (need_ovlan_action) {
1636                 memset(&fm_op, 0, sizeof(fm_op));
1637                 fm_op.fa_op = FMOP_SET_OVLAN;
1638                 fm_op.ovlan.vlan = ovlan;
1639                 ret = enic_fm_append_action_op(fm, &fm_op, error);
1640                 if (ret)
1641                         return ret;
1642         }
1643         /* Add steer op for PORT_ID without QUEUE */
1644         if ((overlap & PORT_ID) && !steer && ingress) {
1645                 memset(&fm_op, 0, sizeof(fm_op));
1646                 /* Always to queue 0 for now as generic RSS is not available */
1647                 fm_op.fa_op = FMOP_RQ_STEER;
1648                 fm_op.rq_steer.rq_index = 0;
1649                 fm_op.rq_steer.vnic_handle = vnic_h;
1650                 ret = enic_fm_append_action_op(fm, &fm_op, error);
1651                 if (ret)
1652                         return ret;
1653                 ENICPMD_LOG(DEBUG, "add implicit steer op");
1654         }
1655         /* Add required END */
1656         memset(&fm_op, 0, sizeof(fm_op));
1657         fm_op.fa_op = FMOP_END;
1658         ret = enic_fm_append_action_op(fm, &fm_op, error);
1659         if (ret)
1660                 return ret;
1661         enic_fm_reorder_action_op(fm);
1662         return 0;
1663
1664 unsupported:
1665         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
1666                                   NULL, "enic: unsupported action");
1667 }
1668
1669 /** Check if the action is supported */
1670 static int
1671 enic_fm_match_action(const struct rte_flow_action *action,
1672                      const enum rte_flow_action_type *supported_actions)
1673 {
1674         for (; *supported_actions != RTE_FLOW_ACTION_TYPE_END;
1675              supported_actions++) {
1676                 if (action->type == *supported_actions)
1677                         return 1;
1678         }
1679         return 0;
1680 }
1681
1682 /* Debug function to dump internal NIC action structure. */
1683 static void
1684 enic_fm_dump_tcam_actions(const struct fm_action *fm_action)
1685 {
1686         /* Manually keep in sync with FMOP commands */
1687         const char *fmop_str[FMOP_OP_MAX] = {
1688                 [FMOP_END] = "end",
1689                 [FMOP_DROP] = "drop",
1690                 [FMOP_RQ_STEER] = "steer",
1691                 [FMOP_EXACT_MATCH] = "exmatch",
1692                 [FMOP_MARK] = "mark",
1693                 [FMOP_EXT_MARK] = "ext_mark",
1694                 [FMOP_TAG] = "tag",
1695                 [FMOP_EG_HAIRPIN] = "eg_hairpin",
1696                 [FMOP_IG_HAIRPIN] = "ig_hairpin",
1697                 [FMOP_ENCAP_IVLAN] = "encap_ivlan",
1698                 [FMOP_ENCAP_NOIVLAN] = "encap_noivlan",
1699                 [FMOP_ENCAP] = "encap",
1700                 [FMOP_SET_OVLAN] = "set_ovlan",
1701                 [FMOP_DECAP_NOSTRIP] = "decap_nostrip",
1702                 [FMOP_DECAP_STRIP] = "decap_strip",
1703                 [FMOP_POP_VLAN] = "pop_vlan",
1704                 [FMOP_SET_EGPORT] = "set_egport",
1705                 [FMOP_RQ_STEER_ONLY] = "rq_steer_only",
1706                 [FMOP_SET_ENCAP_VLAN] = "set_encap_vlan",
1707                 [FMOP_EMIT] = "emit",
1708                 [FMOP_MODIFY] = "modify",
1709         };
1710         const struct fm_action_op *op = &fm_action->fma_action_ops[0];
1711         char buf[128], *bp = buf;
1712         const char *op_str;
1713         int i, n, buf_len;
1714
1715         buf[0] = '\0';
1716         buf_len = sizeof(buf);
1717         for (i = 0; i < FM_ACTION_OP_MAX; i++) {
1718                 if (op->fa_op == FMOP_END)
1719                         break;
1720                 if (op->fa_op >= FMOP_OP_MAX)
1721                         op_str = "unknown";
1722                 else
1723                         op_str = fmop_str[op->fa_op];
1724                 n = snprintf(bp, buf_len, "%s,", op_str);
1725                 if (n > 0 && n < buf_len) {
1726                         bp += n;
1727                         buf_len -= n;
1728                 }
1729                 op++;
1730         }
1731         /* Remove trailing comma */
1732         if (buf[0])
1733                 *(bp - 1) = '\0';
1734         ENICPMD_LOG(DEBUG, "       Acions: %s", buf);
1735 }
1736
1737 static int
1738 bits_to_str(uint32_t bits, const char *strings[], int max,
1739             char *buf, int buf_len)
1740 {
1741         int i, n = 0, len = 0;
1742
1743         for (i = 0; i < max; i++) {
1744                 if (bits & (1 << i)) {
1745                         n = snprintf(buf, buf_len, "%s,", strings[i]);
1746                         if (n > 0 && n < buf_len) {
1747                                 buf += n;
1748                                 buf_len -= n;
1749                                 len += n;
1750                         }
1751                 }
1752         }
1753         /* Remove trailing comma */
1754         if (len) {
1755                 *(buf - 1) = '\0';
1756                 len--;
1757         }
1758         return len;
1759 }
1760
1761 /* Debug function to dump internal NIC filter structure. */
1762 static void
1763 __enic_fm_dump_tcam_match(const struct fm_header_set *fk_hdrset, char *buf,
1764                           int buf_len)
1765 {
1766         /* Manually keep in sync with FKM_BITS */
1767         const char *fm_fkm_str[FKM_BIT_COUNT] = {
1768                 [FKM_QTAG_BIT] = "qtag",
1769                 [FKM_CMD_BIT] = "cmd",
1770                 [FKM_IPV4_BIT] = "ip4",
1771                 [FKM_IPV6_BIT] = "ip6",
1772                 [FKM_ROCE_BIT] = "roce",
1773                 [FKM_UDP_BIT] = "udp",
1774                 [FKM_TCP_BIT] = "tcp",
1775                 [FKM_TCPORUDP_BIT] = "tcpportudp",
1776                 [FKM_IPFRAG_BIT] = "ipfrag",
1777                 [FKM_NVGRE_BIT] = "nvgre",
1778                 [FKM_VXLAN_BIT] = "vxlan",
1779                 [FKM_GENEVE_BIT] = "geneve",
1780                 [FKM_NSH_BIT] = "nsh",
1781                 [FKM_ROCEV2_BIT] = "rocev2",
1782                 [FKM_VLAN_PRES_BIT] = "vlan_pres",
1783                 [FKM_IPOK_BIT] = "ipok",
1784                 [FKM_L4OK_BIT] = "l4ok",
1785                 [FKM_ROCEOK_BIT] = "roceok",
1786                 [FKM_FCSOK_BIT] = "fcsok",
1787                 [FKM_EG_SPAN_BIT] = "eg_span",
1788                 [FKM_IG_SPAN_BIT] = "ig_span",
1789                 [FKM_EG_HAIRPINNED_BIT] = "eg_hairpinned",
1790         };
1791         /* Manually keep in sync with FKH_BITS */
1792         const char *fm_fkh_str[FKH_BIT_COUNT] = {
1793                 [FKH_ETHER_BIT] = "eth",
1794                 [FKH_QTAG_BIT] = "qtag",
1795                 [FKH_L2RAW_BIT] = "l2raw",
1796                 [FKH_IPV4_BIT] = "ip4",
1797                 [FKH_IPV6_BIT] = "ip6",
1798                 [FKH_L3RAW_BIT] = "l3raw",
1799                 [FKH_UDP_BIT] = "udp",
1800                 [FKH_TCP_BIT] = "tcp",
1801                 [FKH_ICMP_BIT] = "icmp",
1802                 [FKH_VXLAN_BIT] = "vxlan",
1803                 [FKH_L4RAW_BIT] = "l4raw",
1804         };
1805         uint32_t fkh_bits = fk_hdrset->fk_header_select;
1806         uint32_t fkm_bits = fk_hdrset->fk_metadata;
1807         int n;
1808
1809         if (!fkm_bits && !fkh_bits)
1810                 return;
1811         n = snprintf(buf, buf_len, "metadata(");
1812         if (n > 0 && n < buf_len) {
1813                 buf += n;
1814                 buf_len -= n;
1815         }
1816         n = bits_to_str(fkm_bits, fm_fkm_str, FKM_BIT_COUNT, buf, buf_len);
1817         if (n > 0 && n < buf_len) {
1818                 buf += n;
1819                 buf_len -= n;
1820         }
1821         n = snprintf(buf, buf_len, ") valid hdr fields(");
1822         if (n > 0 && n < buf_len) {
1823                 buf += n;
1824                 buf_len -= n;
1825         }
1826         n = bits_to_str(fkh_bits, fm_fkh_str, FKH_BIT_COUNT, buf, buf_len);
1827         if (n > 0 && n < buf_len) {
1828                 buf += n;
1829                 buf_len -= n;
1830         }
1831         snprintf(buf, buf_len, ")");
1832 }
1833
1834 static void
1835 enic_fm_dump_tcam_match(const struct fm_tcam_match_entry *match,
1836                         uint8_t ingress)
1837 {
1838         char buf[256];
1839
1840         memset(buf, 0, sizeof(buf));
1841         __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[0],
1842                                   buf, sizeof(buf));
1843         ENICPMD_LOG(DEBUG, " TCAM %s Outer: %s %scounter position %u",
1844                     (ingress) ? "IG" : "EG", buf,
1845                     (match->ftm_flags & FMEF_COUNTER) ? "" : "no ",
1846                     match->ftm_position);
1847         memset(buf, 0, sizeof(buf));
1848         __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[1],
1849                                   buf, sizeof(buf));
1850         if (buf[0])
1851                 ENICPMD_LOG(DEBUG, "         Inner: %s", buf);
1852 }
1853
1854 /* Debug function to dump internal NIC flow structures. */
1855 static void
1856 enic_fm_dump_tcam_entry(const struct fm_tcam_match_entry *fm_match,
1857                         const struct fm_action *fm_action,
1858                         uint8_t ingress)
1859 {
1860         if (!rte_log_can_log(enic_pmd_logtype, RTE_LOG_DEBUG))
1861                 return;
1862         enic_fm_dump_tcam_match(fm_match, ingress);
1863         enic_fm_dump_tcam_actions(fm_action);
1864 }
1865
1866 static int
1867 enic_fm_flow_parse(struct enic_flowman *fm,
1868                    const struct rte_flow_attr *attrs,
1869                    const struct rte_flow_item pattern[],
1870                    const struct rte_flow_action actions[],
1871                    struct rte_flow_error *error)
1872 {
1873         const struct rte_flow_action *action;
1874         unsigned int ret;
1875         static const enum rte_flow_action_type *sa;
1876
1877         ENICPMD_FUNC_TRACE();
1878         ret = 0;
1879         if (!pattern) {
1880                 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM,
1881                                    NULL, "no pattern specified");
1882                 return -rte_errno;
1883         }
1884
1885         if (!actions) {
1886                 rte_flow_error_set(error, EINVAL,
1887                                    RTE_FLOW_ERROR_TYPE_ACTION_NUM,
1888                                    NULL, "no action specified");
1889                 return -rte_errno;
1890         }
1891
1892         if (attrs) {
1893                 if (attrs->group != FM_TCAM_RTE_GROUP && attrs->priority) {
1894                         rte_flow_error_set(error, ENOTSUP,
1895                                            RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1896                                            NULL,
1897                                            "priorities are not supported for non-default (0) groups");
1898                         return -rte_errno;
1899                 } else if (!fm->owner_enic->switchdev_mode && attrs->transfer) {
1900                         rte_flow_error_set(error, ENOTSUP,
1901                                            RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1902                                            NULL,
1903                                            "transfer is not supported");
1904                         return -rte_errno;
1905                 } else if (attrs->ingress && attrs->egress) {
1906                         rte_flow_error_set(error, ENOTSUP,
1907                                            RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1908                                            NULL,
1909                                            "bidirectional rules not supported");
1910                         return -rte_errno;
1911                 }
1912
1913         } else {
1914                 rte_flow_error_set(error, EINVAL,
1915                                    RTE_FLOW_ERROR_TYPE_ATTR,
1916                                    NULL, "no attribute specified");
1917                 return -rte_errno;
1918         }
1919
1920         /* Verify Actions. */
1921         sa = (attrs->ingress) ? enic_fm_supported_ig_actions :
1922              enic_fm_supported_eg_actions;
1923         for (action = &actions[0]; action->type != RTE_FLOW_ACTION_TYPE_END;
1924              action++) {
1925                 if (action->type == RTE_FLOW_ACTION_TYPE_VOID)
1926                         continue;
1927                 else if (!enic_fm_match_action(action, sa))
1928                         break;
1929         }
1930         if (action->type != RTE_FLOW_ACTION_TYPE_END) {
1931                 rte_flow_error_set(error, EPERM, RTE_FLOW_ERROR_TYPE_ACTION,
1932                                    action, "invalid action");
1933                 return -rte_errno;
1934         }
1935         ret = enic_fm_copy_entry(fm, pattern, error);
1936         if (ret)
1937                 return ret;
1938         ret = enic_fm_copy_action(fm, actions, attrs->ingress, error);
1939         return ret;
1940 }
1941
1942 static void
1943 enic_fm_counter_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
1944 {
1945         if (!fm_flow->counter_valid)
1946                 return;
1947         SLIST_INSERT_HEAD(&fm->counters, fm_flow->counter, next);
1948         fm_flow->counter_valid = false;
1949 }
1950
1951 static int
1952 enic_fm_more_counters(struct enic_flowman *fm)
1953 {
1954         struct enic_fm_counter *new_stack;
1955         struct enic_fm_counter *ctrs;
1956         int i, rc;
1957         uint64_t args[2];
1958
1959         ENICPMD_FUNC_TRACE();
1960         new_stack = rte_realloc(fm->counter_stack, (fm->counters_alloced +
1961                                 FM_COUNTERS_EXPAND) *
1962                                 sizeof(struct enic_fm_counter), 0);
1963         if (new_stack == NULL) {
1964                 ENICPMD_LOG(ERR, "cannot alloc counter memory");
1965                 return -ENOMEM;
1966         }
1967         fm->counter_stack = new_stack;
1968
1969         args[0] = FM_COUNTER_BRK;
1970         args[1] = fm->counters_alloced + FM_COUNTERS_EXPAND;
1971         rc = flowman_cmd(fm, args, 2);
1972         if (rc != 0) {
1973                 ENICPMD_LOG(ERR, "cannot alloc counters rc=%d", rc);
1974                 return rc;
1975         }
1976         ctrs = (struct enic_fm_counter *)fm->counter_stack +
1977                 fm->counters_alloced;
1978         for (i = 0; i < FM_COUNTERS_EXPAND; i++, ctrs++) {
1979                 ctrs->handle = fm->counters_alloced + i;
1980                 SLIST_INSERT_HEAD(&fm->counters, ctrs, next);
1981         }
1982         fm->counters_alloced += FM_COUNTERS_EXPAND;
1983         ENICPMD_LOG(DEBUG, "%u counters allocated, total: %u",
1984                     FM_COUNTERS_EXPAND, fm->counters_alloced);
1985         return 0;
1986 }
1987
1988 static int
1989 enic_fm_counter_zero(struct enic_flowman *fm, struct enic_fm_counter *c)
1990 {
1991         uint64_t args[3];
1992         int ret;
1993
1994         ENICPMD_FUNC_TRACE();
1995         args[0] = FM_COUNTER_QUERY;
1996         args[1] = c->handle;
1997         args[2] = 1; /* clear */
1998         ret = flowman_cmd(fm, args, 3);
1999         if (ret) {
2000                 ENICPMD_LOG(ERR, "counter init: rc=%d handle=0x%x",
2001                             ret, c->handle);
2002                 return ret;
2003         }
2004         return 0;
2005 }
2006
2007 static int
2008 enic_fm_counter_alloc(struct enic_flowman *fm, struct rte_flow_error *error,
2009                       struct enic_fm_counter **ctr)
2010 {
2011         struct enic_fm_counter *c;
2012         int ret;
2013
2014         ENICPMD_FUNC_TRACE();
2015         *ctr = NULL;
2016         if (SLIST_EMPTY(&fm->counters)) {
2017                 ret = enic_fm_more_counters(fm);
2018                 if (ret)
2019                         return rte_flow_error_set(error, -ret,
2020                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2021                                 NULL, "enic: out of counters");
2022         }
2023         c = SLIST_FIRST(&fm->counters);
2024         SLIST_REMOVE_HEAD(&fm->counters, next);
2025         *ctr = c;
2026         return 0;
2027 }
2028
2029 static int
2030 enic_fm_action_free(struct enic_flowman *fm, struct enic_fm_action *ah)
2031 {
2032         uint64_t args[2];
2033         int ret = 0;
2034
2035         ENICPMD_FUNC_TRACE();
2036         RTE_ASSERT(ah->ref > 0);
2037         ah->ref--;
2038         if (ah->ref == 0) {
2039                 args[0] = FM_ACTION_FREE;
2040                 args[1] = ah->handle;
2041                 ret = flowman_cmd(fm, args, 2);
2042                 if (ret)
2043                         /* This is a "should never happen" error. */
2044                         ENICPMD_LOG(ERR, "freeing action rc=%d handle=0x%"
2045                                     PRIx64, ret, ah->handle);
2046                 rte_hash_del_key(fm->action_hash, (const void *)&ah->key);
2047                 free(ah);
2048         }
2049         return ret;
2050 }
2051
2052 static int
2053 enic_fm_entry_free(struct enic_flowman *fm, uint64_t handle)
2054 {
2055         uint64_t args[2];
2056         int rc;
2057
2058         ENICPMD_FUNC_TRACE();
2059         args[0] = FM_MATCH_ENTRY_REMOVE;
2060         args[1] = handle;
2061         rc = flowman_cmd(fm, args, 2);
2062         if (rc)
2063                 ENICPMD_LOG(ERR, "cannot free match entry: rc=%d"
2064                             " handle=0x%" PRIx64, rc, handle);
2065         return rc;
2066 }
2067
2068 static struct enic_fm_jump_flow *
2069 find_jump_flow(struct enic_flowman *fm, uint32_t group)
2070 {
2071         struct enic_fm_jump_flow *j;
2072
2073         ENICPMD_FUNC_TRACE();
2074         TAILQ_FOREACH(j, &fm->jump_list, list) {
2075                 if (j->group == group)
2076                         return j;
2077         }
2078         return NULL;
2079 }
2080
2081 static void
2082 remove_jump_flow(struct enic_flowman *fm, struct rte_flow *flow)
2083 {
2084         struct enic_fm_jump_flow *j;
2085
2086         ENICPMD_FUNC_TRACE();
2087         TAILQ_FOREACH(j, &fm->jump_list, list) {
2088                 if (j->flow == flow) {
2089                         TAILQ_REMOVE(&fm->jump_list, j, list);
2090                         free(j);
2091                         return;
2092                 }
2093         }
2094 }
2095
2096 static int
2097 save_jump_flow(struct enic_flowman *fm,
2098                struct rte_flow *flow,
2099                uint32_t group,
2100                struct fm_tcam_match_entry *match,
2101                struct fm_action *action)
2102 {
2103         struct enic_fm_jump_flow *j;
2104
2105         ENICPMD_FUNC_TRACE();
2106         j = calloc(1, sizeof(struct enic_fm_jump_flow));
2107         if (j == NULL)
2108                 return -ENOMEM;
2109         j->flow = flow;
2110         j->group = group;
2111         j->match = *match;
2112         j->action = *action;
2113         TAILQ_INSERT_HEAD(&fm->jump_list, j, list);
2114         ENICPMD_LOG(DEBUG, "saved jump flow: flow=%p group=%u", flow, group);
2115         return 0;
2116 }
2117
2118 static void
2119 __enic_fm_flow_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
2120 {
2121         if (fm_flow->entry_handle != FM_INVALID_HANDLE) {
2122                 enic_fm_entry_free(fm, fm_flow->entry_handle);
2123                 fm_flow->entry_handle = FM_INVALID_HANDLE;
2124         }
2125         if (fm_flow->action != NULL) {
2126                 enic_fm_action_free(fm, fm_flow->action);
2127                 fm_flow->action = NULL;
2128         }
2129         enic_fm_counter_free(fm, fm_flow);
2130         if (fm_flow->fet) {
2131                 enic_fet_put(fm, fm_flow->fet);
2132                 fm_flow->fet = NULL;
2133         }
2134 }
2135
2136 static void
2137 enic_fm_flow_free(struct enic_flowman *fm, struct rte_flow *flow)
2138 {
2139         struct enic_fm_flow *steer = flow->fm->hairpin_steer_flow;
2140
2141         if (flow->fm->fet && flow->fm->fet->default_key)
2142                 remove_jump_flow(fm, flow);
2143         __enic_fm_flow_free(fm, flow->fm);
2144         if (steer) {
2145                 __enic_fm_flow_free(fm, steer);
2146                 free(steer);
2147         }
2148         free(flow->fm);
2149         free(flow);
2150 }
2151
2152 static int
2153 enic_fm_add_tcam_entry(struct enic_flowman *fm,
2154                        struct fm_tcam_match_entry *match_in,
2155                        uint64_t *entry_handle,
2156                        uint8_t ingress,
2157                        struct rte_flow_error *error)
2158 {
2159         struct fm_tcam_match_entry *ftm;
2160         uint64_t args[3];
2161         int ret;
2162
2163         ENICPMD_FUNC_TRACE();
2164         /* Copy entry to the command buffer */
2165         ftm = &fm->cmd.va->fm_tcam_match_entry;
2166         memcpy(ftm, match_in, sizeof(*ftm));
2167         /* Add TCAM entry */
2168         args[0] = FM_TCAM_ENTRY_INSTALL;
2169         args[1] = ingress ? fm->ig_tcam_hndl : fm->eg_tcam_hndl;
2170         args[2] = fm->cmd.pa;
2171         ret = flowman_cmd(fm, args, 3);
2172         if (ret != 0) {
2173                 ENICPMD_LOG(ERR, "cannot add %s TCAM entry: rc=%d",
2174                             ingress ? "ingress" : "egress", ret);
2175                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2176                         NULL, "enic: devcmd(tcam-entry-install)");
2177                 return ret;
2178         }
2179         ENICPMD_LOG(DEBUG, "installed %s TCAM entry: handle=0x%" PRIx64,
2180                     ingress ? "ingress" : "egress", (uint64_t)args[0]);
2181         *entry_handle = args[0];
2182         return 0;
2183 }
2184
2185 static int
2186 enic_fm_add_exact_entry(struct enic_flowman *fm,
2187                         struct fm_tcam_match_entry *match_in,
2188                         uint64_t *entry_handle,
2189                         struct enic_fm_fet *fet,
2190                         struct rte_flow_error *error)
2191 {
2192         struct fm_exact_match_entry *fem;
2193         uint64_t args[3];
2194         int ret;
2195
2196         ENICPMD_FUNC_TRACE();
2197         /* The new entry must have the table's key */
2198         if (memcmp(fet->key.fk_hdrset, match_in->ftm_mask.fk_hdrset,
2199                    sizeof(struct fm_header_set) * FM_HDRSET_MAX)) {
2200                 return rte_flow_error_set(error, EINVAL,
2201                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2202                         "enic: key does not match group's key");
2203         }
2204
2205         /* Copy entry to the command buffer */
2206         fem = &fm->cmd.va->fm_exact_match_entry;
2207         /*
2208          * Translate TCAM entry to exact entry. As is only need to drop
2209          * position and mask. The mask is part of the exact match table.
2210          * Position (aka priority) is not supported in the exact match table.
2211          */
2212         fem->fem_data = match_in->ftm_data;
2213         fem->fem_flags = match_in->ftm_flags;
2214         fem->fem_action = match_in->ftm_action;
2215         fem->fem_counter = match_in->ftm_counter;
2216
2217         /* Add exact entry */
2218         args[0] = FM_EXACT_ENTRY_INSTALL;
2219         args[1] = fet->handle;
2220         args[2] = fm->cmd.pa;
2221         ret = flowman_cmd(fm, args, 3);
2222         if (ret != 0) {
2223                 ENICPMD_LOG(ERR, "cannot add %s exact entry: group=%u",
2224                             fet->ingress ? "ingress" : "egress", fet->group);
2225                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2226                         NULL, "enic: devcmd(exact-entry-install)");
2227                 return ret;
2228         }
2229         ENICPMD_LOG(DEBUG, "installed %s exact entry: group=%u"
2230                     " handle=0x%" PRIx64,
2231                     fet->ingress ? "ingress" : "egress", fet->group,
2232                     (uint64_t)args[0]);
2233         *entry_handle = args[0];
2234         return 0;
2235 }
2236
2237 static int
2238 enic_action_handle_get(struct enic_flowman *fm, struct fm_action *action_in,
2239                        struct rte_flow_error *error,
2240                        struct enic_fm_action **ah_o)
2241 {
2242         struct enic_fm_action *ah;
2243         struct fm_action *fma;
2244         uint64_t args[2];
2245         int ret = 0;
2246
2247         ret = rte_hash_lookup_data(fm->action_hash, action_in,
2248                                    (void **)&ah);
2249         if (ret < 0 && ret != -ENOENT)
2250                 return rte_flow_error_set(error, -ret,
2251                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2252                                    NULL, "enic: rte_hash_lookup(aciton)");
2253
2254         if (ret == -ENOENT) {
2255                 /* Allocate a new action on the NIC. */
2256                 fma = &fm->cmd.va->fm_action;
2257                 memcpy(fma, action_in, sizeof(*fma));
2258
2259                 ah = calloc(1, sizeof(*ah));
2260                 memcpy(&ah->key, action_in, sizeof(struct fm_action));
2261                 if (ah == NULL)
2262                         return rte_flow_error_set(error, ENOMEM,
2263                                            RTE_FLOW_ERROR_TYPE_HANDLE,
2264                                            NULL, "enic: calloc(fm-action)");
2265                 args[0] = FM_ACTION_ALLOC;
2266                 args[1] = fm->cmd.pa;
2267                 ret = flowman_cmd(fm, args, 2);
2268                 if (ret != 0) {
2269                         rte_flow_error_set(error, -ret,
2270                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2271                                            NULL, "enic: devcmd(action-alloc)");
2272                         goto error_with_ah;
2273                 }
2274                 ah->handle = args[0];
2275                 ret = rte_hash_add_key_data(fm->action_hash,
2276                                             (const void *)action_in,
2277                                             (void *)ah);
2278                 if (ret != 0) {
2279                         rte_flow_error_set(error, -ret,
2280                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2281                                            NULL,
2282                                            "enic: rte_hash_add_key_data(actn)");
2283                         goto error_with_action_handle;
2284                 }
2285                 ENICPMD_LOG(DEBUG, "action allocated: handle=0x%" PRIx64,
2286                             ah->handle);
2287         }
2288
2289         /* Action handle struct is valid, increment reference count. */
2290         ah->ref++;
2291         *ah_o = ah;
2292         return 0;
2293 error_with_action_handle:
2294         args[0] = FM_ACTION_FREE;
2295         args[1] = ah->handle;
2296         ret = flowman_cmd(fm, args, 2);
2297         if (ret != 0)
2298                 rte_flow_error_set(error, -ret,
2299                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2300                                    NULL, "enic: devcmd(action-free)");
2301 error_with_ah:
2302         free(ah);
2303         return ret;
2304 }
2305
2306 /* Push match-action to the NIC. */
2307 static int
2308 __enic_fm_flow_add_entry(struct enic_flowman *fm,
2309                          struct enic_fm_flow *fm_flow,
2310                          struct fm_tcam_match_entry *match_in,
2311                          struct fm_action *action_in,
2312                          uint32_t group,
2313                          uint8_t ingress,
2314                          struct rte_flow_error *error)
2315 {
2316         struct enic_fm_counter *ctr;
2317         struct enic_fm_action *ah = NULL;
2318         uint64_t entry_h;
2319         int ret;
2320
2321         ENICPMD_FUNC_TRACE();
2322
2323         /* Get or create an aciton handle. */
2324         ret = enic_action_handle_get(fm, action_in, error, &ah);
2325         if (ret)
2326                 return ret;
2327         match_in->ftm_action = ah->handle;
2328         fm_flow->action = ah;
2329
2330         /* Allocate counter if requested. */
2331         if (match_in->ftm_flags & FMEF_COUNTER) {
2332                 ret = enic_fm_counter_alloc(fm, error, &ctr);
2333                 if (ret) /* error has been filled in */
2334                         return ret;
2335                 fm_flow->counter_valid = true;
2336                 fm_flow->counter = ctr;
2337                 match_in->ftm_counter = ctr->handle;
2338         }
2339
2340         /*
2341          * Get the group's table (either TCAM or exact match table) and
2342          * add entry to it. If we use the exact match table, the handler
2343          * will translate the TCAM entry (match_in) to the appropriate
2344          * exact match entry and use that instead.
2345          */
2346         entry_h = FM_INVALID_HANDLE;
2347         if (group == FM_TCAM_RTE_GROUP) {
2348                 ret = enic_fm_add_tcam_entry(fm, match_in, &entry_h, ingress,
2349                                              error);
2350                 if (ret)
2351                         return ret;
2352                 /* Jump action might have a ref to fet */
2353                 fm_flow->fet = fm->fet;
2354                 fm->fet = NULL;
2355         } else {
2356                 struct enic_fm_fet *fet = NULL;
2357
2358                 ret = enic_fet_get(fm, group, ingress,
2359                                    &match_in->ftm_mask, &fet, error);
2360                 if (ret)
2361                         return ret;
2362                 fm_flow->fet = fet;
2363                 ret = enic_fm_add_exact_entry(fm, match_in, &entry_h, fet,
2364                                               error);
2365                 if (ret)
2366                         return ret;
2367         }
2368         /* Clear counter after adding entry, as it requires in-use counter */
2369         if (fm_flow->counter_valid) {
2370                 ret = enic_fm_counter_zero(fm, fm_flow->counter);
2371                 if (ret)
2372                         return ret;
2373         }
2374         fm_flow->entry_handle = entry_h;
2375         return 0;
2376 }
2377
2378 /* Push match-action to the NIC. */
2379 static struct rte_flow *
2380 enic_fm_flow_add_entry(struct enic_flowman *fm,
2381                        struct fm_tcam_match_entry *match_in,
2382                        struct fm_action *action_in,
2383                        const struct rte_flow_attr *attrs,
2384                        struct rte_flow_error *error)
2385 {
2386         struct enic_fm_flow *fm_flow;
2387         struct rte_flow *flow;
2388
2389         ENICPMD_FUNC_TRACE();
2390         match_in->ftm_position = attrs->priority;
2391         enic_fm_dump_tcam_entry(match_in, action_in, attrs->ingress);
2392         flow = calloc(1, sizeof(*flow));
2393         fm_flow = calloc(1, sizeof(*fm_flow));
2394         if (flow == NULL || fm_flow == NULL) {
2395                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
2396                         NULL, "enic: cannot allocate rte_flow");
2397                 free(flow);
2398                 free(fm_flow);
2399                 return NULL;
2400         }
2401         flow->fm = fm_flow;
2402         fm_flow->action = NULL;
2403         fm_flow->entry_handle = FM_INVALID_HANDLE;
2404         if (__enic_fm_flow_add_entry(fm, fm_flow, match_in, action_in,
2405                                      attrs->group, attrs->ingress, error)) {
2406                 enic_fm_flow_free(fm, flow);
2407                 return NULL;
2408         }
2409         return flow;
2410 }
2411
2412 static void
2413 convert_jump_flows(struct enic_flowman *fm, struct enic_fm_fet *fet,
2414                    struct rte_flow_error *error)
2415 {
2416         struct enic_fm_flow *fm_flow;
2417         struct enic_fm_jump_flow *j;
2418         struct fm_action *fma;
2419         uint32_t group;
2420
2421         ENICPMD_FUNC_TRACE();
2422         /*
2423          * Find the saved flows that should jump to the new table (fet).
2424          * Then delete the old TCAM entry that jumps to the default table,
2425          * and add a new one that jumps to the new table.
2426          */
2427         group = fet->group;
2428         j = find_jump_flow(fm, group);
2429         while (j) {
2430                 ENICPMD_LOG(DEBUG, "convert jump flow: flow=%p group=%u",
2431                             j->flow, group);
2432                 /* Delete old entry */
2433                 fm_flow = j->flow->fm;
2434                 __enic_fm_flow_free(fm, fm_flow);
2435
2436                 /* Add new entry */
2437                 fma = &j->action;
2438                 fma->fma_action_ops[0].exact.handle = fet->handle;
2439                 if (__enic_fm_flow_add_entry(fm, fm_flow, &j->match, fma,
2440                         FM_TCAM_RTE_GROUP, fet->ingress, error)) {
2441                         /* Cannot roll back changes at the moment */
2442                         ENICPMD_LOG(ERR, "cannot convert jump flow: flow=%p",
2443                                     j->flow);
2444                 } else {
2445                         fm_flow->fet = fet;
2446                         fet->ref++;
2447                         ENICPMD_LOG(DEBUG, "convert ok: group=%u ref=%u",
2448                                     fet->group, fet->ref);
2449                 }
2450
2451                 TAILQ_REMOVE(&fm->jump_list, j, list);
2452                 free(j);
2453                 j = find_jump_flow(fm, group);
2454         }
2455 }
2456
2457 static int
2458 add_hairpin_steer(struct enic_flowman *fm, struct rte_flow *flow,
2459                   struct rte_flow_error *error)
2460 {
2461         struct fm_tcam_match_entry *fm_tcam_entry;
2462         struct enic_fm_flow *fm_flow;
2463         struct fm_action *fm_action;
2464         struct fm_action_op fm_op;
2465         int ret;
2466
2467         ENICPMD_FUNC_TRACE();
2468         fm_flow = calloc(1, sizeof(*fm_flow));
2469         if (fm_flow == NULL) {
2470                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
2471                         NULL, "enic: cannot allocate rte_flow");
2472                 return -ENOMEM;
2473         }
2474         /* Original egress hairpin flow */
2475         fm_tcam_entry = &fm->tcam_entry;
2476         fm_action = &fm->action;
2477         /* Use the match pattern of the egress flow as is, without counters */
2478         fm_tcam_entry->ftm_flags &= ~FMEF_COUNTER;
2479         /* The only action is steer to vnic */
2480         fm->action_op_count = 0;
2481         memset(fm_action, 0, sizeof(*fm_action));
2482         memset(&fm_op, 0, sizeof(fm_op));
2483         /* Always to queue 0 for now */
2484         fm_op.fa_op = FMOP_RQ_STEER;
2485         fm_op.rq_steer.rq_index = 0;
2486         fm_op.rq_steer.vnic_handle = fm->hairpin_steer_vnic_h;
2487         ret = enic_fm_append_action_op(fm, &fm_op, error);
2488         if (ret)
2489                 goto error_with_flow;
2490         ENICPMD_LOG(DEBUG, "add steer op");
2491         /* Add required END */
2492         memset(&fm_op, 0, sizeof(fm_op));
2493         fm_op.fa_op = FMOP_END;
2494         ret = enic_fm_append_action_op(fm, &fm_op, error);
2495         if (ret)
2496                 goto error_with_flow;
2497         /* Add the ingress flow */
2498         fm_flow->action = NULL;
2499         fm_flow->entry_handle = FM_INVALID_HANDLE;
2500         ret = __enic_fm_flow_add_entry(fm, fm_flow, fm_tcam_entry, fm_action,
2501                                        FM_TCAM_RTE_GROUP, 1 /* ingress */, error);
2502         if (ret) {
2503                 ENICPMD_LOG(ERR, "cannot add hairpin-steer flow");
2504                 goto error_with_flow;
2505         }
2506         /* The new flow is now the egress flow's paired flow */
2507         flow->fm->hairpin_steer_flow = fm_flow;
2508         return 0;
2509
2510 error_with_flow:
2511         free(fm_flow);
2512         return ret;
2513 }
2514
2515 static void
2516 enic_fm_open_scratch(struct enic_flowman *fm)
2517 {
2518         fm->action_op_count = 0;
2519         fm->fet = NULL;
2520         fm->need_hairpin_steer = 0;
2521         fm->hairpin_steer_vnic_h = 0;
2522         memset(&fm->tcam_entry, 0, sizeof(fm->tcam_entry));
2523         memset(&fm->action, 0, sizeof(fm->action));
2524 }
2525
2526 static void
2527 enic_fm_close_scratch(struct enic_flowman *fm)
2528 {
2529         if (fm->fet) {
2530                 enic_fet_put(fm, fm->fet);
2531                 fm->fet = NULL;
2532         }
2533         fm->action_op_count = 0;
2534 }
2535
2536 static int
2537 enic_fm_flow_validate(struct rte_eth_dev *dev,
2538                       const struct rte_flow_attr *attrs,
2539                       const struct rte_flow_item pattern[],
2540                       const struct rte_flow_action actions[],
2541                       struct rte_flow_error *error)
2542 {
2543         struct fm_tcam_match_entry *fm_tcam_entry;
2544         struct fm_action *fm_action;
2545         struct enic_flowman *fm;
2546         int ret;
2547
2548         ENICPMD_FUNC_TRACE();
2549         fm = begin_fm(pmd_priv(dev));
2550         if (fm == NULL)
2551                 return -ENOTSUP;
2552         enic_fm_open_scratch(fm);
2553         ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2554         if (!ret) {
2555                 fm_tcam_entry = &fm->tcam_entry;
2556                 fm_action = &fm->action;
2557                 enic_fm_dump_tcam_entry(fm_tcam_entry, fm_action,
2558                                         attrs->ingress);
2559         }
2560         enic_fm_close_scratch(fm);
2561         end_fm(fm);
2562         return ret;
2563 }
2564
2565 static int
2566 enic_fm_flow_query_count(struct rte_eth_dev *dev,
2567                          struct rte_flow *flow, void *data,
2568                          struct rte_flow_error *error)
2569 {
2570         struct rte_flow_query_count *query;
2571         struct enic_fm_flow *fm_flow;
2572         struct enic_flowman *fm;
2573         uint64_t args[3];
2574         int rc;
2575
2576         ENICPMD_FUNC_TRACE();
2577         fm = begin_fm(pmd_priv(dev));
2578         query = data;
2579         fm_flow = flow->fm;
2580         if (!fm_flow->counter_valid) {
2581                 rc = rte_flow_error_set(error, ENOTSUP,
2582                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2583                         "enic: flow does not have counter");
2584                 goto exit;
2585         }
2586
2587         args[0] = FM_COUNTER_QUERY;
2588         args[1] = fm_flow->counter->handle;
2589         args[2] = query->reset;
2590         rc = flowman_cmd(fm, args, 3);
2591         if (rc) {
2592                 ENICPMD_LOG(ERR, "cannot query counter: rc=%d handle=0x%x",
2593                             rc, fm_flow->counter->handle);
2594                 goto exit;
2595         }
2596         query->hits_set = 1;
2597         query->hits = args[0];
2598         query->bytes_set = 1;
2599         query->bytes = args[1];
2600         rc = 0;
2601 exit:
2602         end_fm(fm);
2603         return rc;
2604 }
2605
2606 static int
2607 enic_fm_flow_query(struct rte_eth_dev *dev,
2608                    struct rte_flow *flow,
2609                    const struct rte_flow_action *actions,
2610                    void *data,
2611                    struct rte_flow_error *error)
2612 {
2613         int ret = 0;
2614
2615         ENICPMD_FUNC_TRACE();
2616         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2617                 switch (actions->type) {
2618                 case RTE_FLOW_ACTION_TYPE_VOID:
2619                         break;
2620                 case RTE_FLOW_ACTION_TYPE_COUNT:
2621                         ret = enic_fm_flow_query_count(dev, flow, data, error);
2622                         break;
2623                 default:
2624                         return rte_flow_error_set(error, ENOTSUP,
2625                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2626                                                   actions,
2627                                                   "action not supported");
2628                 }
2629                 if (ret < 0)
2630                         return ret;
2631         }
2632         return 0;
2633 }
2634
2635 static struct rte_flow *
2636 enic_fm_flow_create(struct rte_eth_dev *dev,
2637                     const struct rte_flow_attr *attrs,
2638                     const struct rte_flow_item pattern[],
2639                     const struct rte_flow_action actions[],
2640                     struct rte_flow_error *error)
2641 {
2642         struct fm_tcam_match_entry *fm_tcam_entry;
2643         struct fm_action *fm_action;
2644         struct enic_flowman *fm;
2645         struct enic_fm_fet *fet;
2646         struct rte_flow *flow;
2647         struct enic *enic;
2648         int ret;
2649
2650         ENICPMD_FUNC_TRACE();
2651         enic = pmd_priv(dev);
2652         fm = begin_fm(enic);
2653         if (fm == NULL) {
2654                 rte_flow_error_set(error, ENOTSUP,
2655                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2656                         "flowman is not initialized");
2657                 return NULL;
2658         }
2659         enic_fm_open_scratch(fm);
2660         flow = NULL;
2661         ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2662         if (ret < 0)
2663                 goto error_with_scratch;
2664         fm_tcam_entry = &fm->tcam_entry;
2665         fm_action = &fm->action;
2666         flow = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2667                                       attrs, error);
2668         if (flow) {
2669                 /* Add ingress rule that pairs with hairpin rule */
2670                 if (fm->need_hairpin_steer) {
2671                         ret = add_hairpin_steer(fm, flow, error);
2672                         if (ret) {
2673                                 enic_fm_flow_free(fm, flow);
2674                                 flow = NULL;
2675                                 goto error_with_scratch;
2676                         }
2677                 }
2678                 LIST_INSERT_HEAD(&enic->flows, flow, next);
2679                 fet = flow->fm->fet;
2680                 if (fet && fet->default_key) {
2681                         /*
2682                          * Jump to non-existent group? Save the relevant info
2683                          * so we can convert this flow when that group
2684                          * materializes.
2685                          */
2686                         save_jump_flow(fm, flow, fet->group,
2687                                        fm_tcam_entry, fm_action);
2688                 } else if (fet && fet->ref == 1) {
2689                         /*
2690                          * A new table is created. Convert the saved flows
2691                          * that should jump to this group.
2692                          */
2693                         convert_jump_flows(fm, fet, error);
2694                 }
2695         }
2696
2697 error_with_scratch:
2698         enic_fm_close_scratch(fm);
2699         end_fm(fm);
2700         return flow;
2701 }
2702
2703 static int
2704 enic_fm_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
2705                      __rte_unused struct rte_flow_error *error)
2706 {
2707         struct enic *enic = pmd_priv(dev);
2708         struct enic_flowman *fm;
2709
2710         ENICPMD_FUNC_TRACE();
2711         fm = begin_fm(enic);
2712         if (fm == NULL)
2713                 return 0;
2714         LIST_REMOVE(flow, next);
2715         enic_fm_flow_free(fm, flow);
2716         end_fm(fm);
2717         return 0;
2718 }
2719
2720 static int
2721 enic_fm_flow_flush(struct rte_eth_dev *dev,
2722                    __rte_unused struct rte_flow_error *error)
2723 {
2724         LIST_HEAD(enic_flows, rte_flow) internal;
2725         struct enic_fm_flow *fm_flow;
2726         struct enic_flowman *fm;
2727         struct rte_flow *flow;
2728         struct enic *enic = pmd_priv(dev);
2729
2730         ENICPMD_FUNC_TRACE();
2731
2732         fm = begin_fm(enic);
2733         if (fm == NULL)
2734                 return 0;
2735         /* Destroy all non-internal flows */
2736         LIST_INIT(&internal);
2737         while (!LIST_EMPTY(&enic->flows)) {
2738                 flow = LIST_FIRST(&enic->flows);
2739                 fm_flow = flow->fm;
2740                 LIST_REMOVE(flow, next);
2741                 if (flow->internal) {
2742                         LIST_INSERT_HEAD(&internal, flow, next);
2743                         continue;
2744                 }
2745                 /*
2746                  * If tables are null, then vNIC is closing, and the firmware
2747                  * has already cleaned up flowman state. So do not try to free
2748                  * resources, as it only causes errors.
2749                  */
2750                 if (fm->ig_tcam_hndl == FM_INVALID_HANDLE) {
2751                         fm_flow->entry_handle = FM_INVALID_HANDLE;
2752                         fm_flow->action = NULL;
2753                         fm_flow->fet = NULL;
2754                 }
2755                 enic_fm_flow_free(fm, flow);
2756         }
2757         while (!LIST_EMPTY(&internal)) {
2758                 flow = LIST_FIRST(&internal);
2759                 LIST_REMOVE(flow, next);
2760                 LIST_INSERT_HEAD(&enic->flows, flow, next);
2761         }
2762         end_fm(fm);
2763         return 0;
2764 }
2765
2766 static int
2767 enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle)
2768 {
2769         uint64_t args[2];
2770         int rc;
2771
2772         args[0] = FM_MATCH_TABLE_FREE;
2773         args[1] = handle;
2774         rc = flowman_cmd(fm, args, 2);
2775         if (rc)
2776                 ENICPMD_LOG(ERR, "cannot free table: rc=%d handle=0x%" PRIx64,
2777                             rc, handle);
2778         return rc;
2779 }
2780
2781 static int
2782 enic_fm_tcam_tbl_alloc(struct enic_flowman *fm, uint32_t direction,
2783                         uint32_t max_entries, uint64_t *handle)
2784 {
2785         struct fm_tcam_match_table *tcam_tbl;
2786         uint64_t args[2];
2787         int rc;
2788
2789         ENICPMD_FUNC_TRACE();
2790         tcam_tbl = &fm->cmd.va->fm_tcam_match_table;
2791         tcam_tbl->ftt_direction = direction;
2792         tcam_tbl->ftt_stage = FM_STAGE_LAST;
2793         tcam_tbl->ftt_max_entries = max_entries;
2794         args[0] = FM_TCAM_TABLE_ALLOC;
2795         args[1] = fm->cmd.pa;
2796         rc = flowman_cmd(fm, args, 2);
2797         if (rc) {
2798                 ENICPMD_LOG(ERR, "cannot alloc %s TCAM table: rc=%d",
2799                             (direction == FM_INGRESS) ? "IG" : "EG", rc);
2800                 return rc;
2801         }
2802         *handle = args[0];
2803         ENICPMD_LOG(DEBUG, "%s TCAM table allocated, handle=0x%" PRIx64,
2804                     (direction == FM_INGRESS) ? "IG" : "EG", *handle);
2805         return 0;
2806 }
2807
2808 static int
2809 enic_fm_init_actions(struct enic_flowman *fm)
2810 {
2811         struct rte_hash *a_hash;
2812         char name[RTE_HASH_NAMESIZE];
2813         struct rte_hash_parameters params = {
2814                 .entries = FM_MAX_ACTION_TABLE_SIZE,
2815                 .key_len = sizeof(struct fm_action),
2816                 .hash_func = rte_jhash,
2817                 .hash_func_init_val = 0,
2818                 .socket_id = rte_socket_id(),
2819         };
2820
2821         ENICPMD_FUNC_TRACE();
2822         snprintf((char *)name, sizeof(name), "fm-ah-%s",
2823                  fm->owner_enic->bdf_name);
2824         params.name = name;
2825
2826         a_hash = rte_hash_create(&params);
2827         if (a_hash == NULL)
2828                 return -rte_errno;
2829         fm->action_hash = a_hash;
2830         return 0;
2831 }
2832
2833 static int
2834 enic_fm_init_counters(struct enic_flowman *fm)
2835 {
2836         ENICPMD_FUNC_TRACE();
2837         SLIST_INIT(&fm->counters);
2838         return enic_fm_more_counters(fm);
2839 }
2840
2841 static void
2842 enic_fm_free_all_counters(struct enic_flowman *fm)
2843 {
2844         uint64_t args[2];
2845         int rc;
2846
2847         args[0] = FM_COUNTER_BRK;
2848         args[1] = 0;
2849         rc = flowman_cmd(fm, args, 2);
2850         if (rc != 0)
2851                 ENICPMD_LOG(ERR, "cannot free counters: rc=%d", rc);
2852         rte_free(fm->counter_stack);
2853 }
2854
2855 static int
2856 enic_fm_alloc_tcam_tables(struct enic_flowman *fm)
2857 {
2858         int rc;
2859
2860         ENICPMD_FUNC_TRACE();
2861         rc = enic_fm_tcam_tbl_alloc(fm, FM_INGRESS, FM_MAX_TCAM_TABLE_SIZE,
2862                                     &fm->ig_tcam_hndl);
2863         if (rc)
2864                 return rc;
2865         rc = enic_fm_tcam_tbl_alloc(fm, FM_EGRESS, FM_MAX_TCAM_TABLE_SIZE,
2866                                     &fm->eg_tcam_hndl);
2867         return rc;
2868 }
2869
2870 static void
2871 enic_fm_free_tcam_tables(struct enic_flowman *fm)
2872 {
2873         ENICPMD_FUNC_TRACE();
2874         if (fm->ig_tcam_hndl) {
2875                 ENICPMD_LOG(DEBUG, "free IG TCAM table handle=0x%" PRIx64,
2876                             fm->ig_tcam_hndl);
2877                 enic_fm_tbl_free(fm, fm->ig_tcam_hndl);
2878                 fm->ig_tcam_hndl = FM_INVALID_HANDLE;
2879         }
2880         if (fm->eg_tcam_hndl) {
2881                 ENICPMD_LOG(DEBUG, "free EG TCAM table handle=0x%" PRIx64,
2882                             fm->eg_tcam_hndl);
2883                 enic_fm_tbl_free(fm, fm->eg_tcam_hndl);
2884                 fm->eg_tcam_hndl = FM_INVALID_HANDLE;
2885         }
2886 }
2887
2888 int
2889 enic_fm_init(struct enic *enic)
2890 {
2891         const struct rte_pci_addr *addr;
2892         struct enic_flowman *fm;
2893         uint8_t name[RTE_MEMZONE_NAMESIZE];
2894         int rc;
2895
2896         if (enic->flow_filter_mode != FILTER_FLOWMAN)
2897                 return 0;
2898         ENICPMD_FUNC_TRACE();
2899         /* Get vnic handle and save for port-id action */
2900         if (enic_is_vf_rep(enic))
2901                 addr = &VF_ENIC_TO_VF_REP(enic)->bdf;
2902         else
2903                 addr = &RTE_ETH_DEV_TO_PCI(enic->rte_dev)->addr;
2904         rc = enic_fm_find_vnic(enic, addr, &enic->fm_vnic_handle);
2905         if (rc) {
2906                 ENICPMD_LOG(ERR, "cannot find vnic handle for %x:%x:%x",
2907                             addr->bus, addr->devid, addr->function);
2908                 return rc;
2909         }
2910         /* Save UIF for egport action */
2911         enic->fm_vnic_uif = vnic_dev_uif(enic->vdev);
2912         ENICPMD_LOG(DEBUG, "uif %u", enic->fm_vnic_uif);
2913         /* Nothing else to do for representor. It will share the PF flowman */
2914         if (enic_is_vf_rep(enic))
2915                 return 0;
2916         fm = calloc(1, sizeof(*fm));
2917         if (fm == NULL) {
2918                 ENICPMD_LOG(ERR, "cannot alloc flowman struct");
2919                 return -ENOMEM;
2920         }
2921         fm->owner_enic = enic;
2922         rte_spinlock_init(&fm->lock);
2923         TAILQ_INIT(&fm->fet_list);
2924         TAILQ_INIT(&fm->jump_list);
2925         /* Allocate host memory for flowman commands */
2926         snprintf((char *)name, sizeof(name), "fm-cmd-%s", enic->bdf_name);
2927         fm->cmd.va = enic_alloc_consistent(enic,
2928                 sizeof(union enic_flowman_cmd_mem), &fm->cmd.pa, name);
2929         if (!fm->cmd.va) {
2930                 ENICPMD_LOG(ERR, "cannot allocate flowman command memory");
2931                 rc = -ENOMEM;
2932                 goto error_fm;
2933         }
2934         /* Allocate TCAM tables upfront as they are the main tables */
2935         rc = enic_fm_alloc_tcam_tables(fm);
2936         if (rc) {
2937                 ENICPMD_LOG(ERR, "cannot alloc TCAM tables");
2938                 goto error_cmd;
2939         }
2940         /* Then a number of counters */
2941         rc = enic_fm_init_counters(fm);
2942         if (rc) {
2943                 ENICPMD_LOG(ERR, "cannot alloc counters");
2944                 goto error_tables;
2945         }
2946         /* set up action handle hash */
2947         rc = enic_fm_init_actions(fm);
2948         if (rc) {
2949                 ENICPMD_LOG(ERR, "cannot create action hash, error:%d", rc);
2950                 goto error_counters;
2951         }
2952         /*
2953          * One default exact match table for each direction. We hold onto
2954          * it until close.
2955          */
2956         rc = enic_fet_alloc(fm, 1, NULL, 128, &fm->default_ig_fet);
2957         if (rc) {
2958                 ENICPMD_LOG(ERR, "cannot alloc default IG exact match table");
2959                 goto error_actions;
2960         }
2961         fm->default_ig_fet->ref = 1;
2962         rc = enic_fet_alloc(fm, 0, NULL, 128, &fm->default_eg_fet);
2963         if (rc) {
2964                 ENICPMD_LOG(ERR, "cannot alloc default EG exact match table");
2965                 goto error_ig_fet;
2966         }
2967         fm->default_eg_fet->ref = 1;
2968         fm->vf_rep_tag = FM_VF_REP_TAG;
2969         enic->fm = fm;
2970         return 0;
2971
2972 error_ig_fet:
2973         enic_fet_free(fm, fm->default_ig_fet);
2974 error_actions:
2975         rte_hash_free(fm->action_hash);
2976 error_counters:
2977         enic_fm_free_all_counters(fm);
2978 error_tables:
2979         enic_fm_free_tcam_tables(fm);
2980 error_cmd:
2981         enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
2982                 fm->cmd.va, fm->cmd.pa);
2983 error_fm:
2984         free(fm);
2985         return rc;
2986 }
2987
2988 void
2989 enic_fm_destroy(struct enic *enic)
2990 {
2991         struct enic_flowman *fm;
2992         struct enic_fm_fet *fet;
2993
2994         ENICPMD_FUNC_TRACE();
2995         if (enic_is_vf_rep(enic)) {
2996                 delete_rep_flows(enic);
2997                 return;
2998         }
2999         if (enic->fm == NULL)
3000                 return;
3001         fm = enic->fm;
3002         enic_fm_flow_flush(enic->rte_dev, NULL);
3003         enic_fet_free(fm, fm->default_eg_fet);
3004         enic_fet_free(fm, fm->default_ig_fet);
3005         /* Free all exact match tables still open */
3006         while (!TAILQ_EMPTY(&fm->fet_list)) {
3007                 fet = TAILQ_FIRST(&fm->fet_list);
3008                 enic_fet_free(fm, fet);
3009         }
3010         enic_fm_free_tcam_tables(fm);
3011         enic_fm_free_all_counters(fm);
3012         rte_hash_free(fm->action_hash);
3013         enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
3014                 fm->cmd.va, fm->cmd.pa);
3015         fm->cmd.va = NULL;
3016         free(fm);
3017         enic->fm = NULL;
3018 }
3019
3020 int
3021 enic_fm_allocate_switch_domain(struct enic *pf)
3022 {
3023         const struct rte_pci_addr *cur_a, *prev_a;
3024         struct rte_eth_dev *dev;
3025         struct enic *cur, *prev;
3026         uint16_t domain_id;
3027         uint64_t vnic_h;
3028         uint16_t pid;
3029         int ret;
3030
3031         ENICPMD_FUNC_TRACE();
3032         if (enic_is_vf_rep(pf))
3033                 return -EINVAL;
3034         cur = pf;
3035         cur_a = &RTE_ETH_DEV_TO_PCI(cur->rte_dev)->addr;
3036         /* Go through ports and find another PF that is on the same adapter */
3037         RTE_ETH_FOREACH_DEV(pid) {
3038                 dev = &rte_eth_devices[pid];
3039                 if (!dev_is_enic(dev))
3040                         continue;
3041                 if (dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)
3042                         continue;
3043                 if (dev == cur->rte_dev)
3044                         continue;
3045                 /* dev is another PF. Is it on the same adapter? */
3046                 prev = pmd_priv(dev);
3047                 prev_a = &RTE_ETH_DEV_TO_PCI(dev)->addr;
3048                 if (!enic_fm_find_vnic(cur, prev_a, &vnic_h)) {
3049                         ENICPMD_LOG(DEBUG, "Port %u (PF BDF %x:%x:%x) and port %u (PF BDF %x:%x:%x domain %u) are on the same VIC",
3050                                 cur->rte_dev->data->port_id,
3051                                 cur_a->bus, cur_a->devid, cur_a->function,
3052                                 dev->data->port_id,
3053                                 prev_a->bus, prev_a->devid, prev_a->function,
3054                                 prev->switch_domain_id);
3055                         cur->switch_domain_id = prev->switch_domain_id;
3056                         return 0;
3057                 }
3058         }
3059         ret = rte_eth_switch_domain_alloc(&domain_id);
3060         if (ret) {
3061                 ENICPMD_LOG(WARNING, "failed to allocate switch domain for device %d",
3062                             ret);
3063         }
3064         cur->switch_domain_id = domain_id;
3065         ENICPMD_LOG(DEBUG, "Port %u (PF BDF %x:%x:%x) is the 1st PF on the VIC. Allocated switch domain id %u",
3066                     cur->rte_dev->data->port_id,
3067                     cur_a->bus, cur_a->devid, cur_a->function,
3068                     domain_id);
3069         return ret;
3070 }
3071
3072 const struct rte_flow_ops enic_fm_flow_ops = {
3073         .validate = enic_fm_flow_validate,
3074         .create = enic_fm_flow_create,
3075         .destroy = enic_fm_flow_destroy,
3076         .flush = enic_fm_flow_flush,
3077         .query = enic_fm_flow_query,
3078 };
3079
3080 /* Add a high priority flow that loops representor packets to VF */
3081 int
3082 enic_fm_add_rep2vf_flow(struct enic_vf_representor *vf)
3083 {
3084         struct fm_tcam_match_entry *fm_tcam_entry;
3085         struct rte_flow *flow0, *flow1;
3086         struct fm_action *fm_action;
3087         struct rte_flow_error error;
3088         struct rte_flow_attr attrs;
3089         struct fm_action_op fm_op;
3090         struct enic_flowman *fm;
3091         struct enic *pf;
3092         uint8_t tag;
3093
3094         pf = vf->pf;
3095         fm = pf->fm;
3096         tag = fm->vf_rep_tag;
3097         enic_fm_open_scratch(fm);
3098         fm_tcam_entry = &fm->tcam_entry;
3099         fm_action = &fm->action;
3100         /* Egress rule: match WQ ID and tag+hairpin */
3101         fm_tcam_entry->ftm_data.fk_wq_id = vf->pf_wq_idx;
3102         fm_tcam_entry->ftm_mask.fk_wq_id = 0xffff;
3103         fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3104         memset(&fm_op, 0, sizeof(fm_op));
3105         fm_op.fa_op = FMOP_TAG;
3106         fm_op.tag.tag = tag;
3107         enic_fm_append_action_op(fm, &fm_op, &error);
3108         memset(&fm_op, 0, sizeof(fm_op));
3109         fm_op.fa_op = FMOP_EG_HAIRPIN;
3110         enic_fm_append_action_op(fm, &fm_op, &error);
3111         memset(&fm_op, 0, sizeof(fm_op));
3112         fm_op.fa_op = FMOP_END;
3113         enic_fm_append_action_op(fm, &fm_op, &error);
3114         attrs.group = 0;
3115         attrs.ingress = 0;
3116         attrs.egress = 1;
3117         attrs.priority = FM_HIGHEST_PRIORITY;
3118         flow0 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3119                                        &attrs, &error);
3120         enic_fm_close_scratch(fm);
3121         if (flow0 == NULL) {
3122                 ENICPMD_LOG(ERR, "Cannot create flow 0 for representor->VF");
3123                 return -EINVAL;
3124         }
3125         LIST_INSERT_HEAD(&pf->flows, flow0, next);
3126         /* Make this flow internal, so the user app cannot delete it */
3127         flow0->internal = 1;
3128         ENICPMD_LOG(DEBUG, "representor->VF %d flow created: wq %d -> tag %d hairpin",
3129                     vf->vf_id, vf->pf_wq_idx, tag);
3130
3131         /* Ingress: steer hairpinned to VF RQ 0 */
3132         enic_fm_open_scratch(fm);
3133         fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3134         fm_tcam_entry->ftm_data.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3135         fm_tcam_entry->ftm_mask.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3136         fm_tcam_entry->ftm_data.fk_packet_tag = tag;
3137         fm_tcam_entry->ftm_mask.fk_packet_tag = 0xff;
3138         memset(&fm_op, 0, sizeof(fm_op));
3139         fm_op.fa_op = FMOP_RQ_STEER;
3140         fm_op.rq_steer.rq_index = 0;
3141         fm_op.rq_steer.vnic_handle = vf->enic.fm_vnic_handle;
3142         enic_fm_append_action_op(fm, &fm_op, &error);
3143         memset(&fm_op, 0, sizeof(fm_op));
3144         fm_op.fa_op = FMOP_END;
3145         enic_fm_append_action_op(fm, &fm_op, &error);
3146         attrs.group = 0;
3147         attrs.ingress = 1;
3148         attrs.egress = 0;
3149         attrs.priority = FM_HIGHEST_PRIORITY;
3150         flow1 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3151                                        &attrs, &error);
3152         enic_fm_close_scratch(fm);
3153         if (flow1 == NULL) {
3154                 ENICPMD_LOG(ERR, "Cannot create flow 1 for representor->VF");
3155                 enic_fm_flow_destroy(pf->rte_dev, flow0, &error);
3156                 return -EINVAL;
3157         }
3158         LIST_INSERT_HEAD(&pf->flows, flow1, next);
3159         flow1->internal = 1;
3160         ENICPMD_LOG(DEBUG, "representor->VF %d flow created: tag %d hairpinned -> VF RQ %d",
3161                     vf->vf_id, tag, fm_op.rq_steer.rq_index);
3162         vf->rep2vf_flow[0] = flow0;
3163         vf->rep2vf_flow[1] = flow1;
3164         /* Done with this tag, use a different one next time */
3165         fm->vf_rep_tag++;
3166         return 0;
3167 }
3168
3169 /*
3170  * Add a low priority flow that matches all packets from VF and loops them
3171  * back to the representor.
3172  */
3173 int
3174 enic_fm_add_vf2rep_flow(struct enic_vf_representor *vf)
3175 {
3176         struct fm_tcam_match_entry *fm_tcam_entry;
3177         struct rte_flow *flow0, *flow1;
3178         struct fm_action *fm_action;
3179         struct rte_flow_error error;
3180         struct rte_flow_attr attrs;
3181         struct fm_action_op fm_op;
3182         struct enic_flowman *fm;
3183         struct enic *pf;
3184         uint8_t tag;
3185
3186         pf = vf->pf;
3187         fm = pf->fm;
3188         tag = fm->vf_rep_tag;
3189         enic_fm_open_scratch(fm);
3190         fm_tcam_entry = &fm->tcam_entry;
3191         fm_action = &fm->action;
3192         /* Egress rule: match-any and tag+hairpin */
3193         fm_tcam_entry->ftm_data.fk_wq_id = 0;
3194         fm_tcam_entry->ftm_mask.fk_wq_id = 0xffff;
3195         fm_tcam_entry->ftm_data.fk_wq_vnic = vf->enic.fm_vnic_handle;
3196         fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3197         memset(&fm_op, 0, sizeof(fm_op));
3198         fm_op.fa_op = FMOP_TAG;
3199         fm_op.tag.tag = tag;
3200         enic_fm_append_action_op(fm, &fm_op, &error);
3201         memset(&fm_op, 0, sizeof(fm_op));
3202         fm_op.fa_op = FMOP_EG_HAIRPIN;
3203         enic_fm_append_action_op(fm, &fm_op, &error);
3204         memset(&fm_op, 0, sizeof(fm_op));
3205         fm_op.fa_op = FMOP_END;
3206         enic_fm_append_action_op(fm, &fm_op, &error);
3207         attrs.group = 0;
3208         attrs.ingress = 0;
3209         attrs.egress = 1;
3210         attrs.priority = FM_LOWEST_PRIORITY;
3211         flow0 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3212                                        &attrs, &error);
3213         enic_fm_close_scratch(fm);
3214         if (flow0 == NULL) {
3215                 ENICPMD_LOG(ERR, "Cannot create flow 0 for VF->representor");
3216                 return -EINVAL;
3217         }
3218         LIST_INSERT_HEAD(&pf->flows, flow0, next);
3219         /* Make this flow internal, so the user app cannot delete it */
3220         flow0->internal = 1;
3221         ENICPMD_LOG(DEBUG, "VF %d->representor flow created: wq %d (low prio) -> tag %d hairpin",
3222                     vf->vf_id, fm_tcam_entry->ftm_data.fk_wq_id, tag);
3223
3224         /* Ingress: steer hairpinned to VF rep RQ */
3225         enic_fm_open_scratch(fm);
3226         fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3227         fm_tcam_entry->ftm_data.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3228         fm_tcam_entry->ftm_mask.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3229         fm_tcam_entry->ftm_data.fk_packet_tag = tag;
3230         fm_tcam_entry->ftm_mask.fk_packet_tag = 0xff;
3231         memset(&fm_op, 0, sizeof(fm_op));
3232         fm_op.fa_op = FMOP_RQ_STEER;
3233         fm_op.rq_steer.rq_index = vf->pf_rq_sop_idx;
3234         fm_op.rq_steer.vnic_handle = pf->fm_vnic_handle;
3235         enic_fm_append_action_op(fm, &fm_op, &error);
3236         memset(&fm_op, 0, sizeof(fm_op));
3237         fm_op.fa_op = FMOP_END;
3238         enic_fm_append_action_op(fm, &fm_op, &error);
3239         attrs.group = 0;
3240         attrs.ingress = 1;
3241         attrs.egress = 0;
3242         attrs.priority = FM_HIGHEST_PRIORITY;
3243         flow1 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3244                                        &attrs, &error);
3245         enic_fm_close_scratch(fm);
3246         if (flow1 == NULL) {
3247                 ENICPMD_LOG(ERR, "Cannot create flow 1 for VF->representor");
3248                 enic_fm_flow_destroy(pf->rte_dev, flow0, &error);
3249                 return -EINVAL;
3250         }
3251         LIST_INSERT_HEAD(&pf->flows, flow1, next);
3252         flow1->internal = 1;
3253         ENICPMD_LOG(DEBUG, "VF %d->representor flow created: tag %d hairpinned -> PF RQ %d",
3254                     vf->vf_id, tag, vf->pf_rq_sop_idx);
3255         vf->vf2rep_flow[0] = flow0;
3256         vf->vf2rep_flow[1] = flow1;
3257         /* Done with this tag, use a different one next time */
3258         fm->vf_rep_tag++;
3259         return 0;
3260 }
3261
3262 /* Destroy representor flows created by enic_fm_add_{rep2vf,vf2rep}_flow */
3263 static void
3264 delete_rep_flows(struct enic *enic)
3265 {
3266         struct enic_vf_representor *vf;
3267         struct rte_flow_error error;
3268         struct rte_eth_dev *dev;
3269         uint32_t i;
3270
3271         RTE_ASSERT(enic_is_vf_rep(enic));
3272         vf = VF_ENIC_TO_VF_REP(enic);
3273         dev = vf->pf->rte_dev;
3274         for (i = 0; i < ARRAY_SIZE(vf->vf2rep_flow); i++) {
3275                 if (vf->vf2rep_flow[i])
3276                         enic_fm_flow_destroy(dev, vf->vf2rep_flow[i], &error);
3277         }
3278         for (i = 0; i < ARRAY_SIZE(vf->rep2vf_flow); i++) {
3279                 if (vf->rep2vf_flow[i])
3280                         enic_fm_flow_destroy(dev, vf->rep2vf_flow[i], &error);
3281         }
3282 }
3283
3284 static struct enic_flowman *
3285 begin_fm(struct enic *enic)
3286 {
3287         struct enic_vf_representor *vf;
3288         struct enic_flowman *fm;
3289
3290         /* Representor uses PF flowman */
3291         if (enic_is_vf_rep(enic)) {
3292                 vf = VF_ENIC_TO_VF_REP(enic);
3293                 fm = vf->pf->fm;
3294         } else {
3295                 fm = enic->fm;
3296         }
3297         /* Save the API caller and lock if representors exist */
3298         if (fm) {
3299                 if (fm->owner_enic->switchdev_mode)
3300                         rte_spinlock_lock(&fm->lock);
3301                 fm->user_enic = enic;
3302         }
3303         return fm;
3304 }
3305
3306 static void
3307 end_fm(struct enic_flowman *fm)
3308 {
3309         fm->user_enic = NULL;
3310         if (fm->owner_enic->switchdev_mode)
3311                 rte_spinlock_unlock(&fm->lock);
3312 }