drivers/net/enic/enic_fm_flow.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2008-2019 Cisco Systems, Inc.  All rights reserved.
   3  */
   4
   5 #include <errno.h>
   6 #include <stdint.h>
   7 #include <rte_log.h>
   8 #include <rte_ethdev_driver.h>
   9 #include <rte_flow_driver.h>
  10 #include <rte_ether.h>
  11 #include <rte_ip.h>
  12 #include <rte_udp.h>
  13 #include <rte_memzone.h>
  14
  15 #include "enic_compat.h"
  16 #include "enic.h"
  17 #include "vnic_dev.h"
  18 #include "vnic_nic.h"
  19
  20 #define IP_DEFTTL  64   /* from RFC 1340. */
  21 #define IP6_VTC_FLOW 0x60000000
  22
  23 /* Highest Item type supported by Flowman */
  24 #define FM_MAX_ITEM_TYPE RTE_FLOW_ITEM_TYPE_VXLAN
  25
  26 /* Up to 1024 TCAM entries */
  27 #define FM_MAX_TCAM_TABLE_SIZE 1024
  28
  29 /* Up to 4096 entries per exact match table */
  30 #define FM_MAX_EXACT_TABLE_SIZE 4096
  31
  32 /* Number of counters to increase on for each increment */
  33 #define FM_COUNTERS_EXPAND  100
  34
  35 #define FM_INVALID_HANDLE 0
  36
  37 /*
  38  * Flow exact match tables (FET) in the VIC and rte_flow groups.
  39  * Use a simple scheme to map groups to tables.
  40  * Group 0 uses the single TCAM tables, one for each direction.
  41  * Group 1, 2, ... uses its own exact match table.
  42  *
  43  * The TCAM tables are allocated upfront during init.
  44  *
  45  * Exact match tables are allocated on demand. 3 paths that lead allocations.
  46  *
  47  * 1. Add a flow that jumps from group 0 to group N.
  48  *
  49  * If N does not exist, we allocate an exact match table for it, using
  50  * a dummy key. A key is required for the table.
  51  *
  52  * 2. Add a flow that uses group N.
  53  *
  54  * If N does not exist, we allocate an exact match table for it, using
  55  * the flow's key. Subsequent flows to the same group all should have
  56  * the same key.
  57  *
  58  * Without a jump flow to N, N is not reachable in hardware. No packets
  59  * reach N and match.
  60  *
  61  * 3. Add a flow to an empty group N.
  62  *
  63  * N has been created via (1) and the dummy key. We free that table, allocate
  64  * a new table using the new flow's key. Also re-do the existing jump flow to
  65  * point to the new table.
  66  */
  67 #define FM_TCAM_RTE_GROUP 0
  68
  69 struct enic_fm_fet {
  70         TAILQ_ENTRY(enic_fm_fet) list;
  71         uint32_t group; /* rte_flow group ID */
  72         uint64_t handle; /* Exact match table handle from flowman */
  73         uint8_t ingress;
  74         uint8_t default_key;
  75         int ref; /* Reference count via get/put */
  76         struct fm_key_template key; /* Key associated with the table */
  77 };
  78
  79 struct enic_fm_counter {
  80         SLIST_ENTRY(enic_fm_counter) next;
  81         uint32_t handle;
  82 };
  83
  84 /* rte_flow.fm */
  85 struct enic_fm_flow {
  86         bool counter_valid;
  87         uint64_t entry_handle;
  88         uint64_t action_handle;
  89         struct enic_fm_counter *counter;
  90         struct enic_fm_fet *fet;
  91 };
  92
  93 struct enic_fm_jump_flow {
  94         TAILQ_ENTRY(enic_fm_jump_flow) list;
  95         struct rte_flow *flow;
  96         uint32_t group;
  97         struct fm_tcam_match_entry match;
  98         struct fm_action action;
  99 };
 100
 101 /*
 102  * Flowman uses host memory for commands. This structure is allocated
 103  * in DMA-able memory.
 104  */
 105 union enic_flowman_cmd_mem {
 106         struct fm_tcam_match_table fm_tcam_match_table;
 107         struct fm_exact_match_table fm_exact_match_table;
 108         struct fm_tcam_match_entry fm_tcam_match_entry;
 109         struct fm_exact_match_entry fm_exact_match_entry;
 110         struct fm_action fm_action;
 111 };
 112
 113 struct enic_flowman {
 114         struct enic *enic;
 115         /* Command buffer */
 116         struct {
 117                 union enic_flowman_cmd_mem *va;
 118                 dma_addr_t pa;
 119         } cmd;
 120         /* TCAM tables allocated upfront, used for group 0 */
 121         uint64_t ig_tcam_hndl;
 122         uint64_t eg_tcam_hndl;
 123         /* Counters */
 124         SLIST_HEAD(enic_free_counters, enic_fm_counter) counters;
 125         void *counter_stack;
 126         uint32_t counters_alloced;
 127         /* Exact match tables for groups != 0, dynamically allocated */
 128         TAILQ_HEAD(fet_list, enic_fm_fet) fet_list;
 129         /*
 130          * Default exact match tables used for jump actions to
 131          * non-existent groups.
 132          */
 133         struct enic_fm_fet *default_eg_fet;
 134         struct enic_fm_fet *default_ig_fet;
 135         /* Flows that jump to the default table above */
 136         TAILQ_HEAD(jump_flow_list, enic_fm_jump_flow) jump_list;
 137         /*
 138          * Scratch data used during each invocation of flow_create
 139          * and flow_validate.
 140          */
 141         struct enic_fm_fet *fet;
 142         struct fm_tcam_match_entry tcam_entry;
 143         struct fm_action action;
 144         struct fm_action action_tmp; /* enic_fm_reorder_action_op */
 145         int action_op_count;
 146 };
 147
 148 static int enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle);
 149
 150 /*
 151  * Common arguments passed to copy_item functions. Use this structure
 152  * so we can easily add new arguments.
 153  * item: Item specification.
 154  * fm_tcam_entry: Flowman TCAM match entry.
 155  * header_level: 0 for outer header, 1 for inner header.
 156  */
 157 struct copy_item_args {
 158         const struct rte_flow_item *item;
 159         struct fm_tcam_match_entry *fm_tcam_entry;
 160         uint8_t header_level;
 161 };
 162
 163 /* functions for copying items into flowman match */
 164 typedef int (enic_copy_item_fn)(struct copy_item_args *arg);
 165
 166 /* Info about how to copy items into flowman match */
 167 struct enic_fm_items {
 168         /* Function for copying and validating an item. */
 169         enic_copy_item_fn * const copy_item;
 170         /* List of valid previous items. */
 171         const enum rte_flow_item_type * const prev_items;
 172         /*
 173          * True if it's OK for this item to be the first item. For some NIC
 174          * versions, it's invalid to start the stack above layer 3.
 175          */
 176         const uint8_t valid_start_item;
 177 };
 178
 179 static enic_copy_item_fn enic_fm_copy_item_eth;
 180 static enic_copy_item_fn enic_fm_copy_item_ipv4;
 181 static enic_copy_item_fn enic_fm_copy_item_ipv6;
 182 static enic_copy_item_fn enic_fm_copy_item_raw;
 183 static enic_copy_item_fn enic_fm_copy_item_sctp;
 184 static enic_copy_item_fn enic_fm_copy_item_tcp;
 185 static enic_copy_item_fn enic_fm_copy_item_udp;
 186 static enic_copy_item_fn enic_fm_copy_item_vlan;
 187 static enic_copy_item_fn enic_fm_copy_item_vxlan;
 188
 189 /* Ingress actions */
 190 static const enum rte_flow_action_type enic_fm_supported_ig_actions[] = {
 191         RTE_FLOW_ACTION_TYPE_COUNT,
 192         RTE_FLOW_ACTION_TYPE_DROP,
 193         RTE_FLOW_ACTION_TYPE_FLAG,
 194         RTE_FLOW_ACTION_TYPE_JUMP,
 195         RTE_FLOW_ACTION_TYPE_MARK,
 196         RTE_FLOW_ACTION_TYPE_OF_POP_VLAN,
 197         RTE_FLOW_ACTION_TYPE_PORT_ID,
 198         RTE_FLOW_ACTION_TYPE_PASSTHRU,
 199         RTE_FLOW_ACTION_TYPE_QUEUE,
 200         RTE_FLOW_ACTION_TYPE_RSS,
 201         RTE_FLOW_ACTION_TYPE_VOID,
 202         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
 203         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP,
 204         RTE_FLOW_ACTION_TYPE_END, /* END must be the last entry */
 205 };
 206
 207 /* Egress actions */
 208 static const enum rte_flow_action_type enic_fm_supported_eg_actions[] = {
 209         RTE_FLOW_ACTION_TYPE_COUNT,
 210         RTE_FLOW_ACTION_TYPE_DROP,
 211         RTE_FLOW_ACTION_TYPE_JUMP,
 212         RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN,
 213         RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP,
 214         RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID,
 215         RTE_FLOW_ACTION_TYPE_PASSTHRU,
 216         RTE_FLOW_ACTION_TYPE_VOID,
 217         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
 218         RTE_FLOW_ACTION_TYPE_END,
 219 };
 220
 221 static const struct enic_fm_items enic_fm_items[] = {
 222         [RTE_FLOW_ITEM_TYPE_RAW] = {
 223                 .copy_item = enic_fm_copy_item_raw,
 224                 .valid_start_item = 0,
 225                 .prev_items = (const enum rte_flow_item_type[]) {
 226                                RTE_FLOW_ITEM_TYPE_UDP,
 227                                RTE_FLOW_ITEM_TYPE_END,
 228                 },
 229         },
 230         [RTE_FLOW_ITEM_TYPE_ETH] = {
 231                 .copy_item = enic_fm_copy_item_eth,
 232                 .valid_start_item = 1,
 233                 .prev_items = (const enum rte_flow_item_type[]) {
 234                                RTE_FLOW_ITEM_TYPE_END,
 235                 },
 236         },
 237         [RTE_FLOW_ITEM_TYPE_VLAN] = {
 238                 .copy_item = enic_fm_copy_item_vlan,
 239                 .valid_start_item = 1,
 240                 .prev_items = (const enum rte_flow_item_type[]) {
 241                                RTE_FLOW_ITEM_TYPE_ETH,
 242                                RTE_FLOW_ITEM_TYPE_END,
 243                 },
 244         },
 245         [RTE_FLOW_ITEM_TYPE_IPV4] = {
 246                 .copy_item = enic_fm_copy_item_ipv4,
 247                 .valid_start_item = 1,
 248                 .prev_items = (const enum rte_flow_item_type[]) {
 249                                RTE_FLOW_ITEM_TYPE_ETH,
 250                                RTE_FLOW_ITEM_TYPE_VLAN,
 251                                RTE_FLOW_ITEM_TYPE_END,
 252                 },
 253         },
 254         [RTE_FLOW_ITEM_TYPE_IPV6] = {
 255                 .copy_item = enic_fm_copy_item_ipv6,
 256                 .valid_start_item = 1,
 257                 .prev_items = (const enum rte_flow_item_type[]) {
 258                                RTE_FLOW_ITEM_TYPE_ETH,
 259                                RTE_FLOW_ITEM_TYPE_VLAN,
 260                                RTE_FLOW_ITEM_TYPE_END,
 261                 },
 262         },
 263         [RTE_FLOW_ITEM_TYPE_UDP] = {
 264                 .copy_item = enic_fm_copy_item_udp,
 265                 .valid_start_item = 1,
 266                 .prev_items = (const enum rte_flow_item_type[]) {
 267                                RTE_FLOW_ITEM_TYPE_IPV4,
 268                                RTE_FLOW_ITEM_TYPE_IPV6,
 269                                RTE_FLOW_ITEM_TYPE_END,
 270                 },
 271         },
 272         [RTE_FLOW_ITEM_TYPE_TCP] = {
 273                 .copy_item = enic_fm_copy_item_tcp,
 274                 .valid_start_item = 1,
 275                 .prev_items = (const enum rte_flow_item_type[]) {
 276                                RTE_FLOW_ITEM_TYPE_IPV4,
 277                                RTE_FLOW_ITEM_TYPE_IPV6,
 278                                RTE_FLOW_ITEM_TYPE_END,
 279                 },
 280         },
 281         [RTE_FLOW_ITEM_TYPE_SCTP] = {
 282                 .copy_item = enic_fm_copy_item_sctp,
 283                 .valid_start_item = 0,
 284                 .prev_items = (const enum rte_flow_item_type[]) {
 285                                RTE_FLOW_ITEM_TYPE_IPV4,
 286                                RTE_FLOW_ITEM_TYPE_IPV6,
 287                                RTE_FLOW_ITEM_TYPE_END,
 288                 },
 289         },
 290         [RTE_FLOW_ITEM_TYPE_VXLAN] = {
 291                 .copy_item = enic_fm_copy_item_vxlan,
 292                 .valid_start_item = 1,
 293                 .prev_items = (const enum rte_flow_item_type[]) {
 294                                RTE_FLOW_ITEM_TYPE_UDP,
 295                                RTE_FLOW_ITEM_TYPE_END,
 296                 },
 297         },
 298 };
 299
 300 static int
 301 enic_fm_copy_item_eth(struct copy_item_args *arg)
 302 {
 303         const struct rte_flow_item *item = arg->item;
 304         const struct rte_flow_item_eth *spec = item->spec;
 305         const struct rte_flow_item_eth *mask = item->mask;
 306         const uint8_t lvl = arg->header_level;
 307         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 308         struct fm_header_set *fm_data, *fm_mask;
 309
 310         ENICPMD_FUNC_TRACE();
 311         /* Match all if no spec */
 312         if (!spec)
 313                 return 0;
 314         if (!mask)
 315                 mask = &rte_flow_item_eth_mask;
 316         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 317         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 318         fm_data->fk_header_select |= FKH_ETHER;
 319         fm_mask->fk_header_select |= FKH_ETHER;
 320         memcpy(&fm_data->l2.eth, spec, sizeof(*spec));
 321         memcpy(&fm_mask->l2.eth, mask, sizeof(*mask));
 322         return 0;
 323 }
 324
 325 static int
 326 enic_fm_copy_item_vlan(struct copy_item_args *arg)
 327 {
 328         const struct rte_flow_item *item = arg->item;
 329         const struct rte_flow_item_vlan *spec = item->spec;
 330         const struct rte_flow_item_vlan *mask = item->mask;
 331         const uint8_t lvl = arg->header_level;
 332         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 333         struct fm_header_set *fm_data, *fm_mask;
 334         struct rte_ether_hdr *eth_mask;
 335         struct rte_ether_hdr *eth_val;
 336         uint32_t meta;
 337
 338         ENICPMD_FUNC_TRACE();
 339         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 340         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 341         /* Outer and inner packet vlans need different flags */
 342         meta = FKM_VLAN_PRES;
 343         if (lvl > 0)
 344                 meta = FKM_QTAG;
 345         fm_data->fk_metadata |= meta;
 346         fm_mask->fk_metadata |= meta;
 347
 348         /* Match all if no spec */
 349         if (!spec)
 350                 return 0;
 351         if (!mask)
 352                 mask = &rte_flow_item_vlan_mask;
 353
 354         eth_mask = (void *)&fm_mask->l2.eth;
 355         eth_val = (void *)&fm_data->l2.eth;
 356
 357         /* Outer TPID cannot be matched */
 358         if (eth_mask->ether_type)
 359                 return -ENOTSUP;
 360
 361         /*
 362          * When packet matching, the VIC always compares vlan-stripped
 363          * L2, regardless of vlan stripping settings. So, the inner type
 364          * from vlan becomes the ether type of the eth header.
 365          */
 366         eth_mask->ether_type = mask->inner_type;
 367         eth_val->ether_type = spec->inner_type;
 368         fm_data->fk_header_select |= FKH_ETHER | FKH_QTAG;
 369         fm_mask->fk_header_select |= FKH_ETHER | FKH_QTAG;
 370         fm_data->fk_vlan = rte_be_to_cpu_16(spec->tci);
 371         fm_mask->fk_vlan = rte_be_to_cpu_16(mask->tci);
 372         return 0;
 373 }
 374
 375 static int
 376 enic_fm_copy_item_ipv4(struct copy_item_args *arg)
 377 {
 378         const struct rte_flow_item *item = arg->item;
 379         const struct rte_flow_item_ipv4 *spec = item->spec;
 380         const struct rte_flow_item_ipv4 *mask = item->mask;
 381         const uint8_t lvl = arg->header_level;
 382         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 383         struct fm_header_set *fm_data, *fm_mask;
 384
 385         ENICPMD_FUNC_TRACE();
 386         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 387         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 388         fm_data->fk_metadata |= FKM_IPV4;
 389         fm_mask->fk_metadata |= FKM_IPV4;
 390
 391         if (!spec)
 392                 return 0;
 393         if (!mask)
 394                 mask = &rte_flow_item_ipv4_mask;
 395
 396         fm_data->fk_header_select |= FKH_IPV4;
 397         fm_mask->fk_header_select |= FKH_IPV4;
 398         memcpy(&fm_data->l3.ip4, spec, sizeof(*spec));
 399         memcpy(&fm_mask->l3.ip4, mask, sizeof(*mask));
 400         return 0;
 401 }
 402
 403 static int
 404 enic_fm_copy_item_ipv6(struct copy_item_args *arg)
 405 {
 406         const struct rte_flow_item *item = arg->item;
 407         const struct rte_flow_item_ipv6 *spec = item->spec;
 408         const struct rte_flow_item_ipv6 *mask = item->mask;
 409         const uint8_t lvl = arg->header_level;
 410         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 411         struct fm_header_set *fm_data, *fm_mask;
 412
 413         ENICPMD_FUNC_TRACE();
 414         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 415         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 416         fm_data->fk_metadata |= FKM_IPV6;
 417         fm_mask->fk_metadata |= FKM_IPV6;
 418
 419         if (!spec)
 420                 return 0;
 421         if (!mask)
 422                 mask = &rte_flow_item_ipv6_mask;
 423
 424         fm_data->fk_header_select |= FKH_IPV6;
 425         fm_mask->fk_header_select |= FKH_IPV6;
 426         memcpy(&fm_data->l3.ip6, spec, sizeof(*spec));
 427         memcpy(&fm_mask->l3.ip6, mask, sizeof(*mask));
 428         return 0;
 429 }
 430
 431 static int
 432 enic_fm_copy_item_udp(struct copy_item_args *arg)
 433 {
 434         const struct rte_flow_item *item = arg->item;
 435         const struct rte_flow_item_udp *spec = item->spec;
 436         const struct rte_flow_item_udp *mask = item->mask;
 437         const uint8_t lvl = arg->header_level;
 438         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 439         struct fm_header_set *fm_data, *fm_mask;
 440
 441         ENICPMD_FUNC_TRACE();
 442         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 443         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 444         fm_data->fk_metadata |= FKM_UDP;
 445         fm_mask->fk_metadata |= FKM_UDP;
 446
 447         if (!spec)
 448                 return 0;
 449         if (!mask)
 450                 mask = &rte_flow_item_udp_mask;
 451
 452         fm_data->fk_header_select |= FKH_UDP;
 453         fm_mask->fk_header_select |= FKH_UDP;
 454         memcpy(&fm_data->l4.udp, spec, sizeof(*spec));
 455         memcpy(&fm_mask->l4.udp, mask, sizeof(*mask));
 456         return 0;
 457 }
 458
 459 static int
 460 enic_fm_copy_item_tcp(struct copy_item_args *arg)
 461 {
 462         const struct rte_flow_item *item = arg->item;
 463         const struct rte_flow_item_tcp *spec = item->spec;
 464         const struct rte_flow_item_tcp *mask = item->mask;
 465         const uint8_t lvl = arg->header_level;
 466         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 467         struct fm_header_set *fm_data, *fm_mask;
 468
 469         ENICPMD_FUNC_TRACE();
 470         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 471         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 472         fm_data->fk_metadata |= FKM_TCP;
 473         fm_mask->fk_metadata |= FKM_TCP;
 474
 475         if (!spec)
 476                 return 0;
 477         if (!mask)
 478                 mask = &rte_flow_item_tcp_mask;
 479
 480         fm_data->fk_header_select |= FKH_TCP;
 481         fm_mask->fk_header_select |= FKH_TCP;
 482         memcpy(&fm_data->l4.tcp, spec, sizeof(*spec));
 483         memcpy(&fm_mask->l4.tcp, mask, sizeof(*mask));
 484         return 0;
 485 }
 486
 487 static int
 488 enic_fm_copy_item_sctp(struct copy_item_args *arg)
 489 {
 490         const struct rte_flow_item *item = arg->item;
 491         const struct rte_flow_item_sctp *spec = item->spec;
 492         const struct rte_flow_item_sctp *mask = item->mask;
 493         const uint8_t lvl = arg->header_level;
 494         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 495         struct fm_header_set *fm_data, *fm_mask;
 496         uint8_t *ip_proto_mask = NULL;
 497         uint8_t *ip_proto = NULL;
 498         uint32_t l3_fkh;
 499
 500         ENICPMD_FUNC_TRACE();
 501         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 502         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 503         /*
 504          * The NIC filter API has no flags for "match sctp", so explicitly
 505          * set the protocol number in the IP pattern.
 506          */
 507         if (fm_data->fk_metadata & FKM_IPV4) {
 508                 struct rte_ipv4_hdr *ip;
 509                 ip = (struct rte_ipv4_hdr *)&fm_mask->l3.ip4;
 510                 ip_proto_mask = &ip->next_proto_id;
 511                 ip = (struct rte_ipv4_hdr *)&fm_data->l3.ip4;
 512                 ip_proto = &ip->next_proto_id;
 513                 l3_fkh = FKH_IPV4;
 514         } else if (fm_data->fk_metadata & FKM_IPV6) {
 515                 struct rte_ipv6_hdr *ip;
 516                 ip = (struct rte_ipv6_hdr *)&fm_mask->l3.ip6;
 517                 ip_proto_mask = &ip->proto;
 518                 ip = (struct rte_ipv6_hdr *)&fm_data->l3.ip6;
 519                 ip_proto = &ip->proto;
 520                 l3_fkh = FKH_IPV6;
 521         } else {
 522                 /* Need IPv4/IPv6 pattern first */
 523                 return -EINVAL;
 524         }
 525         *ip_proto = IPPROTO_SCTP;
 526         *ip_proto_mask = 0xff;
 527         fm_data->fk_header_select |= l3_fkh;
 528         fm_mask->fk_header_select |= l3_fkh;
 529
 530         if (!spec)
 531                 return 0;
 532         if (!mask)
 533                 mask = &rte_flow_item_sctp_mask;
 534
 535         fm_data->fk_header_select |= FKH_L4RAW;
 536         fm_mask->fk_header_select |= FKH_L4RAW;
 537         memcpy(fm_data->l4.rawdata, spec, sizeof(*spec));
 538         memcpy(fm_mask->l4.rawdata, mask, sizeof(*mask));
 539         return 0;
 540 }
 541
 542 static int
 543 enic_fm_copy_item_vxlan(struct copy_item_args *arg)
 544 {
 545         const struct rte_flow_item *item = arg->item;
 546         const struct rte_flow_item_vxlan *spec = item->spec;
 547         const struct rte_flow_item_vxlan *mask = item->mask;
 548         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 549         struct fm_header_set *fm_data, *fm_mask;
 550
 551         ENICPMD_FUNC_TRACE();
 552         /* Only 2 header levels (outer and inner) allowed */
 553         if (arg->header_level > 0)
 554                 return -EINVAL;
 555
 556         fm_data = &entry->ftm_data.fk_hdrset[0];
 557         fm_mask = &entry->ftm_mask.fk_hdrset[0];
 558         fm_data->fk_metadata |= FKM_VXLAN;
 559         fm_mask->fk_metadata |= FKM_VXLAN;
 560         /* items from here on out are inner header items */
 561         arg->header_level = 1;
 562
 563         /* Match all if no spec */
 564         if (!spec)
 565                 return 0;
 566         if (!mask)
 567                 mask = &rte_flow_item_vxlan_mask;
 568
 569         fm_data->fk_header_select |= FKH_VXLAN;
 570         fm_mask->fk_header_select |= FKH_VXLAN;
 571         memcpy(&fm_data->vxlan, spec, sizeof(*spec));
 572         memcpy(&fm_mask->vxlan, mask, sizeof(*mask));
 573         return 0;
 574 }
 575
 576 /*
 577  * Currently, raw pattern match is very limited. It is intended for matching
 578  * UDP tunnel header (e.g. vxlan or geneve).
 579  */
 580 static int
 581 enic_fm_copy_item_raw(struct copy_item_args *arg)
 582 {
 583         const struct rte_flow_item *item = arg->item;
 584         const struct rte_flow_item_raw *spec = item->spec;
 585         const struct rte_flow_item_raw *mask = item->mask;
 586         const uint8_t lvl = arg->header_level;
 587         struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
 588         struct fm_header_set *fm_data, *fm_mask;
 589
 590         ENICPMD_FUNC_TRACE();
 591         /* Cannot be used for inner packet */
 592         if (lvl > 0)
 593                 return -EINVAL;
 594         /* Need both spec and mask */
 595         if (!spec || !mask)
 596                 return -EINVAL;
 597         /* Only supports relative with offset 0 */
 598         if (!spec->relative || spec->offset != 0 || spec->search ||
 599             spec->limit)
 600                 return -EINVAL;
 601         /* Need non-null pattern that fits within the NIC's filter pattern */
 602         if (spec->length == 0 ||
 603             spec->length + sizeof(struct rte_udp_hdr) > FM_LAYER_SIZE ||
 604             !spec->pattern || !mask->pattern)
 605                 return -EINVAL;
 606         /*
 607          * Mask fields, including length, are often set to zero. Assume that
 608          * means "same as spec" to avoid breaking existing apps. If length
 609          * is not zero, then it should be >= spec length.
 610          *
 611          * No more pattern follows this, so append to the L4 layer instead of
 612          * L5 to work with both recent and older VICs.
 613          */
 614         if (mask->length != 0 && mask->length < spec->length)
 615                 return -EINVAL;
 616
 617         fm_data = &entry->ftm_data.fk_hdrset[lvl];
 618         fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
 619         fm_data->fk_header_select |= FKH_L4RAW;
 620         fm_mask->fk_header_select |= FKH_L4RAW;
 621         fm_data->fk_header_select &= ~FKH_UDP;
 622         fm_mask->fk_header_select &= ~FKH_UDP;
 623         memcpy(fm_data->l4.rawdata + sizeof(struct rte_udp_hdr),
 624                spec->pattern, spec->length);
 625         memcpy(fm_mask->l4.rawdata + sizeof(struct rte_udp_hdr),
 626                mask->pattern, spec->length);
 627         return 0;
 628 }
 629
 630 static int
 631 enic_fet_alloc(struct enic_flowman *fm, uint8_t ingress,
 632                struct fm_key_template *key, int entries,
 633                struct enic_fm_fet **fet_out)
 634 {
 635         struct fm_exact_match_table *cmd;
 636         struct fm_header_set *hdr;
 637         struct enic_fm_fet *fet;
 638         uint64_t args[3];
 639         int ret;
 640
 641         ENICPMD_FUNC_TRACE();
 642         fet = calloc(1, sizeof(struct enic_fm_fet));
 643         if (fet == NULL)
 644                 return -ENOMEM;
 645         cmd = &fm->cmd.va->fm_exact_match_table;
 646         memset(cmd, 0, sizeof(*cmd));
 647         cmd->fet_direction = ingress ? FM_INGRESS : FM_EGRESS;
 648         cmd->fet_stage = FM_STAGE_LAST;
 649         cmd->fet_max_entries = entries ? entries : FM_MAX_EXACT_TABLE_SIZE;
 650         if (key == NULL) {
 651                 hdr = &cmd->fet_key.fk_hdrset[0];
 652                 memset(hdr, 0, sizeof(*hdr));
 653                 hdr->fk_header_select = FKH_IPV4 | FKH_UDP;
 654                 hdr->l3.ip4.fk_saddr = 0xFFFFFFFF;
 655                 hdr->l3.ip4.fk_daddr = 0xFFFFFFFF;
 656                 hdr->l4.udp.fk_source = 0xFFFF;
 657                 hdr->l4.udp.fk_dest = 0xFFFF;
 658                 fet->default_key = 1;
 659         } else {
 660                 memcpy(&cmd->fet_key, key, sizeof(*key));
 661                 memcpy(&fet->key, key, sizeof(*key));
 662                 fet->default_key = 0;
 663         }
 664         cmd->fet_key.fk_packet_tag = 1;
 665
 666         args[0] = FM_EXACT_TABLE_ALLOC;
 667         args[1] = fm->cmd.pa;
 668         ret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
 669         if (ret) {
 670                 ENICPMD_LOG(ERR, "cannot alloc exact match table: rc=%d", ret);
 671                 free(fet);
 672                 return ret;
 673         }
 674         fet->handle = args[0];
 675         fet->ingress = ingress;
 676         ENICPMD_LOG(DEBUG, "allocated exact match table: handle=0x%" PRIx64,
 677                     fet->handle);
 678         *fet_out = fet;
 679         return 0;
 680 }
 681
 682 static void
 683 enic_fet_free(struct enic_flowman *fm, struct enic_fm_fet *fet)
 684 {
 685         ENICPMD_FUNC_TRACE();
 686         enic_fm_tbl_free(fm, fet->handle);
 687         if (!fet->default_key)
 688                 TAILQ_REMOVE(&fm->fet_list, fet, list);
 689         free(fet);
 690 }
 691
 692 /*
 693  * Get the exact match table for the given combination of
 694  * <group, ingress, key>. Allocate one on the fly as necessary.
 695  */
 696 static int
 697 enic_fet_get(struct enic_flowman *fm,
 698              uint32_t group,
 699              uint8_t ingress,
 700              struct fm_key_template *key,
 701              struct enic_fm_fet **fet_out,
 702              struct rte_flow_error *error)
 703 {
 704         struct enic_fm_fet *fet;
 705
 706         ENICPMD_FUNC_TRACE();
 707         /* See if we already have this table open */
 708         TAILQ_FOREACH(fet, &fm->fet_list, list) {
 709                 if (fet->group == group && fet->ingress == ingress)
 710                         break;
 711         }
 712         if (fet == NULL) {
 713                 /* Jumping to a non-existing group? Use the default table */
 714                 if (key == NULL) {
 715                         fet = ingress ? fm->default_ig_fet : fm->default_eg_fet;
 716                 } else if (enic_fet_alloc(fm, ingress, key, 0, &fet)) {
 717                         return rte_flow_error_set(error, EINVAL,
 718                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 719                                 NULL, "enic: cannot get exact match table");
 720                 }
 721                 fet->group = group;
 722                 /* Default table is never on the open table list */
 723                 if (!fet->default_key)
 724                         TAILQ_INSERT_HEAD(&fm->fet_list, fet, list);
 725         }
 726         fet->ref++;
 727         *fet_out = fet;
 728         ENICPMD_LOG(DEBUG, "fet_get: %s %s group=%u ref=%u",
 729                     fet->default_key ? "default" : "",
 730                     fet->ingress ? "ingress" : "egress",
 731                     fet->group, fet->ref);
 732         return 0;
 733 }
 734
 735 static void
 736 enic_fet_put(struct enic_flowman *fm, struct enic_fm_fet *fet)
 737 {
 738         ENICPMD_FUNC_TRACE();
 739         RTE_ASSERT(fet->ref > 0);
 740         fet->ref--;
 741         ENICPMD_LOG(DEBUG, "fet_put: %s %s group=%u ref=%u",
 742                     fet->default_key ? "default" : "",
 743                     fet->ingress ? "ingress" : "egress",
 744                     fet->group, fet->ref);
 745         if (fet->ref == 0)
 746                 enic_fet_free(fm, fet);
 747 }
 748
 749 /* Return 1 if current item is valid on top of the previous one. */
 750 static int
 751 fm_item_stacking_valid(enum rte_flow_item_type prev_item,
 752                        const struct enic_fm_items *item_info,
 753                        uint8_t is_first_item)
 754 {
 755         enum rte_flow_item_type const *allowed_items = item_info->prev_items;
 756
 757         ENICPMD_FUNC_TRACE();
 758         for (; *allowed_items != RTE_FLOW_ITEM_TYPE_END; allowed_items++) {
 759                 if (prev_item == *allowed_items)
 760                         return 1;
 761         }
 762
 763         /* This is the first item in the stack. Check if that's cool */
 764         if (is_first_item && item_info->valid_start_item)
 765                 return 1;
 766         return 0;
 767 }
 768
 769 /*
 770  * Build the flow manager match entry structure from the provided pattern.
 771  * The pattern is validated as the items are copied.
 772  */
 773 static int
 774 enic_fm_copy_entry(struct enic_flowman *fm,
 775                    const struct rte_flow_item pattern[],
 776                    struct rte_flow_error *error)
 777 {
 778         const struct enic_fm_items *item_info;
 779         enum rte_flow_item_type prev_item;
 780         const struct rte_flow_item *item;
 781         struct copy_item_args args;
 782         uint8_t prev_header_level;
 783         uint8_t is_first_item;
 784         int ret;
 785
 786         ENICPMD_FUNC_TRACE();
 787         item = pattern;
 788         is_first_item = 1;
 789         prev_item = RTE_FLOW_ITEM_TYPE_END;
 790
 791         args.fm_tcam_entry = &fm->tcam_entry;
 792         args.header_level = 0;
 793         prev_header_level = 0;
 794         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 795                 /*
 796                  * Get info about how to validate and copy the item. If NULL
 797                  * is returned the nic does not support the item.
 798                  */
 799                 if (item->type == RTE_FLOW_ITEM_TYPE_VOID)
 800                         continue;
 801
 802                 item_info = &enic_fm_items[item->type];
 803
 804                 if (item->type > FM_MAX_ITEM_TYPE ||
 805                     item_info->copy_item == NULL) {
 806                         return rte_flow_error_set(error, ENOTSUP,
 807                                 RTE_FLOW_ERROR_TYPE_ITEM,
 808                                 NULL, "enic: unsupported item");
 809                 }
 810
 811                 /* check to see if item stacking is valid */
 812                 if (!fm_item_stacking_valid(prev_item, item_info,
 813                                             is_first_item))
 814                         goto stacking_error;
 815
 816                 args.item = item;
 817                 ret = item_info->copy_item(&args);
 818                 if (ret)
 819                         goto item_not_supported;
 820                 /* Going from outer to inner? Treat it as a new packet start */
 821                 if (prev_header_level != args.header_level) {
 822                         prev_item = RTE_FLOW_ITEM_TYPE_END;
 823                         is_first_item = 1;
 824                 } else {
 825                         prev_item = item->type;
 826                         is_first_item = 0;
 827                 }
 828                 prev_header_level = args.header_level;
 829         }
 830         return 0;
 831
 832 item_not_supported:
 833         return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,
 834                                   NULL, "enic: unsupported item type");
 835
 836 stacking_error:
 837         return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
 838                                   item, "enic: unsupported item stack");
 839 }
 840
 841 static void
 842 flow_item_skip_void(const struct rte_flow_item **item)
 843 {
 844         for ( ; ; (*item)++)
 845                 if ((*item)->type != RTE_FLOW_ITEM_TYPE_VOID)
 846                         return;
 847 }
 848
 849 static void
 850 append_template(void **template, uint8_t *off, const void *data, int len)
 851 {
 852         memcpy(*template, data, len);
 853         *template = (char *)*template + len;
 854         *off = *off + len;
 855 }
 856
 857 static int
 858 enic_fm_append_action_op(struct enic_flowman *fm,
 859                          struct fm_action_op *fm_op,
 860                          struct rte_flow_error *error)
 861 {
 862         int count;
 863
 864         count = fm->action_op_count;
 865         ENICPMD_LOG(DEBUG, "append action op: idx=%d op=%u",
 866                     count, fm_op->fa_op);
 867         if (count == FM_ACTION_OP_MAX) {
 868                 return rte_flow_error_set(error, EINVAL,
 869                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 870                         "too many action operations");
 871         }
 872         fm->action.fma_action_ops[count] = *fm_op;
 873         fm->action_op_count = count + 1;
 874         return 0;
 875 }
 876
 877 /* NIC requires that 1st steer appear before decap.
 878  * Correct example: steer, decap, steer, steer, ...
 879  */
 880 static void
 881 enic_fm_reorder_action_op(struct enic_flowman *fm)
 882 {
 883         struct fm_action_op *op, *steer, *decap;
 884         struct fm_action_op tmp_op;
 885
 886         ENICPMD_FUNC_TRACE();
 887         /* Find 1st steer and decap */
 888         op = fm->action.fma_action_ops;
 889         steer = NULL;
 890         decap = NULL;
 891         while (op->fa_op != FMOP_END) {
 892                 if (!decap && op->fa_op == FMOP_DECAP_NOSTRIP)
 893                         decap = op;
 894                 else if (!steer && op->fa_op == FMOP_RQ_STEER)
 895                         steer = op;
 896                 op++;
 897         }
 898         /* If decap is before steer, swap */
 899         if (steer && decap && decap < steer) {
 900                 op = fm->action.fma_action_ops;
 901                 ENICPMD_LOG(DEBUG, "swap decap %ld <-> steer %ld",
 902                             (long)(decap - op), (long)(steer - op));
 903                 tmp_op = *decap;
 904                 *decap = *steer;
 905                 *steer = tmp_op;
 906         }
 907 }
 908
 909 /* VXLAN decap is done via flowman compound action */
 910 static int
 911 enic_fm_copy_vxlan_decap(struct enic_flowman *fm,
 912                          struct fm_tcam_match_entry *fmt,
 913                          const struct rte_flow_action *action,
 914                          struct rte_flow_error *error)
 915 {
 916         struct fm_header_set *fm_data;
 917         struct fm_action_op fm_op;
 918
 919         ENICPMD_FUNC_TRACE();
 920         fm_data = &fmt->ftm_data.fk_hdrset[0];
 921         if (!(fm_data->fk_metadata & FKM_VXLAN)) {
 922                 return rte_flow_error_set(error, EINVAL,
 923                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 924                         "vxlan-decap: vxlan must be in pattern");
 925         }
 926
 927         memset(&fm_op, 0, sizeof(fm_op));
 928         fm_op.fa_op = FMOP_DECAP_NOSTRIP;
 929         return enic_fm_append_action_op(fm, &fm_op, error);
 930 }
 931
 932 /* VXLAN encap is done via flowman compound action */
 933 static int
 934 enic_fm_copy_vxlan_encap(struct enic_flowman *fm,
 935                          const struct rte_flow_item *item,
 936                          struct rte_flow_error *error)
 937 {
 938         struct fm_action_op fm_op;
 939         struct rte_ether_hdr *eth;
 940         uint16_t *ethertype;
 941         void *template;
 942         uint8_t off;
 943
 944         ENICPMD_FUNC_TRACE();
 945         memset(&fm_op, 0, sizeof(fm_op));
 946         fm_op.fa_op = FMOP_ENCAP;
 947         template = fm->action.fma_data;
 948         off = 0;
 949         /*
 950          * Copy flow items to the flowman template starting L2.
 951          * L2 must be ethernet.
 952          */
 953         flow_item_skip_void(&item);
 954         if (item->type != RTE_FLOW_ITEM_TYPE_ETH)
 955                 return rte_flow_error_set(error, EINVAL,
 956                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 957                         "vxlan-encap: first item should be ethernet");
 958         eth = (struct rte_ether_hdr *)template;
 959         ethertype = &eth->ether_type;
 960         append_template(&template, &off, item->spec,
 961                         sizeof(struct rte_flow_item_eth));
 962         item++;
 963         flow_item_skip_void(&item);
 964         /* Optional VLAN */
 965         if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
 966                 const struct rte_flow_item_vlan *spec;
 967
 968                 ENICPMD_LOG(DEBUG, "vxlan-encap: vlan");
 969                 spec = item->spec;
 970                 fm_op.encap.outer_vlan = rte_be_to_cpu_16(spec->tci);
 971                 item++;
 972                 flow_item_skip_void(&item);
 973         }
 974         /* L3 must be IPv4, IPv6 */
 975         switch (item->type) {
 976         case RTE_FLOW_ITEM_TYPE_IPV4:
 977         {
 978                 struct rte_ipv4_hdr *ip4;
 979
 980                 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv4");
 981                 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
 982                 ip4 = (struct rte_ipv4_hdr *)template;
 983                 /*
 984                  * Offset of IPv4 length field and its initial value
 985                  * (IP + UDP + VXLAN) are specified in the action. The NIC
 986                  * will add inner packet length.
 987                  */
 988                 fm_op.encap.len1_offset = off +
 989                         offsetof(struct rte_ipv4_hdr, total_length);
 990                 fm_op.encap.len1_delta = sizeof(struct rte_ipv4_hdr) +
 991                         sizeof(struct rte_udp_hdr) +
 992                         sizeof(struct rte_vxlan_hdr);
 993                 append_template(&template, &off, item->spec,
 994                                 sizeof(struct rte_ipv4_hdr));
 995                 ip4->version_ihl = RTE_IPV4_VHL_DEF;
 996                 if (ip4->time_to_live == 0)
 997                         ip4->time_to_live = IP_DEFTTL;
 998                 ip4->next_proto_id = IPPROTO_UDP;
 999                 break;
1000         }
1001         case RTE_FLOW_ITEM_TYPE_IPV6:
1002         {
1003                 struct rte_ipv6_hdr *ip6;
1004
1005                 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv6");
1006                 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
1007                 ip6 = (struct rte_ipv6_hdr *)template;
1008                 fm_op.encap.len1_offset = off +
1009                         offsetof(struct rte_ipv6_hdr, payload_len);
1010                 fm_op.encap.len1_delta = sizeof(struct rte_udp_hdr) +
1011                         sizeof(struct rte_vxlan_hdr);
1012                 append_template(&template, &off, item->spec,
1013                                 sizeof(struct rte_ipv6_hdr));
1014                 ip6->vtc_flow |= rte_cpu_to_be_32(IP6_VTC_FLOW);
1015                 if (ip6->hop_limits == 0)
1016                         ip6->hop_limits = IP_DEFTTL;
1017                 ip6->proto = IPPROTO_UDP;
1018                 break;
1019         }
1020         default:
1021                 return rte_flow_error_set(error,
1022                         EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1023                         "vxlan-encap: L3 must be IPv4/IPv6");
1024         }
1025         item++;
1026         flow_item_skip_void(&item);
1027
1028         /* L4 is UDP */
1029         if (item->type != RTE_FLOW_ITEM_TYPE_UDP)
1030                 return rte_flow_error_set(error, EINVAL,
1031                         RTE_FLOW_ERROR_TYPE_ITEM, item,
1032                         "vxlan-encap: UDP must follow IPv4/IPv6");
1033         /* UDP length = UDP + VXLAN. NIC will add inner packet length. */
1034         fm_op.encap.len2_offset =
1035                 off + offsetof(struct rte_udp_hdr, dgram_len);
1036         fm_op.encap.len2_delta =
1037                 sizeof(struct rte_udp_hdr) + sizeof(struct rte_vxlan_hdr);
1038         append_template(&template, &off, item->spec,
1039                         sizeof(struct rte_udp_hdr));
1040         item++;
1041         flow_item_skip_void(&item);
1042
1043         /* Finally VXLAN */
1044         if (item->type != RTE_FLOW_ITEM_TYPE_VXLAN)
1045                 return rte_flow_error_set(error,
1046                         EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1047                         "vxlan-encap: VXLAN must follow UDP");
1048         append_template(&template, &off, item->spec,
1049                         sizeof(struct rte_flow_item_vxlan));
1050
1051         /*
1052          * Fill in the rest of the action structure.
1053          * Indicate that we want to encap with vxlan at packet start.
1054          */
1055         fm_op.encap.template_offset = 0;
1056         fm_op.encap.template_len = off;
1057         return enic_fm_append_action_op(fm, &fm_op, error);
1058 }
1059
1060 static int
1061 enic_fm_find_vnic(struct enic *enic, const struct rte_pci_addr *addr,
1062                   uint64_t *handle)
1063 {
1064         uint32_t bdf;
1065         uint64_t args[2];
1066         int rc;
1067
1068         ENICPMD_FUNC_TRACE();
1069         ENICPMD_LOG(DEBUG, "bdf=%x:%x:%x", addr->bus, addr->devid,
1070                     addr->function);
1071         bdf = addr->bus << 8 | addr->devid << 3 | addr->function;
1072         args[0] = FM_VNIC_FIND;
1073         args[1] = bdf;
1074         rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
1075         if (rc != 0) {
1076                 /* Expected to fail if BDF is not on the adapter */
1077                 ENICPMD_LOG(DEBUG, "cannot find vnic handle: rc=%d", rc);
1078                 return rc;
1079         }
1080         *handle = args[0];
1081         ENICPMD_LOG(DEBUG, "found vnic: handle=0x%" PRIx64, *handle);
1082         return 0;
1083 }
1084
1085 /* Translate flow actions to flowman TCAM entry actions */
1086 static int
1087 enic_fm_copy_action(struct enic_flowman *fm,
1088                     const struct rte_flow_action actions[],
1089                     uint8_t ingress,
1090                     struct rte_flow_error *error)
1091 {
1092         enum {
1093                 FATE = 1 << 0,
1094                 DECAP = 1 << 1,
1095                 PASSTHRU = 1 << 2,
1096                 COUNT = 1 << 3,
1097                 ENCAP = 1 << 4,
1098                 PUSH_VLAN = 1 << 5,
1099         };
1100         struct fm_tcam_match_entry *fmt;
1101         struct fm_action_op fm_op;
1102         bool need_ovlan_action;
1103         struct enic *enic;
1104         uint32_t overlap;
1105         uint64_t vnic_h;
1106         uint16_t ovlan;
1107         bool first_rq;
1108         int ret;
1109
1110         ENICPMD_FUNC_TRACE();
1111         fmt = &fm->tcam_entry;
1112         need_ovlan_action = false;
1113         ovlan = 0;
1114         first_rq = true;
1115         enic = fm->enic;
1116         overlap = 0;
1117         vnic_h = 0; /* 0 = current vNIC */
1118         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1119                 switch (actions->type) {
1120                 case RTE_FLOW_ACTION_TYPE_VOID:
1121                         continue;
1122                 case RTE_FLOW_ACTION_TYPE_PASSTHRU: {
1123                         if (overlap & PASSTHRU)
1124                                 goto unsupported;
1125                         overlap |= PASSTHRU;
1126                         break;
1127                 }
1128                 case RTE_FLOW_ACTION_TYPE_JUMP: {
1129                         const struct rte_flow_action_jump *jump =
1130                                 actions->conf;
1131                         struct enic_fm_fet *fet;
1132
1133                         if (overlap & FATE)
1134                                 goto unsupported;
1135                         ret = enic_fet_get(fm, jump->group, ingress, NULL,
1136                                            &fet, error);
1137                         if (ret)
1138                                 return ret;
1139                         overlap |= FATE;
1140                         memset(&fm_op, 0, sizeof(fm_op));
1141                         fm_op.fa_op = FMOP_EXACT_MATCH;
1142                         fm_op.exact.handle = fet->handle;
1143                         fm->fet = fet;
1144                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1145                         if (ret)
1146                                 return ret;
1147                         break;
1148                 }
1149                 case RTE_FLOW_ACTION_TYPE_MARK: {
1150                         const struct rte_flow_action_mark *mark =
1151                                 actions->conf;
1152
1153                         if (mark->id >= ENIC_MAGIC_FILTER_ID - 1)
1154                                 return rte_flow_error_set(error, EINVAL,
1155                                         RTE_FLOW_ERROR_TYPE_ACTION,
1156                                         NULL, "invalid mark id");
1157                         memset(&fm_op, 0, sizeof(fm_op));
1158                         fm_op.fa_op = FMOP_MARK;
1159                         fm_op.mark.mark = mark->id + 1;
1160                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1161                         if (ret)
1162                                 return ret;
1163                         break;
1164                 }
1165                 case RTE_FLOW_ACTION_TYPE_FLAG: {
1166                         /* ENIC_MAGIC_FILTER_ID is reserved for flagging */
1167                         memset(&fm_op, 0, sizeof(fm_op));
1168                         fm_op.fa_op = FMOP_MARK;
1169                         fm_op.mark.mark = ENIC_MAGIC_FILTER_ID;
1170                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1171                         if (ret)
1172                                 return ret;
1173                         break;
1174                 }
1175                 case RTE_FLOW_ACTION_TYPE_QUEUE: {
1176                         const struct rte_flow_action_queue *queue =
1177                                 actions->conf;
1178
1179                         /*
1180                          * If fate other than QUEUE or RSS, fail. Multiple
1181                          * rss and queue actions are ok.
1182                          */
1183                         if ((overlap & FATE) && first_rq)
1184                                 goto unsupported;
1185                         first_rq = false;
1186                         overlap |= FATE;
1187                         memset(&fm_op, 0, sizeof(fm_op));
1188                         fm_op.fa_op = FMOP_RQ_STEER;
1189                         fm_op.rq_steer.rq_index =
1190                                 enic_rte_rq_idx_to_sop_idx(queue->index);
1191                         fm_op.rq_steer.rq_count = 1;
1192                         fm_op.rq_steer.vnic_handle = vnic_h;
1193                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1194                         if (ret)
1195                                 return ret;
1196                         ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1197                                     fm_op.rq_steer.rq_index);
1198                         break;
1199                 }
1200                 case RTE_FLOW_ACTION_TYPE_DROP: {
1201                         if (overlap & FATE)
1202                                 goto unsupported;
1203                         overlap |= FATE;
1204                         memset(&fm_op, 0, sizeof(fm_op));
1205                         fm_op.fa_op = FMOP_DROP;
1206                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1207                         if (ret)
1208                                 return ret;
1209                         ENICPMD_LOG(DEBUG, "create DROP action");
1210                         break;
1211                 }
1212                 case RTE_FLOW_ACTION_TYPE_COUNT: {
1213                         if (overlap & COUNT)
1214                                 goto unsupported;
1215                         overlap |= COUNT;
1216                         /* Count is associated with entry not action on VIC. */
1217                         fmt->ftm_flags |= FMEF_COUNTER;
1218                         break;
1219                 }
1220                 case RTE_FLOW_ACTION_TYPE_RSS: {
1221                         const struct rte_flow_action_rss *rss = actions->conf;
1222                         bool allow;
1223                         uint16_t i;
1224
1225                         /*
1226                          * If fate other than QUEUE or RSS, fail. Multiple
1227                          * rss and queue actions are ok.
1228                          */
1229                         if ((overlap & FATE) && first_rq)
1230                                 goto unsupported;
1231                         first_rq = false;
1232                         overlap |= FATE;
1233
1234                         /*
1235                          * Hardware only supports RSS actions on outer level
1236                          * with default type and function. Queues must be
1237                          * sequential.
1238                          */
1239                         allow = rss->func == RTE_ETH_HASH_FUNCTION_DEFAULT &&
1240                                 rss->level == 0 && (rss->types == 0 ||
1241                                 rss->types == enic->rss_hf) &&
1242                                 rss->queue_num <= enic->rq_count &&
1243                                 rss->queue[rss->queue_num - 1] < enic->rq_count;
1244
1245
1246                         /* Identity queue map needs to be sequential */
1247                         for (i = 1; i < rss->queue_num; i++)
1248                                 allow = allow && (rss->queue[i] ==
1249                                         rss->queue[i - 1] + 1);
1250                         if (!allow)
1251                                 goto unsupported;
1252
1253                         memset(&fm_op, 0, sizeof(fm_op));
1254                         fm_op.fa_op = FMOP_RQ_STEER;
1255                         fm_op.rq_steer.rq_index =
1256                                 enic_rte_rq_idx_to_sop_idx(rss->queue[0]);
1257                         fm_op.rq_steer.rq_count = rss->queue_num;
1258                         fm_op.rq_steer.vnic_handle = vnic_h;
1259                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1260                         if (ret)
1261                                 return ret;
1262                         ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1263                                     fm_op.rq_steer.rq_index);
1264                         break;
1265                 }
1266                 case RTE_FLOW_ACTION_TYPE_PORT_ID: {
1267                         const struct rte_flow_action_port_id *port;
1268                         struct rte_pci_device *pdev;
1269                         struct rte_eth_dev *dev;
1270
1271                         port = actions->conf;
1272                         if (port->original) {
1273                                 vnic_h = 0; /* This port */
1274                                 break;
1275                         }
1276                         ENICPMD_LOG(DEBUG, "port id %u", port->id);
1277                         if (!rte_eth_dev_is_valid_port(port->id)) {
1278                                 return rte_flow_error_set(error, EINVAL,
1279                                         RTE_FLOW_ERROR_TYPE_ACTION,
1280                                         NULL, "invalid port_id");
1281                         }
1282                         dev = &rte_eth_devices[port->id];
1283                         if (!dev_is_enic(dev)) {
1284                                 return rte_flow_error_set(error, EINVAL,
1285                                         RTE_FLOW_ERROR_TYPE_ACTION,
1286                                         NULL, "port_id is not enic");
1287                         }
1288                         pdev = RTE_ETH_DEV_TO_PCI(dev);
1289                         if (enic_fm_find_vnic(enic, &pdev->addr, &vnic_h)) {
1290                                 return rte_flow_error_set(error, EINVAL,
1291                                         RTE_FLOW_ERROR_TYPE_ACTION,
1292                                         NULL, "port_id is not vnic");
1293                         }
1294                         break;
1295                 }
1296                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: {
1297                         if (overlap & DECAP)
1298                                 goto unsupported;
1299                         overlap |= DECAP;
1300
1301                         ret = enic_fm_copy_vxlan_decap(fm, fmt, actions,
1302                                 error);
1303                         if (ret != 0)
1304                                 return ret;
1305                         break;
1306                 }
1307                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: {
1308                         const struct rte_flow_action_vxlan_encap *encap;
1309
1310                         encap = actions->conf;
1311                         if (overlap & ENCAP)
1312                                 goto unsupported;
1313                         overlap |= ENCAP;
1314                         ret = enic_fm_copy_vxlan_encap(fm, encap->definition,
1315                                 error);
1316                         if (ret != 0)
1317                                 return ret;
1318                         break;
1319                 }
1320                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: {
1321                         memset(&fm_op, 0, sizeof(fm_op));
1322                         fm_op.fa_op = FMOP_POP_VLAN;
1323                         ret = enic_fm_append_action_op(fm, &fm_op, error);
1324                         if (ret)
1325                                 return ret;
1326                         break;
1327                 }
1328                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: {
1329                         const struct rte_flow_action_of_push_vlan *vlan;
1330
1331                         if (overlap & PASSTHRU)
1332                                 goto unsupported;
1333                         vlan = actions->conf;
1334                         if (vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN)) {
1335                                 return rte_flow_error_set(error, EINVAL,
1336                                         RTE_FLOW_ERROR_TYPE_ACTION,
1337                                         NULL, "unexpected push_vlan ethertype");
1338                         }
1339                         overlap |= PUSH_VLAN;
1340                         need_ovlan_action = true;
1341                         break;
1342                 }
1343                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: {
1344                         const struct rte_flow_action_of_set_vlan_pcp *pcp;
1345
1346                         pcp = actions->conf;
1347                         if (pcp->vlan_pcp > 7) {
1348                                 return rte_flow_error_set(error, EINVAL,
1349                                         RTE_FLOW_ERROR_TYPE_ACTION,
1350                                         NULL, "invalid vlan_pcp");
1351                         }
1352                         need_ovlan_action = true;
1353                         ovlan |= ((uint16_t)pcp->vlan_pcp) << 13;
1354                         break;
1355                 }
1356                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: {
1357                         const struct rte_flow_action_of_set_vlan_vid *vid;
1358
1359                         vid = actions->conf;
1360                         need_ovlan_action = true;
1361                         ovlan |= rte_be_to_cpu_16(vid->vlan_vid);
1362                         break;
1363                 }
1364                 default:
1365                         goto unsupported;
1366                 }
1367         }
1368
1369         if (!(overlap & (FATE | PASSTHRU | COUNT)))
1370                 goto unsupported;
1371         if (need_ovlan_action) {
1372                 memset(&fm_op, 0, sizeof(fm_op));
1373                 fm_op.fa_op = FMOP_SET_OVLAN;
1374                 fm_op.ovlan.vlan = ovlan;
1375                 ret = enic_fm_append_action_op(fm, &fm_op, error);
1376                 if (ret)
1377                         return ret;
1378         }
1379         memset(&fm_op, 0, sizeof(fm_op));
1380         fm_op.fa_op = FMOP_END;
1381         ret = enic_fm_append_action_op(fm, &fm_op, error);
1382         if (ret)
1383                 return ret;
1384         enic_fm_reorder_action_op(fm);
1385         return 0;
1386
1387 unsupported:
1388         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
1389                                   NULL, "enic: unsupported action");
1390 }
1391
1392 /** Check if the action is supported */
1393 static int
1394 enic_fm_match_action(const struct rte_flow_action *action,
1395                      const enum rte_flow_action_type *supported_actions)
1396 {
1397         for (; *supported_actions != RTE_FLOW_ACTION_TYPE_END;
1398              supported_actions++) {
1399                 if (action->type == *supported_actions)
1400                         return 1;
1401         }
1402         return 0;
1403 }
1404
1405 /* Debug function to dump internal NIC action structure. */
1406 static void
1407 enic_fm_dump_tcam_actions(const struct fm_action *fm_action)
1408 {
1409         /* Manually keep in sync with FMOP commands */
1410         const char *fmop_str[FMOP_OP_MAX] = {
1411                 [FMOP_END] = "end",
1412                 [FMOP_DROP] = "drop",
1413                 [FMOP_RQ_STEER] = "steer",
1414                 [FMOP_EXACT_MATCH] = "exmatch",
1415                 [FMOP_MARK] = "mark",
1416                 [FMOP_EXT_MARK] = "ext_mark",
1417                 [FMOP_TAG] = "tag",
1418                 [FMOP_EG_HAIRPIN] = "eg_hairpin",
1419                 [FMOP_IG_HAIRPIN] = "ig_hairpin",
1420                 [FMOP_ENCAP_IVLAN] = "encap_ivlan",
1421                 [FMOP_ENCAP_NOIVLAN] = "encap_noivlan",
1422                 [FMOP_ENCAP] = "encap",
1423                 [FMOP_SET_OVLAN] = "set_ovlan",
1424                 [FMOP_DECAP_NOSTRIP] = "decap_nostrip",
1425                 [FMOP_DECAP_STRIP] = "decap_strip",
1426                 [FMOP_POP_VLAN] = "pop_vlan",
1427                 [FMOP_SET_EGPORT] = "set_egport",
1428                 [FMOP_RQ_STEER_ONLY] = "rq_steer_only",
1429                 [FMOP_SET_ENCAP_VLAN] = "set_encap_vlan",
1430                 [FMOP_EMIT] = "emit",
1431                 [FMOP_MODIFY] = "modify",
1432         };
1433         const struct fm_action_op *op = &fm_action->fma_action_ops[0];
1434         char buf[128], *bp = buf;
1435         const char *op_str;
1436         int i, n, buf_len;
1437
1438         buf[0] = '\0';
1439         buf_len = sizeof(buf);
1440         for (i = 0; i < FM_ACTION_OP_MAX; i++) {
1441                 if (op->fa_op == FMOP_END)
1442                         break;
1443                 if (op->fa_op >= FMOP_OP_MAX)
1444                         op_str = "unknown";
1445                 else
1446                         op_str = fmop_str[op->fa_op];
1447                 n = snprintf(bp, buf_len, "%s,", op_str);
1448                 if (n > 0 && n < buf_len) {
1449                         bp += n;
1450                         buf_len -= n;
1451                 }
1452                 op++;
1453         }
1454         /* Remove trailing comma */
1455         if (buf[0])
1456                 *(bp - 1) = '\0';
1457         ENICPMD_LOG(DEBUG, "       Acions: %s", buf);
1458 }
1459
1460 static int
1461 bits_to_str(uint32_t bits, const char *strings[], int max,
1462             char *buf, int buf_len)
1463 {
1464         int i, n = 0, len = 0;
1465
1466         for (i = 0; i < max; i++) {
1467                 if (bits & (1 << i)) {
1468                         n = snprintf(buf, buf_len, "%s,", strings[i]);
1469                         if (n > 0 && n < buf_len) {
1470                                 buf += n;
1471                                 buf_len -= n;
1472                                 len += n;
1473                         }
1474                 }
1475         }
1476         /* Remove trailing comma */
1477         if (len) {
1478                 *(buf - 1) = '\0';
1479                 len--;
1480         }
1481         return len;
1482 }
1483
1484 /* Debug function to dump internal NIC filter structure. */
1485 static void
1486 __enic_fm_dump_tcam_match(const struct fm_header_set *fk_hdrset, char *buf,
1487                           int buf_len)
1488 {
1489         /* Manually keep in sync with FKM_BITS */
1490         const char *fm_fkm_str[FKM_BIT_COUNT] = {
1491                 [FKM_QTAG_BIT] = "qtag",
1492                 [FKM_CMD_BIT] = "cmd",
1493                 [FKM_IPV4_BIT] = "ip4",
1494                 [FKM_IPV6_BIT] = "ip6",
1495                 [FKM_ROCE_BIT] = "roce",
1496                 [FKM_UDP_BIT] = "udp",
1497                 [FKM_TCP_BIT] = "tcp",
1498                 [FKM_TCPORUDP_BIT] = "tcpportudp",
1499                 [FKM_IPFRAG_BIT] = "ipfrag",
1500                 [FKM_NVGRE_BIT] = "nvgre",
1501                 [FKM_VXLAN_BIT] = "vxlan",
1502                 [FKM_GENEVE_BIT] = "geneve",
1503                 [FKM_NSH_BIT] = "nsh",
1504                 [FKM_ROCEV2_BIT] = "rocev2",
1505                 [FKM_VLAN_PRES_BIT] = "vlan_pres",
1506                 [FKM_IPOK_BIT] = "ipok",
1507                 [FKM_L4OK_BIT] = "l4ok",
1508                 [FKM_ROCEOK_BIT] = "roceok",
1509                 [FKM_FCSOK_BIT] = "fcsok",
1510                 [FKM_EG_SPAN_BIT] = "eg_span",
1511                 [FKM_IG_SPAN_BIT] = "ig_span",
1512                 [FKM_EG_HAIRPINNED_BIT] = "eg_hairpinned",
1513         };
1514         /* Manually keep in sync with FKH_BITS */
1515         const char *fm_fkh_str[FKH_BIT_COUNT] = {
1516                 [FKH_ETHER_BIT] = "eth",
1517                 [FKH_QTAG_BIT] = "qtag",
1518                 [FKH_L2RAW_BIT] = "l2raw",
1519                 [FKH_IPV4_BIT] = "ip4",
1520                 [FKH_IPV6_BIT] = "ip6",
1521                 [FKH_L3RAW_BIT] = "l3raw",
1522                 [FKH_UDP_BIT] = "udp",
1523                 [FKH_TCP_BIT] = "tcp",
1524                 [FKH_ICMP_BIT] = "icmp",
1525                 [FKH_VXLAN_BIT] = "vxlan",
1526                 [FKH_L4RAW_BIT] = "l4raw",
1527         };
1528         uint32_t fkh_bits = fk_hdrset->fk_header_select;
1529         uint32_t fkm_bits = fk_hdrset->fk_metadata;
1530         int n;
1531
1532         if (!fkm_bits && !fkh_bits)
1533                 return;
1534         n = snprintf(buf, buf_len, "metadata(");
1535         if (n > 0 && n < buf_len) {
1536                 buf += n;
1537                 buf_len -= n;
1538         }
1539         n = bits_to_str(fkm_bits, fm_fkm_str, FKM_BIT_COUNT, buf, buf_len);
1540         if (n > 0 && n < buf_len) {
1541                 buf += n;
1542                 buf_len -= n;
1543         }
1544         n = snprintf(buf, buf_len, ") valid hdr fields(");
1545         if (n > 0 && n < buf_len) {
1546                 buf += n;
1547                 buf_len -= n;
1548         }
1549         n = bits_to_str(fkh_bits, fm_fkh_str, FKH_BIT_COUNT, buf, buf_len);
1550         if (n > 0 && n < buf_len) {
1551                 buf += n;
1552                 buf_len -= n;
1553         }
1554         snprintf(buf, buf_len, ")");
1555 }
1556
1557 static void
1558 enic_fm_dump_tcam_match(const struct fm_tcam_match_entry *match,
1559                         uint8_t ingress)
1560 {
1561         char buf[256];
1562
1563         memset(buf, 0, sizeof(buf));
1564         __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[0],
1565                                   buf, sizeof(buf));
1566         ENICPMD_LOG(DEBUG, " TCAM %s Outer: %s %scounter",
1567                     (ingress) ? "IG" : "EG", buf,
1568                     (match->ftm_flags & FMEF_COUNTER) ? "" : "no ");
1569         memset(buf, 0, sizeof(buf));
1570         __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[1],
1571                                   buf, sizeof(buf));
1572         if (buf[0])
1573                 ENICPMD_LOG(DEBUG, "         Inner: %s", buf);
1574 }
1575
1576 /* Debug function to dump internal NIC flow structures. */
1577 static void
1578 enic_fm_dump_tcam_entry(const struct fm_tcam_match_entry *fm_match,
1579                         const struct fm_action *fm_action,
1580                         uint8_t ingress)
1581 {
1582         if (!rte_log_can_log(enic_pmd_logtype, RTE_LOG_DEBUG))
1583                 return;
1584         enic_fm_dump_tcam_match(fm_match, ingress);
1585         enic_fm_dump_tcam_actions(fm_action);
1586 }
1587
1588 static int
1589 enic_fm_flow_parse(struct enic_flowman *fm,
1590                    const struct rte_flow_attr *attrs,
1591                    const struct rte_flow_item pattern[],
1592                    const struct rte_flow_action actions[],
1593                    struct rte_flow_error *error)
1594 {
1595         const struct rte_flow_action *action;
1596         unsigned int ret;
1597         static const enum rte_flow_action_type *sa;
1598
1599         ENICPMD_FUNC_TRACE();
1600         ret = 0;
1601         if (!pattern) {
1602                 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM,
1603                                    NULL, "no pattern specified");
1604                 return -rte_errno;
1605         }
1606
1607         if (!actions) {
1608                 rte_flow_error_set(error, EINVAL,
1609                                    RTE_FLOW_ERROR_TYPE_ACTION_NUM,
1610                                    NULL, "no action specified");
1611                 return -rte_errno;
1612         }
1613
1614         if (attrs) {
1615                 if (attrs->priority) {
1616                         rte_flow_error_set(error, ENOTSUP,
1617                                            RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1618                                            NULL,
1619                                            "priorities are not supported");
1620                         return -rte_errno;
1621                 } else if (attrs->transfer) {
1622                         rte_flow_error_set(error, ENOTSUP,
1623                                            RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1624                                            NULL,
1625                                            "transfer is not supported");
1626                         return -rte_errno;
1627                 } else if (attrs->ingress && attrs->egress) {
1628                         rte_flow_error_set(error, ENOTSUP,
1629                                            RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1630                                            NULL,
1631                                            "bidirectional rules not supported");
1632                         return -rte_errno;
1633                 }
1634
1635         } else {
1636                 rte_flow_error_set(error, EINVAL,
1637                                    RTE_FLOW_ERROR_TYPE_ATTR,
1638                                    NULL, "no attribute specified");
1639                 return -rte_errno;
1640         }
1641
1642         /* Verify Actions. */
1643         sa = (attrs->ingress) ? enic_fm_supported_ig_actions :
1644              enic_fm_supported_eg_actions;
1645         for (action = &actions[0]; action->type != RTE_FLOW_ACTION_TYPE_END;
1646              action++) {
1647                 if (action->type == RTE_FLOW_ACTION_TYPE_VOID)
1648                         continue;
1649                 else if (!enic_fm_match_action(action, sa))
1650                         break;
1651         }
1652         if (action->type != RTE_FLOW_ACTION_TYPE_END) {
1653                 rte_flow_error_set(error, EPERM, RTE_FLOW_ERROR_TYPE_ACTION,
1654                                    action, "invalid action");
1655                 return -rte_errno;
1656         }
1657         ret = enic_fm_copy_entry(fm, pattern, error);
1658         if (ret)
1659                 return ret;
1660         ret = enic_fm_copy_action(fm, actions, attrs->ingress, error);
1661         return ret;
1662 }
1663
1664 static void
1665 enic_fm_counter_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
1666 {
1667         if (!fm_flow->counter_valid)
1668                 return;
1669         SLIST_INSERT_HEAD(&fm->counters, fm_flow->counter, next);
1670         fm_flow->counter_valid = false;
1671 }
1672
1673 static int
1674 enic_fm_more_counters(struct enic_flowman *fm)
1675 {
1676         struct enic_fm_counter *new_stack;
1677         struct enic_fm_counter *ctrs;
1678         struct enic *enic;
1679         int i, rc;
1680         uint64_t args[2];
1681
1682         ENICPMD_FUNC_TRACE();
1683         enic = fm->enic;
1684         new_stack = rte_realloc(fm->counter_stack, (fm->counters_alloced +
1685                                 FM_COUNTERS_EXPAND) *
1686                                 sizeof(struct enic_fm_counter), 0);
1687         if (new_stack == NULL) {
1688                 ENICPMD_LOG(ERR, "cannot alloc counter memory");
1689                 return -ENOMEM;
1690         }
1691         fm->counter_stack = new_stack;
1692
1693         args[0] = FM_COUNTER_BRK;
1694         args[1] = fm->counters_alloced + FM_COUNTERS_EXPAND;
1695         rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
1696         if (rc != 0) {
1697                 ENICPMD_LOG(ERR, "cannot alloc counters rc=%d", rc);
1698                 return rc;
1699         }
1700         ctrs = (struct enic_fm_counter *)fm->counter_stack +
1701                 fm->counters_alloced;
1702         for (i = 0; i < FM_COUNTERS_EXPAND; i++, ctrs++) {
1703                 ctrs->handle = fm->counters_alloced + i;
1704                 SLIST_INSERT_HEAD(&fm->counters, ctrs, next);
1705         }
1706         fm->counters_alloced += FM_COUNTERS_EXPAND;
1707         ENICPMD_LOG(DEBUG, "%u counters allocated, total: %u",
1708                     FM_COUNTERS_EXPAND, fm->counters_alloced);
1709         return 0;
1710 }
1711
1712 static int
1713 enic_fm_counter_zero(struct enic_flowman *fm, struct enic_fm_counter *c)
1714 {
1715         struct enic *enic;
1716         uint64_t args[3];
1717         int ret;
1718
1719         ENICPMD_FUNC_TRACE();
1720         enic = fm->enic;
1721         args[0] = FM_COUNTER_QUERY;
1722         args[1] = c->handle;
1723         args[2] = 1; /* clear */
1724         ret = vnic_dev_flowman_cmd(enic->vdev, args, 3);
1725         if (ret) {
1726                 ENICPMD_LOG(ERR, "counter init: rc=%d handle=0x%x",
1727                             ret, c->handle);
1728                 return ret;
1729         }
1730         return 0;
1731 }
1732
1733 static int
1734 enic_fm_counter_alloc(struct enic_flowman *fm, struct rte_flow_error *error,
1735                       struct enic_fm_counter **ctr)
1736 {
1737         struct enic_fm_counter *c;
1738         int ret;
1739
1740         ENICPMD_FUNC_TRACE();
1741         *ctr = NULL;
1742         if (SLIST_EMPTY(&fm->counters)) {
1743                 ret = enic_fm_more_counters(fm);
1744                 if (ret)
1745                         return rte_flow_error_set(error, -ret,
1746                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1747                                 NULL, "enic: out of counters");
1748         }
1749         c = SLIST_FIRST(&fm->counters);
1750         SLIST_REMOVE_HEAD(&fm->counters, next);
1751         *ctr = c;
1752         return 0;
1753 }
1754
1755 static int
1756 enic_fm_action_free(struct enic_flowman *fm, uint64_t handle)
1757 {
1758         uint64_t args[2];
1759         int rc;
1760
1761         ENICPMD_FUNC_TRACE();
1762         args[0] = FM_ACTION_FREE;
1763         args[1] = handle;
1764         rc = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
1765         if (rc)
1766                 ENICPMD_LOG(ERR, "cannot free action: rc=%d handle=0x%" PRIx64,
1767                             rc, handle);
1768         return rc;
1769 }
1770
1771 static int
1772 enic_fm_entry_free(struct enic_flowman *fm, uint64_t handle)
1773 {
1774         uint64_t args[2];
1775         int rc;
1776
1777         ENICPMD_FUNC_TRACE();
1778         args[0] = FM_MATCH_ENTRY_REMOVE;
1779         args[1] = handle;
1780         rc = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
1781         if (rc)
1782                 ENICPMD_LOG(ERR, "cannot free match entry: rc=%d"
1783                             " handle=0x%" PRIx64, rc, handle);
1784         return rc;
1785 }
1786
1787 static struct enic_fm_jump_flow *
1788 find_jump_flow(struct enic_flowman *fm, uint32_t group)
1789 {
1790         struct enic_fm_jump_flow *j;
1791
1792         ENICPMD_FUNC_TRACE();
1793         TAILQ_FOREACH(j, &fm->jump_list, list) {
1794                 if (j->group == group)
1795                         return j;
1796         }
1797         return NULL;
1798 }
1799
1800 static void
1801 remove_jump_flow(struct enic_flowman *fm, struct rte_flow *flow)
1802 {
1803         struct enic_fm_jump_flow *j;
1804
1805         ENICPMD_FUNC_TRACE();
1806         TAILQ_FOREACH(j, &fm->jump_list, list) {
1807                 if (j->flow == flow) {
1808                         TAILQ_REMOVE(&fm->jump_list, j, list);
1809                         free(j);
1810                         return;
1811                 }
1812         }
1813 }
1814
1815 static int
1816 save_jump_flow(struct enic_flowman *fm,
1817                struct rte_flow *flow,
1818                uint32_t group,
1819                struct fm_tcam_match_entry *match,
1820                struct fm_action *action)
1821 {
1822         struct enic_fm_jump_flow *j;
1823
1824         ENICPMD_FUNC_TRACE();
1825         j = calloc(1, sizeof(struct enic_fm_jump_flow));
1826         if (j == NULL)
1827                 return -ENOMEM;
1828         j->flow = flow;
1829         j->group = group;
1830         j->match = *match;
1831         j->action = *action;
1832         TAILQ_INSERT_HEAD(&fm->jump_list, j, list);
1833         ENICPMD_LOG(DEBUG, "saved jump flow: flow=%p group=%u", flow, group);
1834         return 0;
1835 }
1836
1837 static void
1838 __enic_fm_flow_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
1839 {
1840         if (fm_flow->entry_handle != FM_INVALID_HANDLE) {
1841                 enic_fm_entry_free(fm, fm_flow->entry_handle);
1842                 fm_flow->entry_handle = FM_INVALID_HANDLE;
1843         }
1844         if (fm_flow->action_handle != FM_INVALID_HANDLE) {
1845                 enic_fm_action_free(fm, fm_flow->action_handle);
1846                 fm_flow->action_handle = FM_INVALID_HANDLE;
1847         }
1848         enic_fm_counter_free(fm, fm_flow);
1849         if (fm_flow->fet) {
1850                 enic_fet_put(fm, fm_flow->fet);
1851                 fm_flow->fet = NULL;
1852         }
1853 }
1854
1855 static void
1856 enic_fm_flow_free(struct enic_flowman *fm, struct rte_flow *flow)
1857 {
1858         if (flow->fm->fet && flow->fm->fet->default_key)
1859                 remove_jump_flow(fm, flow);
1860         __enic_fm_flow_free(fm, flow->fm);
1861         free(flow->fm);
1862         free(flow);
1863 }
1864
1865 static int
1866 enic_fm_add_tcam_entry(struct enic_flowman *fm,
1867                        struct fm_tcam_match_entry *match_in,
1868                        uint64_t *entry_handle,
1869                        uint8_t ingress,
1870                        struct rte_flow_error *error)
1871 {
1872         struct fm_tcam_match_entry *ftm;
1873         uint64_t args[3];
1874         int ret;
1875
1876         ENICPMD_FUNC_TRACE();
1877         /* Copy entry to the command buffer */
1878         ftm = &fm->cmd.va->fm_tcam_match_entry;
1879         memcpy(ftm, match_in, sizeof(*ftm));
1880         /* Add TCAM entry */
1881         args[0] = FM_TCAM_ENTRY_INSTALL;
1882         args[1] = ingress ? fm->ig_tcam_hndl : fm->eg_tcam_hndl;
1883         args[2] = fm->cmd.pa;
1884         ret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 3);
1885         if (ret != 0) {
1886                 ENICPMD_LOG(ERR, "cannot add %s TCAM entry: rc=%d",
1887                             ingress ? "ingress" : "egress", ret);
1888                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1889                         NULL, "enic: devcmd(tcam-entry-install)");
1890                 return ret;
1891         }
1892         ENICPMD_LOG(DEBUG, "installed %s TCAM entry: handle=0x%" PRIx64,
1893                     ingress ? "ingress" : "egress", (uint64_t)args[0]);
1894         *entry_handle = args[0];
1895         return 0;
1896 }
1897
1898 static int
1899 enic_fm_add_exact_entry(struct enic_flowman *fm,
1900                         struct fm_tcam_match_entry *match_in,
1901                         uint64_t *entry_handle,
1902                         struct enic_fm_fet *fet,
1903                         struct rte_flow_error *error)
1904 {
1905         struct fm_exact_match_entry *fem;
1906         uint64_t args[3];
1907         int ret;
1908
1909         ENICPMD_FUNC_TRACE();
1910         /* The new entry must have the table's key */
1911         if (memcmp(fet->key.fk_hdrset, match_in->ftm_mask.fk_hdrset,
1912                    sizeof(struct fm_header_set) * FM_HDRSET_MAX)) {
1913                 return rte_flow_error_set(error, EINVAL,
1914                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1915                         "enic: key does not match group's key");
1916         }
1917
1918         /* Copy entry to the command buffer */
1919         fem = &fm->cmd.va->fm_exact_match_entry;
1920         /*
1921          * Translate TCAM entry to exact entry. As is only need to drop
1922          * position and mask. The mask is part of the exact match table.
1923          * Position (aka priority) is not supported in the exact match table.
1924          */
1925         fem->fem_data = match_in->ftm_data;
1926         fem->fem_flags = match_in->ftm_flags;
1927         fem->fem_action = match_in->ftm_action;
1928         fem->fem_counter = match_in->ftm_counter;
1929
1930         /* Add exact entry */
1931         args[0] = FM_EXACT_ENTRY_INSTALL;
1932         args[1] = fet->handle;
1933         args[2] = fm->cmd.pa;
1934         ret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 3);
1935         if (ret != 0) {
1936                 ENICPMD_LOG(ERR, "cannot add %s exact entry: group=%u",
1937                             fet->ingress ? "ingress" : "egress", fet->group);
1938                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1939                         NULL, "enic: devcmd(exact-entry-install)");
1940                 return ret;
1941         }
1942         ENICPMD_LOG(DEBUG, "installed %s exact entry: group=%u"
1943                     " handle=0x%" PRIx64,
1944                     fet->ingress ? "ingress" : "egress", fet->group,
1945                     (uint64_t)args[0]);
1946         *entry_handle = args[0];
1947         return 0;
1948 }
1949
1950 /* Push match-action to the NIC. */
1951 static int
1952 __enic_fm_flow_add_entry(struct enic_flowman *fm,
1953                          struct enic_fm_flow *fm_flow,
1954                          struct fm_tcam_match_entry *match_in,
1955                          struct fm_action *action_in,
1956                          uint32_t group,
1957                          uint8_t ingress,
1958                          struct rte_flow_error *error)
1959 {
1960         struct enic_fm_counter *ctr;
1961         struct fm_action *fma;
1962         uint64_t action_h;
1963         uint64_t entry_h;
1964         uint64_t args[3];
1965         int ret;
1966
1967         ENICPMD_FUNC_TRACE();
1968         /* Allocate action. */
1969         fma = &fm->cmd.va->fm_action;
1970         memcpy(fma, action_in, sizeof(*fma));
1971         args[0] = FM_ACTION_ALLOC;
1972         args[1] = fm->cmd.pa;
1973         ret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
1974         if (ret != 0) {
1975                 ENICPMD_LOG(ERR, "allocating TCAM table action rc=%d", ret);
1976                 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1977                         NULL, "enic: devcmd(action-alloc)");
1978                 return ret;
1979         }
1980         action_h = args[0];
1981         fm_flow->action_handle = action_h;
1982         match_in->ftm_action = action_h;
1983         ENICPMD_LOG(DEBUG, "action allocated: handle=0x%" PRIx64, action_h);
1984
1985         /* Allocate counter if requested. */
1986         if (match_in->ftm_flags & FMEF_COUNTER) {
1987                 ret = enic_fm_counter_alloc(fm, error, &ctr);
1988                 if (ret) /* error has been filled in */
1989                         return ret;
1990                 fm_flow->counter_valid = true;
1991                 fm_flow->counter = ctr;
1992                 match_in->ftm_counter = ctr->handle;
1993         }
1994
1995         /*
1996          * Get the group's table (either TCAM or exact match table) and
1997          * add entry to it. If we use the exact match table, the handler
1998          * will translate the TCAM entry (match_in) to the appropriate
1999          * exact match entry and use that instead.
2000          */
2001         entry_h = FM_INVALID_HANDLE;
2002         if (group == FM_TCAM_RTE_GROUP) {
2003                 ret = enic_fm_add_tcam_entry(fm, match_in, &entry_h, ingress,
2004                                              error);
2005                 if (ret)
2006                         return ret;
2007                 /* Jump action might have a ref to fet */
2008                 fm_flow->fet = fm->fet;
2009                 fm->fet = NULL;
2010         } else {
2011                 struct enic_fm_fet *fet = NULL;
2012
2013                 ret = enic_fet_get(fm, group, ingress,
2014                                    &match_in->ftm_mask, &fet, error);
2015                 if (ret)
2016                         return ret;
2017                 fm_flow->fet = fet;
2018                 ret = enic_fm_add_exact_entry(fm, match_in, &entry_h, fet,
2019                                               error);
2020                 if (ret)
2021                         return ret;
2022         }
2023         /* Clear counter after adding entry, as it requires in-use counter */
2024         if (fm_flow->counter_valid) {
2025                 ret = enic_fm_counter_zero(fm, fm_flow->counter);
2026                 if (ret)
2027                         return ret;
2028         }
2029         fm_flow->entry_handle = entry_h;
2030         return 0;
2031 }
2032
2033 /* Push match-action to the NIC. */
2034 static struct rte_flow *
2035 enic_fm_flow_add_entry(struct enic_flowman *fm,
2036                        struct fm_tcam_match_entry *match_in,
2037                        struct fm_action *action_in,
2038                        const struct rte_flow_attr *attrs,
2039                        struct rte_flow_error *error)
2040 {
2041         struct enic_fm_flow *fm_flow;
2042         struct rte_flow *flow;
2043
2044         ENICPMD_FUNC_TRACE();
2045         enic_fm_dump_tcam_entry(match_in, action_in, attrs->ingress);
2046         flow = calloc(1, sizeof(*flow));
2047         fm_flow = calloc(1, sizeof(*fm_flow));
2048         if (flow == NULL || fm_flow == NULL) {
2049                 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
2050                         NULL, "enic: cannot allocate rte_flow");
2051                 free(flow);
2052                 free(fm_flow);
2053                 return NULL;
2054         }
2055         flow->fm = fm_flow;
2056         fm_flow->action_handle = FM_INVALID_HANDLE;
2057         fm_flow->entry_handle = FM_INVALID_HANDLE;
2058         if (__enic_fm_flow_add_entry(fm, fm_flow, match_in, action_in,
2059                                      attrs->group, attrs->ingress, error)) {
2060                 enic_fm_flow_free(fm, flow);
2061                 return NULL;
2062         }
2063         return flow;
2064 }
2065
2066 static void
2067 convert_jump_flows(struct enic_flowman *fm, struct enic_fm_fet *fet,
2068                    struct rte_flow_error *error)
2069 {
2070         struct enic_fm_flow *fm_flow;
2071         struct enic_fm_jump_flow *j;
2072         struct fm_action *fma;
2073         uint32_t group;
2074
2075         ENICPMD_FUNC_TRACE();
2076         /*
2077          * Find the saved flows that should jump to the new table (fet).
2078          * Then delete the old TCAM entry that jumps to the default table,
2079          * and add a new one that jumps to the new table.
2080          */
2081         group = fet->group;
2082         j = find_jump_flow(fm, group);
2083         while (j) {
2084                 ENICPMD_LOG(DEBUG, "convert jump flow: flow=%p group=%u",
2085                             j->flow, group);
2086                 /* Delete old entry */
2087                 fm_flow = j->flow->fm;
2088                 __enic_fm_flow_free(fm, fm_flow);
2089
2090                 /* Add new entry */
2091                 fma = &j->action;
2092                 fma->fma_action_ops[0].exact.handle = fet->handle;
2093                 if (__enic_fm_flow_add_entry(fm, fm_flow, &j->match, fma,
2094                         FM_TCAM_RTE_GROUP, fet->ingress, error)) {
2095                         /* Cannot roll back changes at the moment */
2096                         ENICPMD_LOG(ERR, "cannot convert jump flow: flow=%p",
2097                                     j->flow);
2098                 } else {
2099                         fm_flow->fet = fet;
2100                         fet->ref++;
2101                         ENICPMD_LOG(DEBUG, "convert ok: group=%u ref=%u",
2102                                     fet->group, fet->ref);
2103                 }
2104
2105                 TAILQ_REMOVE(&fm->jump_list, j, list);
2106                 free(j);
2107                 j = find_jump_flow(fm, group);
2108         }
2109 }
2110
2111 static void
2112 enic_fm_open_scratch(struct enic_flowman *fm)
2113 {
2114         fm->action_op_count = 0;
2115         fm->fet = NULL;
2116         memset(&fm->tcam_entry, 0, sizeof(fm->tcam_entry));
2117         memset(&fm->action, 0, sizeof(fm->action));
2118 }
2119
2120 static void
2121 enic_fm_close_scratch(struct enic_flowman *fm)
2122 {
2123         if (fm->fet) {
2124                 enic_fet_put(fm, fm->fet);
2125                 fm->fet = NULL;
2126         }
2127         fm->action_op_count = 0;
2128 }
2129
2130 static int
2131 enic_fm_flow_validate(struct rte_eth_dev *dev,
2132                       const struct rte_flow_attr *attrs,
2133                       const struct rte_flow_item pattern[],
2134                       const struct rte_flow_action actions[],
2135                       struct rte_flow_error *error)
2136 {
2137         struct fm_tcam_match_entry *fm_tcam_entry;
2138         struct fm_action *fm_action;
2139         struct enic_flowman *fm;
2140         int ret;
2141
2142         ENICPMD_FUNC_TRACE();
2143         fm = pmd_priv(dev)->fm;
2144         if (fm == NULL)
2145                 return -ENOTSUP;
2146         enic_fm_open_scratch(fm);
2147         ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2148         if (!ret) {
2149                 fm_tcam_entry = &fm->tcam_entry;
2150                 fm_action = &fm->action;
2151                 enic_fm_dump_tcam_entry(fm_tcam_entry, fm_action,
2152                                         attrs->ingress);
2153         }
2154         enic_fm_close_scratch(fm);
2155         return ret;
2156 }
2157
2158 static int
2159 enic_fm_flow_query_count(struct rte_eth_dev *dev,
2160                          struct rte_flow *flow, void *data,
2161                          struct rte_flow_error *error)
2162 {
2163         struct rte_flow_query_count *query;
2164         struct enic_fm_flow *fm_flow;
2165         struct enic *enic;
2166         uint64_t args[3];
2167         int rc;
2168
2169         ENICPMD_FUNC_TRACE();
2170         enic = pmd_priv(dev);
2171         query = data;
2172         fm_flow = flow->fm;
2173         if (!fm_flow->counter_valid)
2174                 return rte_flow_error_set(error, ENOTSUP,
2175                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2176                         "enic: flow does not have counter");
2177
2178         args[0] = FM_COUNTER_QUERY;
2179         args[1] = fm_flow->counter->handle;
2180         args[2] = query->reset;
2181         rc = vnic_dev_flowman_cmd(enic->vdev, args, 3);
2182         if (rc) {
2183                 ENICPMD_LOG(ERR, "cannot query counter: rc=%d handle=0x%x",
2184                             rc, fm_flow->counter->handle);
2185                 return rc;
2186         }
2187         query->hits_set = 1;
2188         query->hits = args[0];
2189         query->bytes_set = 1;
2190         query->bytes = args[1];
2191         return 0;
2192 }
2193
2194 static int
2195 enic_fm_flow_query(struct rte_eth_dev *dev,
2196                    struct rte_flow *flow,
2197                    const struct rte_flow_action *actions,
2198                    void *data,
2199                    struct rte_flow_error *error)
2200 {
2201         int ret = 0;
2202
2203         ENICPMD_FUNC_TRACE();
2204         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2205                 switch (actions->type) {
2206                 case RTE_FLOW_ACTION_TYPE_VOID:
2207                         break;
2208                 case RTE_FLOW_ACTION_TYPE_COUNT:
2209                         ret = enic_fm_flow_query_count(dev, flow, data, error);
2210                         break;
2211                 default:
2212                         return rte_flow_error_set(error, ENOTSUP,
2213                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2214                                                   actions,
2215                                                   "action not supported");
2216                 }
2217                 if (ret < 0)
2218                         return ret;
2219         }
2220         return 0;
2221 }
2222
2223 static struct rte_flow *
2224 enic_fm_flow_create(struct rte_eth_dev *dev,
2225                     const struct rte_flow_attr *attrs,
2226                     const struct rte_flow_item pattern[],
2227                     const struct rte_flow_action actions[],
2228                     struct rte_flow_error *error)
2229 {
2230         struct fm_tcam_match_entry *fm_tcam_entry;
2231         struct fm_action *fm_action;
2232         struct enic_flowman *fm;
2233         struct enic_fm_fet *fet;
2234         struct rte_flow *flow;
2235         struct enic *enic;
2236         int ret;
2237
2238         ENICPMD_FUNC_TRACE();
2239         enic = pmd_priv(dev);
2240         fm = enic->fm;
2241         if (fm == NULL) {
2242                 rte_flow_error_set(error, ENOTSUP,
2243                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2244                         "flowman is not initialized");
2245                 return NULL;
2246         }
2247         enic_fm_open_scratch(fm);
2248         flow = NULL;
2249         ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2250         if (ret < 0)
2251                 goto error_with_scratch;
2252         fm_tcam_entry = &fm->tcam_entry;
2253         fm_action = &fm->action;
2254         flow = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2255                                       attrs, error);
2256         if (flow) {
2257                 LIST_INSERT_HEAD(&enic->flows, flow, next);
2258                 fet = flow->fm->fet;
2259                 if (fet && fet->default_key) {
2260                         /*
2261                          * Jump to non-existent group? Save the relevant info
2262                          * so we can convert this flow when that group
2263                          * materializes.
2264                          */
2265                         save_jump_flow(fm, flow, fet->group,
2266                                        fm_tcam_entry, fm_action);
2267                 } else if (fet && fet->ref == 1) {
2268                         /*
2269                          * A new table is created. Convert the saved flows
2270                          * that should jump to this group.
2271                          */
2272                         convert_jump_flows(fm, fet, error);
2273                 }
2274         }
2275
2276 error_with_scratch:
2277         enic_fm_close_scratch(fm);
2278         return flow;
2279 }
2280
2281 static int
2282 enic_fm_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
2283                      __rte_unused struct rte_flow_error *error)
2284 {
2285         struct enic *enic = pmd_priv(dev);
2286
2287         ENICPMD_FUNC_TRACE();
2288         if (enic->fm == NULL)
2289                 return 0;
2290         LIST_REMOVE(flow, next);
2291         enic_fm_flow_free(enic->fm, flow);
2292         return 0;
2293 }
2294
2295 static int
2296 enic_fm_flow_flush(struct rte_eth_dev *dev,
2297                    __rte_unused struct rte_flow_error *error)
2298 {
2299         struct enic_fm_flow *fm_flow;
2300         struct enic_flowman *fm;
2301         struct rte_flow *flow;
2302         struct enic *enic = pmd_priv(dev);
2303
2304         ENICPMD_FUNC_TRACE();
2305         if (enic->fm == NULL)
2306                 return 0;
2307         fm = enic->fm;
2308         while (!LIST_EMPTY(&enic->flows)) {
2309                 flow = LIST_FIRST(&enic->flows);
2310                 fm_flow = flow->fm;
2311                 LIST_REMOVE(flow, next);
2312                 /*
2313                  * If tables are null, then vNIC is closing, and the firmware
2314                  * has already cleaned up flowman state. So do not try to free
2315                  * resources, as it only causes errors.
2316                  */
2317                 if (fm->ig_tcam_hndl == FM_INVALID_HANDLE) {
2318                         fm_flow->entry_handle = FM_INVALID_HANDLE;
2319                         fm_flow->action_handle = FM_INVALID_HANDLE;
2320                         fm_flow->fet = NULL;
2321                 }
2322                 enic_fm_flow_free(fm, flow);
2323         }
2324         return 0;
2325 }
2326
2327 static int
2328 enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle)
2329 {
2330         uint64_t args[2];
2331         int rc;
2332
2333         args[0] = FM_MATCH_TABLE_FREE;
2334         args[1] = handle;
2335         rc = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
2336         if (rc)
2337                 ENICPMD_LOG(ERR, "cannot free table: rc=%d handle=0x%" PRIx64,
2338                             rc, handle);
2339         return rc;
2340 }
2341
2342 static int
2343 enic_fm_tcam_tbl_alloc(struct enic_flowman *fm, uint32_t direction,
2344                         uint32_t max_entries, uint64_t *handle)
2345 {
2346         struct fm_tcam_match_table *tcam_tbl;
2347         struct enic *enic;
2348         uint64_t args[2];
2349         int rc;
2350
2351         ENICPMD_FUNC_TRACE();
2352         enic = fm->enic;
2353         tcam_tbl = &fm->cmd.va->fm_tcam_match_table;
2354         tcam_tbl->ftt_direction = direction;
2355         tcam_tbl->ftt_stage = FM_STAGE_LAST;
2356         tcam_tbl->ftt_max_entries = max_entries;
2357         args[0] = FM_TCAM_TABLE_ALLOC;
2358         args[1] = fm->cmd.pa;
2359         rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
2360         if (rc) {
2361                 ENICPMD_LOG(ERR, "cannot alloc %s TCAM table: rc=%d",
2362                             (direction == FM_INGRESS) ? "IG" : "EG", rc);
2363                 return rc;
2364         }
2365         *handle = args[0];
2366         ENICPMD_LOG(DEBUG, "%s TCAM table allocated, handle=0x%" PRIx64,
2367                     (direction == FM_INGRESS) ? "IG" : "EG", *handle);
2368         return 0;
2369 }
2370
2371 static int
2372 enic_fm_init_counters(struct enic_flowman *fm)
2373 {
2374         ENICPMD_FUNC_TRACE();
2375         SLIST_INIT(&fm->counters);
2376         return enic_fm_more_counters(fm);
2377 }
2378
2379 static void
2380 enic_fm_free_all_counters(struct enic_flowman *fm)
2381 {
2382         struct enic *enic;
2383         uint64_t args[2];
2384         int rc;
2385
2386         enic = fm->enic;
2387         args[0] = FM_COUNTER_BRK;
2388         args[1] = 0;
2389         rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
2390         if (rc != 0)
2391                 ENICPMD_LOG(ERR, "cannot free counters: rc=%d", rc);
2392         rte_free(fm->counter_stack);
2393 }
2394
2395 static int
2396 enic_fm_alloc_tcam_tables(struct enic_flowman *fm)
2397 {
2398         int rc;
2399
2400         ENICPMD_FUNC_TRACE();
2401         rc = enic_fm_tcam_tbl_alloc(fm, FM_INGRESS, FM_MAX_TCAM_TABLE_SIZE,
2402                                     &fm->ig_tcam_hndl);
2403         if (rc)
2404                 return rc;
2405         rc = enic_fm_tcam_tbl_alloc(fm, FM_EGRESS, FM_MAX_TCAM_TABLE_SIZE,
2406                                     &fm->eg_tcam_hndl);
2407         return rc;
2408 }
2409
2410 static void
2411 enic_fm_free_tcam_tables(struct enic_flowman *fm)
2412 {
2413         ENICPMD_FUNC_TRACE();
2414         if (fm->ig_tcam_hndl) {
2415                 ENICPMD_LOG(DEBUG, "free IG TCAM table handle=0x%" PRIx64,
2416                             fm->ig_tcam_hndl);
2417                 enic_fm_tbl_free(fm, fm->ig_tcam_hndl);
2418                 fm->ig_tcam_hndl = FM_INVALID_HANDLE;
2419         }
2420         if (fm->eg_tcam_hndl) {
2421                 ENICPMD_LOG(DEBUG, "free EG TCAM table handle=0x%" PRIx64,
2422                             fm->eg_tcam_hndl);
2423                 enic_fm_tbl_free(fm, fm->eg_tcam_hndl);
2424                 fm->eg_tcam_hndl = FM_INVALID_HANDLE;
2425         }
2426 }
2427
2428 int
2429 enic_fm_init(struct enic *enic)
2430 {
2431         struct enic_flowman *fm;
2432         uint8_t name[RTE_MEMZONE_NAMESIZE];
2433         int rc;
2434
2435         if (enic->flow_filter_mode != FILTER_FLOWMAN)
2436                 return 0;
2437         ENICPMD_FUNC_TRACE();
2438         fm = calloc(1, sizeof(*fm));
2439         if (fm == NULL) {
2440                 ENICPMD_LOG(ERR, "cannot alloc flowman struct");
2441                 return -ENOMEM;
2442         }
2443         fm->enic = enic;
2444         TAILQ_INIT(&fm->fet_list);
2445         TAILQ_INIT(&fm->jump_list);
2446         /* Allocate host memory for flowman commands */
2447         snprintf((char *)name, sizeof(name), "fm-cmd-%s", enic->bdf_name);
2448         fm->cmd.va = enic_alloc_consistent(enic,
2449                 sizeof(union enic_flowman_cmd_mem), &fm->cmd.pa, name);
2450         if (!fm->cmd.va) {
2451                 ENICPMD_LOG(ERR, "cannot allocate flowman command memory");
2452                 rc = -ENOMEM;
2453                 goto error_fm;
2454         }
2455         /* Allocate TCAM tables upfront as they are the main tables */
2456         rc = enic_fm_alloc_tcam_tables(fm);
2457         if (rc) {
2458                 ENICPMD_LOG(ERR, "cannot alloc TCAM tables");
2459                 goto error_cmd;
2460         }
2461         /* Then a number of counters */
2462         rc = enic_fm_init_counters(fm);
2463         if (rc) {
2464                 ENICPMD_LOG(ERR, "cannot alloc counters");
2465                 goto error_tables;
2466         }
2467         /*
2468          * One default exact match table for each direction. We hold onto
2469          * it until close.
2470          */
2471         rc = enic_fet_alloc(fm, 1, NULL, 128, &fm->default_ig_fet);
2472         if (rc) {
2473                 ENICPMD_LOG(ERR, "cannot alloc default IG exact match table");
2474                 goto error_counters;
2475         }
2476         fm->default_ig_fet->ref = 1;
2477         rc = enic_fet_alloc(fm, 0, NULL, 128, &fm->default_eg_fet);
2478         if (rc) {
2479                 ENICPMD_LOG(ERR, "cannot alloc default EG exact match table");
2480                 goto error_ig_fet;
2481         }
2482         fm->default_eg_fet->ref = 1;
2483         enic->fm = fm;
2484         return 0;
2485
2486 error_ig_fet:
2487         enic_fet_free(fm, fm->default_ig_fet);
2488 error_counters:
2489         enic_fm_free_all_counters(fm);
2490 error_tables:
2491         enic_fm_free_tcam_tables(fm);
2492 error_cmd:
2493         enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
2494                 fm->cmd.va, fm->cmd.pa);
2495 error_fm:
2496         free(fm);
2497         return rc;
2498 }
2499
2500 void
2501 enic_fm_destroy(struct enic *enic)
2502 {
2503         struct enic_flowman *fm;
2504         struct enic_fm_fet *fet;
2505
2506         if (enic->fm == NULL)
2507                 return;
2508         ENICPMD_FUNC_TRACE();
2509         fm = enic->fm;
2510         enic_fet_free(fm, fm->default_eg_fet);
2511         enic_fet_free(fm, fm->default_ig_fet);
2512         /* Free all exact match tables still open */
2513         while (!TAILQ_EMPTY(&fm->fet_list)) {
2514                 fet = TAILQ_FIRST(&fm->fet_list);
2515                 enic_fet_free(fm, fet);
2516         }
2517         enic_fm_free_tcam_tables(fm);
2518         enic_fm_free_all_counters(fm);
2519         enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
2520                 fm->cmd.va, fm->cmd.pa);
2521         fm->cmd.va = NULL;
2522         free(fm);
2523         enic->fm = NULL;
2524 }
2525
2526 int
2527 enic_fm_allocate_switch_domain(struct enic *pf)
2528 {
2529         const struct rte_pci_addr *cur_a, *prev_a;
2530         struct rte_eth_dev *dev;
2531         struct enic *cur, *prev;
2532         uint16_t domain_id;
2533         uint64_t vnic_h;
2534         uint16_t pid;
2535         int ret;
2536
2537         ENICPMD_FUNC_TRACE();
2538         if (enic_is_vf_rep(pf))
2539                 return -EINVAL;
2540         cur = pf;
2541         cur_a = &RTE_ETH_DEV_TO_PCI(cur->rte_dev)->addr;
2542         /* Go through ports and find another PF that is on the same adapter */
2543         RTE_ETH_FOREACH_DEV(pid) {
2544                 dev = &rte_eth_devices[pid];
2545                 if (!dev_is_enic(dev))
2546                         continue;
2547                 if (dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)
2548                         continue;
2549                 if (dev == cur->rte_dev)
2550                         continue;
2551                 /* dev is another PF. Is it on the same adapter? */
2552                 prev = pmd_priv(dev);
2553                 prev_a = &RTE_ETH_DEV_TO_PCI(dev)->addr;
2554                 if (!enic_fm_find_vnic(cur, prev_a, &vnic_h)) {
2555                         ENICPMD_LOG(DEBUG, "Port %u (PF BDF %x:%x:%x) and port %u (PF BDF %x:%x:%x domain %u) are on the same VIC",
2556                                 cur->rte_dev->data->port_id,
2557                                 cur_a->bus, cur_a->devid, cur_a->function,
2558                                 dev->data->port_id,
2559                                 prev_a->bus, prev_a->devid, prev_a->function,
2560                                 prev->switch_domain_id);
2561                         cur->switch_domain_id = prev->switch_domain_id;
2562                         return 0;
2563                 }
2564         }
2565         ret = rte_eth_switch_domain_alloc(&domain_id);
2566         if (ret) {
2567                 ENICPMD_LOG(WARNING, "failed to allocate switch domain for device %d",
2568                             ret);
2569         }
2570         cur->switch_domain_id = domain_id;
2571         ENICPMD_LOG(DEBUG, "Port %u (PF BDF %x:%x:%x) is the 1st PF on the VIC. Allocated switch domain id %u",
2572                     cur->rte_dev->data->port_id,
2573                     cur_a->bus, cur_a->devid, cur_a->function,
2574                     domain_id);
2575         return ret;
2576 }
2577
2578 const struct rte_flow_ops enic_fm_flow_ops = {
2579         .validate = enic_fm_flow_validate,
2580         .create = enic_fm_flow_create,
2581         .destroy = enic_fm_flow_destroy,
2582         .flush = enic_fm_flow_flush,
2583         .query = enic_fm_flow_query,
2584 };