1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2008-2019 Cisco Systems, Inc. All rights reserved.
8 #include <rte_ethdev_driver.h>
9 #include <rte_flow_driver.h>
10 #include <rte_ether.h>
13 #include <rte_memzone.h>
15 #include "enic_compat.h"
20 #define IP_DEFTTL 64 /* from RFC 1340. */
21 #define IP6_VTC_FLOW 0x60000000
23 /* Highest Item type supported by Flowman */
24 #define FM_MAX_ITEM_TYPE RTE_FLOW_ITEM_TYPE_VXLAN
26 /* Up to 1024 TCAM entries */
27 #define FM_MAX_TCAM_TABLE_SIZE 1024
29 /* Up to 4096 entries per exact match table */
30 #define FM_MAX_EXACT_TABLE_SIZE 4096
32 /* Number of counters to increase on for each increment */
33 #define FM_COUNTERS_EXPAND 100
35 #define FM_INVALID_HANDLE 0
37 /* Low priority used for implicit VF -> representor flow */
38 #define FM_LOWEST_PRIORITY 100000
40 /* High priority used for implicit representor -> VF flow */
41 #define FM_HIGHEST_PRIORITY 0
43 /* Tag used for implicit VF <-> representor flows */
44 #define FM_VF_REP_TAG 1
47 * Flow exact match tables (FET) in the VIC and rte_flow groups.
48 * Use a simple scheme to map groups to tables.
49 * Group 0 uses the single TCAM tables, one for each direction.
50 * Group 1, 2, ... uses its own exact match table.
52 * The TCAM tables are allocated upfront during init.
54 * Exact match tables are allocated on demand. 3 paths that lead allocations.
56 * 1. Add a flow that jumps from group 0 to group N.
58 * If N does not exist, we allocate an exact match table for it, using
59 * a dummy key. A key is required for the table.
61 * 2. Add a flow that uses group N.
63 * If N does not exist, we allocate an exact match table for it, using
64 * the flow's key. Subsequent flows to the same group all should have
67 * Without a jump flow to N, N is not reachable in hardware. No packets
70 * 3. Add a flow to an empty group N.
72 * N has been created via (1) and the dummy key. We free that table, allocate
73 * a new table using the new flow's key. Also re-do the existing jump flow to
74 * point to the new table.
76 #define FM_TCAM_RTE_GROUP 0
79 TAILQ_ENTRY(enic_fm_fet) list;
80 uint32_t group; /* rte_flow group ID */
81 uint64_t handle; /* Exact match table handle from flowman */
84 int ref; /* Reference count via get/put */
85 struct fm_key_template key; /* Key associated with the table */
88 struct enic_fm_counter {
89 SLIST_ENTRY(enic_fm_counter) next;
96 uint64_t entry_handle;
97 uint64_t action_handle;
98 struct enic_fm_counter *counter;
99 struct enic_fm_fet *fet;
102 struct enic_fm_jump_flow {
103 TAILQ_ENTRY(enic_fm_jump_flow) list;
104 struct rte_flow *flow;
106 struct fm_tcam_match_entry match;
107 struct fm_action action;
111 * Flowman uses host memory for commands. This structure is allocated
112 * in DMA-able memory.
114 union enic_flowman_cmd_mem {
115 struct fm_tcam_match_table fm_tcam_match_table;
116 struct fm_exact_match_table fm_exact_match_table;
117 struct fm_tcam_match_entry fm_tcam_match_entry;
118 struct fm_exact_match_entry fm_exact_match_entry;
119 struct fm_action fm_action;
123 * PF has a flowman instance, and VF representors share it with PF.
124 * PF allocates this structure and owns it. VF representors borrow
125 * the PF's structure during API calls (e.g. create, query).
127 struct enic_flowman {
128 struct enic *owner_enic; /* PF */
129 struct enic *user_enic; /* API caller (PF or representor) */
131 * Representors and PF share the same underlying flowman.
132 * Lock API calls to serialize accesses from them. Only used
133 * when VF representors are present.
138 union enic_flowman_cmd_mem *va;
141 /* TCAM tables allocated upfront, used for group 0 */
142 uint64_t ig_tcam_hndl;
143 uint64_t eg_tcam_hndl;
145 SLIST_HEAD(enic_free_counters, enic_fm_counter) counters;
147 uint32_t counters_alloced;
148 /* Exact match tables for groups != 0, dynamically allocated */
149 TAILQ_HEAD(fet_list, enic_fm_fet) fet_list;
151 * Default exact match tables used for jump actions to
152 * non-existent groups.
154 struct enic_fm_fet *default_eg_fet;
155 struct enic_fm_fet *default_ig_fet;
156 /* Flows that jump to the default table above */
157 TAILQ_HEAD(jump_flow_list, enic_fm_jump_flow) jump_list;
159 * Scratch data used during each invocation of flow_create
162 struct enic_fm_fet *fet;
163 struct fm_tcam_match_entry tcam_entry;
164 struct fm_action action;
165 struct fm_action action_tmp; /* enic_fm_reorder_action_op */
167 /* Tags used for representor flows */
171 static int enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle);
173 * API functions (create, destroy, validate, flush) call begin_fm()
174 * upon entering to save the caller enic (PF or VF representor) and
175 * lock. Upon exit, they call end_fm() to unlock.
177 static struct enic_flowman *begin_fm(struct enic *enic);
178 static void end_fm(struct enic_flowman *fm);
179 /* Delete internal flows created for representor paths */
180 static void delete_rep_flows(struct enic *enic);
183 * Common arguments passed to copy_item functions. Use this structure
184 * so we can easily add new arguments.
185 * item: Item specification.
186 * fm_tcam_entry: Flowman TCAM match entry.
187 * header_level: 0 for outer header, 1 for inner header.
189 struct copy_item_args {
190 const struct rte_flow_item *item;
191 struct fm_tcam_match_entry *fm_tcam_entry;
192 uint8_t header_level;
195 /* functions for copying items into flowman match */
196 typedef int (enic_copy_item_fn)(struct copy_item_args *arg);
198 /* Info about how to copy items into flowman match */
199 struct enic_fm_items {
200 /* Function for copying and validating an item. */
201 enic_copy_item_fn * const copy_item;
202 /* List of valid previous items. */
203 const enum rte_flow_item_type * const prev_items;
205 * True if it's OK for this item to be the first item. For some NIC
206 * versions, it's invalid to start the stack above layer 3.
208 const uint8_t valid_start_item;
211 static enic_copy_item_fn enic_fm_copy_item_eth;
212 static enic_copy_item_fn enic_fm_copy_item_ipv4;
213 static enic_copy_item_fn enic_fm_copy_item_ipv6;
214 static enic_copy_item_fn enic_fm_copy_item_raw;
215 static enic_copy_item_fn enic_fm_copy_item_sctp;
216 static enic_copy_item_fn enic_fm_copy_item_tcp;
217 static enic_copy_item_fn enic_fm_copy_item_udp;
218 static enic_copy_item_fn enic_fm_copy_item_vlan;
219 static enic_copy_item_fn enic_fm_copy_item_vxlan;
221 /* Ingress actions */
222 static const enum rte_flow_action_type enic_fm_supported_ig_actions[] = {
223 RTE_FLOW_ACTION_TYPE_COUNT,
224 RTE_FLOW_ACTION_TYPE_DROP,
225 RTE_FLOW_ACTION_TYPE_FLAG,
226 RTE_FLOW_ACTION_TYPE_JUMP,
227 RTE_FLOW_ACTION_TYPE_MARK,
228 RTE_FLOW_ACTION_TYPE_OF_POP_VLAN,
229 RTE_FLOW_ACTION_TYPE_PORT_ID,
230 RTE_FLOW_ACTION_TYPE_PASSTHRU,
231 RTE_FLOW_ACTION_TYPE_QUEUE,
232 RTE_FLOW_ACTION_TYPE_RSS,
233 RTE_FLOW_ACTION_TYPE_VOID,
234 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
235 RTE_FLOW_ACTION_TYPE_VXLAN_DECAP,
236 RTE_FLOW_ACTION_TYPE_END, /* END must be the last entry */
240 static const enum rte_flow_action_type enic_fm_supported_eg_actions[] = {
241 RTE_FLOW_ACTION_TYPE_COUNT,
242 RTE_FLOW_ACTION_TYPE_DROP,
243 RTE_FLOW_ACTION_TYPE_JUMP,
244 RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN,
245 RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP,
246 RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID,
247 RTE_FLOW_ACTION_TYPE_PASSTHRU,
248 RTE_FLOW_ACTION_TYPE_VOID,
249 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
250 RTE_FLOW_ACTION_TYPE_END,
253 static const struct enic_fm_items enic_fm_items[] = {
254 [RTE_FLOW_ITEM_TYPE_RAW] = {
255 .copy_item = enic_fm_copy_item_raw,
256 .valid_start_item = 0,
257 .prev_items = (const enum rte_flow_item_type[]) {
258 RTE_FLOW_ITEM_TYPE_UDP,
259 RTE_FLOW_ITEM_TYPE_END,
262 [RTE_FLOW_ITEM_TYPE_ETH] = {
263 .copy_item = enic_fm_copy_item_eth,
264 .valid_start_item = 1,
265 .prev_items = (const enum rte_flow_item_type[]) {
266 RTE_FLOW_ITEM_TYPE_END,
269 [RTE_FLOW_ITEM_TYPE_VLAN] = {
270 .copy_item = enic_fm_copy_item_vlan,
271 .valid_start_item = 1,
272 .prev_items = (const enum rte_flow_item_type[]) {
273 RTE_FLOW_ITEM_TYPE_ETH,
274 RTE_FLOW_ITEM_TYPE_END,
277 [RTE_FLOW_ITEM_TYPE_IPV4] = {
278 .copy_item = enic_fm_copy_item_ipv4,
279 .valid_start_item = 1,
280 .prev_items = (const enum rte_flow_item_type[]) {
281 RTE_FLOW_ITEM_TYPE_ETH,
282 RTE_FLOW_ITEM_TYPE_VLAN,
283 RTE_FLOW_ITEM_TYPE_END,
286 [RTE_FLOW_ITEM_TYPE_IPV6] = {
287 .copy_item = enic_fm_copy_item_ipv6,
288 .valid_start_item = 1,
289 .prev_items = (const enum rte_flow_item_type[]) {
290 RTE_FLOW_ITEM_TYPE_ETH,
291 RTE_FLOW_ITEM_TYPE_VLAN,
292 RTE_FLOW_ITEM_TYPE_END,
295 [RTE_FLOW_ITEM_TYPE_UDP] = {
296 .copy_item = enic_fm_copy_item_udp,
297 .valid_start_item = 1,
298 .prev_items = (const enum rte_flow_item_type[]) {
299 RTE_FLOW_ITEM_TYPE_IPV4,
300 RTE_FLOW_ITEM_TYPE_IPV6,
301 RTE_FLOW_ITEM_TYPE_END,
304 [RTE_FLOW_ITEM_TYPE_TCP] = {
305 .copy_item = enic_fm_copy_item_tcp,
306 .valid_start_item = 1,
307 .prev_items = (const enum rte_flow_item_type[]) {
308 RTE_FLOW_ITEM_TYPE_IPV4,
309 RTE_FLOW_ITEM_TYPE_IPV6,
310 RTE_FLOW_ITEM_TYPE_END,
313 [RTE_FLOW_ITEM_TYPE_SCTP] = {
314 .copy_item = enic_fm_copy_item_sctp,
315 .valid_start_item = 0,
316 .prev_items = (const enum rte_flow_item_type[]) {
317 RTE_FLOW_ITEM_TYPE_IPV4,
318 RTE_FLOW_ITEM_TYPE_IPV6,
319 RTE_FLOW_ITEM_TYPE_END,
322 [RTE_FLOW_ITEM_TYPE_VXLAN] = {
323 .copy_item = enic_fm_copy_item_vxlan,
324 .valid_start_item = 1,
325 .prev_items = (const enum rte_flow_item_type[]) {
326 RTE_FLOW_ITEM_TYPE_UDP,
327 RTE_FLOW_ITEM_TYPE_END,
333 enic_fm_copy_item_eth(struct copy_item_args *arg)
335 const struct rte_flow_item *item = arg->item;
336 const struct rte_flow_item_eth *spec = item->spec;
337 const struct rte_flow_item_eth *mask = item->mask;
338 const uint8_t lvl = arg->header_level;
339 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
340 struct fm_header_set *fm_data, *fm_mask;
342 ENICPMD_FUNC_TRACE();
343 /* Match all if no spec */
347 mask = &rte_flow_item_eth_mask;
348 fm_data = &entry->ftm_data.fk_hdrset[lvl];
349 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
350 fm_data->fk_header_select |= FKH_ETHER;
351 fm_mask->fk_header_select |= FKH_ETHER;
352 memcpy(&fm_data->l2.eth, spec, sizeof(*spec));
353 memcpy(&fm_mask->l2.eth, mask, sizeof(*mask));
358 enic_fm_copy_item_vlan(struct copy_item_args *arg)
360 const struct rte_flow_item *item = arg->item;
361 const struct rte_flow_item_vlan *spec = item->spec;
362 const struct rte_flow_item_vlan *mask = item->mask;
363 const uint8_t lvl = arg->header_level;
364 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
365 struct fm_header_set *fm_data, *fm_mask;
366 struct rte_ether_hdr *eth_mask;
367 struct rte_ether_hdr *eth_val;
370 ENICPMD_FUNC_TRACE();
371 fm_data = &entry->ftm_data.fk_hdrset[lvl];
372 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
373 /* Outer and inner packet vlans need different flags */
374 meta = FKM_VLAN_PRES;
377 fm_data->fk_metadata |= meta;
378 fm_mask->fk_metadata |= meta;
380 /* Match all if no spec */
384 mask = &rte_flow_item_vlan_mask;
386 eth_mask = (void *)&fm_mask->l2.eth;
387 eth_val = (void *)&fm_data->l2.eth;
389 /* Outer TPID cannot be matched */
390 if (eth_mask->ether_type)
394 * When packet matching, the VIC always compares vlan-stripped
395 * L2, regardless of vlan stripping settings. So, the inner type
396 * from vlan becomes the ether type of the eth header.
398 eth_mask->ether_type = mask->inner_type;
399 eth_val->ether_type = spec->inner_type;
400 fm_data->fk_header_select |= FKH_ETHER | FKH_QTAG;
401 fm_mask->fk_header_select |= FKH_ETHER | FKH_QTAG;
402 fm_data->fk_vlan = rte_be_to_cpu_16(spec->tci);
403 fm_mask->fk_vlan = rte_be_to_cpu_16(mask->tci);
408 enic_fm_copy_item_ipv4(struct copy_item_args *arg)
410 const struct rte_flow_item *item = arg->item;
411 const struct rte_flow_item_ipv4 *spec = item->spec;
412 const struct rte_flow_item_ipv4 *mask = item->mask;
413 const uint8_t lvl = arg->header_level;
414 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
415 struct fm_header_set *fm_data, *fm_mask;
417 ENICPMD_FUNC_TRACE();
418 fm_data = &entry->ftm_data.fk_hdrset[lvl];
419 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
420 fm_data->fk_metadata |= FKM_IPV4;
421 fm_mask->fk_metadata |= FKM_IPV4;
426 mask = &rte_flow_item_ipv4_mask;
428 fm_data->fk_header_select |= FKH_IPV4;
429 fm_mask->fk_header_select |= FKH_IPV4;
430 memcpy(&fm_data->l3.ip4, spec, sizeof(*spec));
431 memcpy(&fm_mask->l3.ip4, mask, sizeof(*mask));
436 enic_fm_copy_item_ipv6(struct copy_item_args *arg)
438 const struct rte_flow_item *item = arg->item;
439 const struct rte_flow_item_ipv6 *spec = item->spec;
440 const struct rte_flow_item_ipv6 *mask = item->mask;
441 const uint8_t lvl = arg->header_level;
442 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
443 struct fm_header_set *fm_data, *fm_mask;
445 ENICPMD_FUNC_TRACE();
446 fm_data = &entry->ftm_data.fk_hdrset[lvl];
447 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
448 fm_data->fk_metadata |= FKM_IPV6;
449 fm_mask->fk_metadata |= FKM_IPV6;
454 mask = &rte_flow_item_ipv6_mask;
456 fm_data->fk_header_select |= FKH_IPV6;
457 fm_mask->fk_header_select |= FKH_IPV6;
458 memcpy(&fm_data->l3.ip6, spec, sizeof(*spec));
459 memcpy(&fm_mask->l3.ip6, mask, sizeof(*mask));
464 enic_fm_copy_item_udp(struct copy_item_args *arg)
466 const struct rte_flow_item *item = arg->item;
467 const struct rte_flow_item_udp *spec = item->spec;
468 const struct rte_flow_item_udp *mask = item->mask;
469 const uint8_t lvl = arg->header_level;
470 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
471 struct fm_header_set *fm_data, *fm_mask;
473 ENICPMD_FUNC_TRACE();
474 fm_data = &entry->ftm_data.fk_hdrset[lvl];
475 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
476 fm_data->fk_metadata |= FKM_UDP;
477 fm_mask->fk_metadata |= FKM_UDP;
482 mask = &rte_flow_item_udp_mask;
484 fm_data->fk_header_select |= FKH_UDP;
485 fm_mask->fk_header_select |= FKH_UDP;
486 memcpy(&fm_data->l4.udp, spec, sizeof(*spec));
487 memcpy(&fm_mask->l4.udp, mask, sizeof(*mask));
492 enic_fm_copy_item_tcp(struct copy_item_args *arg)
494 const struct rte_flow_item *item = arg->item;
495 const struct rte_flow_item_tcp *spec = item->spec;
496 const struct rte_flow_item_tcp *mask = item->mask;
497 const uint8_t lvl = arg->header_level;
498 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
499 struct fm_header_set *fm_data, *fm_mask;
501 ENICPMD_FUNC_TRACE();
502 fm_data = &entry->ftm_data.fk_hdrset[lvl];
503 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
504 fm_data->fk_metadata |= FKM_TCP;
505 fm_mask->fk_metadata |= FKM_TCP;
510 mask = &rte_flow_item_tcp_mask;
512 fm_data->fk_header_select |= FKH_TCP;
513 fm_mask->fk_header_select |= FKH_TCP;
514 memcpy(&fm_data->l4.tcp, spec, sizeof(*spec));
515 memcpy(&fm_mask->l4.tcp, mask, sizeof(*mask));
520 enic_fm_copy_item_sctp(struct copy_item_args *arg)
522 const struct rte_flow_item *item = arg->item;
523 const struct rte_flow_item_sctp *spec = item->spec;
524 const struct rte_flow_item_sctp *mask = item->mask;
525 const uint8_t lvl = arg->header_level;
526 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
527 struct fm_header_set *fm_data, *fm_mask;
528 uint8_t *ip_proto_mask = NULL;
529 uint8_t *ip_proto = NULL;
532 ENICPMD_FUNC_TRACE();
533 fm_data = &entry->ftm_data.fk_hdrset[lvl];
534 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
536 * The NIC filter API has no flags for "match sctp", so explicitly
537 * set the protocol number in the IP pattern.
539 if (fm_data->fk_metadata & FKM_IPV4) {
540 struct rte_ipv4_hdr *ip;
541 ip = (struct rte_ipv4_hdr *)&fm_mask->l3.ip4;
542 ip_proto_mask = &ip->next_proto_id;
543 ip = (struct rte_ipv4_hdr *)&fm_data->l3.ip4;
544 ip_proto = &ip->next_proto_id;
546 } else if (fm_data->fk_metadata & FKM_IPV6) {
547 struct rte_ipv6_hdr *ip;
548 ip = (struct rte_ipv6_hdr *)&fm_mask->l3.ip6;
549 ip_proto_mask = &ip->proto;
550 ip = (struct rte_ipv6_hdr *)&fm_data->l3.ip6;
551 ip_proto = &ip->proto;
554 /* Need IPv4/IPv6 pattern first */
557 *ip_proto = IPPROTO_SCTP;
558 *ip_proto_mask = 0xff;
559 fm_data->fk_header_select |= l3_fkh;
560 fm_mask->fk_header_select |= l3_fkh;
565 mask = &rte_flow_item_sctp_mask;
567 fm_data->fk_header_select |= FKH_L4RAW;
568 fm_mask->fk_header_select |= FKH_L4RAW;
569 memcpy(fm_data->l4.rawdata, spec, sizeof(*spec));
570 memcpy(fm_mask->l4.rawdata, mask, sizeof(*mask));
575 enic_fm_copy_item_vxlan(struct copy_item_args *arg)
577 const struct rte_flow_item *item = arg->item;
578 const struct rte_flow_item_vxlan *spec = item->spec;
579 const struct rte_flow_item_vxlan *mask = item->mask;
580 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
581 struct fm_header_set *fm_data, *fm_mask;
583 ENICPMD_FUNC_TRACE();
584 /* Only 2 header levels (outer and inner) allowed */
585 if (arg->header_level > 0)
588 fm_data = &entry->ftm_data.fk_hdrset[0];
589 fm_mask = &entry->ftm_mask.fk_hdrset[0];
590 fm_data->fk_metadata |= FKM_VXLAN;
591 fm_mask->fk_metadata |= FKM_VXLAN;
592 /* items from here on out are inner header items */
593 arg->header_level = 1;
595 /* Match all if no spec */
599 mask = &rte_flow_item_vxlan_mask;
601 fm_data->fk_header_select |= FKH_VXLAN;
602 fm_mask->fk_header_select |= FKH_VXLAN;
603 memcpy(&fm_data->vxlan, spec, sizeof(*spec));
604 memcpy(&fm_mask->vxlan, mask, sizeof(*mask));
609 * Currently, raw pattern match is very limited. It is intended for matching
610 * UDP tunnel header (e.g. vxlan or geneve).
613 enic_fm_copy_item_raw(struct copy_item_args *arg)
615 const struct rte_flow_item *item = arg->item;
616 const struct rte_flow_item_raw *spec = item->spec;
617 const struct rte_flow_item_raw *mask = item->mask;
618 const uint8_t lvl = arg->header_level;
619 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
620 struct fm_header_set *fm_data, *fm_mask;
622 ENICPMD_FUNC_TRACE();
623 /* Cannot be used for inner packet */
626 /* Need both spec and mask */
629 /* Only supports relative with offset 0 */
630 if (!spec->relative || spec->offset != 0 || spec->search ||
633 /* Need non-null pattern that fits within the NIC's filter pattern */
634 if (spec->length == 0 ||
635 spec->length + sizeof(struct rte_udp_hdr) > FM_LAYER_SIZE ||
636 !spec->pattern || !mask->pattern)
639 * Mask fields, including length, are often set to zero. Assume that
640 * means "same as spec" to avoid breaking existing apps. If length
641 * is not zero, then it should be >= spec length.
643 * No more pattern follows this, so append to the L4 layer instead of
644 * L5 to work with both recent and older VICs.
646 if (mask->length != 0 && mask->length < spec->length)
649 fm_data = &entry->ftm_data.fk_hdrset[lvl];
650 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
651 fm_data->fk_header_select |= FKH_L4RAW;
652 fm_mask->fk_header_select |= FKH_L4RAW;
653 fm_data->fk_header_select &= ~FKH_UDP;
654 fm_mask->fk_header_select &= ~FKH_UDP;
655 memcpy(fm_data->l4.rawdata + sizeof(struct rte_udp_hdr),
656 spec->pattern, spec->length);
657 memcpy(fm_mask->l4.rawdata + sizeof(struct rte_udp_hdr),
658 mask->pattern, spec->length);
663 flowman_cmd(struct enic_flowman *fm, uint64_t *args, int nargs)
665 return vnic_dev_flowman_cmd(fm->owner_enic->vdev, args, nargs);
669 enic_fet_alloc(struct enic_flowman *fm, uint8_t ingress,
670 struct fm_key_template *key, int entries,
671 struct enic_fm_fet **fet_out)
673 struct fm_exact_match_table *cmd;
674 struct fm_header_set *hdr;
675 struct enic_fm_fet *fet;
679 ENICPMD_FUNC_TRACE();
680 fet = calloc(1, sizeof(struct enic_fm_fet));
683 cmd = &fm->cmd.va->fm_exact_match_table;
684 memset(cmd, 0, sizeof(*cmd));
685 cmd->fet_direction = ingress ? FM_INGRESS : FM_EGRESS;
686 cmd->fet_stage = FM_STAGE_LAST;
687 cmd->fet_max_entries = entries ? entries : FM_MAX_EXACT_TABLE_SIZE;
689 hdr = &cmd->fet_key.fk_hdrset[0];
690 memset(hdr, 0, sizeof(*hdr));
691 hdr->fk_header_select = FKH_IPV4 | FKH_UDP;
692 hdr->l3.ip4.fk_saddr = 0xFFFFFFFF;
693 hdr->l3.ip4.fk_daddr = 0xFFFFFFFF;
694 hdr->l4.udp.fk_source = 0xFFFF;
695 hdr->l4.udp.fk_dest = 0xFFFF;
696 fet->default_key = 1;
698 memcpy(&cmd->fet_key, key, sizeof(*key));
699 memcpy(&fet->key, key, sizeof(*key));
700 fet->default_key = 0;
702 cmd->fet_key.fk_packet_tag = 1;
704 args[0] = FM_EXACT_TABLE_ALLOC;
705 args[1] = fm->cmd.pa;
706 ret = flowman_cmd(fm, args, 2);
708 ENICPMD_LOG(ERR, "cannot alloc exact match table: rc=%d", ret);
712 fet->handle = args[0];
713 fet->ingress = ingress;
714 ENICPMD_LOG(DEBUG, "allocated exact match table: handle=0x%" PRIx64,
721 enic_fet_free(struct enic_flowman *fm, struct enic_fm_fet *fet)
723 ENICPMD_FUNC_TRACE();
724 enic_fm_tbl_free(fm, fet->handle);
725 if (!fet->default_key)
726 TAILQ_REMOVE(&fm->fet_list, fet, list);
731 * Get the exact match table for the given combination of
732 * <group, ingress, key>. Allocate one on the fly as necessary.
735 enic_fet_get(struct enic_flowman *fm,
738 struct fm_key_template *key,
739 struct enic_fm_fet **fet_out,
740 struct rte_flow_error *error)
742 struct enic_fm_fet *fet;
744 ENICPMD_FUNC_TRACE();
745 /* See if we already have this table open */
746 TAILQ_FOREACH(fet, &fm->fet_list, list) {
747 if (fet->group == group && fet->ingress == ingress)
751 /* Jumping to a non-existing group? Use the default table */
753 fet = ingress ? fm->default_ig_fet : fm->default_eg_fet;
754 } else if (enic_fet_alloc(fm, ingress, key, 0, &fet)) {
755 return rte_flow_error_set(error, EINVAL,
756 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
757 NULL, "enic: cannot get exact match table");
760 /* Default table is never on the open table list */
761 if (!fet->default_key)
762 TAILQ_INSERT_HEAD(&fm->fet_list, fet, list);
766 ENICPMD_LOG(DEBUG, "fet_get: %s %s group=%u ref=%u",
767 fet->default_key ? "default" : "",
768 fet->ingress ? "ingress" : "egress",
769 fet->group, fet->ref);
774 enic_fet_put(struct enic_flowman *fm, struct enic_fm_fet *fet)
776 ENICPMD_FUNC_TRACE();
777 RTE_ASSERT(fet->ref > 0);
779 ENICPMD_LOG(DEBUG, "fet_put: %s %s group=%u ref=%u",
780 fet->default_key ? "default" : "",
781 fet->ingress ? "ingress" : "egress",
782 fet->group, fet->ref);
784 enic_fet_free(fm, fet);
787 /* Return 1 if current item is valid on top of the previous one. */
789 fm_item_stacking_valid(enum rte_flow_item_type prev_item,
790 const struct enic_fm_items *item_info,
791 uint8_t is_first_item)
793 enum rte_flow_item_type const *allowed_items = item_info->prev_items;
795 ENICPMD_FUNC_TRACE();
796 for (; *allowed_items != RTE_FLOW_ITEM_TYPE_END; allowed_items++) {
797 if (prev_item == *allowed_items)
801 /* This is the first item in the stack. Check if that's cool */
802 if (is_first_item && item_info->valid_start_item)
808 * Build the flow manager match entry structure from the provided pattern.
809 * The pattern is validated as the items are copied.
812 enic_fm_copy_entry(struct enic_flowman *fm,
813 const struct rte_flow_item pattern[],
814 struct rte_flow_error *error)
816 const struct enic_fm_items *item_info;
817 enum rte_flow_item_type prev_item;
818 const struct rte_flow_item *item;
819 struct copy_item_args args;
820 uint8_t prev_header_level;
821 uint8_t is_first_item;
824 ENICPMD_FUNC_TRACE();
827 prev_item = RTE_FLOW_ITEM_TYPE_END;
829 args.fm_tcam_entry = &fm->tcam_entry;
830 args.header_level = 0;
831 prev_header_level = 0;
832 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
834 * Get info about how to validate and copy the item. If NULL
835 * is returned the nic does not support the item.
837 if (item->type == RTE_FLOW_ITEM_TYPE_VOID)
840 item_info = &enic_fm_items[item->type];
842 if (item->type > FM_MAX_ITEM_TYPE ||
843 item_info->copy_item == NULL) {
844 return rte_flow_error_set(error, ENOTSUP,
845 RTE_FLOW_ERROR_TYPE_ITEM,
846 NULL, "enic: unsupported item");
849 /* check to see if item stacking is valid */
850 if (!fm_item_stacking_valid(prev_item, item_info,
855 ret = item_info->copy_item(&args);
857 goto item_not_supported;
858 /* Going from outer to inner? Treat it as a new packet start */
859 if (prev_header_level != args.header_level) {
860 prev_item = RTE_FLOW_ITEM_TYPE_END;
863 prev_item = item->type;
866 prev_header_level = args.header_level;
871 return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,
872 NULL, "enic: unsupported item type");
875 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
876 item, "enic: unsupported item stack");
880 flow_item_skip_void(const struct rte_flow_item **item)
883 if ((*item)->type != RTE_FLOW_ITEM_TYPE_VOID)
888 append_template(void **template, uint8_t *off, const void *data, int len)
890 memcpy(*template, data, len);
891 *template = (char *)*template + len;
896 enic_fm_append_action_op(struct enic_flowman *fm,
897 struct fm_action_op *fm_op,
898 struct rte_flow_error *error)
902 count = fm->action_op_count;
903 ENICPMD_LOG(DEBUG, "append action op: idx=%d op=%u",
904 count, fm_op->fa_op);
905 if (count == FM_ACTION_OP_MAX) {
906 return rte_flow_error_set(error, EINVAL,
907 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
908 "too many action operations");
910 fm->action.fma_action_ops[count] = *fm_op;
911 fm->action_op_count = count + 1;
915 /* NIC requires that 1st steer appear before decap.
916 * Correct example: steer, decap, steer, steer, ...
919 enic_fm_reorder_action_op(struct enic_flowman *fm)
921 struct fm_action_op *op, *steer, *decap;
922 struct fm_action_op tmp_op;
924 ENICPMD_FUNC_TRACE();
925 /* Find 1st steer and decap */
926 op = fm->action.fma_action_ops;
929 while (op->fa_op != FMOP_END) {
930 if (!decap && op->fa_op == FMOP_DECAP_NOSTRIP)
932 else if (!steer && op->fa_op == FMOP_RQ_STEER)
936 /* If decap is before steer, swap */
937 if (steer && decap && decap < steer) {
938 op = fm->action.fma_action_ops;
939 ENICPMD_LOG(DEBUG, "swap decap %ld <-> steer %ld",
940 (long)(decap - op), (long)(steer - op));
947 /* VXLAN decap is done via flowman compound action */
949 enic_fm_copy_vxlan_decap(struct enic_flowman *fm,
950 struct fm_tcam_match_entry *fmt,
951 const struct rte_flow_action *action,
952 struct rte_flow_error *error)
954 struct fm_header_set *fm_data;
955 struct fm_action_op fm_op;
957 ENICPMD_FUNC_TRACE();
958 fm_data = &fmt->ftm_data.fk_hdrset[0];
959 if (!(fm_data->fk_metadata & FKM_VXLAN)) {
960 return rte_flow_error_set(error, EINVAL,
961 RTE_FLOW_ERROR_TYPE_ACTION, action,
962 "vxlan-decap: vxlan must be in pattern");
965 memset(&fm_op, 0, sizeof(fm_op));
966 fm_op.fa_op = FMOP_DECAP_NOSTRIP;
967 return enic_fm_append_action_op(fm, &fm_op, error);
970 /* VXLAN encap is done via flowman compound action */
972 enic_fm_copy_vxlan_encap(struct enic_flowman *fm,
973 const struct rte_flow_item *item,
974 struct rte_flow_error *error)
976 struct fm_action_op fm_op;
977 struct rte_ether_hdr *eth;
982 ENICPMD_FUNC_TRACE();
983 memset(&fm_op, 0, sizeof(fm_op));
984 fm_op.fa_op = FMOP_ENCAP;
985 template = fm->action.fma_data;
988 * Copy flow items to the flowman template starting L2.
989 * L2 must be ethernet.
991 flow_item_skip_void(&item);
992 if (item->type != RTE_FLOW_ITEM_TYPE_ETH)
993 return rte_flow_error_set(error, EINVAL,
994 RTE_FLOW_ERROR_TYPE_ITEM, item,
995 "vxlan-encap: first item should be ethernet");
996 eth = (struct rte_ether_hdr *)template;
997 ethertype = ð->ether_type;
998 append_template(&template, &off, item->spec,
999 sizeof(struct rte_flow_item_eth));
1001 flow_item_skip_void(&item);
1003 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
1004 const struct rte_flow_item_vlan *spec;
1006 ENICPMD_LOG(DEBUG, "vxlan-encap: vlan");
1008 fm_op.encap.outer_vlan = rte_be_to_cpu_16(spec->tci);
1010 flow_item_skip_void(&item);
1012 /* L3 must be IPv4, IPv6 */
1013 switch (item->type) {
1014 case RTE_FLOW_ITEM_TYPE_IPV4:
1016 struct rte_ipv4_hdr *ip4;
1018 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv4");
1019 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
1020 ip4 = (struct rte_ipv4_hdr *)template;
1022 * Offset of IPv4 length field and its initial value
1023 * (IP + UDP + VXLAN) are specified in the action. The NIC
1024 * will add inner packet length.
1026 fm_op.encap.len1_offset = off +
1027 offsetof(struct rte_ipv4_hdr, total_length);
1028 fm_op.encap.len1_delta = sizeof(struct rte_ipv4_hdr) +
1029 sizeof(struct rte_udp_hdr) +
1030 sizeof(struct rte_vxlan_hdr);
1031 append_template(&template, &off, item->spec,
1032 sizeof(struct rte_ipv4_hdr));
1033 ip4->version_ihl = RTE_IPV4_VHL_DEF;
1034 if (ip4->time_to_live == 0)
1035 ip4->time_to_live = IP_DEFTTL;
1036 ip4->next_proto_id = IPPROTO_UDP;
1039 case RTE_FLOW_ITEM_TYPE_IPV6:
1041 struct rte_ipv6_hdr *ip6;
1043 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv6");
1044 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
1045 ip6 = (struct rte_ipv6_hdr *)template;
1046 fm_op.encap.len1_offset = off +
1047 offsetof(struct rte_ipv6_hdr, payload_len);
1048 fm_op.encap.len1_delta = sizeof(struct rte_udp_hdr) +
1049 sizeof(struct rte_vxlan_hdr);
1050 append_template(&template, &off, item->spec,
1051 sizeof(struct rte_ipv6_hdr));
1052 ip6->vtc_flow |= rte_cpu_to_be_32(IP6_VTC_FLOW);
1053 if (ip6->hop_limits == 0)
1054 ip6->hop_limits = IP_DEFTTL;
1055 ip6->proto = IPPROTO_UDP;
1059 return rte_flow_error_set(error,
1060 EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1061 "vxlan-encap: L3 must be IPv4/IPv6");
1064 flow_item_skip_void(&item);
1067 if (item->type != RTE_FLOW_ITEM_TYPE_UDP)
1068 return rte_flow_error_set(error, EINVAL,
1069 RTE_FLOW_ERROR_TYPE_ITEM, item,
1070 "vxlan-encap: UDP must follow IPv4/IPv6");
1071 /* UDP length = UDP + VXLAN. NIC will add inner packet length. */
1072 fm_op.encap.len2_offset =
1073 off + offsetof(struct rte_udp_hdr, dgram_len);
1074 fm_op.encap.len2_delta =
1075 sizeof(struct rte_udp_hdr) + sizeof(struct rte_vxlan_hdr);
1076 append_template(&template, &off, item->spec,
1077 sizeof(struct rte_udp_hdr));
1079 flow_item_skip_void(&item);
1082 if (item->type != RTE_FLOW_ITEM_TYPE_VXLAN)
1083 return rte_flow_error_set(error,
1084 EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1085 "vxlan-encap: VXLAN must follow UDP");
1086 append_template(&template, &off, item->spec,
1087 sizeof(struct rte_flow_item_vxlan));
1090 * Fill in the rest of the action structure.
1091 * Indicate that we want to encap with vxlan at packet start.
1093 fm_op.encap.template_offset = 0;
1094 fm_op.encap.template_len = off;
1095 return enic_fm_append_action_op(fm, &fm_op, error);
1099 enic_fm_find_vnic(struct enic *enic, const struct rte_pci_addr *addr,
1106 ENICPMD_FUNC_TRACE();
1107 ENICPMD_LOG(DEBUG, "bdf=%x:%x:%x", addr->bus, addr->devid,
1109 bdf = addr->bus << 8 | addr->devid << 3 | addr->function;
1110 args[0] = FM_VNIC_FIND;
1112 rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
1114 /* Expected to fail if BDF is not on the adapter */
1115 ENICPMD_LOG(DEBUG, "cannot find vnic handle: rc=%d", rc);
1119 ENICPMD_LOG(DEBUG, "found vnic: handle=0x%" PRIx64, *handle);
1123 /* Translate flow actions to flowman TCAM entry actions */
1125 enic_fm_copy_action(struct enic_flowman *fm,
1126 const struct rte_flow_action actions[],
1128 struct rte_flow_error *error)
1139 struct fm_tcam_match_entry *fmt;
1140 struct fm_action_op fm_op;
1141 bool need_ovlan_action;
1150 ENICPMD_FUNC_TRACE();
1151 fmt = &fm->tcam_entry;
1152 need_ovlan_action = false;
1156 enic = fm->user_enic;
1158 vnic_h = enic->fm_vnic_handle;
1160 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1161 switch (actions->type) {
1162 case RTE_FLOW_ACTION_TYPE_VOID:
1164 case RTE_FLOW_ACTION_TYPE_PASSTHRU: {
1165 if (overlap & PASSTHRU)
1167 overlap |= PASSTHRU;
1170 case RTE_FLOW_ACTION_TYPE_JUMP: {
1171 const struct rte_flow_action_jump *jump =
1173 struct enic_fm_fet *fet;
1177 ret = enic_fet_get(fm, jump->group, ingress, NULL,
1182 memset(&fm_op, 0, sizeof(fm_op));
1183 fm_op.fa_op = FMOP_EXACT_MATCH;
1184 fm_op.exact.handle = fet->handle;
1186 ret = enic_fm_append_action_op(fm, &fm_op, error);
1191 case RTE_FLOW_ACTION_TYPE_MARK: {
1192 const struct rte_flow_action_mark *mark =
1195 if (mark->id >= ENIC_MAGIC_FILTER_ID - 1)
1196 return rte_flow_error_set(error, EINVAL,
1197 RTE_FLOW_ERROR_TYPE_ACTION,
1198 NULL, "invalid mark id");
1199 memset(&fm_op, 0, sizeof(fm_op));
1200 fm_op.fa_op = FMOP_MARK;
1201 fm_op.mark.mark = mark->id + 1;
1202 ret = enic_fm_append_action_op(fm, &fm_op, error);
1207 case RTE_FLOW_ACTION_TYPE_FLAG: {
1208 /* ENIC_MAGIC_FILTER_ID is reserved for flagging */
1209 memset(&fm_op, 0, sizeof(fm_op));
1210 fm_op.fa_op = FMOP_MARK;
1211 fm_op.mark.mark = ENIC_MAGIC_FILTER_ID;
1212 ret = enic_fm_append_action_op(fm, &fm_op, error);
1217 case RTE_FLOW_ACTION_TYPE_QUEUE: {
1218 const struct rte_flow_action_queue *queue =
1222 * If fate other than QUEUE or RSS, fail. Multiple
1223 * rss and queue actions are ok.
1225 if ((overlap & FATE) && first_rq)
1229 memset(&fm_op, 0, sizeof(fm_op));
1230 fm_op.fa_op = FMOP_RQ_STEER;
1231 fm_op.rq_steer.rq_index =
1232 enic_rte_rq_idx_to_sop_idx(queue->index);
1233 fm_op.rq_steer.rq_count = 1;
1234 fm_op.rq_steer.vnic_handle = vnic_h;
1235 ret = enic_fm_append_action_op(fm, &fm_op, error);
1238 ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1239 fm_op.rq_steer.rq_index);
1243 case RTE_FLOW_ACTION_TYPE_DROP: {
1247 memset(&fm_op, 0, sizeof(fm_op));
1248 fm_op.fa_op = FMOP_DROP;
1249 ret = enic_fm_append_action_op(fm, &fm_op, error);
1252 ENICPMD_LOG(DEBUG, "create DROP action");
1255 case RTE_FLOW_ACTION_TYPE_COUNT: {
1256 if (overlap & COUNT)
1259 /* Count is associated with entry not action on VIC. */
1260 fmt->ftm_flags |= FMEF_COUNTER;
1263 case RTE_FLOW_ACTION_TYPE_RSS: {
1264 const struct rte_flow_action_rss *rss = actions->conf;
1269 * If fate other than QUEUE or RSS, fail. Multiple
1270 * rss and queue actions are ok.
1272 if ((overlap & FATE) && first_rq)
1278 * Hardware only supports RSS actions on outer level
1279 * with default type and function. Queues must be
1282 allow = rss->func == RTE_ETH_HASH_FUNCTION_DEFAULT &&
1283 rss->level == 0 && (rss->types == 0 ||
1284 rss->types == enic->rss_hf) &&
1285 rss->queue_num <= enic->rq_count &&
1286 rss->queue[rss->queue_num - 1] < enic->rq_count;
1289 /* Identity queue map needs to be sequential */
1290 for (i = 1; i < rss->queue_num; i++)
1291 allow = allow && (rss->queue[i] ==
1292 rss->queue[i - 1] + 1);
1296 memset(&fm_op, 0, sizeof(fm_op));
1297 fm_op.fa_op = FMOP_RQ_STEER;
1298 fm_op.rq_steer.rq_index =
1299 enic_rte_rq_idx_to_sop_idx(rss->queue[0]);
1300 fm_op.rq_steer.rq_count = rss->queue_num;
1301 fm_op.rq_steer.vnic_handle = vnic_h;
1302 ret = enic_fm_append_action_op(fm, &fm_op, error);
1305 ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1306 fm_op.rq_steer.rq_index);
1310 case RTE_FLOW_ACTION_TYPE_PORT_ID: {
1311 const struct rte_flow_action_port_id *port;
1312 struct rte_eth_dev *dev;
1314 port = actions->conf;
1315 if (port->original) {
1316 vnic_h = enic->fm_vnic_handle; /* This port */
1319 ENICPMD_LOG(DEBUG, "port id %u", port->id);
1320 if (!rte_eth_dev_is_valid_port(port->id)) {
1321 return rte_flow_error_set(error, EINVAL,
1322 RTE_FLOW_ERROR_TYPE_ACTION,
1323 NULL, "invalid port_id");
1325 dev = &rte_eth_devices[port->id];
1326 if (!dev_is_enic(dev)) {
1327 return rte_flow_error_set(error, EINVAL,
1328 RTE_FLOW_ERROR_TYPE_ACTION,
1329 NULL, "port_id is not enic");
1331 if (enic->switch_domain_id !=
1332 pmd_priv(dev)->switch_domain_id) {
1333 return rte_flow_error_set(error, EINVAL,
1334 RTE_FLOW_ERROR_TYPE_ACTION,
1335 NULL, "destination and source ports are not in the same switch domain");
1337 vnic_h = pmd_priv(dev)->fm_vnic_handle;
1340 * Ingress. Nothing more to do. We add an implicit
1341 * steer at the end if needed.
1345 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: {
1346 if (overlap & DECAP)
1350 ret = enic_fm_copy_vxlan_decap(fm, fmt, actions,
1356 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: {
1357 const struct rte_flow_action_vxlan_encap *encap;
1359 encap = actions->conf;
1360 if (overlap & ENCAP)
1363 ret = enic_fm_copy_vxlan_encap(fm, encap->definition,
1369 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: {
1370 memset(&fm_op, 0, sizeof(fm_op));
1371 fm_op.fa_op = FMOP_POP_VLAN;
1372 ret = enic_fm_append_action_op(fm, &fm_op, error);
1377 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: {
1378 const struct rte_flow_action_of_push_vlan *vlan;
1380 if (overlap & PASSTHRU)
1382 vlan = actions->conf;
1383 if (vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN)) {
1384 return rte_flow_error_set(error, EINVAL,
1385 RTE_FLOW_ERROR_TYPE_ACTION,
1386 NULL, "unexpected push_vlan ethertype");
1388 overlap |= PUSH_VLAN;
1389 need_ovlan_action = true;
1392 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: {
1393 const struct rte_flow_action_of_set_vlan_pcp *pcp;
1395 pcp = actions->conf;
1396 if (pcp->vlan_pcp > 7) {
1397 return rte_flow_error_set(error, EINVAL,
1398 RTE_FLOW_ERROR_TYPE_ACTION,
1399 NULL, "invalid vlan_pcp");
1401 need_ovlan_action = true;
1402 ovlan |= ((uint16_t)pcp->vlan_pcp) << 13;
1405 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: {
1406 const struct rte_flow_action_of_set_vlan_vid *vid;
1408 vid = actions->conf;
1409 need_ovlan_action = true;
1410 ovlan |= rte_be_to_cpu_16(vid->vlan_vid);
1418 if (!(overlap & (FATE | PASSTHRU | COUNT | PORT_ID)))
1420 /* Egress from VF: need implicit WQ match */
1421 if (enic_is_vf_rep(enic) && !ingress) {
1422 fmt->ftm_data.fk_wq_id = 0;
1423 fmt->ftm_mask.fk_wq_id = 0xffff;
1424 fmt->ftm_data.fk_wq_vnic = enic->fm_vnic_handle;
1425 ENICPMD_LOG(DEBUG, "add implicit wq id match for vf %d",
1426 VF_ENIC_TO_VF_REP(enic)->vf_id);
1428 if (need_ovlan_action) {
1429 memset(&fm_op, 0, sizeof(fm_op));
1430 fm_op.fa_op = FMOP_SET_OVLAN;
1431 fm_op.ovlan.vlan = ovlan;
1432 ret = enic_fm_append_action_op(fm, &fm_op, error);
1436 /* Add steer op for PORT_ID without QUEUE */
1437 if ((overlap & PORT_ID) && !steer && ingress) {
1438 memset(&fm_op, 0, sizeof(fm_op));
1439 /* Always to queue 0 for now as generic RSS is not available */
1440 fm_op.fa_op = FMOP_RQ_STEER;
1441 fm_op.rq_steer.rq_index = 0;
1442 fm_op.rq_steer.vnic_handle = vnic_h;
1443 ret = enic_fm_append_action_op(fm, &fm_op, error);
1446 ENICPMD_LOG(DEBUG, "add implicit steer op");
1448 /* Add required END */
1449 memset(&fm_op, 0, sizeof(fm_op));
1450 fm_op.fa_op = FMOP_END;
1451 ret = enic_fm_append_action_op(fm, &fm_op, error);
1454 enic_fm_reorder_action_op(fm);
1458 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
1459 NULL, "enic: unsupported action");
1462 /** Check if the action is supported */
1464 enic_fm_match_action(const struct rte_flow_action *action,
1465 const enum rte_flow_action_type *supported_actions)
1467 for (; *supported_actions != RTE_FLOW_ACTION_TYPE_END;
1468 supported_actions++) {
1469 if (action->type == *supported_actions)
1475 /* Debug function to dump internal NIC action structure. */
1477 enic_fm_dump_tcam_actions(const struct fm_action *fm_action)
1479 /* Manually keep in sync with FMOP commands */
1480 const char *fmop_str[FMOP_OP_MAX] = {
1482 [FMOP_DROP] = "drop",
1483 [FMOP_RQ_STEER] = "steer",
1484 [FMOP_EXACT_MATCH] = "exmatch",
1485 [FMOP_MARK] = "mark",
1486 [FMOP_EXT_MARK] = "ext_mark",
1488 [FMOP_EG_HAIRPIN] = "eg_hairpin",
1489 [FMOP_IG_HAIRPIN] = "ig_hairpin",
1490 [FMOP_ENCAP_IVLAN] = "encap_ivlan",
1491 [FMOP_ENCAP_NOIVLAN] = "encap_noivlan",
1492 [FMOP_ENCAP] = "encap",
1493 [FMOP_SET_OVLAN] = "set_ovlan",
1494 [FMOP_DECAP_NOSTRIP] = "decap_nostrip",
1495 [FMOP_DECAP_STRIP] = "decap_strip",
1496 [FMOP_POP_VLAN] = "pop_vlan",
1497 [FMOP_SET_EGPORT] = "set_egport",
1498 [FMOP_RQ_STEER_ONLY] = "rq_steer_only",
1499 [FMOP_SET_ENCAP_VLAN] = "set_encap_vlan",
1500 [FMOP_EMIT] = "emit",
1501 [FMOP_MODIFY] = "modify",
1503 const struct fm_action_op *op = &fm_action->fma_action_ops[0];
1504 char buf[128], *bp = buf;
1509 buf_len = sizeof(buf);
1510 for (i = 0; i < FM_ACTION_OP_MAX; i++) {
1511 if (op->fa_op == FMOP_END)
1513 if (op->fa_op >= FMOP_OP_MAX)
1516 op_str = fmop_str[op->fa_op];
1517 n = snprintf(bp, buf_len, "%s,", op_str);
1518 if (n > 0 && n < buf_len) {
1524 /* Remove trailing comma */
1527 ENICPMD_LOG(DEBUG, " Acions: %s", buf);
1531 bits_to_str(uint32_t bits, const char *strings[], int max,
1532 char *buf, int buf_len)
1534 int i, n = 0, len = 0;
1536 for (i = 0; i < max; i++) {
1537 if (bits & (1 << i)) {
1538 n = snprintf(buf, buf_len, "%s,", strings[i]);
1539 if (n > 0 && n < buf_len) {
1546 /* Remove trailing comma */
1554 /* Debug function to dump internal NIC filter structure. */
1556 __enic_fm_dump_tcam_match(const struct fm_header_set *fk_hdrset, char *buf,
1559 /* Manually keep in sync with FKM_BITS */
1560 const char *fm_fkm_str[FKM_BIT_COUNT] = {
1561 [FKM_QTAG_BIT] = "qtag",
1562 [FKM_CMD_BIT] = "cmd",
1563 [FKM_IPV4_BIT] = "ip4",
1564 [FKM_IPV6_BIT] = "ip6",
1565 [FKM_ROCE_BIT] = "roce",
1566 [FKM_UDP_BIT] = "udp",
1567 [FKM_TCP_BIT] = "tcp",
1568 [FKM_TCPORUDP_BIT] = "tcpportudp",
1569 [FKM_IPFRAG_BIT] = "ipfrag",
1570 [FKM_NVGRE_BIT] = "nvgre",
1571 [FKM_VXLAN_BIT] = "vxlan",
1572 [FKM_GENEVE_BIT] = "geneve",
1573 [FKM_NSH_BIT] = "nsh",
1574 [FKM_ROCEV2_BIT] = "rocev2",
1575 [FKM_VLAN_PRES_BIT] = "vlan_pres",
1576 [FKM_IPOK_BIT] = "ipok",
1577 [FKM_L4OK_BIT] = "l4ok",
1578 [FKM_ROCEOK_BIT] = "roceok",
1579 [FKM_FCSOK_BIT] = "fcsok",
1580 [FKM_EG_SPAN_BIT] = "eg_span",
1581 [FKM_IG_SPAN_BIT] = "ig_span",
1582 [FKM_EG_HAIRPINNED_BIT] = "eg_hairpinned",
1584 /* Manually keep in sync with FKH_BITS */
1585 const char *fm_fkh_str[FKH_BIT_COUNT] = {
1586 [FKH_ETHER_BIT] = "eth",
1587 [FKH_QTAG_BIT] = "qtag",
1588 [FKH_L2RAW_BIT] = "l2raw",
1589 [FKH_IPV4_BIT] = "ip4",
1590 [FKH_IPV6_BIT] = "ip6",
1591 [FKH_L3RAW_BIT] = "l3raw",
1592 [FKH_UDP_BIT] = "udp",
1593 [FKH_TCP_BIT] = "tcp",
1594 [FKH_ICMP_BIT] = "icmp",
1595 [FKH_VXLAN_BIT] = "vxlan",
1596 [FKH_L4RAW_BIT] = "l4raw",
1598 uint32_t fkh_bits = fk_hdrset->fk_header_select;
1599 uint32_t fkm_bits = fk_hdrset->fk_metadata;
1602 if (!fkm_bits && !fkh_bits)
1604 n = snprintf(buf, buf_len, "metadata(");
1605 if (n > 0 && n < buf_len) {
1609 n = bits_to_str(fkm_bits, fm_fkm_str, FKM_BIT_COUNT, buf, buf_len);
1610 if (n > 0 && n < buf_len) {
1614 n = snprintf(buf, buf_len, ") valid hdr fields(");
1615 if (n > 0 && n < buf_len) {
1619 n = bits_to_str(fkh_bits, fm_fkh_str, FKH_BIT_COUNT, buf, buf_len);
1620 if (n > 0 && n < buf_len) {
1624 snprintf(buf, buf_len, ")");
1628 enic_fm_dump_tcam_match(const struct fm_tcam_match_entry *match,
1633 memset(buf, 0, sizeof(buf));
1634 __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[0],
1636 ENICPMD_LOG(DEBUG, " TCAM %s Outer: %s %scounter",
1637 (ingress) ? "IG" : "EG", buf,
1638 (match->ftm_flags & FMEF_COUNTER) ? "" : "no ");
1639 memset(buf, 0, sizeof(buf));
1640 __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[1],
1643 ENICPMD_LOG(DEBUG, " Inner: %s", buf);
1646 /* Debug function to dump internal NIC flow structures. */
1648 enic_fm_dump_tcam_entry(const struct fm_tcam_match_entry *fm_match,
1649 const struct fm_action *fm_action,
1652 if (!rte_log_can_log(enic_pmd_logtype, RTE_LOG_DEBUG))
1654 enic_fm_dump_tcam_match(fm_match, ingress);
1655 enic_fm_dump_tcam_actions(fm_action);
1659 enic_fm_flow_parse(struct enic_flowman *fm,
1660 const struct rte_flow_attr *attrs,
1661 const struct rte_flow_item pattern[],
1662 const struct rte_flow_action actions[],
1663 struct rte_flow_error *error)
1665 const struct rte_flow_action *action;
1667 static const enum rte_flow_action_type *sa;
1669 ENICPMD_FUNC_TRACE();
1672 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM,
1673 NULL, "no pattern specified");
1678 rte_flow_error_set(error, EINVAL,
1679 RTE_FLOW_ERROR_TYPE_ACTION_NUM,
1680 NULL, "no action specified");
1685 if (attrs->priority) {
1686 rte_flow_error_set(error, ENOTSUP,
1687 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1689 "priorities are not supported");
1691 } else if (!fm->owner_enic->switchdev_mode && attrs->transfer) {
1692 rte_flow_error_set(error, ENOTSUP,
1693 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1695 "transfer is not supported");
1697 } else if (attrs->ingress && attrs->egress) {
1698 rte_flow_error_set(error, ENOTSUP,
1699 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1701 "bidirectional rules not supported");
1706 rte_flow_error_set(error, EINVAL,
1707 RTE_FLOW_ERROR_TYPE_ATTR,
1708 NULL, "no attribute specified");
1712 /* Verify Actions. */
1713 sa = (attrs->ingress) ? enic_fm_supported_ig_actions :
1714 enic_fm_supported_eg_actions;
1715 for (action = &actions[0]; action->type != RTE_FLOW_ACTION_TYPE_END;
1717 if (action->type == RTE_FLOW_ACTION_TYPE_VOID)
1719 else if (!enic_fm_match_action(action, sa))
1722 if (action->type != RTE_FLOW_ACTION_TYPE_END) {
1723 rte_flow_error_set(error, EPERM, RTE_FLOW_ERROR_TYPE_ACTION,
1724 action, "invalid action");
1727 ret = enic_fm_copy_entry(fm, pattern, error);
1730 ret = enic_fm_copy_action(fm, actions, attrs->ingress, error);
1735 enic_fm_counter_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
1737 if (!fm_flow->counter_valid)
1739 SLIST_INSERT_HEAD(&fm->counters, fm_flow->counter, next);
1740 fm_flow->counter_valid = false;
1744 enic_fm_more_counters(struct enic_flowman *fm)
1746 struct enic_fm_counter *new_stack;
1747 struct enic_fm_counter *ctrs;
1751 ENICPMD_FUNC_TRACE();
1752 new_stack = rte_realloc(fm->counter_stack, (fm->counters_alloced +
1753 FM_COUNTERS_EXPAND) *
1754 sizeof(struct enic_fm_counter), 0);
1755 if (new_stack == NULL) {
1756 ENICPMD_LOG(ERR, "cannot alloc counter memory");
1759 fm->counter_stack = new_stack;
1761 args[0] = FM_COUNTER_BRK;
1762 args[1] = fm->counters_alloced + FM_COUNTERS_EXPAND;
1763 rc = flowman_cmd(fm, args, 2);
1765 ENICPMD_LOG(ERR, "cannot alloc counters rc=%d", rc);
1768 ctrs = (struct enic_fm_counter *)fm->counter_stack +
1769 fm->counters_alloced;
1770 for (i = 0; i < FM_COUNTERS_EXPAND; i++, ctrs++) {
1771 ctrs->handle = fm->counters_alloced + i;
1772 SLIST_INSERT_HEAD(&fm->counters, ctrs, next);
1774 fm->counters_alloced += FM_COUNTERS_EXPAND;
1775 ENICPMD_LOG(DEBUG, "%u counters allocated, total: %u",
1776 FM_COUNTERS_EXPAND, fm->counters_alloced);
1781 enic_fm_counter_zero(struct enic_flowman *fm, struct enic_fm_counter *c)
1786 ENICPMD_FUNC_TRACE();
1787 args[0] = FM_COUNTER_QUERY;
1788 args[1] = c->handle;
1789 args[2] = 1; /* clear */
1790 ret = flowman_cmd(fm, args, 3);
1792 ENICPMD_LOG(ERR, "counter init: rc=%d handle=0x%x",
1800 enic_fm_counter_alloc(struct enic_flowman *fm, struct rte_flow_error *error,
1801 struct enic_fm_counter **ctr)
1803 struct enic_fm_counter *c;
1806 ENICPMD_FUNC_TRACE();
1808 if (SLIST_EMPTY(&fm->counters)) {
1809 ret = enic_fm_more_counters(fm);
1811 return rte_flow_error_set(error, -ret,
1812 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1813 NULL, "enic: out of counters");
1815 c = SLIST_FIRST(&fm->counters);
1816 SLIST_REMOVE_HEAD(&fm->counters, next);
1822 enic_fm_action_free(struct enic_flowman *fm, uint64_t handle)
1827 ENICPMD_FUNC_TRACE();
1828 args[0] = FM_ACTION_FREE;
1830 rc = flowman_cmd(fm, args, 2);
1832 ENICPMD_LOG(ERR, "cannot free action: rc=%d handle=0x%" PRIx64,
1838 enic_fm_entry_free(struct enic_flowman *fm, uint64_t handle)
1843 ENICPMD_FUNC_TRACE();
1844 args[0] = FM_MATCH_ENTRY_REMOVE;
1846 rc = flowman_cmd(fm, args, 2);
1848 ENICPMD_LOG(ERR, "cannot free match entry: rc=%d"
1849 " handle=0x%" PRIx64, rc, handle);
1853 static struct enic_fm_jump_flow *
1854 find_jump_flow(struct enic_flowman *fm, uint32_t group)
1856 struct enic_fm_jump_flow *j;
1858 ENICPMD_FUNC_TRACE();
1859 TAILQ_FOREACH(j, &fm->jump_list, list) {
1860 if (j->group == group)
1867 remove_jump_flow(struct enic_flowman *fm, struct rte_flow *flow)
1869 struct enic_fm_jump_flow *j;
1871 ENICPMD_FUNC_TRACE();
1872 TAILQ_FOREACH(j, &fm->jump_list, list) {
1873 if (j->flow == flow) {
1874 TAILQ_REMOVE(&fm->jump_list, j, list);
1882 save_jump_flow(struct enic_flowman *fm,
1883 struct rte_flow *flow,
1885 struct fm_tcam_match_entry *match,
1886 struct fm_action *action)
1888 struct enic_fm_jump_flow *j;
1890 ENICPMD_FUNC_TRACE();
1891 j = calloc(1, sizeof(struct enic_fm_jump_flow));
1897 j->action = *action;
1898 TAILQ_INSERT_HEAD(&fm->jump_list, j, list);
1899 ENICPMD_LOG(DEBUG, "saved jump flow: flow=%p group=%u", flow, group);
1904 __enic_fm_flow_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
1906 if (fm_flow->entry_handle != FM_INVALID_HANDLE) {
1907 enic_fm_entry_free(fm, fm_flow->entry_handle);
1908 fm_flow->entry_handle = FM_INVALID_HANDLE;
1910 if (fm_flow->action_handle != FM_INVALID_HANDLE) {
1911 enic_fm_action_free(fm, fm_flow->action_handle);
1912 fm_flow->action_handle = FM_INVALID_HANDLE;
1914 enic_fm_counter_free(fm, fm_flow);
1916 enic_fet_put(fm, fm_flow->fet);
1917 fm_flow->fet = NULL;
1922 enic_fm_flow_free(struct enic_flowman *fm, struct rte_flow *flow)
1924 if (flow->fm->fet && flow->fm->fet->default_key)
1925 remove_jump_flow(fm, flow);
1926 __enic_fm_flow_free(fm, flow->fm);
1932 enic_fm_add_tcam_entry(struct enic_flowman *fm,
1933 struct fm_tcam_match_entry *match_in,
1934 uint64_t *entry_handle,
1936 struct rte_flow_error *error)
1938 struct fm_tcam_match_entry *ftm;
1942 ENICPMD_FUNC_TRACE();
1943 /* Copy entry to the command buffer */
1944 ftm = &fm->cmd.va->fm_tcam_match_entry;
1945 memcpy(ftm, match_in, sizeof(*ftm));
1946 /* Add TCAM entry */
1947 args[0] = FM_TCAM_ENTRY_INSTALL;
1948 args[1] = ingress ? fm->ig_tcam_hndl : fm->eg_tcam_hndl;
1949 args[2] = fm->cmd.pa;
1950 ret = flowman_cmd(fm, args, 3);
1952 ENICPMD_LOG(ERR, "cannot add %s TCAM entry: rc=%d",
1953 ingress ? "ingress" : "egress", ret);
1954 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1955 NULL, "enic: devcmd(tcam-entry-install)");
1958 ENICPMD_LOG(DEBUG, "installed %s TCAM entry: handle=0x%" PRIx64,
1959 ingress ? "ingress" : "egress", (uint64_t)args[0]);
1960 *entry_handle = args[0];
1965 enic_fm_add_exact_entry(struct enic_flowman *fm,
1966 struct fm_tcam_match_entry *match_in,
1967 uint64_t *entry_handle,
1968 struct enic_fm_fet *fet,
1969 struct rte_flow_error *error)
1971 struct fm_exact_match_entry *fem;
1975 ENICPMD_FUNC_TRACE();
1976 /* The new entry must have the table's key */
1977 if (memcmp(fet->key.fk_hdrset, match_in->ftm_mask.fk_hdrset,
1978 sizeof(struct fm_header_set) * FM_HDRSET_MAX)) {
1979 return rte_flow_error_set(error, EINVAL,
1980 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1981 "enic: key does not match group's key");
1984 /* Copy entry to the command buffer */
1985 fem = &fm->cmd.va->fm_exact_match_entry;
1987 * Translate TCAM entry to exact entry. As is only need to drop
1988 * position and mask. The mask is part of the exact match table.
1989 * Position (aka priority) is not supported in the exact match table.
1991 fem->fem_data = match_in->ftm_data;
1992 fem->fem_flags = match_in->ftm_flags;
1993 fem->fem_action = match_in->ftm_action;
1994 fem->fem_counter = match_in->ftm_counter;
1996 /* Add exact entry */
1997 args[0] = FM_EXACT_ENTRY_INSTALL;
1998 args[1] = fet->handle;
1999 args[2] = fm->cmd.pa;
2000 ret = flowman_cmd(fm, args, 3);
2002 ENICPMD_LOG(ERR, "cannot add %s exact entry: group=%u",
2003 fet->ingress ? "ingress" : "egress", fet->group);
2004 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2005 NULL, "enic: devcmd(exact-entry-install)");
2008 ENICPMD_LOG(DEBUG, "installed %s exact entry: group=%u"
2009 " handle=0x%" PRIx64,
2010 fet->ingress ? "ingress" : "egress", fet->group,
2012 *entry_handle = args[0];
2016 /* Push match-action to the NIC. */
2018 __enic_fm_flow_add_entry(struct enic_flowman *fm,
2019 struct enic_fm_flow *fm_flow,
2020 struct fm_tcam_match_entry *match_in,
2021 struct fm_action *action_in,
2024 struct rte_flow_error *error)
2026 struct enic_fm_counter *ctr;
2027 struct fm_action *fma;
2033 ENICPMD_FUNC_TRACE();
2034 /* Allocate action. */
2035 fma = &fm->cmd.va->fm_action;
2036 memcpy(fma, action_in, sizeof(*fma));
2037 args[0] = FM_ACTION_ALLOC;
2038 args[1] = fm->cmd.pa;
2039 ret = flowman_cmd(fm, args, 2);
2041 ENICPMD_LOG(ERR, "allocating TCAM table action rc=%d", ret);
2042 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2043 NULL, "enic: devcmd(action-alloc)");
2047 fm_flow->action_handle = action_h;
2048 match_in->ftm_action = action_h;
2049 ENICPMD_LOG(DEBUG, "action allocated: handle=0x%" PRIx64, action_h);
2051 /* Allocate counter if requested. */
2052 if (match_in->ftm_flags & FMEF_COUNTER) {
2053 ret = enic_fm_counter_alloc(fm, error, &ctr);
2054 if (ret) /* error has been filled in */
2056 fm_flow->counter_valid = true;
2057 fm_flow->counter = ctr;
2058 match_in->ftm_counter = ctr->handle;
2062 * Get the group's table (either TCAM or exact match table) and
2063 * add entry to it. If we use the exact match table, the handler
2064 * will translate the TCAM entry (match_in) to the appropriate
2065 * exact match entry and use that instead.
2067 entry_h = FM_INVALID_HANDLE;
2068 if (group == FM_TCAM_RTE_GROUP) {
2069 ret = enic_fm_add_tcam_entry(fm, match_in, &entry_h, ingress,
2073 /* Jump action might have a ref to fet */
2074 fm_flow->fet = fm->fet;
2077 struct enic_fm_fet *fet = NULL;
2079 ret = enic_fet_get(fm, group, ingress,
2080 &match_in->ftm_mask, &fet, error);
2084 ret = enic_fm_add_exact_entry(fm, match_in, &entry_h, fet,
2089 /* Clear counter after adding entry, as it requires in-use counter */
2090 if (fm_flow->counter_valid) {
2091 ret = enic_fm_counter_zero(fm, fm_flow->counter);
2095 fm_flow->entry_handle = entry_h;
2099 /* Push match-action to the NIC. */
2100 static struct rte_flow *
2101 enic_fm_flow_add_entry(struct enic_flowman *fm,
2102 struct fm_tcam_match_entry *match_in,
2103 struct fm_action *action_in,
2104 const struct rte_flow_attr *attrs,
2105 struct rte_flow_error *error)
2107 struct enic_fm_flow *fm_flow;
2108 struct rte_flow *flow;
2110 ENICPMD_FUNC_TRACE();
2111 enic_fm_dump_tcam_entry(match_in, action_in, attrs->ingress);
2112 flow = calloc(1, sizeof(*flow));
2113 fm_flow = calloc(1, sizeof(*fm_flow));
2114 if (flow == NULL || fm_flow == NULL) {
2115 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
2116 NULL, "enic: cannot allocate rte_flow");
2122 fm_flow->action_handle = FM_INVALID_HANDLE;
2123 fm_flow->entry_handle = FM_INVALID_HANDLE;
2124 if (__enic_fm_flow_add_entry(fm, fm_flow, match_in, action_in,
2125 attrs->group, attrs->ingress, error)) {
2126 enic_fm_flow_free(fm, flow);
2133 convert_jump_flows(struct enic_flowman *fm, struct enic_fm_fet *fet,
2134 struct rte_flow_error *error)
2136 struct enic_fm_flow *fm_flow;
2137 struct enic_fm_jump_flow *j;
2138 struct fm_action *fma;
2141 ENICPMD_FUNC_TRACE();
2143 * Find the saved flows that should jump to the new table (fet).
2144 * Then delete the old TCAM entry that jumps to the default table,
2145 * and add a new one that jumps to the new table.
2148 j = find_jump_flow(fm, group);
2150 ENICPMD_LOG(DEBUG, "convert jump flow: flow=%p group=%u",
2152 /* Delete old entry */
2153 fm_flow = j->flow->fm;
2154 __enic_fm_flow_free(fm, fm_flow);
2158 fma->fma_action_ops[0].exact.handle = fet->handle;
2159 if (__enic_fm_flow_add_entry(fm, fm_flow, &j->match, fma,
2160 FM_TCAM_RTE_GROUP, fet->ingress, error)) {
2161 /* Cannot roll back changes at the moment */
2162 ENICPMD_LOG(ERR, "cannot convert jump flow: flow=%p",
2167 ENICPMD_LOG(DEBUG, "convert ok: group=%u ref=%u",
2168 fet->group, fet->ref);
2171 TAILQ_REMOVE(&fm->jump_list, j, list);
2173 j = find_jump_flow(fm, group);
2178 enic_fm_open_scratch(struct enic_flowman *fm)
2180 fm->action_op_count = 0;
2182 memset(&fm->tcam_entry, 0, sizeof(fm->tcam_entry));
2183 memset(&fm->action, 0, sizeof(fm->action));
2187 enic_fm_close_scratch(struct enic_flowman *fm)
2190 enic_fet_put(fm, fm->fet);
2193 fm->action_op_count = 0;
2197 enic_fm_flow_validate(struct rte_eth_dev *dev,
2198 const struct rte_flow_attr *attrs,
2199 const struct rte_flow_item pattern[],
2200 const struct rte_flow_action actions[],
2201 struct rte_flow_error *error)
2203 struct fm_tcam_match_entry *fm_tcam_entry;
2204 struct fm_action *fm_action;
2205 struct enic_flowman *fm;
2208 ENICPMD_FUNC_TRACE();
2209 fm = begin_fm(pmd_priv(dev));
2212 enic_fm_open_scratch(fm);
2213 ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2215 fm_tcam_entry = &fm->tcam_entry;
2216 fm_action = &fm->action;
2217 enic_fm_dump_tcam_entry(fm_tcam_entry, fm_action,
2220 enic_fm_close_scratch(fm);
2226 enic_fm_flow_query_count(struct rte_eth_dev *dev,
2227 struct rte_flow *flow, void *data,
2228 struct rte_flow_error *error)
2230 struct rte_flow_query_count *query;
2231 struct enic_fm_flow *fm_flow;
2232 struct enic_flowman *fm;
2236 ENICPMD_FUNC_TRACE();
2237 fm = begin_fm(pmd_priv(dev));
2240 if (!fm_flow->counter_valid) {
2241 rc = rte_flow_error_set(error, ENOTSUP,
2242 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2243 "enic: flow does not have counter");
2247 args[0] = FM_COUNTER_QUERY;
2248 args[1] = fm_flow->counter->handle;
2249 args[2] = query->reset;
2250 rc = flowman_cmd(fm, args, 3);
2252 ENICPMD_LOG(ERR, "cannot query counter: rc=%d handle=0x%x",
2253 rc, fm_flow->counter->handle);
2256 query->hits_set = 1;
2257 query->hits = args[0];
2258 query->bytes_set = 1;
2259 query->bytes = args[1];
2267 enic_fm_flow_query(struct rte_eth_dev *dev,
2268 struct rte_flow *flow,
2269 const struct rte_flow_action *actions,
2271 struct rte_flow_error *error)
2275 ENICPMD_FUNC_TRACE();
2276 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2277 switch (actions->type) {
2278 case RTE_FLOW_ACTION_TYPE_VOID:
2280 case RTE_FLOW_ACTION_TYPE_COUNT:
2281 ret = enic_fm_flow_query_count(dev, flow, data, error);
2284 return rte_flow_error_set(error, ENOTSUP,
2285 RTE_FLOW_ERROR_TYPE_ACTION,
2287 "action not supported");
2295 static struct rte_flow *
2296 enic_fm_flow_create(struct rte_eth_dev *dev,
2297 const struct rte_flow_attr *attrs,
2298 const struct rte_flow_item pattern[],
2299 const struct rte_flow_action actions[],
2300 struct rte_flow_error *error)
2302 struct fm_tcam_match_entry *fm_tcam_entry;
2303 struct fm_action *fm_action;
2304 struct enic_flowman *fm;
2305 struct enic_fm_fet *fet;
2306 struct rte_flow *flow;
2310 ENICPMD_FUNC_TRACE();
2311 enic = pmd_priv(dev);
2312 fm = begin_fm(enic);
2314 rte_flow_error_set(error, ENOTSUP,
2315 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2316 "flowman is not initialized");
2319 enic_fm_open_scratch(fm);
2321 ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2323 goto error_with_scratch;
2324 fm_tcam_entry = &fm->tcam_entry;
2325 fm_action = &fm->action;
2326 flow = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2329 LIST_INSERT_HEAD(&enic->flows, flow, next);
2330 fet = flow->fm->fet;
2331 if (fet && fet->default_key) {
2333 * Jump to non-existent group? Save the relevant info
2334 * so we can convert this flow when that group
2337 save_jump_flow(fm, flow, fet->group,
2338 fm_tcam_entry, fm_action);
2339 } else if (fet && fet->ref == 1) {
2341 * A new table is created. Convert the saved flows
2342 * that should jump to this group.
2344 convert_jump_flows(fm, fet, error);
2349 enic_fm_close_scratch(fm);
2355 enic_fm_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
2356 __rte_unused struct rte_flow_error *error)
2358 struct enic *enic = pmd_priv(dev);
2359 struct enic_flowman *fm;
2361 ENICPMD_FUNC_TRACE();
2362 fm = begin_fm(enic);
2365 LIST_REMOVE(flow, next);
2366 enic_fm_flow_free(fm, flow);
2372 enic_fm_flow_flush(struct rte_eth_dev *dev,
2373 __rte_unused struct rte_flow_error *error)
2375 LIST_HEAD(enic_flows, rte_flow) internal;
2376 struct enic_fm_flow *fm_flow;
2377 struct enic_flowman *fm;
2378 struct rte_flow *flow;
2379 struct enic *enic = pmd_priv(dev);
2381 ENICPMD_FUNC_TRACE();
2383 fm = begin_fm(enic);
2386 /* Destroy all non-internal flows */
2387 LIST_INIT(&internal);
2388 while (!LIST_EMPTY(&enic->flows)) {
2389 flow = LIST_FIRST(&enic->flows);
2391 LIST_REMOVE(flow, next);
2392 if (flow->internal) {
2393 LIST_INSERT_HEAD(&internal, flow, next);
2397 * If tables are null, then vNIC is closing, and the firmware
2398 * has already cleaned up flowman state. So do not try to free
2399 * resources, as it only causes errors.
2401 if (fm->ig_tcam_hndl == FM_INVALID_HANDLE) {
2402 fm_flow->entry_handle = FM_INVALID_HANDLE;
2403 fm_flow->action_handle = FM_INVALID_HANDLE;
2404 fm_flow->fet = NULL;
2406 enic_fm_flow_free(fm, flow);
2408 while (!LIST_EMPTY(&internal)) {
2409 flow = LIST_FIRST(&internal);
2410 LIST_REMOVE(flow, next);
2411 LIST_INSERT_HEAD(&enic->flows, flow, next);
2418 enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle)
2423 args[0] = FM_MATCH_TABLE_FREE;
2425 rc = flowman_cmd(fm, args, 2);
2427 ENICPMD_LOG(ERR, "cannot free table: rc=%d handle=0x%" PRIx64,
2433 enic_fm_tcam_tbl_alloc(struct enic_flowman *fm, uint32_t direction,
2434 uint32_t max_entries, uint64_t *handle)
2436 struct fm_tcam_match_table *tcam_tbl;
2440 ENICPMD_FUNC_TRACE();
2441 tcam_tbl = &fm->cmd.va->fm_tcam_match_table;
2442 tcam_tbl->ftt_direction = direction;
2443 tcam_tbl->ftt_stage = FM_STAGE_LAST;
2444 tcam_tbl->ftt_max_entries = max_entries;
2445 args[0] = FM_TCAM_TABLE_ALLOC;
2446 args[1] = fm->cmd.pa;
2447 rc = flowman_cmd(fm, args, 2);
2449 ENICPMD_LOG(ERR, "cannot alloc %s TCAM table: rc=%d",
2450 (direction == FM_INGRESS) ? "IG" : "EG", rc);
2454 ENICPMD_LOG(DEBUG, "%s TCAM table allocated, handle=0x%" PRIx64,
2455 (direction == FM_INGRESS) ? "IG" : "EG", *handle);
2460 enic_fm_init_counters(struct enic_flowman *fm)
2462 ENICPMD_FUNC_TRACE();
2463 SLIST_INIT(&fm->counters);
2464 return enic_fm_more_counters(fm);
2468 enic_fm_free_all_counters(struct enic_flowman *fm)
2473 args[0] = FM_COUNTER_BRK;
2475 rc = flowman_cmd(fm, args, 2);
2477 ENICPMD_LOG(ERR, "cannot free counters: rc=%d", rc);
2478 rte_free(fm->counter_stack);
2482 enic_fm_alloc_tcam_tables(struct enic_flowman *fm)
2486 ENICPMD_FUNC_TRACE();
2487 rc = enic_fm_tcam_tbl_alloc(fm, FM_INGRESS, FM_MAX_TCAM_TABLE_SIZE,
2491 rc = enic_fm_tcam_tbl_alloc(fm, FM_EGRESS, FM_MAX_TCAM_TABLE_SIZE,
2497 enic_fm_free_tcam_tables(struct enic_flowman *fm)
2499 ENICPMD_FUNC_TRACE();
2500 if (fm->ig_tcam_hndl) {
2501 ENICPMD_LOG(DEBUG, "free IG TCAM table handle=0x%" PRIx64,
2503 enic_fm_tbl_free(fm, fm->ig_tcam_hndl);
2504 fm->ig_tcam_hndl = FM_INVALID_HANDLE;
2506 if (fm->eg_tcam_hndl) {
2507 ENICPMD_LOG(DEBUG, "free EG TCAM table handle=0x%" PRIx64,
2509 enic_fm_tbl_free(fm, fm->eg_tcam_hndl);
2510 fm->eg_tcam_hndl = FM_INVALID_HANDLE;
2515 enic_fm_init(struct enic *enic)
2517 const struct rte_pci_addr *addr;
2518 struct enic_flowman *fm;
2519 uint8_t name[RTE_MEMZONE_NAMESIZE];
2522 if (enic->flow_filter_mode != FILTER_FLOWMAN)
2524 ENICPMD_FUNC_TRACE();
2525 /* Get vnic handle and save for port-id action */
2526 if (enic_is_vf_rep(enic))
2527 addr = &VF_ENIC_TO_VF_REP(enic)->bdf;
2529 addr = &RTE_ETH_DEV_TO_PCI(enic->rte_dev)->addr;
2530 rc = enic_fm_find_vnic(enic, addr, &enic->fm_vnic_handle);
2532 ENICPMD_LOG(ERR, "cannot find vnic handle for %x:%x:%x",
2533 addr->bus, addr->devid, addr->function);
2536 /* Save UIF for egport action */
2537 enic->fm_vnic_uif = vnic_dev_uif(enic->vdev);
2538 ENICPMD_LOG(DEBUG, "uif %u", enic->fm_vnic_uif);
2539 /* Nothing else to do for representor. It will share the PF flowman */
2540 if (enic_is_vf_rep(enic))
2542 fm = calloc(1, sizeof(*fm));
2544 ENICPMD_LOG(ERR, "cannot alloc flowman struct");
2547 fm->owner_enic = enic;
2548 rte_spinlock_init(&fm->lock);
2549 TAILQ_INIT(&fm->fet_list);
2550 TAILQ_INIT(&fm->jump_list);
2551 /* Allocate host memory for flowman commands */
2552 snprintf((char *)name, sizeof(name), "fm-cmd-%s", enic->bdf_name);
2553 fm->cmd.va = enic_alloc_consistent(enic,
2554 sizeof(union enic_flowman_cmd_mem), &fm->cmd.pa, name);
2556 ENICPMD_LOG(ERR, "cannot allocate flowman command memory");
2560 /* Allocate TCAM tables upfront as they are the main tables */
2561 rc = enic_fm_alloc_tcam_tables(fm);
2563 ENICPMD_LOG(ERR, "cannot alloc TCAM tables");
2566 /* Then a number of counters */
2567 rc = enic_fm_init_counters(fm);
2569 ENICPMD_LOG(ERR, "cannot alloc counters");
2573 * One default exact match table for each direction. We hold onto
2576 rc = enic_fet_alloc(fm, 1, NULL, 128, &fm->default_ig_fet);
2578 ENICPMD_LOG(ERR, "cannot alloc default IG exact match table");
2579 goto error_counters;
2581 fm->default_ig_fet->ref = 1;
2582 rc = enic_fet_alloc(fm, 0, NULL, 128, &fm->default_eg_fet);
2584 ENICPMD_LOG(ERR, "cannot alloc default EG exact match table");
2587 fm->default_eg_fet->ref = 1;
2588 fm->vf_rep_tag = FM_VF_REP_TAG;
2593 enic_fet_free(fm, fm->default_ig_fet);
2595 enic_fm_free_all_counters(fm);
2597 enic_fm_free_tcam_tables(fm);
2599 enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
2600 fm->cmd.va, fm->cmd.pa);
2607 enic_fm_destroy(struct enic *enic)
2609 struct enic_flowman *fm;
2610 struct enic_fm_fet *fet;
2612 ENICPMD_FUNC_TRACE();
2613 if (enic_is_vf_rep(enic)) {
2614 delete_rep_flows(enic);
2617 if (enic->fm == NULL)
2620 enic_fet_free(fm, fm->default_eg_fet);
2621 enic_fet_free(fm, fm->default_ig_fet);
2622 /* Free all exact match tables still open */
2623 while (!TAILQ_EMPTY(&fm->fet_list)) {
2624 fet = TAILQ_FIRST(&fm->fet_list);
2625 enic_fet_free(fm, fet);
2627 enic_fm_free_tcam_tables(fm);
2628 enic_fm_free_all_counters(fm);
2629 enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
2630 fm->cmd.va, fm->cmd.pa);
2637 enic_fm_allocate_switch_domain(struct enic *pf)
2639 const struct rte_pci_addr *cur_a, *prev_a;
2640 struct rte_eth_dev *dev;
2641 struct enic *cur, *prev;
2647 ENICPMD_FUNC_TRACE();
2648 if (enic_is_vf_rep(pf))
2651 cur_a = &RTE_ETH_DEV_TO_PCI(cur->rte_dev)->addr;
2652 /* Go through ports and find another PF that is on the same adapter */
2653 RTE_ETH_FOREACH_DEV(pid) {
2654 dev = &rte_eth_devices[pid];
2655 if (!dev_is_enic(dev))
2657 if (dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)
2659 if (dev == cur->rte_dev)
2661 /* dev is another PF. Is it on the same adapter? */
2662 prev = pmd_priv(dev);
2663 prev_a = &RTE_ETH_DEV_TO_PCI(dev)->addr;
2664 if (!enic_fm_find_vnic(cur, prev_a, &vnic_h)) {
2665 ENICPMD_LOG(DEBUG, "Port %u (PF BDF %x:%x:%x) and port %u (PF BDF %x:%x:%x domain %u) are on the same VIC",
2666 cur->rte_dev->data->port_id,
2667 cur_a->bus, cur_a->devid, cur_a->function,
2669 prev_a->bus, prev_a->devid, prev_a->function,
2670 prev->switch_domain_id);
2671 cur->switch_domain_id = prev->switch_domain_id;
2675 ret = rte_eth_switch_domain_alloc(&domain_id);
2677 ENICPMD_LOG(WARNING, "failed to allocate switch domain for device %d",
2680 cur->switch_domain_id = domain_id;
2681 ENICPMD_LOG(DEBUG, "Port %u (PF BDF %x:%x:%x) is the 1st PF on the VIC. Allocated switch domain id %u",
2682 cur->rte_dev->data->port_id,
2683 cur_a->bus, cur_a->devid, cur_a->function,
2688 const struct rte_flow_ops enic_fm_flow_ops = {
2689 .validate = enic_fm_flow_validate,
2690 .create = enic_fm_flow_create,
2691 .destroy = enic_fm_flow_destroy,
2692 .flush = enic_fm_flow_flush,
2693 .query = enic_fm_flow_query,
2696 /* Add a high priority flow that loops representor packets to VF */
2698 enic_fm_add_rep2vf_flow(struct enic_vf_representor *vf)
2700 struct fm_tcam_match_entry *fm_tcam_entry;
2701 struct rte_flow *flow0, *flow1;
2702 struct fm_action *fm_action;
2703 struct rte_flow_error error;
2704 struct rte_flow_attr attrs;
2705 struct fm_action_op fm_op;
2706 struct enic_flowman *fm;
2712 tag = fm->vf_rep_tag;
2713 enic_fm_open_scratch(fm);
2714 fm_tcam_entry = &fm->tcam_entry;
2715 fm_action = &fm->action;
2716 /* Egress rule: match WQ ID and tag+hairpin */
2717 fm_tcam_entry->ftm_data.fk_wq_id = vf->pf_wq_idx;
2718 fm_tcam_entry->ftm_mask.fk_wq_id = 0xffff;
2719 fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
2720 memset(&fm_op, 0, sizeof(fm_op));
2721 fm_op.fa_op = FMOP_TAG;
2722 fm_op.tag.tag = tag;
2723 enic_fm_append_action_op(fm, &fm_op, &error);
2724 memset(&fm_op, 0, sizeof(fm_op));
2725 fm_op.fa_op = FMOP_EG_HAIRPIN;
2726 enic_fm_append_action_op(fm, &fm_op, &error);
2727 memset(&fm_op, 0, sizeof(fm_op));
2728 fm_op.fa_op = FMOP_END;
2729 enic_fm_append_action_op(fm, &fm_op, &error);
2733 attrs.priority = FM_HIGHEST_PRIORITY;
2734 flow0 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2736 enic_fm_close_scratch(fm);
2737 if (flow0 == NULL) {
2738 ENICPMD_LOG(ERR, "Cannot create flow 0 for representor->VF");
2741 LIST_INSERT_HEAD(&pf->flows, flow0, next);
2742 /* Make this flow internal, so the user app cannot delete it */
2743 flow0->internal = 1;
2744 ENICPMD_LOG(DEBUG, "representor->VF %d flow created: wq %d -> tag %d hairpin",
2745 vf->vf_id, vf->pf_wq_idx, tag);
2747 /* Ingress: steer hairpinned to VF RQ 0 */
2748 enic_fm_open_scratch(fm);
2749 fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
2750 fm_tcam_entry->ftm_data.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
2751 fm_tcam_entry->ftm_mask.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
2752 fm_tcam_entry->ftm_data.fk_packet_tag = tag;
2753 fm_tcam_entry->ftm_mask.fk_packet_tag = 0xff;
2754 memset(&fm_op, 0, sizeof(fm_op));
2755 fm_op.fa_op = FMOP_RQ_STEER;
2756 fm_op.rq_steer.rq_index = 0;
2757 fm_op.rq_steer.vnic_handle = vf->enic.fm_vnic_handle;
2758 enic_fm_append_action_op(fm, &fm_op, &error);
2759 memset(&fm_op, 0, sizeof(fm_op));
2760 fm_op.fa_op = FMOP_END;
2761 enic_fm_append_action_op(fm, &fm_op, &error);
2765 attrs.priority = FM_HIGHEST_PRIORITY;
2766 flow1 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2768 enic_fm_close_scratch(fm);
2769 if (flow1 == NULL) {
2770 ENICPMD_LOG(ERR, "Cannot create flow 1 for representor->VF");
2771 enic_fm_flow_destroy(pf->rte_dev, flow0, &error);
2774 LIST_INSERT_HEAD(&pf->flows, flow1, next);
2775 flow1->internal = 1;
2776 ENICPMD_LOG(DEBUG, "representor->VF %d flow created: tag %d hairpinned -> VF RQ %d",
2777 vf->vf_id, tag, fm_op.rq_steer.rq_index);
2778 vf->rep2vf_flow[0] = flow0;
2779 vf->rep2vf_flow[1] = flow1;
2780 /* Done with this tag, use a different one next time */
2786 * Add a low priority flow that matches all packets from VF and loops them
2787 * back to the representor.
2790 enic_fm_add_vf2rep_flow(struct enic_vf_representor *vf)
2792 struct fm_tcam_match_entry *fm_tcam_entry;
2793 struct rte_flow *flow0, *flow1;
2794 struct fm_action *fm_action;
2795 struct rte_flow_error error;
2796 struct rte_flow_attr attrs;
2797 struct fm_action_op fm_op;
2798 struct enic_flowman *fm;
2804 tag = fm->vf_rep_tag;
2805 enic_fm_open_scratch(fm);
2806 fm_tcam_entry = &fm->tcam_entry;
2807 fm_action = &fm->action;
2808 /* Egress rule: match-any and tag+hairpin */
2809 fm_tcam_entry->ftm_data.fk_wq_id = 0;
2810 fm_tcam_entry->ftm_mask.fk_wq_id = 0xffff;
2811 fm_tcam_entry->ftm_data.fk_wq_vnic = vf->enic.fm_vnic_handle;
2812 fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
2813 memset(&fm_op, 0, sizeof(fm_op));
2814 fm_op.fa_op = FMOP_TAG;
2815 fm_op.tag.tag = tag;
2816 enic_fm_append_action_op(fm, &fm_op, &error);
2817 memset(&fm_op, 0, sizeof(fm_op));
2818 fm_op.fa_op = FMOP_EG_HAIRPIN;
2819 enic_fm_append_action_op(fm, &fm_op, &error);
2820 memset(&fm_op, 0, sizeof(fm_op));
2821 fm_op.fa_op = FMOP_END;
2822 enic_fm_append_action_op(fm, &fm_op, &error);
2826 attrs.priority = FM_LOWEST_PRIORITY;
2827 flow0 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2829 enic_fm_close_scratch(fm);
2830 if (flow0 == NULL) {
2831 ENICPMD_LOG(ERR, "Cannot create flow 0 for VF->representor");
2834 LIST_INSERT_HEAD(&pf->flows, flow0, next);
2835 /* Make this flow internal, so the user app cannot delete it */
2836 flow0->internal = 1;
2837 ENICPMD_LOG(DEBUG, "VF %d->representor flow created: wq %d (low prio) -> tag %d hairpin",
2838 vf->vf_id, fm_tcam_entry->ftm_data.fk_wq_id, tag);
2840 /* Ingress: steer hairpinned to VF rep RQ */
2841 enic_fm_open_scratch(fm);
2842 fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
2843 fm_tcam_entry->ftm_data.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
2844 fm_tcam_entry->ftm_mask.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
2845 fm_tcam_entry->ftm_data.fk_packet_tag = tag;
2846 fm_tcam_entry->ftm_mask.fk_packet_tag = 0xff;
2847 memset(&fm_op, 0, sizeof(fm_op));
2848 fm_op.fa_op = FMOP_RQ_STEER;
2849 fm_op.rq_steer.rq_index = vf->pf_rq_sop_idx;
2850 fm_op.rq_steer.vnic_handle = pf->fm_vnic_handle;
2851 enic_fm_append_action_op(fm, &fm_op, &error);
2852 memset(&fm_op, 0, sizeof(fm_op));
2853 fm_op.fa_op = FMOP_END;
2854 enic_fm_append_action_op(fm, &fm_op, &error);
2858 attrs.priority = FM_HIGHEST_PRIORITY;
2859 flow1 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2861 enic_fm_close_scratch(fm);
2862 if (flow1 == NULL) {
2863 ENICPMD_LOG(ERR, "Cannot create flow 1 for VF->representor");
2864 enic_fm_flow_destroy(pf->rte_dev, flow0, &error);
2867 LIST_INSERT_HEAD(&pf->flows, flow1, next);
2868 flow1->internal = 1;
2869 ENICPMD_LOG(DEBUG, "VF %d->representor flow created: tag %d hairpinned -> PF RQ %d",
2870 vf->vf_id, tag, vf->pf_rq_sop_idx);
2871 vf->vf2rep_flow[0] = flow0;
2872 vf->vf2rep_flow[1] = flow1;
2873 /* Done with this tag, use a different one next time */
2878 /* Destroy representor flows created by enic_fm_add_{rep2vf,vf2rep}_flow */
2880 delete_rep_flows(struct enic *enic)
2882 struct enic_vf_representor *vf;
2883 struct rte_flow_error error;
2884 struct rte_eth_dev *dev;
2887 RTE_ASSERT(enic_is_vf_rep(enic));
2888 vf = VF_ENIC_TO_VF_REP(enic);
2889 dev = vf->pf->rte_dev;
2890 for (i = 0; i < ARRAY_SIZE(vf->vf2rep_flow); i++) {
2891 if (vf->vf2rep_flow[i])
2892 enic_fm_flow_destroy(dev, vf->vf2rep_flow[i], &error);
2894 for (i = 0; i < ARRAY_SIZE(vf->rep2vf_flow); i++) {
2895 if (vf->rep2vf_flow[i])
2896 enic_fm_flow_destroy(dev, vf->rep2vf_flow[i], &error);
2900 static struct enic_flowman *
2901 begin_fm(struct enic *enic)
2903 struct enic_vf_representor *vf;
2904 struct enic_flowman *fm;
2906 /* Representor uses PF flowman */
2907 if (enic_is_vf_rep(enic)) {
2908 vf = VF_ENIC_TO_VF_REP(enic);
2913 /* Save the API caller and lock if representors exist */
2915 if (fm->owner_enic->switchdev_mode)
2916 rte_spinlock_lock(&fm->lock);
2917 fm->user_enic = enic;
2923 end_fm(struct enic_flowman *fm)
2925 fm->user_enic = NULL;
2926 if (fm->owner_enic->switchdev_mode)
2927 rte_spinlock_unlock(&fm->lock);