1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2008-2019 Cisco Systems, Inc. All rights reserved.
8 #include <ethdev_driver.h>
9 #include <rte_flow_driver.h>
10 #include <rte_ether.h>
11 #include <rte_geneve.h>
13 #include <rte_jhash.h>
16 #include <rte_memzone.h>
18 #include "enic_compat.h"
23 #define IP_DEFTTL 64 /* from RFC 1340. */
24 #define IP6_VTC_FLOW 0x60000000
26 /* Up to 1024 TCAM entries */
27 #define FM_MAX_TCAM_TABLE_SIZE 1024
29 /* Up to 4096 entries per exact match table */
30 #define FM_MAX_EXACT_TABLE_SIZE 4096
32 /* Number of counters to increase on for each increment */
33 #define FM_COUNTERS_EXPAND 100
35 #define FM_INVALID_HANDLE 0
37 /* Low priority used for implicit VF -> representor flow */
38 #define FM_LOWEST_PRIORITY 100000
40 /* High priority used for implicit representor -> VF flow */
41 #define FM_HIGHEST_PRIORITY 0
43 /* Tag used for implicit VF <-> representor flows */
44 #define FM_VF_REP_TAG 1
46 /* Max number of actions supported by VIC is 2K. Make hash table double that. */
47 #define FM_MAX_ACTION_TABLE_SIZE 4096
50 * Flow exact match tables (FET) in the VIC and rte_flow groups.
51 * Use a simple scheme to map groups to tables.
52 * Group 0 uses the single TCAM tables, one for each direction.
53 * Group 1, 2, ... uses its own exact match table.
55 * The TCAM tables are allocated upfront during init.
57 * Exact match tables are allocated on demand. 3 paths that lead allocations.
59 * 1. Add a flow that jumps from group 0 to group N.
61 * If N does not exist, we allocate an exact match table for it, using
62 * a dummy key. A key is required for the table.
64 * 2. Add a flow that uses group N.
66 * If N does not exist, we allocate an exact match table for it, using
67 * the flow's key. Subsequent flows to the same group all should have
70 * Without a jump flow to N, N is not reachable in hardware. No packets
73 * 3. Add a flow to an empty group N.
75 * N has been created via (1) and the dummy key. We free that table, allocate
76 * a new table using the new flow's key. Also re-do the existing jump flow to
77 * point to the new table.
79 #define FM_TCAM_RTE_GROUP 0
82 TAILQ_ENTRY(enic_fm_fet) list;
83 uint32_t group; /* rte_flow group ID */
84 uint64_t handle; /* Exact match table handle from flowman */
87 int ref; /* Reference count via get/put */
88 struct fm_key_template key; /* Key associated with the table */
91 struct enic_fm_counter {
92 SLIST_ENTRY(enic_fm_counter) next;
96 struct enic_fm_action {
103 struct enic_fm_flow {
105 uint64_t entry_handle;
106 struct enic_fm_action *action;
107 struct enic_fm_counter *counter;
108 struct enic_fm_fet *fet;
109 /* Auto-added steer action for hairpin flows (e.g. vnic->vnic) */
110 struct enic_fm_flow *hairpin_steer_flow;
113 struct enic_fm_jump_flow {
114 TAILQ_ENTRY(enic_fm_jump_flow) list;
115 struct rte_flow *flow;
117 struct fm_tcam_match_entry match;
118 struct fm_action action;
122 * Flowman uses host memory for commands. This structure is allocated
123 * in DMA-able memory.
125 union enic_flowman_cmd_mem {
126 struct fm_tcam_match_table fm_tcam_match_table;
127 struct fm_exact_match_table fm_exact_match_table;
128 struct fm_tcam_match_entry fm_tcam_match_entry;
129 struct fm_exact_match_entry fm_exact_match_entry;
130 struct fm_action fm_action;
134 * PF has a flowman instance, and VF representors share it with PF.
135 * PF allocates this structure and owns it. VF representors borrow
136 * the PF's structure during API calls (e.g. create, query).
138 struct enic_flowman {
139 struct enic *owner_enic; /* PF */
140 struct enic *user_enic; /* API caller (PF or representor) */
142 * Representors and PF share the same underlying flowman.
143 * Lock API calls to serialize accesses from them. Only used
144 * when VF representors are present.
149 union enic_flowman_cmd_mem *va;
152 /* TCAM tables allocated upfront, used for group 0 */
153 uint64_t ig_tcam_hndl;
154 uint64_t eg_tcam_hndl;
156 SLIST_HEAD(enic_free_counters, enic_fm_counter) counters;
158 uint32_t counters_alloced;
159 /* Exact match tables for groups != 0, dynamically allocated */
160 TAILQ_HEAD(fet_list, enic_fm_fet) fet_list;
162 * Default exact match tables used for jump actions to
163 * non-existent groups.
165 struct enic_fm_fet *default_eg_fet;
166 struct enic_fm_fet *default_ig_fet;
167 /* hash table for Action reuse */
168 struct rte_hash *action_hash;
169 /* Flows that jump to the default table above */
170 TAILQ_HEAD(jump_flow_list, enic_fm_jump_flow) jump_list;
172 * Scratch data used during each invocation of flow_create
175 struct enic_fm_fet *fet;
176 struct fm_tcam_match_entry tcam_entry;
177 struct fm_action action;
178 struct fm_action action_tmp; /* enic_fm_reorder_action_op */
180 /* Tags used for representor flows */
182 /* For auto-added steer action for hairpin */
183 int need_hairpin_steer;
184 uint64_t hairpin_steer_vnic_h;
187 static int enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle);
189 * API functions (create, destroy, validate, flush) call begin_fm()
190 * upon entering to save the caller enic (PF or VF representor) and
191 * lock. Upon exit, they call end_fm() to unlock.
193 static struct enic_flowman *begin_fm(struct enic *enic);
194 static void end_fm(struct enic_flowman *fm);
195 /* Delete internal flows created for representor paths */
196 static void delete_rep_flows(struct enic *enic);
199 * Common arguments passed to copy_item functions. Use this structure
200 * so we can easily add new arguments.
201 * item: Item specification.
202 * fm_tcam_entry: Flowman TCAM match entry.
203 * header_level: 0 for outer header, 1 for inner header.
205 struct copy_item_args {
206 const struct rte_flow_item *item;
207 struct fm_tcam_match_entry *fm_tcam_entry;
208 uint8_t header_level;
209 struct rte_flow_error *error;
212 /* functions for copying items into flowman match */
213 typedef int (enic_copy_item_fn)(struct copy_item_args *arg);
215 /* Info about how to copy items into flowman match */
216 struct enic_fm_items {
217 /* Function for copying and validating an item. */
218 enic_copy_item_fn * const copy_item;
219 /* List of valid previous items. */
220 const enum rte_flow_item_type * const prev_items;
222 * True if it's OK for this item to be the first item. For some NIC
223 * versions, it's invalid to start the stack above layer 3.
225 const uint8_t valid_start_item;
228 static enic_copy_item_fn enic_fm_copy_item_eth;
229 static enic_copy_item_fn enic_fm_copy_item_ipv4;
230 static enic_copy_item_fn enic_fm_copy_item_ipv6;
231 static enic_copy_item_fn enic_fm_copy_item_raw;
232 static enic_copy_item_fn enic_fm_copy_item_sctp;
233 static enic_copy_item_fn enic_fm_copy_item_tcp;
234 static enic_copy_item_fn enic_fm_copy_item_udp;
235 static enic_copy_item_fn enic_fm_copy_item_vlan;
236 static enic_copy_item_fn enic_fm_copy_item_vxlan;
237 static enic_copy_item_fn enic_fm_copy_item_gtp;
238 static enic_copy_item_fn enic_fm_copy_item_geneve;
239 static enic_copy_item_fn enic_fm_copy_item_geneve_opt;
241 /* Ingress actions */
242 static const enum rte_flow_action_type enic_fm_supported_ig_actions[] = {
243 RTE_FLOW_ACTION_TYPE_COUNT,
244 RTE_FLOW_ACTION_TYPE_DROP,
245 RTE_FLOW_ACTION_TYPE_FLAG,
246 RTE_FLOW_ACTION_TYPE_JUMP,
247 RTE_FLOW_ACTION_TYPE_MARK,
248 RTE_FLOW_ACTION_TYPE_OF_POP_VLAN,
249 RTE_FLOW_ACTION_TYPE_PORT_ID,
250 RTE_FLOW_ACTION_TYPE_PASSTHRU,
251 RTE_FLOW_ACTION_TYPE_QUEUE,
252 RTE_FLOW_ACTION_TYPE_RSS,
253 RTE_FLOW_ACTION_TYPE_VOID,
254 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
255 RTE_FLOW_ACTION_TYPE_VXLAN_DECAP,
256 RTE_FLOW_ACTION_TYPE_END, /* END must be the last entry */
260 static const enum rte_flow_action_type enic_fm_supported_eg_actions[] = {
261 RTE_FLOW_ACTION_TYPE_COUNT,
262 RTE_FLOW_ACTION_TYPE_DROP,
263 RTE_FLOW_ACTION_TYPE_JUMP,
264 RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN,
265 RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP,
266 RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID,
267 RTE_FLOW_ACTION_TYPE_PORT_ID,
268 RTE_FLOW_ACTION_TYPE_PASSTHRU,
269 RTE_FLOW_ACTION_TYPE_VOID,
270 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
271 RTE_FLOW_ACTION_TYPE_END,
274 static const struct enic_fm_items enic_fm_items[] = {
275 [RTE_FLOW_ITEM_TYPE_RAW] = {
276 .copy_item = enic_fm_copy_item_raw,
277 .valid_start_item = 0,
278 .prev_items = (const enum rte_flow_item_type[]) {
279 RTE_FLOW_ITEM_TYPE_UDP,
280 RTE_FLOW_ITEM_TYPE_END,
283 [RTE_FLOW_ITEM_TYPE_ETH] = {
284 .copy_item = enic_fm_copy_item_eth,
285 .valid_start_item = 1,
286 .prev_items = (const enum rte_flow_item_type[]) {
287 RTE_FLOW_ITEM_TYPE_END,
290 [RTE_FLOW_ITEM_TYPE_VLAN] = {
291 .copy_item = enic_fm_copy_item_vlan,
292 .valid_start_item = 1,
293 .prev_items = (const enum rte_flow_item_type[]) {
294 RTE_FLOW_ITEM_TYPE_ETH,
295 RTE_FLOW_ITEM_TYPE_END,
298 [RTE_FLOW_ITEM_TYPE_IPV4] = {
299 .copy_item = enic_fm_copy_item_ipv4,
300 .valid_start_item = 1,
301 .prev_items = (const enum rte_flow_item_type[]) {
302 RTE_FLOW_ITEM_TYPE_ETH,
303 RTE_FLOW_ITEM_TYPE_VLAN,
304 RTE_FLOW_ITEM_TYPE_END,
307 [RTE_FLOW_ITEM_TYPE_IPV6] = {
308 .copy_item = enic_fm_copy_item_ipv6,
309 .valid_start_item = 1,
310 .prev_items = (const enum rte_flow_item_type[]) {
311 RTE_FLOW_ITEM_TYPE_ETH,
312 RTE_FLOW_ITEM_TYPE_VLAN,
313 RTE_FLOW_ITEM_TYPE_END,
316 [RTE_FLOW_ITEM_TYPE_UDP] = {
317 .copy_item = enic_fm_copy_item_udp,
318 .valid_start_item = 1,
319 .prev_items = (const enum rte_flow_item_type[]) {
320 RTE_FLOW_ITEM_TYPE_IPV4,
321 RTE_FLOW_ITEM_TYPE_IPV6,
322 RTE_FLOW_ITEM_TYPE_END,
325 [RTE_FLOW_ITEM_TYPE_TCP] = {
326 .copy_item = enic_fm_copy_item_tcp,
327 .valid_start_item = 1,
328 .prev_items = (const enum rte_flow_item_type[]) {
329 RTE_FLOW_ITEM_TYPE_IPV4,
330 RTE_FLOW_ITEM_TYPE_IPV6,
331 RTE_FLOW_ITEM_TYPE_END,
334 [RTE_FLOW_ITEM_TYPE_SCTP] = {
335 .copy_item = enic_fm_copy_item_sctp,
336 .valid_start_item = 0,
337 .prev_items = (const enum rte_flow_item_type[]) {
338 RTE_FLOW_ITEM_TYPE_IPV4,
339 RTE_FLOW_ITEM_TYPE_IPV6,
340 RTE_FLOW_ITEM_TYPE_END,
343 [RTE_FLOW_ITEM_TYPE_VXLAN] = {
344 .copy_item = enic_fm_copy_item_vxlan,
345 .valid_start_item = 1,
346 .prev_items = (const enum rte_flow_item_type[]) {
347 RTE_FLOW_ITEM_TYPE_UDP,
348 RTE_FLOW_ITEM_TYPE_END,
351 [RTE_FLOW_ITEM_TYPE_GTP] = {
352 .copy_item = enic_fm_copy_item_gtp,
353 .valid_start_item = 0,
354 .prev_items = (const enum rte_flow_item_type[]) {
355 RTE_FLOW_ITEM_TYPE_UDP,
356 RTE_FLOW_ITEM_TYPE_END,
359 [RTE_FLOW_ITEM_TYPE_GTPC] = {
360 .copy_item = enic_fm_copy_item_gtp,
361 .valid_start_item = 1,
362 .prev_items = (const enum rte_flow_item_type[]) {
363 RTE_FLOW_ITEM_TYPE_UDP,
364 RTE_FLOW_ITEM_TYPE_END,
367 [RTE_FLOW_ITEM_TYPE_GTPU] = {
368 .copy_item = enic_fm_copy_item_gtp,
369 .valid_start_item = 1,
370 .prev_items = (const enum rte_flow_item_type[]) {
371 RTE_FLOW_ITEM_TYPE_UDP,
372 RTE_FLOW_ITEM_TYPE_END,
375 [RTE_FLOW_ITEM_TYPE_GENEVE] = {
376 .copy_item = enic_fm_copy_item_geneve,
377 .valid_start_item = 1,
378 .prev_items = (const enum rte_flow_item_type[]) {
379 RTE_FLOW_ITEM_TYPE_ETH,
380 RTE_FLOW_ITEM_TYPE_IPV4,
381 RTE_FLOW_ITEM_TYPE_IPV6,
382 RTE_FLOW_ITEM_TYPE_UDP,
383 RTE_FLOW_ITEM_TYPE_END,
386 [RTE_FLOW_ITEM_TYPE_GENEVE_OPT] = {
387 .copy_item = enic_fm_copy_item_geneve_opt,
388 .valid_start_item = 1,
389 /* Can match at most 1 option */
390 .prev_items = (const enum rte_flow_item_type[]) {
391 RTE_FLOW_ITEM_TYPE_GENEVE,
392 RTE_FLOW_ITEM_TYPE_END,
398 enic_fm_copy_item_eth(struct copy_item_args *arg)
400 const struct rte_flow_item *item = arg->item;
401 const struct rte_flow_item_eth *spec = item->spec;
402 const struct rte_flow_item_eth *mask = item->mask;
403 const uint8_t lvl = arg->header_level;
404 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
405 struct fm_header_set *fm_data, *fm_mask;
407 ENICPMD_FUNC_TRACE();
408 /* Match all if no spec */
412 mask = &rte_flow_item_eth_mask;
413 fm_data = &entry->ftm_data.fk_hdrset[lvl];
414 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
415 fm_data->fk_header_select |= FKH_ETHER;
416 fm_mask->fk_header_select |= FKH_ETHER;
417 memcpy(&fm_data->l2.eth, spec, sizeof(struct rte_ether_hdr));
418 memcpy(&fm_mask->l2.eth, mask, sizeof(struct rte_ether_hdr));
423 enic_fm_copy_item_vlan(struct copy_item_args *arg)
425 const struct rte_flow_item *item = arg->item;
426 const struct rte_flow_item_vlan *spec = item->spec;
427 const struct rte_flow_item_vlan *mask = item->mask;
428 const uint8_t lvl = arg->header_level;
429 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
430 struct fm_header_set *fm_data, *fm_mask;
431 struct rte_ether_hdr *eth_mask;
432 struct rte_ether_hdr *eth_val;
435 ENICPMD_FUNC_TRACE();
436 fm_data = &entry->ftm_data.fk_hdrset[lvl];
437 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
438 /* Outer and inner packet vlans need different flags */
439 meta = FKM_VLAN_PRES;
442 fm_data->fk_metadata |= meta;
443 fm_mask->fk_metadata |= meta;
445 /* Match all if no spec */
449 mask = &rte_flow_item_vlan_mask;
451 eth_mask = (void *)&fm_mask->l2.eth;
452 eth_val = (void *)&fm_data->l2.eth;
455 * Outer TPID cannot be matched. If inner_type is 0, use what is
458 if (eth_mask->ether_type && mask->inner_type)
462 * When packet matching, the VIC always compares vlan-stripped
463 * L2, regardless of vlan stripping settings. So, the inner type
464 * from vlan becomes the ether type of the eth header.
466 if (mask->inner_type) {
467 eth_mask->ether_type = mask->inner_type;
468 eth_val->ether_type = spec->inner_type;
470 fm_data->fk_header_select |= FKH_ETHER | FKH_QTAG;
471 fm_mask->fk_header_select |= FKH_ETHER | FKH_QTAG;
472 fm_data->fk_vlan = rte_be_to_cpu_16(spec->tci);
473 fm_mask->fk_vlan = rte_be_to_cpu_16(mask->tci);
478 enic_fm_copy_item_ipv4(struct copy_item_args *arg)
480 const struct rte_flow_item *item = arg->item;
481 const struct rte_flow_item_ipv4 *spec = item->spec;
482 const struct rte_flow_item_ipv4 *mask = item->mask;
483 const uint8_t lvl = arg->header_level;
484 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
485 struct fm_header_set *fm_data, *fm_mask;
487 ENICPMD_FUNC_TRACE();
488 fm_data = &entry->ftm_data.fk_hdrset[lvl];
489 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
490 fm_data->fk_metadata |= FKM_IPV4;
491 fm_mask->fk_metadata |= FKM_IPV4;
496 mask = &rte_flow_item_ipv4_mask;
498 fm_data->fk_header_select |= FKH_IPV4;
499 fm_mask->fk_header_select |= FKH_IPV4;
500 memcpy(&fm_data->l3.ip4, spec, sizeof(*spec));
501 memcpy(&fm_mask->l3.ip4, mask, sizeof(*mask));
506 enic_fm_copy_item_ipv6(struct copy_item_args *arg)
508 const struct rte_flow_item *item = arg->item;
509 const struct rte_flow_item_ipv6 *spec = item->spec;
510 const struct rte_flow_item_ipv6 *mask = item->mask;
511 const uint8_t lvl = arg->header_level;
512 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
513 struct fm_header_set *fm_data, *fm_mask;
515 ENICPMD_FUNC_TRACE();
516 fm_data = &entry->ftm_data.fk_hdrset[lvl];
517 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
518 fm_data->fk_metadata |= FKM_IPV6;
519 fm_mask->fk_metadata |= FKM_IPV6;
524 mask = &rte_flow_item_ipv6_mask;
526 fm_data->fk_header_select |= FKH_IPV6;
527 fm_mask->fk_header_select |= FKH_IPV6;
528 memcpy(&fm_data->l3.ip6, spec, sizeof(struct rte_ipv6_hdr));
529 memcpy(&fm_mask->l3.ip6, mask, sizeof(struct rte_ipv6_hdr));
534 enic_fm_copy_item_udp(struct copy_item_args *arg)
536 const struct rte_flow_item *item = arg->item;
537 const struct rte_flow_item_udp *spec = item->spec;
538 const struct rte_flow_item_udp *mask = item->mask;
539 const uint8_t lvl = arg->header_level;
540 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
541 struct fm_header_set *fm_data, *fm_mask;
543 ENICPMD_FUNC_TRACE();
544 fm_data = &entry->ftm_data.fk_hdrset[lvl];
545 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
546 fm_data->fk_metadata |= FKM_UDP;
547 fm_mask->fk_metadata |= FKM_UDP;
552 mask = &rte_flow_item_udp_mask;
554 fm_data->fk_header_select |= FKH_UDP;
555 fm_mask->fk_header_select |= FKH_UDP;
556 memcpy(&fm_data->l4.udp, spec, sizeof(*spec));
557 memcpy(&fm_mask->l4.udp, mask, sizeof(*mask));
562 enic_fm_copy_item_tcp(struct copy_item_args *arg)
564 const struct rte_flow_item *item = arg->item;
565 const struct rte_flow_item_tcp *spec = item->spec;
566 const struct rte_flow_item_tcp *mask = item->mask;
567 const uint8_t lvl = arg->header_level;
568 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
569 struct fm_header_set *fm_data, *fm_mask;
571 ENICPMD_FUNC_TRACE();
572 fm_data = &entry->ftm_data.fk_hdrset[lvl];
573 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
574 fm_data->fk_metadata |= FKM_TCP;
575 fm_mask->fk_metadata |= FKM_TCP;
580 mask = &rte_flow_item_tcp_mask;
582 fm_data->fk_header_select |= FKH_TCP;
583 fm_mask->fk_header_select |= FKH_TCP;
584 memcpy(&fm_data->l4.tcp, spec, sizeof(*spec));
585 memcpy(&fm_mask->l4.tcp, mask, sizeof(*mask));
590 enic_fm_copy_item_sctp(struct copy_item_args *arg)
592 const struct rte_flow_item *item = arg->item;
593 const struct rte_flow_item_sctp *spec = item->spec;
594 const struct rte_flow_item_sctp *mask = item->mask;
595 const uint8_t lvl = arg->header_level;
596 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
597 struct fm_header_set *fm_data, *fm_mask;
598 uint8_t *ip_proto_mask = NULL;
599 uint8_t *ip_proto = NULL;
602 ENICPMD_FUNC_TRACE();
603 fm_data = &entry->ftm_data.fk_hdrset[lvl];
604 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
606 * The NIC filter API has no flags for "match sctp", so explicitly
607 * set the protocol number in the IP pattern.
609 if (fm_data->fk_metadata & FKM_IPV4) {
610 struct rte_ipv4_hdr *ip;
611 ip = (struct rte_ipv4_hdr *)&fm_mask->l3.ip4;
612 ip_proto_mask = &ip->next_proto_id;
613 ip = (struct rte_ipv4_hdr *)&fm_data->l3.ip4;
614 ip_proto = &ip->next_proto_id;
616 } else if (fm_data->fk_metadata & FKM_IPV6) {
617 struct rte_ipv6_hdr *ip;
618 ip = (struct rte_ipv6_hdr *)&fm_mask->l3.ip6;
619 ip_proto_mask = &ip->proto;
620 ip = (struct rte_ipv6_hdr *)&fm_data->l3.ip6;
621 ip_proto = &ip->proto;
624 /* Need IPv4/IPv6 pattern first */
627 *ip_proto = IPPROTO_SCTP;
628 *ip_proto_mask = 0xff;
629 fm_data->fk_header_select |= l3_fkh;
630 fm_mask->fk_header_select |= l3_fkh;
635 mask = &rte_flow_item_sctp_mask;
637 fm_data->fk_header_select |= FKH_L4RAW;
638 fm_mask->fk_header_select |= FKH_L4RAW;
639 memcpy(fm_data->l4.rawdata, spec, sizeof(*spec));
640 memcpy(fm_mask->l4.rawdata, mask, sizeof(*mask));
645 enic_fm_copy_item_vxlan(struct copy_item_args *arg)
647 const struct rte_flow_item *item = arg->item;
648 const struct rte_flow_item_vxlan *spec = item->spec;
649 const struct rte_flow_item_vxlan *mask = item->mask;
650 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
651 struct fm_header_set *fm_data, *fm_mask;
653 ENICPMD_FUNC_TRACE();
654 /* Only 2 header levels (outer and inner) allowed */
655 if (arg->header_level > 0)
658 fm_data = &entry->ftm_data.fk_hdrset[0];
659 fm_mask = &entry->ftm_mask.fk_hdrset[0];
660 fm_data->fk_metadata |= FKM_VXLAN;
661 fm_mask->fk_metadata |= FKM_VXLAN;
662 /* items from here on out are inner header items */
663 arg->header_level = 1;
665 /* Match all if no spec */
669 mask = &rte_flow_item_vxlan_mask;
671 fm_data->fk_header_select |= FKH_VXLAN;
672 fm_mask->fk_header_select |= FKH_VXLAN;
673 memcpy(&fm_data->vxlan, spec, sizeof(*spec));
674 memcpy(&fm_mask->vxlan, mask, sizeof(*mask));
679 enic_fm_copy_item_gtp(struct copy_item_args *arg)
681 const struct rte_flow_item *item = arg->item;
682 const struct rte_flow_item_gtp *spec = item->spec;
683 const struct rte_flow_item_gtp *mask = item->mask;
684 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
685 struct fm_header_set *fm_data, *fm_mask;
687 uint16_t udp_gtp_uc_port_be = 0;
689 ENICPMD_FUNC_TRACE();
690 /* Only 2 header levels (outer and inner) allowed */
691 if (arg->header_level > 0)
694 fm_data = &entry->ftm_data.fk_hdrset[0];
695 fm_mask = &entry->ftm_mask.fk_hdrset[0];
697 switch (item->type) {
698 case RTE_FLOW_ITEM_TYPE_GTP:
700 /* For vanilla GTP, the UDP destination port must be specified
701 * but value of the port is not enforced here.
703 if (!(fm_data->fk_metadata & FKM_UDP) ||
704 !(fm_data->fk_header_select & FKH_UDP) ||
705 fm_data->l4.udp.fk_dest == 0)
707 if (!(fm_mask->fk_metadata & FKM_UDP) ||
708 !(fm_mask->fk_header_select & FKH_UDP) ||
709 fm_mask->l4.udp.fk_dest != 0xFFFF)
713 case RTE_FLOW_ITEM_TYPE_GTPC:
715 udp_gtp_uc_port_be = rte_cpu_to_be_16(RTE_GTPC_UDP_PORT);
718 case RTE_FLOW_ITEM_TYPE_GTPU:
720 udp_gtp_uc_port_be = rte_cpu_to_be_16(RTE_GTPU_UDP_PORT);
727 /* The GTP-C or GTP-U UDP destination port must be matched. */
728 if (udp_gtp_uc_port_be) {
729 if (fm_data->fk_metadata & FKM_UDP &&
730 fm_data->fk_header_select & FKH_UDP &&
731 fm_data->l4.udp.fk_dest != udp_gtp_uc_port_be)
733 if (fm_mask->fk_metadata & FKM_UDP &&
734 fm_mask->fk_header_select & FKH_UDP &&
735 fm_mask->l4.udp.fk_dest != 0xFFFF)
738 /* In any case, add match for GTP-C GTP-U UDP dst port */
739 fm_data->fk_metadata |= FKM_UDP;
740 fm_data->fk_header_select |= FKH_UDP;
741 fm_data->l4.udp.fk_dest = udp_gtp_uc_port_be;
742 fm_mask->fk_metadata |= FKM_UDP;
743 fm_mask->fk_header_select |= FKH_UDP;
744 fm_mask->l4.udp.fk_dest = 0xFFFF;
747 /* NIC does not support GTP tunnels. No Items are allowed after this.
748 * This prevents the specification of further items.
750 arg->header_level = 0;
752 /* Match all if no spec */
756 mask = &rte_flow_item_gtp_mask;
759 * Use the raw L4 buffer to match GTP as fm_header_set does not have
760 * GTP header. UDP dst port must be specific. Using the raw buffer
761 * does not affect such UDP item, since we skip UDP in the raw buffer.
763 fm_data->fk_header_select |= FKH_L4RAW;
764 fm_mask->fk_header_select |= FKH_L4RAW;
765 off = sizeof(fm_data->l4.udp);
766 memcpy(&fm_data->l4.rawdata[off], spec, sizeof(*spec));
767 memcpy(&fm_mask->l4.rawdata[off], mask, sizeof(*mask));
772 enic_fm_copy_item_geneve(struct copy_item_args *arg)
774 const struct rte_flow_item *item = arg->item;
775 const struct rte_flow_item_geneve *spec = item->spec;
776 const struct rte_flow_item_geneve *mask = item->mask;
777 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
778 struct fm_header_set *fm_data, *fm_mask;
781 ENICPMD_FUNC_TRACE();
782 /* Only 2 header levels (outer and inner) allowed */
783 if (arg->header_level > 0)
786 fm_data = &entry->ftm_data.fk_hdrset[0];
787 fm_mask = &entry->ftm_mask.fk_hdrset[0];
788 fm_data->fk_metadata |= FKM_GENEVE;
789 fm_mask->fk_metadata |= FKM_GENEVE;
790 /* items from here on out are inner header items, except options */
791 arg->header_level = 1;
793 /* Match all if no spec */
797 mask = &rte_flow_item_geneve_mask;
800 * Use the raw L4 buffer to match geneve as fm_header_set does
801 * not have geneve header. A UDP item may precede the geneve
802 * item. Using the raw buffer does not affect such UDP item,
803 * since we skip UDP in the raw buffer.
805 fm_data->fk_header_select |= FKH_L4RAW;
806 fm_mask->fk_header_select |= FKH_L4RAW;
807 off = sizeof(fm_data->l4.udp);
808 memcpy(&fm_data->l4.rawdata[off], spec, sizeof(struct rte_geneve_hdr));
809 memcpy(&fm_mask->l4.rawdata[off], mask, sizeof(struct rte_geneve_hdr));
814 enic_fm_copy_item_geneve_opt(struct copy_item_args *arg)
816 const struct rte_flow_item *item = arg->item;
817 const struct rte_flow_item_geneve_opt *spec = item->spec;
818 const struct rte_flow_item_geneve_opt *mask = item->mask;
819 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
820 struct fm_header_set *fm_data, *fm_mask;
821 struct rte_geneve_hdr *geneve;
824 ENICPMD_FUNC_TRACE();
825 fm_data = &entry->ftm_data.fk_hdrset[0];
826 fm_mask = &entry->ftm_mask.fk_hdrset[0];
827 /* Match all if no spec */
831 mask = &rte_flow_item_geneve_opt_mask;
833 if (spec->option_len > 0 &&
834 (spec->data == NULL || mask->data == NULL)) {
835 return rte_flow_error_set(arg->error, EINVAL,
836 RTE_FLOW_ERROR_TYPE_ITEM,
837 NULL, "enic: geneve_opt unexpected null data");
840 * Geneve item must already be in the raw buffer. Append the
841 * option pattern to it. There are two limitations.
842 * (1) Can match only the 1st option, the first one following Geneve
843 * (2) Geneve header must specify option length, as HW does not
844 * have "has Geneve option" flag.
846 RTE_ASSERT((fm_data->fk_header_select & FKH_L4RAW) != 0);
847 RTE_ASSERT((fm_mask->fk_header_select & FKH_L4RAW) != 0);
848 off = sizeof(fm_data->l4.udp);
849 geneve = (struct rte_geneve_hdr *)&fm_data->l4.rawdata[off];
850 if (geneve->opt_len == 0) {
851 return rte_flow_error_set(arg->error, EINVAL,
852 RTE_FLOW_ERROR_TYPE_ITEM,
853 NULL, "enic: geneve_opt requires non-zero geneve option length");
855 geneve = (struct rte_geneve_hdr *)&fm_mask->l4.rawdata[off];
856 if (geneve->opt_len == 0) {
857 return rte_flow_error_set(arg->error, EINVAL,
858 RTE_FLOW_ERROR_TYPE_ITEM,
859 NULL, "enic: geneve_opt requires non-zero geneve option length mask");
861 off = sizeof(fm_data->l4.udp) + sizeof(struct rte_geneve_hdr);
862 if (off + (spec->option_len + 1) * 4 > FM_LAYER_SIZE) {
863 return rte_flow_error_set(arg->error, EINVAL,
864 RTE_FLOW_ERROR_TYPE_ITEM,
865 NULL, "enic: geneve_opt too large");
867 /* Copy option header */
868 memcpy(&fm_data->l4.rawdata[off], spec, 4);
869 memcpy(&fm_mask->l4.rawdata[off], mask, 4);
870 /* Copy option data */
871 if (spec->option_len > 0) {
873 len = spec->option_len * 4;
874 memcpy(&fm_data->l4.rawdata[off], spec->data, len);
875 memcpy(&fm_mask->l4.rawdata[off], mask->data, len);
881 * Currently, raw pattern match is very limited. It is intended for matching
882 * UDP tunnel header (e.g. vxlan or geneve).
885 enic_fm_copy_item_raw(struct copy_item_args *arg)
887 const struct rte_flow_item *item = arg->item;
888 const struct rte_flow_item_raw *spec = item->spec;
889 const struct rte_flow_item_raw *mask = item->mask;
890 const uint8_t lvl = arg->header_level;
891 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
892 struct fm_header_set *fm_data, *fm_mask;
894 ENICPMD_FUNC_TRACE();
895 /* Cannot be used for inner packet */
898 /* Need both spec and mask */
901 /* Only supports relative with offset 0 */
902 if (!spec->relative || spec->offset != 0 || spec->search ||
905 /* Need non-null pattern that fits within the NIC's filter pattern */
906 if (spec->length == 0 ||
907 spec->length + sizeof(struct rte_udp_hdr) > FM_LAYER_SIZE ||
908 !spec->pattern || !mask->pattern)
911 * Mask fields, including length, are often set to zero. Assume that
912 * means "same as spec" to avoid breaking existing apps. If length
913 * is not zero, then it should be >= spec length.
915 * No more pattern follows this, so append to the L4 layer instead of
916 * L5 to work with both recent and older VICs.
918 if (mask->length != 0 && mask->length < spec->length)
921 fm_data = &entry->ftm_data.fk_hdrset[lvl];
922 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
923 fm_data->fk_header_select |= FKH_L4RAW;
924 fm_mask->fk_header_select |= FKH_L4RAW;
925 fm_data->fk_header_select &= ~FKH_UDP;
926 fm_mask->fk_header_select &= ~FKH_UDP;
927 memcpy(fm_data->l4.rawdata + sizeof(struct rte_udp_hdr),
928 spec->pattern, spec->length);
929 memcpy(fm_mask->l4.rawdata + sizeof(struct rte_udp_hdr),
930 mask->pattern, spec->length);
935 flowman_cmd(struct enic_flowman *fm, uint64_t *args, int nargs)
937 return vnic_dev_flowman_cmd(fm->owner_enic->vdev, args, nargs);
941 enic_fet_alloc(struct enic_flowman *fm, uint8_t ingress,
942 struct fm_key_template *key, int entries,
943 struct enic_fm_fet **fet_out)
945 struct fm_exact_match_table *cmd;
946 struct fm_header_set *hdr;
947 struct enic_fm_fet *fet;
951 ENICPMD_FUNC_TRACE();
952 fet = calloc(1, sizeof(struct enic_fm_fet));
955 cmd = &fm->cmd.va->fm_exact_match_table;
956 memset(cmd, 0, sizeof(*cmd));
957 cmd->fet_direction = ingress ? FM_INGRESS : FM_EGRESS;
958 cmd->fet_stage = FM_STAGE_LAST;
959 cmd->fet_max_entries = entries ? entries : FM_MAX_EXACT_TABLE_SIZE;
961 hdr = &cmd->fet_key.fk_hdrset[0];
962 memset(hdr, 0, sizeof(*hdr));
963 hdr->fk_header_select = FKH_IPV4 | FKH_UDP;
964 hdr->l3.ip4.fk_saddr = 0xFFFFFFFF;
965 hdr->l3.ip4.fk_daddr = 0xFFFFFFFF;
966 hdr->l4.udp.fk_source = 0xFFFF;
967 hdr->l4.udp.fk_dest = 0xFFFF;
968 fet->default_key = 1;
970 memcpy(&cmd->fet_key, key, sizeof(*key));
971 memcpy(&fet->key, key, sizeof(*key));
972 fet->default_key = 0;
974 cmd->fet_key.fk_packet_tag = 1;
976 args[0] = FM_EXACT_TABLE_ALLOC;
977 args[1] = fm->cmd.pa;
978 ret = flowman_cmd(fm, args, 2);
980 ENICPMD_LOG(ERR, "cannot alloc exact match table: rc=%d", ret);
984 fet->handle = args[0];
985 fet->ingress = ingress;
986 ENICPMD_LOG(DEBUG, "allocated exact match table: handle=0x%" PRIx64,
993 enic_fet_free(struct enic_flowman *fm, struct enic_fm_fet *fet)
995 ENICPMD_FUNC_TRACE();
996 enic_fm_tbl_free(fm, fet->handle);
997 if (!fet->default_key)
998 TAILQ_REMOVE(&fm->fet_list, fet, list);
1003 * Get the exact match table for the given combination of
1004 * <group, ingress, key>. Allocate one on the fly as necessary.
1007 enic_fet_get(struct enic_flowman *fm,
1010 struct fm_key_template *key,
1011 struct enic_fm_fet **fet_out,
1012 struct rte_flow_error *error)
1014 struct enic_fm_fet *fet;
1016 ENICPMD_FUNC_TRACE();
1017 /* See if we already have this table open */
1018 TAILQ_FOREACH(fet, &fm->fet_list, list) {
1019 if (fet->group == group && fet->ingress == ingress)
1023 /* Jumping to a non-existing group? Use the default table */
1025 fet = ingress ? fm->default_ig_fet : fm->default_eg_fet;
1026 } else if (enic_fet_alloc(fm, ingress, key, 0, &fet)) {
1027 return rte_flow_error_set(error, EINVAL,
1028 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1029 NULL, "enic: cannot get exact match table");
1032 /* Default table is never on the open table list */
1033 if (!fet->default_key)
1034 TAILQ_INSERT_HEAD(&fm->fet_list, fet, list);
1038 ENICPMD_LOG(DEBUG, "fet_get: %s %s group=%u ref=%u",
1039 fet->default_key ? "default" : "",
1040 fet->ingress ? "ingress" : "egress",
1041 fet->group, fet->ref);
1046 enic_fet_put(struct enic_flowman *fm, struct enic_fm_fet *fet)
1048 ENICPMD_FUNC_TRACE();
1049 RTE_ASSERT(fet->ref > 0);
1051 ENICPMD_LOG(DEBUG, "fet_put: %s %s group=%u ref=%u",
1052 fet->default_key ? "default" : "",
1053 fet->ingress ? "ingress" : "egress",
1054 fet->group, fet->ref);
1056 enic_fet_free(fm, fet);
1059 /* Return 1 if current item is valid on top of the previous one. */
1061 fm_item_stacking_valid(enum rte_flow_item_type prev_item,
1062 const struct enic_fm_items *item_info,
1063 uint8_t is_first_item)
1065 enum rte_flow_item_type const *allowed_items = item_info->prev_items;
1067 ENICPMD_FUNC_TRACE();
1068 for (; *allowed_items != RTE_FLOW_ITEM_TYPE_END; allowed_items++) {
1069 if (prev_item == *allowed_items)
1073 /* This is the first item in the stack. Check if that's cool */
1074 if (is_first_item && item_info->valid_start_item)
1080 * Build the flow manager match entry structure from the provided pattern.
1081 * The pattern is validated as the items are copied.
1084 enic_fm_copy_entry(struct enic_flowman *fm,
1085 const struct rte_flow_item pattern[],
1086 struct rte_flow_error *error)
1088 const struct enic_fm_items *item_info;
1089 enum rte_flow_item_type prev_item;
1090 const struct rte_flow_item *item;
1091 struct copy_item_args args;
1092 uint8_t prev_header_level;
1093 uint8_t is_first_item;
1096 ENICPMD_FUNC_TRACE();
1099 prev_item = RTE_FLOW_ITEM_TYPE_END;
1101 args.fm_tcam_entry = &fm->tcam_entry;
1102 args.header_level = 0;
1103 prev_header_level = 0;
1104 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
1106 * Get info about how to validate and copy the item. If NULL
1107 * is returned the nic does not support the item.
1109 if (item->type == RTE_FLOW_ITEM_TYPE_VOID)
1112 item_info = &enic_fm_items[item->type];
1114 if (item->type >= RTE_DIM(enic_fm_items) ||
1115 item_info->copy_item == NULL) {
1116 return rte_flow_error_set(error, ENOTSUP,
1117 RTE_FLOW_ERROR_TYPE_ITEM,
1118 NULL, "enic: unsupported item");
1121 * Check vNIC feature dependencies. Geneve item needs
1122 * Geneve offload feature
1124 if (item->type == RTE_FLOW_ITEM_TYPE_GENEVE &&
1125 !fm->user_enic->geneve) {
1126 return rte_flow_error_set(error, ENOTSUP,
1127 RTE_FLOW_ERROR_TYPE_ITEM,
1128 NULL, "enic: geneve not supported");
1130 /* check to see if item stacking is valid */
1131 if (!fm_item_stacking_valid(prev_item, item_info,
1133 goto stacking_error;
1138 error->type = RTE_FLOW_ERROR_TYPE_NONE;
1139 ret = item_info->copy_item(&args);
1141 /* If copy_item set the error, return that */
1142 if (error->type != RTE_FLOW_ERROR_TYPE_NONE)
1144 goto item_not_supported;
1146 /* Going from outer to inner? Treat it as a new packet start */
1147 if (prev_header_level != args.header_level) {
1148 prev_item = RTE_FLOW_ITEM_TYPE_END;
1151 prev_item = item->type;
1154 prev_header_level = args.header_level;
1159 return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,
1160 NULL, "enic: unsupported item type");
1163 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
1164 item, "enic: unsupported item stack");
1168 flow_item_skip_void(const struct rte_flow_item **item)
1170 for ( ; ; (*item)++)
1171 if ((*item)->type != RTE_FLOW_ITEM_TYPE_VOID)
1176 append_template(void **template, uint8_t *off, const void *data, int len)
1178 memcpy(*template, data, len);
1179 *template = (char *)*template + len;
1184 enic_fm_append_action_op(struct enic_flowman *fm,
1185 struct fm_action_op *fm_op,
1186 struct rte_flow_error *error)
1190 count = fm->action_op_count;
1191 ENICPMD_LOG(DEBUG, "append action op: idx=%d op=%u",
1192 count, fm_op->fa_op);
1193 if (count == FM_ACTION_OP_MAX) {
1194 return rte_flow_error_set(error, EINVAL,
1195 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1196 "too many action operations");
1198 fm->action.fma_action_ops[count] = *fm_op;
1199 fm->action_op_count = count + 1;
1203 static struct fm_action_op *
1204 find_prev_action_op(struct enic_flowman *fm, uint32_t opcode)
1206 struct fm_action_op *op;
1209 for (i = 0; i < fm->action_op_count; i++) {
1210 op = &fm->action.fma_action_ops[i];
1211 if (op->fa_op == opcode)
1217 /* NIC requires that 1st steer appear before decap.
1218 * Correct example: steer, decap, steer, steer, ...
1221 enic_fm_reorder_action_op(struct enic_flowman *fm)
1223 struct fm_action_op *op, *steer, *decap;
1224 struct fm_action_op tmp_op;
1226 ENICPMD_FUNC_TRACE();
1227 /* Find 1st steer and decap */
1228 op = fm->action.fma_action_ops;
1231 while (op->fa_op != FMOP_END) {
1232 if (!decap && (op->fa_op == FMOP_DECAP_NOSTRIP ||
1233 op->fa_op == FMOP_DECAP_STRIP))
1235 else if (!steer && op->fa_op == FMOP_RQ_STEER)
1239 /* If decap is before steer, swap */
1240 if (steer && decap && decap < steer) {
1241 op = fm->action.fma_action_ops;
1242 ENICPMD_LOG(DEBUG, "swap decap %ld <-> steer %ld",
1243 (long)(decap - op), (long)(steer - op));
1250 /* VXLAN decap is done via flowman compound action */
1252 enic_fm_copy_vxlan_decap(struct enic_flowman *fm,
1253 struct fm_tcam_match_entry *fmt,
1254 const struct rte_flow_action *action,
1255 struct rte_flow_error *error)
1257 struct fm_header_set *fm_data;
1258 struct fm_action_op fm_op;
1260 ENICPMD_FUNC_TRACE();
1261 fm_data = &fmt->ftm_data.fk_hdrset[0];
1262 if (!(fm_data->fk_metadata & FKM_VXLAN)) {
1263 return rte_flow_error_set(error, EINVAL,
1264 RTE_FLOW_ERROR_TYPE_ACTION, action,
1265 "vxlan-decap: vxlan must be in pattern");
1268 memset(&fm_op, 0, sizeof(fm_op));
1269 fm_op.fa_op = FMOP_DECAP_NOSTRIP;
1270 return enic_fm_append_action_op(fm, &fm_op, error);
1273 /* Generate a reasonable source port number */
1277 /* Min/max below are the default values in OVS-DPDK and Linux */
1278 uint16_t p = rte_rand();
1279 p = RTE_MAX(p, 32768);
1280 p = RTE_MIN(p, 61000);
1281 return rte_cpu_to_be_16(p);
1284 /* VXLAN encap is done via flowman compound action */
1286 enic_fm_copy_vxlan_encap(struct enic_flowman *fm,
1287 const struct rte_flow_item *item,
1288 struct rte_flow_error *error)
1290 struct fm_action_op fm_op;
1291 struct rte_ether_hdr *eth;
1292 struct rte_udp_hdr *udp;
1293 uint16_t *ethertype;
1297 ENICPMD_FUNC_TRACE();
1298 memset(&fm_op, 0, sizeof(fm_op));
1299 fm_op.fa_op = FMOP_ENCAP;
1300 template = fm->action.fma_data;
1303 * Copy flow items to the flowman template starting L2.
1304 * L2 must be ethernet.
1306 flow_item_skip_void(&item);
1307 if (item->type != RTE_FLOW_ITEM_TYPE_ETH)
1308 return rte_flow_error_set(error, EINVAL,
1309 RTE_FLOW_ERROR_TYPE_ITEM, item,
1310 "vxlan-encap: first item should be ethernet");
1311 eth = (struct rte_ether_hdr *)template;
1312 ethertype = ð->ether_type;
1313 append_template(&template, &off, item->spec,
1314 sizeof(struct rte_ether_hdr));
1316 flow_item_skip_void(&item);
1318 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
1319 const struct rte_flow_item_vlan *spec;
1321 ENICPMD_LOG(DEBUG, "vxlan-encap: vlan");
1323 fm_op.encap.outer_vlan = rte_be_to_cpu_16(spec->tci);
1325 flow_item_skip_void(&item);
1327 /* L3 must be IPv4, IPv6 */
1328 switch (item->type) {
1329 case RTE_FLOW_ITEM_TYPE_IPV4:
1331 struct rte_ipv4_hdr *ip4;
1333 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv4");
1334 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
1335 ip4 = (struct rte_ipv4_hdr *)template;
1337 * Offset of IPv4 length field and its initial value
1338 * (IP + UDP + VXLAN) are specified in the action. The NIC
1339 * will add inner packet length.
1341 fm_op.encap.len1_offset = off +
1342 offsetof(struct rte_ipv4_hdr, total_length);
1343 fm_op.encap.len1_delta = sizeof(struct rte_ipv4_hdr) +
1344 sizeof(struct rte_udp_hdr) +
1345 sizeof(struct rte_vxlan_hdr);
1346 append_template(&template, &off, item->spec,
1347 sizeof(struct rte_ipv4_hdr));
1348 ip4->version_ihl = RTE_IPV4_VHL_DEF;
1349 if (ip4->time_to_live == 0)
1350 ip4->time_to_live = IP_DEFTTL;
1351 ip4->next_proto_id = IPPROTO_UDP;
1354 case RTE_FLOW_ITEM_TYPE_IPV6:
1356 struct rte_ipv6_hdr *ip6;
1358 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv6");
1359 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
1360 ip6 = (struct rte_ipv6_hdr *)template;
1361 fm_op.encap.len1_offset = off +
1362 offsetof(struct rte_ipv6_hdr, payload_len);
1363 fm_op.encap.len1_delta = sizeof(struct rte_udp_hdr) +
1364 sizeof(struct rte_vxlan_hdr);
1365 append_template(&template, &off, item->spec,
1366 sizeof(struct rte_ipv6_hdr));
1367 ip6->vtc_flow |= rte_cpu_to_be_32(IP6_VTC_FLOW);
1368 if (ip6->hop_limits == 0)
1369 ip6->hop_limits = IP_DEFTTL;
1370 ip6->proto = IPPROTO_UDP;
1374 return rte_flow_error_set(error,
1375 EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1376 "vxlan-encap: L3 must be IPv4/IPv6");
1379 flow_item_skip_void(&item);
1382 if (item->type != RTE_FLOW_ITEM_TYPE_UDP)
1383 return rte_flow_error_set(error, EINVAL,
1384 RTE_FLOW_ERROR_TYPE_ITEM, item,
1385 "vxlan-encap: UDP must follow IPv4/IPv6");
1386 /* UDP length = UDP + VXLAN. NIC will add inner packet length. */
1387 fm_op.encap.len2_offset =
1388 off + offsetof(struct rte_udp_hdr, dgram_len);
1389 fm_op.encap.len2_delta =
1390 sizeof(struct rte_udp_hdr) + sizeof(struct rte_vxlan_hdr);
1391 udp = (struct rte_udp_hdr *)template;
1392 append_template(&template, &off, item->spec,
1393 sizeof(struct rte_udp_hdr));
1395 * Firmware does not hash/fill source port yet. Generate a
1396 * random port, as there is *usually* one rte_flow for the
1397 * given inner packet stream (i.e. a single stream has one
1400 if (udp->src_port == 0)
1401 udp->src_port = gen_src_port();
1403 flow_item_skip_void(&item);
1406 if (item->type != RTE_FLOW_ITEM_TYPE_VXLAN)
1407 return rte_flow_error_set(error,
1408 EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1409 "vxlan-encap: VXLAN must follow UDP");
1410 append_template(&template, &off, item->spec,
1411 sizeof(struct rte_flow_item_vxlan));
1414 * Fill in the rest of the action structure.
1415 * Indicate that we want to encap with vxlan at packet start.
1417 fm_op.encap.template_offset = 0;
1418 fm_op.encap.template_len = off;
1419 return enic_fm_append_action_op(fm, &fm_op, error);
1423 enic_fm_find_vnic(struct enic *enic, const struct rte_pci_addr *addr,
1430 ENICPMD_FUNC_TRACE();
1431 ENICPMD_LOG(DEBUG, "bdf=%x:%x:%x", addr->bus, addr->devid,
1433 bdf = addr->bus << 8 | addr->devid << 3 | addr->function;
1434 args[0] = FM_VNIC_FIND;
1436 rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
1438 /* Expected to fail if BDF is not on the adapter */
1439 ENICPMD_LOG(DEBUG, "cannot find vnic handle: rc=%d", rc);
1443 ENICPMD_LOG(DEBUG, "found vnic: handle=0x%" PRIx64, *handle);
1448 * Egress: target port should be either PF uplink or VF.
1450 * 1. VF egress -> PF uplink
1451 * PF may be this VF's PF, or another PF, as long as they are on the same VIC.
1452 * 2. VF egress -> VF
1455 * 1. PF egress -> VF
1456 * App should be using representor to pass packets to VF
1459 vf_egress_port_id_action(struct enic_flowman *fm,
1460 struct rte_eth_dev *dst_dev,
1461 uint64_t dst_vnic_h,
1462 struct fm_action_op *fm_op,
1463 struct rte_flow_error *error)
1465 struct enic *src_enic, *dst_enic;
1466 struct enic_vf_representor *vf;
1470 ENICPMD_FUNC_TRACE();
1471 src_enic = fm->user_enic;
1472 dst_enic = pmd_priv(dst_dev);
1473 if (!(src_enic->rte_dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)) {
1474 return rte_flow_error_set(error, EINVAL,
1475 RTE_FLOW_ERROR_TYPE_ACTION,
1476 NULL, "source port is not VF representor");
1479 /* VF -> PF uplink. dst is not VF representor */
1480 if (!(dst_dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)) {
1481 /* PF is the VF's PF? Then nothing to do */
1482 vf = VF_ENIC_TO_VF_REP(src_enic);
1483 if (vf->pf == dst_enic) {
1484 ENICPMD_LOG(DEBUG, "destination port is VF's PF");
1487 /* If not, steer to the remote PF's uplink */
1488 uif = dst_enic->fm_vnic_uif;
1489 ENICPMD_LOG(DEBUG, "steer to uplink %u", uif);
1490 memset(fm_op, 0, sizeof(*fm_op));
1491 fm_op->fa_op = FMOP_SET_EGPORT;
1492 fm_op->set_egport.egport = uif;
1493 ret = enic_fm_append_action_op(fm, fm_op, error);
1497 /* VF -> VF loopback. Hairpin and steer to vnic */
1498 memset(fm_op, 0, sizeof(*fm_op));
1499 fm_op->fa_op = FMOP_EG_HAIRPIN;
1500 ret = enic_fm_append_action_op(fm, fm_op, error);
1503 ENICPMD_LOG(DEBUG, "egress hairpin");
1504 fm->hairpin_steer_vnic_h = dst_vnic_h;
1505 fm->need_hairpin_steer = 1;
1510 enic_fm_check_transfer_dst(struct enic *enic, uint16_t dst_port_id,
1511 struct rte_eth_dev **dst_dev,
1512 struct rte_flow_error *error)
1514 struct rte_eth_dev *dev;
1516 ENICPMD_LOG(DEBUG, "port id %u", dst_port_id);
1517 if (!rte_eth_dev_is_valid_port(dst_port_id)) {
1518 return rte_flow_error_set(error, EINVAL,
1519 RTE_FLOW_ERROR_TYPE_ACTION,
1520 NULL, "invalid port_id");
1522 dev = &rte_eth_devices[dst_port_id];
1523 if (!dev_is_enic(dev)) {
1524 return rte_flow_error_set(error, EINVAL,
1525 RTE_FLOW_ERROR_TYPE_ACTION,
1526 NULL, "port_id is not enic");
1528 if (enic->switch_domain_id != pmd_priv(dev)->switch_domain_id) {
1529 return rte_flow_error_set(error, EINVAL,
1530 RTE_FLOW_ERROR_TYPE_ACTION,
1531 NULL, "destination and source ports are not in the same switch domain");
1538 /* Translate flow actions to flowman TCAM entry actions */
1540 enic_fm_copy_action(struct enic_flowman *fm,
1541 const struct rte_flow_action actions[],
1543 struct rte_flow_error *error)
1554 struct fm_tcam_match_entry *fmt;
1555 struct fm_action_op fm_op;
1556 bool need_ovlan_action;
1565 ENICPMD_FUNC_TRACE();
1566 fmt = &fm->tcam_entry;
1567 need_ovlan_action = false;
1571 enic = fm->user_enic;
1573 vnic_h = enic->fm_vnic_handle;
1575 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1576 switch (actions->type) {
1577 case RTE_FLOW_ACTION_TYPE_VOID:
1579 case RTE_FLOW_ACTION_TYPE_PASSTHRU: {
1580 if (overlap & PASSTHRU)
1582 overlap |= PASSTHRU;
1585 case RTE_FLOW_ACTION_TYPE_JUMP: {
1586 const struct rte_flow_action_jump *jump =
1588 struct enic_fm_fet *fet;
1592 ret = enic_fet_get(fm, jump->group, ingress, NULL,
1597 memset(&fm_op, 0, sizeof(fm_op));
1598 fm_op.fa_op = FMOP_EXACT_MATCH;
1599 fm_op.exact.handle = fet->handle;
1601 ret = enic_fm_append_action_op(fm, &fm_op, error);
1606 case RTE_FLOW_ACTION_TYPE_MARK: {
1607 const struct rte_flow_action_mark *mark =
1610 if (enic->use_noscatter_vec_rx_handler)
1612 if (mark->id >= ENIC_MAGIC_FILTER_ID - 1)
1613 return rte_flow_error_set(error, EINVAL,
1614 RTE_FLOW_ERROR_TYPE_ACTION,
1615 NULL, "invalid mark id");
1616 memset(&fm_op, 0, sizeof(fm_op));
1617 fm_op.fa_op = FMOP_MARK;
1618 fm_op.mark.mark = mark->id + 1;
1619 ret = enic_fm_append_action_op(fm, &fm_op, error);
1624 case RTE_FLOW_ACTION_TYPE_FLAG: {
1625 if (enic->use_noscatter_vec_rx_handler)
1627 /* ENIC_MAGIC_FILTER_ID is reserved for flagging */
1628 memset(&fm_op, 0, sizeof(fm_op));
1629 fm_op.fa_op = FMOP_MARK;
1630 fm_op.mark.mark = ENIC_MAGIC_FILTER_ID;
1631 ret = enic_fm_append_action_op(fm, &fm_op, error);
1636 case RTE_FLOW_ACTION_TYPE_QUEUE: {
1637 const struct rte_flow_action_queue *queue =
1641 * If fate other than QUEUE or RSS, fail. Multiple
1642 * rss and queue actions are ok.
1644 if ((overlap & FATE) && first_rq)
1648 memset(&fm_op, 0, sizeof(fm_op));
1649 fm_op.fa_op = FMOP_RQ_STEER;
1650 fm_op.rq_steer.rq_index =
1651 enic_rte_rq_idx_to_sop_idx(queue->index);
1652 fm_op.rq_steer.rq_count = 1;
1653 fm_op.rq_steer.vnic_handle = vnic_h;
1654 ret = enic_fm_append_action_op(fm, &fm_op, error);
1657 ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1658 fm_op.rq_steer.rq_index);
1662 case RTE_FLOW_ACTION_TYPE_DROP: {
1666 memset(&fm_op, 0, sizeof(fm_op));
1667 fm_op.fa_op = FMOP_DROP;
1668 ret = enic_fm_append_action_op(fm, &fm_op, error);
1671 ENICPMD_LOG(DEBUG, "create DROP action");
1674 case RTE_FLOW_ACTION_TYPE_COUNT: {
1675 if (overlap & COUNT)
1678 /* Count is associated with entry not action on VIC. */
1679 fmt->ftm_flags |= FMEF_COUNTER;
1682 case RTE_FLOW_ACTION_TYPE_RSS: {
1683 const struct rte_flow_action_rss *rss = actions->conf;
1688 * If fate other than QUEUE or RSS, fail. Multiple
1689 * rss and queue actions are ok.
1691 if ((overlap & FATE) && first_rq)
1697 * Hardware only supports RSS actions on outer level
1698 * with default type and function. Queues must be
1701 allow = rss->func == RTE_ETH_HASH_FUNCTION_DEFAULT &&
1702 rss->level == 0 && (rss->types == 0 ||
1703 rss->types == enic->rss_hf) &&
1704 rss->queue_num <= enic->rq_count &&
1705 rss->queue[rss->queue_num - 1] < enic->rq_count;
1708 /* Identity queue map needs to be sequential */
1709 for (i = 1; i < rss->queue_num; i++)
1710 allow = allow && (rss->queue[i] ==
1711 rss->queue[i - 1] + 1);
1715 memset(&fm_op, 0, sizeof(fm_op));
1716 fm_op.fa_op = FMOP_RQ_STEER;
1717 fm_op.rq_steer.rq_index =
1718 enic_rte_rq_idx_to_sop_idx(rss->queue[0]);
1719 fm_op.rq_steer.rq_count = rss->queue_num;
1720 fm_op.rq_steer.vnic_handle = vnic_h;
1721 ret = enic_fm_append_action_op(fm, &fm_op, error);
1724 ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1725 fm_op.rq_steer.rq_index);
1729 case RTE_FLOW_ACTION_TYPE_PORT_ID: {
1730 const struct rte_flow_action_port_id *port;
1731 struct rte_eth_dev *dev = NULL;
1733 if (!ingress && (overlap & PORT_ID)) {
1734 ENICPMD_LOG(DEBUG, "cannot have multiple egress PORT_ID actions");
1737 port = actions->conf;
1738 if (port->original) {
1739 vnic_h = enic->fm_vnic_handle; /* This port */
1742 ret = enic_fm_check_transfer_dst(enic, port->id, &dev,
1746 vnic_h = pmd_priv(dev)->fm_vnic_handle;
1749 * Ingress. Nothing more to do. We add an implicit
1750 * steer at the end if needed.
1755 ret = vf_egress_port_id_action(fm, dev, vnic_h, &fm_op,
1761 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: {
1762 if (overlap & DECAP)
1766 ret = enic_fm_copy_vxlan_decap(fm, fmt, actions,
1772 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: {
1773 const struct rte_flow_action_vxlan_encap *encap;
1775 encap = actions->conf;
1776 if (overlap & ENCAP)
1779 ret = enic_fm_copy_vxlan_encap(fm, encap->definition,
1785 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: {
1786 struct fm_action_op *decap;
1789 * If decap-nostrip appears before pop vlan, this pop
1790 * applies to the inner packet vlan. Turn it into
1793 decap = find_prev_action_op(fm, FMOP_DECAP_NOSTRIP);
1795 ENICPMD_LOG(DEBUG, "pop-vlan inner: decap-nostrip => decap-strip");
1796 decap->fa_op = FMOP_DECAP_STRIP;
1799 memset(&fm_op, 0, sizeof(fm_op));
1800 fm_op.fa_op = FMOP_POP_VLAN;
1801 ret = enic_fm_append_action_op(fm, &fm_op, error);
1806 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: {
1807 const struct rte_flow_action_of_push_vlan *vlan;
1809 if (overlap & PASSTHRU)
1811 vlan = actions->conf;
1812 if (vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN)) {
1813 return rte_flow_error_set(error, EINVAL,
1814 RTE_FLOW_ERROR_TYPE_ACTION,
1815 NULL, "unexpected push_vlan ethertype");
1817 overlap |= PUSH_VLAN;
1818 need_ovlan_action = true;
1821 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: {
1822 const struct rte_flow_action_of_set_vlan_pcp *pcp;
1824 pcp = actions->conf;
1825 if (pcp->vlan_pcp > 7) {
1826 return rte_flow_error_set(error, EINVAL,
1827 RTE_FLOW_ERROR_TYPE_ACTION,
1828 NULL, "invalid vlan_pcp");
1830 need_ovlan_action = true;
1831 ovlan |= ((uint16_t)pcp->vlan_pcp) << 13;
1834 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: {
1835 const struct rte_flow_action_of_set_vlan_vid *vid;
1837 vid = actions->conf;
1838 need_ovlan_action = true;
1839 ovlan |= rte_be_to_cpu_16(vid->vlan_vid);
1842 case RTE_FLOW_ACTION_TYPE_PORT_REPRESENTOR: {
1843 const struct rte_flow_action_ethdev *ethdev;
1844 struct rte_eth_dev *dev = NULL;
1846 ethdev = actions->conf;
1847 ret = enic_fm_check_transfer_dst(enic, ethdev->port_id,
1851 vnic_h = pmd_priv(dev)->fm_vnic_handle;
1854 * Action PORT_REPRESENTOR implies ingress destination.
1855 * Noting to do. We add an implicit stree at the
1861 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: {
1862 const struct rte_flow_action_ethdev *ethdev;
1863 struct rte_eth_dev *dev = NULL;
1865 if (overlap & PORT_ID) {
1866 ENICPMD_LOG(DEBUG, "cannot have multiple egress PORT_ID actions");
1869 ethdev = actions->conf;
1870 ret = enic_fm_check_transfer_dst(enic, ethdev->port_id,
1874 vnic_h = pmd_priv(dev)->fm_vnic_handle;
1876 /* Action REPRESENTED_PORT: always egress destination */
1878 ret = vf_egress_port_id_action(fm, dev, vnic_h, &fm_op,
1889 if (!(overlap & (FATE | PASSTHRU | COUNT | PORT_ID)))
1891 /* Egress from VF: need implicit WQ match */
1892 if (enic_is_vf_rep(enic) && !ingress) {
1893 fmt->ftm_data.fk_wq_id = 0;
1894 fmt->ftm_mask.fk_wq_id = 0xffff;
1895 fmt->ftm_data.fk_wq_vnic = enic->fm_vnic_handle;
1896 ENICPMD_LOG(DEBUG, "add implicit wq id match for vf %d",
1897 VF_ENIC_TO_VF_REP(enic)->vf_id);
1899 if (need_ovlan_action) {
1900 memset(&fm_op, 0, sizeof(fm_op));
1901 fm_op.fa_op = FMOP_SET_OVLAN;
1902 fm_op.ovlan.vlan = ovlan;
1903 ret = enic_fm_append_action_op(fm, &fm_op, error);
1907 /* Add steer op for PORT_ID without QUEUE */
1908 if ((overlap & PORT_ID) && !steer && ingress) {
1909 memset(&fm_op, 0, sizeof(fm_op));
1910 /* Always to queue 0 for now as generic RSS is not available */
1911 fm_op.fa_op = FMOP_RQ_STEER;
1912 fm_op.rq_steer.rq_index = 0;
1913 fm_op.rq_steer.vnic_handle = vnic_h;
1914 ret = enic_fm_append_action_op(fm, &fm_op, error);
1917 ENICPMD_LOG(DEBUG, "add implicit steer op");
1919 /* Add required END */
1920 memset(&fm_op, 0, sizeof(fm_op));
1921 fm_op.fa_op = FMOP_END;
1922 ret = enic_fm_append_action_op(fm, &fm_op, error);
1925 enic_fm_reorder_action_op(fm);
1929 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
1930 NULL, "enic: unsupported action");
1933 /** Check if the action is supported */
1935 enic_fm_match_action(const struct rte_flow_action *action,
1936 const enum rte_flow_action_type *supported_actions)
1938 for (; *supported_actions != RTE_FLOW_ACTION_TYPE_END;
1939 supported_actions++) {
1940 if (action->type == *supported_actions)
1946 /* Debug function to dump internal NIC action structure. */
1948 enic_fm_dump_tcam_actions(const struct fm_action *fm_action)
1950 /* Manually keep in sync with FMOP commands */
1951 const char *fmop_str[FMOP_OP_MAX] = {
1953 [FMOP_DROP] = "drop",
1954 [FMOP_RQ_STEER] = "steer",
1955 [FMOP_EXACT_MATCH] = "exmatch",
1956 [FMOP_MARK] = "mark",
1957 [FMOP_EXT_MARK] = "ext_mark",
1959 [FMOP_EG_HAIRPIN] = "eg_hairpin",
1960 [FMOP_IG_HAIRPIN] = "ig_hairpin",
1961 [FMOP_ENCAP_IVLAN] = "encap_ivlan",
1962 [FMOP_ENCAP_NOIVLAN] = "encap_noivlan",
1963 [FMOP_ENCAP] = "encap",
1964 [FMOP_SET_OVLAN] = "set_ovlan",
1965 [FMOP_DECAP_NOSTRIP] = "decap_nostrip",
1966 [FMOP_DECAP_STRIP] = "decap_strip",
1967 [FMOP_POP_VLAN] = "pop_vlan",
1968 [FMOP_SET_EGPORT] = "set_egport",
1969 [FMOP_RQ_STEER_ONLY] = "rq_steer_only",
1970 [FMOP_SET_ENCAP_VLAN] = "set_encap_vlan",
1971 [FMOP_EMIT] = "emit",
1972 [FMOP_MODIFY] = "modify",
1974 const struct fm_action_op *op = &fm_action->fma_action_ops[0];
1975 char buf[128], *bp = buf;
1980 buf_len = sizeof(buf);
1981 for (i = 0; i < FM_ACTION_OP_MAX; i++) {
1982 if (op->fa_op == FMOP_END)
1984 if (op->fa_op >= FMOP_OP_MAX)
1987 op_str = fmop_str[op->fa_op];
1988 n = snprintf(bp, buf_len, "%s,", op_str);
1989 if (n > 0 && n < buf_len) {
1995 /* Remove trailing comma */
1998 ENICPMD_LOG(DEBUG, " Actions: %s", buf);
2002 bits_to_str(uint32_t bits, const char *strings[], int max,
2003 char *buf, int buf_len)
2005 int i, n = 0, len = 0;
2007 for (i = 0; i < max; i++) {
2008 if (bits & (1 << i)) {
2009 n = snprintf(buf, buf_len, "%s,", strings[i]);
2010 if (n > 0 && n < buf_len) {
2017 /* Remove trailing comma */
2025 /* Debug function to dump internal NIC filter structure. */
2027 __enic_fm_dump_tcam_match(const struct fm_header_set *fk_hdrset, char *buf,
2030 /* Manually keep in sync with FKM_BITS */
2031 const char *fm_fkm_str[FKM_BIT_COUNT] = {
2032 [FKM_QTAG_BIT] = "qtag",
2033 [FKM_CMD_BIT] = "cmd",
2034 [FKM_IPV4_BIT] = "ip4",
2035 [FKM_IPV6_BIT] = "ip6",
2036 [FKM_ROCE_BIT] = "roce",
2037 [FKM_UDP_BIT] = "udp",
2038 [FKM_TCP_BIT] = "tcp",
2039 [FKM_TCPORUDP_BIT] = "tcpportudp",
2040 [FKM_IPFRAG_BIT] = "ipfrag",
2041 [FKM_NVGRE_BIT] = "nvgre",
2042 [FKM_VXLAN_BIT] = "vxlan",
2043 [FKM_GENEVE_BIT] = "geneve",
2044 [FKM_NSH_BIT] = "nsh",
2045 [FKM_ROCEV2_BIT] = "rocev2",
2046 [FKM_VLAN_PRES_BIT] = "vlan_pres",
2047 [FKM_IPOK_BIT] = "ipok",
2048 [FKM_L4OK_BIT] = "l4ok",
2049 [FKM_ROCEOK_BIT] = "roceok",
2050 [FKM_FCSOK_BIT] = "fcsok",
2051 [FKM_EG_SPAN_BIT] = "eg_span",
2052 [FKM_IG_SPAN_BIT] = "ig_span",
2053 [FKM_EG_HAIRPINNED_BIT] = "eg_hairpinned",
2055 /* Manually keep in sync with FKH_BITS */
2056 const char *fm_fkh_str[FKH_BIT_COUNT] = {
2057 [FKH_ETHER_BIT] = "eth",
2058 [FKH_QTAG_BIT] = "qtag",
2059 [FKH_L2RAW_BIT] = "l2raw",
2060 [FKH_IPV4_BIT] = "ip4",
2061 [FKH_IPV6_BIT] = "ip6",
2062 [FKH_L3RAW_BIT] = "l3raw",
2063 [FKH_UDP_BIT] = "udp",
2064 [FKH_TCP_BIT] = "tcp",
2065 [FKH_ICMP_BIT] = "icmp",
2066 [FKH_VXLAN_BIT] = "vxlan",
2067 [FKH_L4RAW_BIT] = "l4raw",
2069 uint32_t fkh_bits = fk_hdrset->fk_header_select;
2070 uint32_t fkm_bits = fk_hdrset->fk_metadata;
2073 if (!fkm_bits && !fkh_bits)
2075 n = snprintf(buf, buf_len, "metadata(");
2076 if (n > 0 && n < buf_len) {
2080 n = bits_to_str(fkm_bits, fm_fkm_str, FKM_BIT_COUNT, buf, buf_len);
2081 if (n > 0 && n < buf_len) {
2085 n = snprintf(buf, buf_len, ") valid hdr fields(");
2086 if (n > 0 && n < buf_len) {
2090 n = bits_to_str(fkh_bits, fm_fkh_str, FKH_BIT_COUNT, buf, buf_len);
2091 if (n > 0 && n < buf_len) {
2095 snprintf(buf, buf_len, ")");
2099 enic_fm_dump_tcam_match(const struct fm_tcam_match_entry *match,
2104 memset(buf, 0, sizeof(buf));
2105 __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[0],
2107 ENICPMD_LOG(DEBUG, " TCAM %s Outer: %s %scounter position %u",
2108 (ingress) ? "IG" : "EG", buf,
2109 (match->ftm_flags & FMEF_COUNTER) ? "" : "no ",
2110 match->ftm_position);
2111 memset(buf, 0, sizeof(buf));
2112 __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[1],
2115 ENICPMD_LOG(DEBUG, " Inner: %s", buf);
2118 /* Debug function to dump internal NIC flow structures. */
2120 enic_fm_dump_tcam_entry(const struct fm_tcam_match_entry *fm_match,
2121 const struct fm_action *fm_action,
2124 if (!rte_log_can_log(enic_pmd_logtype, RTE_LOG_DEBUG))
2126 enic_fm_dump_tcam_match(fm_match, ingress);
2127 enic_fm_dump_tcam_actions(fm_action);
2131 enic_fm_flow_parse(struct enic_flowman *fm,
2132 const struct rte_flow_attr *attrs,
2133 const struct rte_flow_item pattern[],
2134 const struct rte_flow_action actions[],
2135 struct rte_flow_error *error)
2137 const struct rte_flow_action *action;
2139 static const enum rte_flow_action_type *sa;
2141 ENICPMD_FUNC_TRACE();
2144 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM,
2145 NULL, "no pattern specified");
2150 rte_flow_error_set(error, EINVAL,
2151 RTE_FLOW_ERROR_TYPE_ACTION_NUM,
2152 NULL, "no action specified");
2157 if (attrs->group != FM_TCAM_RTE_GROUP && attrs->priority) {
2158 rte_flow_error_set(error, ENOTSUP,
2159 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
2161 "priorities are not supported for non-default (0) groups");
2163 } else if (!fm->owner_enic->switchdev_mode && attrs->transfer) {
2164 rte_flow_error_set(error, ENOTSUP,
2165 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
2167 "transfer is not supported");
2169 } else if (attrs->ingress && attrs->egress) {
2170 rte_flow_error_set(error, ENOTSUP,
2171 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
2173 "bidirectional rules not supported");
2178 rte_flow_error_set(error, EINVAL,
2179 RTE_FLOW_ERROR_TYPE_ATTR,
2180 NULL, "no attribute specified");
2184 /* Verify Actions. */
2185 sa = (attrs->ingress) ? enic_fm_supported_ig_actions :
2186 enic_fm_supported_eg_actions;
2187 for (action = &actions[0]; action->type != RTE_FLOW_ACTION_TYPE_END;
2189 if (action->type == RTE_FLOW_ACTION_TYPE_VOID)
2191 else if (!enic_fm_match_action(action, sa))
2194 if (action->type != RTE_FLOW_ACTION_TYPE_END) {
2195 rte_flow_error_set(error, EPERM, RTE_FLOW_ERROR_TYPE_ACTION,
2196 action, "invalid action");
2199 ret = enic_fm_copy_entry(fm, pattern, error);
2202 ret = enic_fm_copy_action(fm, actions, attrs->ingress, error);
2207 enic_fm_counter_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
2209 if (!fm_flow->counter_valid)
2211 SLIST_INSERT_HEAD(&fm->counters, fm_flow->counter, next);
2212 fm_flow->counter_valid = false;
2216 enic_fm_more_counters(struct enic_flowman *fm)
2218 struct enic_fm_counter *new_stack;
2219 struct enic_fm_counter *ctrs;
2223 ENICPMD_FUNC_TRACE();
2224 new_stack = rte_realloc(fm->counter_stack, (fm->counters_alloced +
2225 FM_COUNTERS_EXPAND) *
2226 sizeof(struct enic_fm_counter), 0);
2227 if (new_stack == NULL) {
2228 ENICPMD_LOG(ERR, "cannot alloc counter memory");
2231 fm->counter_stack = new_stack;
2233 args[0] = FM_COUNTER_BRK;
2234 args[1] = fm->counters_alloced + FM_COUNTERS_EXPAND;
2235 rc = flowman_cmd(fm, args, 2);
2237 ENICPMD_LOG(ERR, "cannot alloc counters rc=%d", rc);
2240 ctrs = (struct enic_fm_counter *)fm->counter_stack +
2241 fm->counters_alloced;
2242 for (i = 0; i < FM_COUNTERS_EXPAND; i++, ctrs++) {
2243 ctrs->handle = fm->counters_alloced + i;
2244 SLIST_INSERT_HEAD(&fm->counters, ctrs, next);
2246 fm->counters_alloced += FM_COUNTERS_EXPAND;
2247 ENICPMD_LOG(DEBUG, "%u counters allocated, total: %u",
2248 FM_COUNTERS_EXPAND, fm->counters_alloced);
2253 enic_fm_counter_zero(struct enic_flowman *fm, struct enic_fm_counter *c)
2258 ENICPMD_FUNC_TRACE();
2259 args[0] = FM_COUNTER_QUERY;
2260 args[1] = c->handle;
2261 args[2] = 1; /* clear */
2262 ret = flowman_cmd(fm, args, 3);
2264 ENICPMD_LOG(ERR, "counter init: rc=%d handle=0x%x",
2272 enic_fm_counter_alloc(struct enic_flowman *fm, struct rte_flow_error *error,
2273 struct enic_fm_counter **ctr)
2275 struct enic_fm_counter *c;
2278 ENICPMD_FUNC_TRACE();
2280 if (SLIST_EMPTY(&fm->counters)) {
2281 ret = enic_fm_more_counters(fm);
2283 return rte_flow_error_set(error, -ret,
2284 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2285 NULL, "enic: out of counters");
2287 c = SLIST_FIRST(&fm->counters);
2288 SLIST_REMOVE_HEAD(&fm->counters, next);
2294 enic_fm_action_free(struct enic_flowman *fm, struct enic_fm_action *ah)
2299 ENICPMD_FUNC_TRACE();
2300 RTE_ASSERT(ah->ref > 0);
2303 args[0] = FM_ACTION_FREE;
2304 args[1] = ah->handle;
2305 ret = flowman_cmd(fm, args, 2);
2307 /* This is a "should never happen" error. */
2308 ENICPMD_LOG(ERR, "freeing action rc=%d handle=0x%"
2309 PRIx64, ret, ah->handle);
2310 rte_hash_del_key(fm->action_hash, (const void *)&ah->key);
2317 enic_fm_entry_free(struct enic_flowman *fm, uint64_t handle)
2322 ENICPMD_FUNC_TRACE();
2323 args[0] = FM_MATCH_ENTRY_REMOVE;
2325 rc = flowman_cmd(fm, args, 2);
2327 ENICPMD_LOG(ERR, "cannot free match entry: rc=%d"
2328 " handle=0x%" PRIx64, rc, handle);
2332 static struct enic_fm_jump_flow *
2333 find_jump_flow(struct enic_flowman *fm, uint32_t group)
2335 struct enic_fm_jump_flow *j;
2337 ENICPMD_FUNC_TRACE();
2338 TAILQ_FOREACH(j, &fm->jump_list, list) {
2339 if (j->group == group)
2346 remove_jump_flow(struct enic_flowman *fm, struct rte_flow *flow)
2348 struct enic_fm_jump_flow *j;
2350 ENICPMD_FUNC_TRACE();
2351 TAILQ_FOREACH(j, &fm->jump_list, list) {
2352 if (j->flow == flow) {
2353 TAILQ_REMOVE(&fm->jump_list, j, list);
2361 save_jump_flow(struct enic_flowman *fm,
2362 struct rte_flow *flow,
2364 struct fm_tcam_match_entry *match,
2365 struct fm_action *action)
2367 struct enic_fm_jump_flow *j;
2369 ENICPMD_FUNC_TRACE();
2370 j = calloc(1, sizeof(struct enic_fm_jump_flow));
2376 j->action = *action;
2377 TAILQ_INSERT_HEAD(&fm->jump_list, j, list);
2378 ENICPMD_LOG(DEBUG, "saved jump flow: flow=%p group=%u", flow, group);
2383 __enic_fm_flow_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
2385 if (fm_flow->entry_handle != FM_INVALID_HANDLE) {
2386 enic_fm_entry_free(fm, fm_flow->entry_handle);
2387 fm_flow->entry_handle = FM_INVALID_HANDLE;
2389 if (fm_flow->action != NULL) {
2390 enic_fm_action_free(fm, fm_flow->action);
2391 fm_flow->action = NULL;
2393 enic_fm_counter_free(fm, fm_flow);
2395 enic_fet_put(fm, fm_flow->fet);
2396 fm_flow->fet = NULL;
2401 enic_fm_flow_free(struct enic_flowman *fm, struct rte_flow *flow)
2403 struct enic_fm_flow *steer = flow->fm->hairpin_steer_flow;
2405 if (flow->fm->fet && flow->fm->fet->default_key)
2406 remove_jump_flow(fm, flow);
2407 __enic_fm_flow_free(fm, flow->fm);
2409 __enic_fm_flow_free(fm, steer);
2417 enic_fm_add_tcam_entry(struct enic_flowman *fm,
2418 struct fm_tcam_match_entry *match_in,
2419 uint64_t *entry_handle,
2421 struct rte_flow_error *error)
2423 struct fm_tcam_match_entry *ftm;
2427 ENICPMD_FUNC_TRACE();
2428 /* Copy entry to the command buffer */
2429 ftm = &fm->cmd.va->fm_tcam_match_entry;
2430 memcpy(ftm, match_in, sizeof(*ftm));
2431 /* Add TCAM entry */
2432 args[0] = FM_TCAM_ENTRY_INSTALL;
2433 args[1] = ingress ? fm->ig_tcam_hndl : fm->eg_tcam_hndl;
2434 args[2] = fm->cmd.pa;
2435 ret = flowman_cmd(fm, args, 3);
2437 ENICPMD_LOG(ERR, "cannot add %s TCAM entry: rc=%d",
2438 ingress ? "ingress" : "egress", ret);
2439 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2440 NULL, "enic: devcmd(tcam-entry-install)");
2443 ENICPMD_LOG(DEBUG, "installed %s TCAM entry: handle=0x%" PRIx64,
2444 ingress ? "ingress" : "egress", (uint64_t)args[0]);
2445 *entry_handle = args[0];
2450 enic_fm_add_exact_entry(struct enic_flowman *fm,
2451 struct fm_tcam_match_entry *match_in,
2452 uint64_t *entry_handle,
2453 struct enic_fm_fet *fet,
2454 struct rte_flow_error *error)
2456 struct fm_exact_match_entry *fem;
2460 ENICPMD_FUNC_TRACE();
2461 /* The new entry must have the table's key */
2462 if (memcmp(fet->key.fk_hdrset, match_in->ftm_mask.fk_hdrset,
2463 sizeof(struct fm_header_set) * FM_HDRSET_MAX)) {
2464 return rte_flow_error_set(error, EINVAL,
2465 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2466 "enic: key does not match group's key");
2469 /* Copy entry to the command buffer */
2470 fem = &fm->cmd.va->fm_exact_match_entry;
2472 * Translate TCAM entry to exact entry. As is only need to drop
2473 * position and mask. The mask is part of the exact match table.
2474 * Position (aka priority) is not supported in the exact match table.
2476 fem->fem_data = match_in->ftm_data;
2477 fem->fem_flags = match_in->ftm_flags;
2478 fem->fem_action = match_in->ftm_action;
2479 fem->fem_counter = match_in->ftm_counter;
2481 /* Add exact entry */
2482 args[0] = FM_EXACT_ENTRY_INSTALL;
2483 args[1] = fet->handle;
2484 args[2] = fm->cmd.pa;
2485 ret = flowman_cmd(fm, args, 3);
2487 ENICPMD_LOG(ERR, "cannot add %s exact entry: group=%u",
2488 fet->ingress ? "ingress" : "egress", fet->group);
2489 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2490 NULL, "enic: devcmd(exact-entry-install)");
2493 ENICPMD_LOG(DEBUG, "installed %s exact entry: group=%u"
2494 " handle=0x%" PRIx64,
2495 fet->ingress ? "ingress" : "egress", fet->group,
2497 *entry_handle = args[0];
2502 enic_action_handle_get(struct enic_flowman *fm, struct fm_action *action_in,
2503 struct rte_flow_error *error,
2504 struct enic_fm_action **ah_o)
2506 struct enic_fm_action *ah;
2507 struct fm_action *fma;
2511 ret = rte_hash_lookup_data(fm->action_hash, action_in,
2513 if (ret < 0 && ret != -ENOENT)
2514 return rte_flow_error_set(error, -ret,
2515 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2516 NULL, "enic: rte_hash_lookup(action)");
2518 if (ret == -ENOENT) {
2519 /* Allocate a new action on the NIC. */
2520 fma = &fm->cmd.va->fm_action;
2521 memcpy(fma, action_in, sizeof(*fma));
2523 ah = calloc(1, sizeof(*ah));
2524 memcpy(&ah->key, action_in, sizeof(struct fm_action));
2526 return rte_flow_error_set(error, ENOMEM,
2527 RTE_FLOW_ERROR_TYPE_HANDLE,
2528 NULL, "enic: calloc(fm-action)");
2529 args[0] = FM_ACTION_ALLOC;
2530 args[1] = fm->cmd.pa;
2531 ret = flowman_cmd(fm, args, 2);
2533 rte_flow_error_set(error, -ret,
2534 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2535 NULL, "enic: devcmd(action-alloc)");
2538 ah->handle = args[0];
2539 ret = rte_hash_add_key_data(fm->action_hash,
2540 (const void *)action_in,
2543 rte_flow_error_set(error, -ret,
2544 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2546 "enic: rte_hash_add_key_data(actn)");
2547 goto error_with_action_handle;
2549 ENICPMD_LOG(DEBUG, "action allocated: handle=0x%" PRIx64,
2553 /* Action handle struct is valid, increment reference count. */
2557 error_with_action_handle:
2558 args[0] = FM_ACTION_FREE;
2559 args[1] = ah->handle;
2560 ret = flowman_cmd(fm, args, 2);
2562 rte_flow_error_set(error, -ret,
2563 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2564 NULL, "enic: devcmd(action-free)");
2570 /* Push match-action to the NIC. */
2572 __enic_fm_flow_add_entry(struct enic_flowman *fm,
2573 struct enic_fm_flow *fm_flow,
2574 struct fm_tcam_match_entry *match_in,
2575 struct fm_action *action_in,
2578 struct rte_flow_error *error)
2580 struct enic_fm_counter *ctr;
2581 struct enic_fm_action *ah = NULL;
2585 ENICPMD_FUNC_TRACE();
2587 /* Get or create an action handle. */
2588 ret = enic_action_handle_get(fm, action_in, error, &ah);
2591 match_in->ftm_action = ah->handle;
2592 fm_flow->action = ah;
2594 /* Allocate counter if requested. */
2595 if (match_in->ftm_flags & FMEF_COUNTER) {
2596 ret = enic_fm_counter_alloc(fm, error, &ctr);
2597 if (ret) /* error has been filled in */
2599 fm_flow->counter_valid = true;
2600 fm_flow->counter = ctr;
2601 match_in->ftm_counter = ctr->handle;
2605 * Get the group's table (either TCAM or exact match table) and
2606 * add entry to it. If we use the exact match table, the handler
2607 * will translate the TCAM entry (match_in) to the appropriate
2608 * exact match entry and use that instead.
2610 entry_h = FM_INVALID_HANDLE;
2611 if (group == FM_TCAM_RTE_GROUP) {
2612 ret = enic_fm_add_tcam_entry(fm, match_in, &entry_h, ingress,
2616 /* Jump action might have a ref to fet */
2617 fm_flow->fet = fm->fet;
2620 struct enic_fm_fet *fet = NULL;
2622 ret = enic_fet_get(fm, group, ingress,
2623 &match_in->ftm_mask, &fet, error);
2627 ret = enic_fm_add_exact_entry(fm, match_in, &entry_h, fet,
2632 /* Clear counter after adding entry, as it requires in-use counter */
2633 if (fm_flow->counter_valid) {
2634 ret = enic_fm_counter_zero(fm, fm_flow->counter);
2638 fm_flow->entry_handle = entry_h;
2642 /* Push match-action to the NIC. */
2643 static struct rte_flow *
2644 enic_fm_flow_add_entry(struct enic_flowman *fm,
2645 struct fm_tcam_match_entry *match_in,
2646 struct fm_action *action_in,
2647 const struct rte_flow_attr *attrs,
2648 struct rte_flow_error *error)
2650 struct enic_fm_flow *fm_flow;
2651 struct rte_flow *flow;
2653 ENICPMD_FUNC_TRACE();
2654 match_in->ftm_position = attrs->priority;
2655 enic_fm_dump_tcam_entry(match_in, action_in, attrs->ingress);
2656 flow = calloc(1, sizeof(*flow));
2657 fm_flow = calloc(1, sizeof(*fm_flow));
2658 if (flow == NULL || fm_flow == NULL) {
2659 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
2660 NULL, "enic: cannot allocate rte_flow");
2666 fm_flow->action = NULL;
2667 fm_flow->entry_handle = FM_INVALID_HANDLE;
2668 if (__enic_fm_flow_add_entry(fm, fm_flow, match_in, action_in,
2669 attrs->group, attrs->ingress, error)) {
2670 enic_fm_flow_free(fm, flow);
2677 convert_jump_flows(struct enic_flowman *fm, struct enic_fm_fet *fet,
2678 struct rte_flow_error *error)
2680 struct enic_fm_flow *fm_flow;
2681 struct enic_fm_jump_flow *j;
2682 struct fm_action *fma;
2685 ENICPMD_FUNC_TRACE();
2687 * Find the saved flows that should jump to the new table (fet).
2688 * Then delete the old TCAM entry that jumps to the default table,
2689 * and add a new one that jumps to the new table.
2692 j = find_jump_flow(fm, group);
2694 ENICPMD_LOG(DEBUG, "convert jump flow: flow=%p group=%u",
2696 /* Delete old entry */
2697 fm_flow = j->flow->fm;
2698 __enic_fm_flow_free(fm, fm_flow);
2702 fma->fma_action_ops[0].exact.handle = fet->handle;
2703 if (__enic_fm_flow_add_entry(fm, fm_flow, &j->match, fma,
2704 FM_TCAM_RTE_GROUP, fet->ingress, error)) {
2705 /* Cannot roll back changes at the moment */
2706 ENICPMD_LOG(ERR, "cannot convert jump flow: flow=%p",
2711 ENICPMD_LOG(DEBUG, "convert ok: group=%u ref=%u",
2712 fet->group, fet->ref);
2715 TAILQ_REMOVE(&fm->jump_list, j, list);
2717 j = find_jump_flow(fm, group);
2722 add_hairpin_steer(struct enic_flowman *fm, struct rte_flow *flow,
2723 struct rte_flow_error *error)
2725 struct fm_tcam_match_entry *fm_tcam_entry;
2726 struct enic_fm_flow *fm_flow;
2727 struct fm_action *fm_action;
2728 struct fm_action_op fm_op;
2731 ENICPMD_FUNC_TRACE();
2732 fm_flow = calloc(1, sizeof(*fm_flow));
2733 if (fm_flow == NULL) {
2734 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
2735 NULL, "enic: cannot allocate rte_flow");
2738 /* Original egress hairpin flow */
2739 fm_tcam_entry = &fm->tcam_entry;
2740 fm_action = &fm->action;
2741 /* Use the match pattern of the egress flow as is, without counters */
2742 fm_tcam_entry->ftm_flags &= ~FMEF_COUNTER;
2743 /* The only action is steer to vnic */
2744 fm->action_op_count = 0;
2745 memset(fm_action, 0, sizeof(*fm_action));
2746 memset(&fm_op, 0, sizeof(fm_op));
2747 /* Always to queue 0 for now */
2748 fm_op.fa_op = FMOP_RQ_STEER;
2749 fm_op.rq_steer.rq_index = 0;
2750 fm_op.rq_steer.vnic_handle = fm->hairpin_steer_vnic_h;
2751 ret = enic_fm_append_action_op(fm, &fm_op, error);
2753 goto error_with_flow;
2754 ENICPMD_LOG(DEBUG, "add steer op");
2755 /* Add required END */
2756 memset(&fm_op, 0, sizeof(fm_op));
2757 fm_op.fa_op = FMOP_END;
2758 ret = enic_fm_append_action_op(fm, &fm_op, error);
2760 goto error_with_flow;
2761 /* Add the ingress flow */
2762 fm_flow->action = NULL;
2763 fm_flow->entry_handle = FM_INVALID_HANDLE;
2764 ret = __enic_fm_flow_add_entry(fm, fm_flow, fm_tcam_entry, fm_action,
2765 FM_TCAM_RTE_GROUP, 1 /* ingress */, error);
2767 ENICPMD_LOG(ERR, "cannot add hairpin-steer flow");
2768 goto error_with_flow;
2770 /* The new flow is now the egress flow's paired flow */
2771 flow->fm->hairpin_steer_flow = fm_flow;
2780 enic_fm_open_scratch(struct enic_flowman *fm)
2782 fm->action_op_count = 0;
2784 fm->need_hairpin_steer = 0;
2785 fm->hairpin_steer_vnic_h = 0;
2786 memset(&fm->tcam_entry, 0, sizeof(fm->tcam_entry));
2787 memset(&fm->action, 0, sizeof(fm->action));
2791 enic_fm_close_scratch(struct enic_flowman *fm)
2794 enic_fet_put(fm, fm->fet);
2797 fm->action_op_count = 0;
2801 enic_fm_flow_validate(struct rte_eth_dev *dev,
2802 const struct rte_flow_attr *attrs,
2803 const struct rte_flow_item pattern[],
2804 const struct rte_flow_action actions[],
2805 struct rte_flow_error *error)
2807 struct fm_tcam_match_entry *fm_tcam_entry;
2808 struct fm_action *fm_action;
2809 struct enic_flowman *fm;
2812 ENICPMD_FUNC_TRACE();
2813 fm = begin_fm(pmd_priv(dev));
2816 enic_fm_open_scratch(fm);
2817 ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2819 fm_tcam_entry = &fm->tcam_entry;
2820 fm_action = &fm->action;
2821 enic_fm_dump_tcam_entry(fm_tcam_entry, fm_action,
2824 enic_fm_close_scratch(fm);
2830 enic_fm_flow_query_count(struct rte_eth_dev *dev,
2831 struct rte_flow *flow, void *data,
2832 struct rte_flow_error *error)
2834 struct rte_flow_query_count *query;
2835 struct enic_fm_flow *fm_flow;
2836 struct enic_flowman *fm;
2840 ENICPMD_FUNC_TRACE();
2841 fm = begin_fm(pmd_priv(dev));
2844 if (!fm_flow->counter_valid) {
2845 rc = rte_flow_error_set(error, ENOTSUP,
2846 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2847 "enic: flow does not have counter");
2851 args[0] = FM_COUNTER_QUERY;
2852 args[1] = fm_flow->counter->handle;
2853 args[2] = query->reset;
2854 rc = flowman_cmd(fm, args, 3);
2856 ENICPMD_LOG(ERR, "cannot query counter: rc=%d handle=0x%x",
2857 rc, fm_flow->counter->handle);
2860 query->hits_set = 1;
2861 query->hits = args[0];
2862 query->bytes_set = 1;
2863 query->bytes = args[1];
2871 enic_fm_flow_query(struct rte_eth_dev *dev,
2872 struct rte_flow *flow,
2873 const struct rte_flow_action *actions,
2875 struct rte_flow_error *error)
2879 ENICPMD_FUNC_TRACE();
2880 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2881 switch (actions->type) {
2882 case RTE_FLOW_ACTION_TYPE_VOID:
2884 case RTE_FLOW_ACTION_TYPE_COUNT:
2885 ret = enic_fm_flow_query_count(dev, flow, data, error);
2888 return rte_flow_error_set(error, ENOTSUP,
2889 RTE_FLOW_ERROR_TYPE_ACTION,
2891 "action not supported");
2899 static struct rte_flow *
2900 enic_fm_flow_create(struct rte_eth_dev *dev,
2901 const struct rte_flow_attr *attrs,
2902 const struct rte_flow_item pattern[],
2903 const struct rte_flow_action actions[],
2904 struct rte_flow_error *error)
2906 struct fm_tcam_match_entry *fm_tcam_entry;
2907 struct fm_action *fm_action;
2908 struct enic_flowman *fm;
2909 struct enic_fm_fet *fet;
2910 struct rte_flow *flow;
2914 ENICPMD_FUNC_TRACE();
2915 enic = pmd_priv(dev);
2916 fm = begin_fm(enic);
2918 rte_flow_error_set(error, ENOTSUP,
2919 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2920 "flowman is not initialized");
2923 enic_fm_open_scratch(fm);
2925 ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2927 goto error_with_scratch;
2928 fm_tcam_entry = &fm->tcam_entry;
2929 fm_action = &fm->action;
2930 flow = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2933 /* Add ingress rule that pairs with hairpin rule */
2934 if (fm->need_hairpin_steer) {
2935 ret = add_hairpin_steer(fm, flow, error);
2937 enic_fm_flow_free(fm, flow);
2939 goto error_with_scratch;
2942 LIST_INSERT_HEAD(&enic->flows, flow, next);
2943 fet = flow->fm->fet;
2944 if (fet && fet->default_key) {
2946 * Jump to non-existent group? Save the relevant info
2947 * so we can convert this flow when that group
2950 save_jump_flow(fm, flow, fet->group,
2951 fm_tcam_entry, fm_action);
2952 } else if (fet && fet->ref == 1) {
2954 * A new table is created. Convert the saved flows
2955 * that should jump to this group.
2957 convert_jump_flows(fm, fet, error);
2962 enic_fm_close_scratch(fm);
2968 enic_fm_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
2969 __rte_unused struct rte_flow_error *error)
2971 struct enic *enic = pmd_priv(dev);
2972 struct enic_flowman *fm;
2974 ENICPMD_FUNC_TRACE();
2975 fm = begin_fm(enic);
2978 LIST_REMOVE(flow, next);
2979 enic_fm_flow_free(fm, flow);
2985 enic_fm_flow_flush(struct rte_eth_dev *dev,
2986 __rte_unused struct rte_flow_error *error)
2988 LIST_HEAD(enic_flows, rte_flow) internal;
2989 struct enic_fm_flow *fm_flow;
2990 struct enic_flowman *fm;
2991 struct rte_flow *flow;
2992 struct enic *enic = pmd_priv(dev);
2994 ENICPMD_FUNC_TRACE();
2996 fm = begin_fm(enic);
2999 /* Destroy all non-internal flows */
3000 LIST_INIT(&internal);
3001 while (!LIST_EMPTY(&enic->flows)) {
3002 flow = LIST_FIRST(&enic->flows);
3004 LIST_REMOVE(flow, next);
3005 if (flow->internal) {
3006 LIST_INSERT_HEAD(&internal, flow, next);
3010 * If tables are null, then vNIC is closing, and the firmware
3011 * has already cleaned up flowman state. So do not try to free
3012 * resources, as it only causes errors.
3014 if (fm->ig_tcam_hndl == FM_INVALID_HANDLE) {
3015 fm_flow->entry_handle = FM_INVALID_HANDLE;
3016 fm_flow->action = NULL;
3017 fm_flow->fet = NULL;
3019 enic_fm_flow_free(fm, flow);
3021 while (!LIST_EMPTY(&internal)) {
3022 flow = LIST_FIRST(&internal);
3023 LIST_REMOVE(flow, next);
3024 LIST_INSERT_HEAD(&enic->flows, flow, next);
3031 enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle)
3036 args[0] = FM_MATCH_TABLE_FREE;
3038 rc = flowman_cmd(fm, args, 2);
3040 ENICPMD_LOG(ERR, "cannot free table: rc=%d handle=0x%" PRIx64,
3046 enic_fm_tcam_tbl_alloc(struct enic_flowman *fm, uint32_t direction,
3047 uint32_t max_entries, uint64_t *handle)
3049 struct fm_tcam_match_table *tcam_tbl;
3053 ENICPMD_FUNC_TRACE();
3054 tcam_tbl = &fm->cmd.va->fm_tcam_match_table;
3055 tcam_tbl->ftt_direction = direction;
3056 tcam_tbl->ftt_stage = FM_STAGE_LAST;
3057 tcam_tbl->ftt_max_entries = max_entries;
3058 args[0] = FM_TCAM_TABLE_ALLOC;
3059 args[1] = fm->cmd.pa;
3060 rc = flowman_cmd(fm, args, 2);
3062 ENICPMD_LOG(ERR, "cannot alloc %s TCAM table: rc=%d",
3063 (direction == FM_INGRESS) ? "IG" : "EG", rc);
3067 ENICPMD_LOG(DEBUG, "%s TCAM table allocated, handle=0x%" PRIx64,
3068 (direction == FM_INGRESS) ? "IG" : "EG", *handle);
3073 enic_fm_init_actions(struct enic_flowman *fm)
3075 struct rte_hash *a_hash;
3076 char name[RTE_HASH_NAMESIZE];
3077 struct rte_hash_parameters params = {
3078 .entries = FM_MAX_ACTION_TABLE_SIZE,
3079 .key_len = sizeof(struct fm_action),
3080 .hash_func = rte_jhash,
3081 .hash_func_init_val = 0,
3082 .socket_id = rte_socket_id(),
3085 ENICPMD_FUNC_TRACE();
3086 snprintf((char *)name, sizeof(name), "fm-ah-%s",
3087 fm->owner_enic->bdf_name);
3090 a_hash = rte_hash_create(¶ms);
3093 fm->action_hash = a_hash;
3098 enic_fm_init_counters(struct enic_flowman *fm)
3100 ENICPMD_FUNC_TRACE();
3101 SLIST_INIT(&fm->counters);
3102 return enic_fm_more_counters(fm);
3106 enic_fm_free_all_counters(struct enic_flowman *fm)
3111 args[0] = FM_COUNTER_BRK;
3113 rc = flowman_cmd(fm, args, 2);
3115 ENICPMD_LOG(ERR, "cannot free counters: rc=%d", rc);
3116 rte_free(fm->counter_stack);
3120 enic_fm_alloc_tcam_tables(struct enic_flowman *fm)
3124 ENICPMD_FUNC_TRACE();
3125 rc = enic_fm_tcam_tbl_alloc(fm, FM_INGRESS, FM_MAX_TCAM_TABLE_SIZE,
3129 rc = enic_fm_tcam_tbl_alloc(fm, FM_EGRESS, FM_MAX_TCAM_TABLE_SIZE,
3135 enic_fm_free_tcam_tables(struct enic_flowman *fm)
3137 ENICPMD_FUNC_TRACE();
3138 if (fm->ig_tcam_hndl) {
3139 ENICPMD_LOG(DEBUG, "free IG TCAM table handle=0x%" PRIx64,
3141 enic_fm_tbl_free(fm, fm->ig_tcam_hndl);
3142 fm->ig_tcam_hndl = FM_INVALID_HANDLE;
3144 if (fm->eg_tcam_hndl) {
3145 ENICPMD_LOG(DEBUG, "free EG TCAM table handle=0x%" PRIx64,
3147 enic_fm_tbl_free(fm, fm->eg_tcam_hndl);
3148 fm->eg_tcam_hndl = FM_INVALID_HANDLE;
3153 enic_fm_init(struct enic *enic)
3155 const struct rte_pci_addr *addr;
3156 struct enic_flowman *fm;
3157 uint8_t name[RTE_MEMZONE_NAMESIZE];
3160 if (enic->flow_filter_mode != FILTER_FLOWMAN)
3162 ENICPMD_FUNC_TRACE();
3163 /* Get vnic handle and save for port-id action */
3164 if (enic_is_vf_rep(enic))
3165 addr = &VF_ENIC_TO_VF_REP(enic)->bdf;
3167 addr = &RTE_ETH_DEV_TO_PCI(enic->rte_dev)->addr;
3168 rc = enic_fm_find_vnic(enic, addr, &enic->fm_vnic_handle);
3170 ENICPMD_LOG(ERR, "cannot find vnic handle for %x:%x:%x",
3171 addr->bus, addr->devid, addr->function);
3174 /* Save UIF for egport action */
3175 enic->fm_vnic_uif = vnic_dev_uif(enic->vdev);
3176 ENICPMD_LOG(DEBUG, "uif %u", enic->fm_vnic_uif);
3177 /* Nothing else to do for representor. It will share the PF flowman */
3178 if (enic_is_vf_rep(enic))
3180 fm = calloc(1, sizeof(*fm));
3182 ENICPMD_LOG(ERR, "cannot alloc flowman struct");
3185 fm->owner_enic = enic;
3186 rte_spinlock_init(&fm->lock);
3187 TAILQ_INIT(&fm->fet_list);
3188 TAILQ_INIT(&fm->jump_list);
3189 /* Allocate host memory for flowman commands */
3190 snprintf((char *)name, sizeof(name), "fm-cmd-%s", enic->bdf_name);
3191 fm->cmd.va = enic_alloc_consistent(enic,
3192 sizeof(union enic_flowman_cmd_mem), &fm->cmd.pa, name);
3194 ENICPMD_LOG(ERR, "cannot allocate flowman command memory");
3198 /* Allocate TCAM tables upfront as they are the main tables */
3199 rc = enic_fm_alloc_tcam_tables(fm);
3201 ENICPMD_LOG(ERR, "cannot alloc TCAM tables");
3204 /* Then a number of counters */
3205 rc = enic_fm_init_counters(fm);
3207 ENICPMD_LOG(ERR, "cannot alloc counters");
3210 /* set up action handle hash */
3211 rc = enic_fm_init_actions(fm);
3213 ENICPMD_LOG(ERR, "cannot create action hash, error:%d", rc);
3214 goto error_counters;
3217 * One default exact match table for each direction. We hold onto
3220 rc = enic_fet_alloc(fm, 1, NULL, 128, &fm->default_ig_fet);
3222 ENICPMD_LOG(ERR, "cannot alloc default IG exact match table");
3225 fm->default_ig_fet->ref = 1;
3226 rc = enic_fet_alloc(fm, 0, NULL, 128, &fm->default_eg_fet);
3228 ENICPMD_LOG(ERR, "cannot alloc default EG exact match table");
3231 fm->default_eg_fet->ref = 1;
3232 fm->vf_rep_tag = FM_VF_REP_TAG;
3237 enic_fet_free(fm, fm->default_ig_fet);
3239 rte_hash_free(fm->action_hash);
3241 enic_fm_free_all_counters(fm);
3243 enic_fm_free_tcam_tables(fm);
3245 enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
3246 fm->cmd.va, fm->cmd.pa);
3253 enic_fm_destroy(struct enic *enic)
3255 struct enic_flowman *fm;
3256 struct enic_fm_fet *fet;
3258 ENICPMD_FUNC_TRACE();
3259 if (enic_is_vf_rep(enic)) {
3260 delete_rep_flows(enic);
3263 if (enic->fm == NULL)
3266 enic_fm_flow_flush(enic->rte_dev, NULL);
3267 enic_fet_free(fm, fm->default_eg_fet);
3268 enic_fet_free(fm, fm->default_ig_fet);
3269 /* Free all exact match tables still open */
3270 while (!TAILQ_EMPTY(&fm->fet_list)) {
3271 fet = TAILQ_FIRST(&fm->fet_list);
3272 enic_fet_free(fm, fet);
3274 enic_fm_free_tcam_tables(fm);
3275 enic_fm_free_all_counters(fm);
3276 rte_hash_free(fm->action_hash);
3277 enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
3278 fm->cmd.va, fm->cmd.pa);
3285 enic_fm_allocate_switch_domain(struct enic *pf)
3287 const struct rte_pci_addr *cur_a, *prev_a;
3288 struct rte_eth_dev *dev;
3289 struct enic *cur, *prev;
3295 ENICPMD_FUNC_TRACE();
3296 if (enic_is_vf_rep(pf))
3299 cur_a = &RTE_ETH_DEV_TO_PCI(cur->rte_dev)->addr;
3300 /* Go through ports and find another PF that is on the same adapter */
3301 RTE_ETH_FOREACH_DEV(pid) {
3302 dev = &rte_eth_devices[pid];
3303 if (!dev_is_enic(dev))
3305 if (dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)
3307 if (dev == cur->rte_dev)
3309 /* dev is another PF. Is it on the same adapter? */
3310 prev = pmd_priv(dev);
3311 prev_a = &RTE_ETH_DEV_TO_PCI(dev)->addr;
3312 if (!enic_fm_find_vnic(cur, prev_a, &vnic_h)) {
3313 ENICPMD_LOG(DEBUG, "Port %u (PF BDF %x:%x:%x) and port %u (PF BDF %x:%x:%x domain %u) are on the same VIC",
3314 cur->rte_dev->data->port_id,
3315 cur_a->bus, cur_a->devid, cur_a->function,
3317 prev_a->bus, prev_a->devid, prev_a->function,
3318 prev->switch_domain_id);
3319 cur->switch_domain_id = prev->switch_domain_id;
3323 ret = rte_eth_switch_domain_alloc(&domain_id);
3325 ENICPMD_LOG(WARNING, "failed to allocate switch domain for device %d",
3328 cur->switch_domain_id = domain_id;
3329 ENICPMD_LOG(DEBUG, "Port %u (PF BDF %x:%x:%x) is the 1st PF on the VIC. Allocated switch domain id %u",
3330 cur->rte_dev->data->port_id,
3331 cur_a->bus, cur_a->devid, cur_a->function,
3336 const struct rte_flow_ops enic_fm_flow_ops = {
3337 .validate = enic_fm_flow_validate,
3338 .create = enic_fm_flow_create,
3339 .destroy = enic_fm_flow_destroy,
3340 .flush = enic_fm_flow_flush,
3341 .query = enic_fm_flow_query,
3344 /* Add a high priority flow that loops representor packets to VF */
3346 enic_fm_add_rep2vf_flow(struct enic_vf_representor *vf)
3348 struct fm_tcam_match_entry *fm_tcam_entry;
3349 struct rte_flow *flow0, *flow1;
3350 struct fm_action *fm_action;
3351 struct rte_flow_error error;
3352 struct rte_flow_attr attrs;
3353 struct fm_action_op fm_op;
3354 struct enic_flowman *fm;
3360 tag = fm->vf_rep_tag;
3361 enic_fm_open_scratch(fm);
3362 fm_tcam_entry = &fm->tcam_entry;
3363 fm_action = &fm->action;
3364 /* Egress rule: match WQ ID and tag+hairpin */
3365 fm_tcam_entry->ftm_data.fk_wq_id = vf->pf_wq_idx;
3366 fm_tcam_entry->ftm_mask.fk_wq_id = 0xffff;
3367 fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3368 memset(&fm_op, 0, sizeof(fm_op));
3369 fm_op.fa_op = FMOP_TAG;
3370 fm_op.tag.tag = tag;
3371 enic_fm_append_action_op(fm, &fm_op, &error);
3372 memset(&fm_op, 0, sizeof(fm_op));
3373 fm_op.fa_op = FMOP_EG_HAIRPIN;
3374 enic_fm_append_action_op(fm, &fm_op, &error);
3375 memset(&fm_op, 0, sizeof(fm_op));
3376 fm_op.fa_op = FMOP_END;
3377 enic_fm_append_action_op(fm, &fm_op, &error);
3381 attrs.priority = FM_HIGHEST_PRIORITY;
3382 flow0 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3384 enic_fm_close_scratch(fm);
3385 if (flow0 == NULL) {
3386 ENICPMD_LOG(ERR, "Cannot create flow 0 for representor->VF");
3389 LIST_INSERT_HEAD(&pf->flows, flow0, next);
3390 /* Make this flow internal, so the user app cannot delete it */
3391 flow0->internal = 1;
3392 ENICPMD_LOG(DEBUG, "representor->VF %d flow created: wq %d -> tag %d hairpin",
3393 vf->vf_id, vf->pf_wq_idx, tag);
3395 /* Ingress: steer hairpinned to VF RQ 0 */
3396 enic_fm_open_scratch(fm);
3397 fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3398 fm_tcam_entry->ftm_data.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3399 fm_tcam_entry->ftm_mask.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3400 fm_tcam_entry->ftm_data.fk_packet_tag = tag;
3401 fm_tcam_entry->ftm_mask.fk_packet_tag = 0xff;
3402 memset(&fm_op, 0, sizeof(fm_op));
3403 fm_op.fa_op = FMOP_RQ_STEER;
3404 fm_op.rq_steer.rq_index = 0;
3405 fm_op.rq_steer.vnic_handle = vf->enic.fm_vnic_handle;
3406 enic_fm_append_action_op(fm, &fm_op, &error);
3407 memset(&fm_op, 0, sizeof(fm_op));
3408 fm_op.fa_op = FMOP_END;
3409 enic_fm_append_action_op(fm, &fm_op, &error);
3413 attrs.priority = FM_HIGHEST_PRIORITY;
3414 flow1 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3416 enic_fm_close_scratch(fm);
3417 if (flow1 == NULL) {
3418 ENICPMD_LOG(ERR, "Cannot create flow 1 for representor->VF");
3419 enic_fm_flow_destroy(pf->rte_dev, flow0, &error);
3422 LIST_INSERT_HEAD(&pf->flows, flow1, next);
3423 flow1->internal = 1;
3424 ENICPMD_LOG(DEBUG, "representor->VF %d flow created: tag %d hairpinned -> VF RQ %d",
3425 vf->vf_id, tag, fm_op.rq_steer.rq_index);
3426 vf->rep2vf_flow[0] = flow0;
3427 vf->rep2vf_flow[1] = flow1;
3428 /* Done with this tag, use a different one next time */
3434 * Add a low priority flow that matches all packets from VF and loops them
3435 * back to the representor.
3438 enic_fm_add_vf2rep_flow(struct enic_vf_representor *vf)
3440 struct fm_tcam_match_entry *fm_tcam_entry;
3441 struct rte_flow *flow0, *flow1;
3442 struct fm_action *fm_action;
3443 struct rte_flow_error error;
3444 struct rte_flow_attr attrs;
3445 struct fm_action_op fm_op;
3446 struct enic_flowman *fm;
3452 tag = fm->vf_rep_tag;
3453 enic_fm_open_scratch(fm);
3454 fm_tcam_entry = &fm->tcam_entry;
3455 fm_action = &fm->action;
3456 /* Egress rule: match-any and tag+hairpin */
3457 fm_tcam_entry->ftm_data.fk_wq_id = 0;
3458 fm_tcam_entry->ftm_mask.fk_wq_id = 0xffff;
3459 fm_tcam_entry->ftm_data.fk_wq_vnic = vf->enic.fm_vnic_handle;
3460 fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3461 memset(&fm_op, 0, sizeof(fm_op));
3462 fm_op.fa_op = FMOP_TAG;
3463 fm_op.tag.tag = tag;
3464 enic_fm_append_action_op(fm, &fm_op, &error);
3465 memset(&fm_op, 0, sizeof(fm_op));
3466 fm_op.fa_op = FMOP_EG_HAIRPIN;
3467 enic_fm_append_action_op(fm, &fm_op, &error);
3468 memset(&fm_op, 0, sizeof(fm_op));
3469 fm_op.fa_op = FMOP_END;
3470 enic_fm_append_action_op(fm, &fm_op, &error);
3474 attrs.priority = FM_LOWEST_PRIORITY;
3475 flow0 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3477 enic_fm_close_scratch(fm);
3478 if (flow0 == NULL) {
3479 ENICPMD_LOG(ERR, "Cannot create flow 0 for VF->representor");
3482 LIST_INSERT_HEAD(&pf->flows, flow0, next);
3483 /* Make this flow internal, so the user app cannot delete it */
3484 flow0->internal = 1;
3485 ENICPMD_LOG(DEBUG, "VF %d->representor flow created: wq %d (low prio) -> tag %d hairpin",
3486 vf->vf_id, fm_tcam_entry->ftm_data.fk_wq_id, tag);
3488 /* Ingress: steer hairpinned to VF rep RQ */
3489 enic_fm_open_scratch(fm);
3490 fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3491 fm_tcam_entry->ftm_data.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3492 fm_tcam_entry->ftm_mask.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3493 fm_tcam_entry->ftm_data.fk_packet_tag = tag;
3494 fm_tcam_entry->ftm_mask.fk_packet_tag = 0xff;
3495 memset(&fm_op, 0, sizeof(fm_op));
3496 fm_op.fa_op = FMOP_RQ_STEER;
3497 fm_op.rq_steer.rq_index = vf->pf_rq_sop_idx;
3498 fm_op.rq_steer.vnic_handle = pf->fm_vnic_handle;
3499 enic_fm_append_action_op(fm, &fm_op, &error);
3500 memset(&fm_op, 0, sizeof(fm_op));
3501 fm_op.fa_op = FMOP_END;
3502 enic_fm_append_action_op(fm, &fm_op, &error);
3506 attrs.priority = FM_HIGHEST_PRIORITY;
3507 flow1 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3509 enic_fm_close_scratch(fm);
3510 if (flow1 == NULL) {
3511 ENICPMD_LOG(ERR, "Cannot create flow 1 for VF->representor");
3512 enic_fm_flow_destroy(pf->rte_dev, flow0, &error);
3515 LIST_INSERT_HEAD(&pf->flows, flow1, next);
3516 flow1->internal = 1;
3517 ENICPMD_LOG(DEBUG, "VF %d->representor flow created: tag %d hairpinned -> PF RQ %d",
3518 vf->vf_id, tag, vf->pf_rq_sop_idx);
3519 vf->vf2rep_flow[0] = flow0;
3520 vf->vf2rep_flow[1] = flow1;
3521 /* Done with this tag, use a different one next time */
3526 /* Destroy representor flows created by enic_fm_add_{rep2vf,vf2rep}_flow */
3528 delete_rep_flows(struct enic *enic)
3530 struct enic_vf_representor *vf;
3531 struct rte_flow_error error;
3532 struct rte_eth_dev *dev;
3535 RTE_ASSERT(enic_is_vf_rep(enic));
3536 vf = VF_ENIC_TO_VF_REP(enic);
3537 dev = vf->pf->rte_dev;
3538 for (i = 0; i < ARRAY_SIZE(vf->vf2rep_flow); i++) {
3539 if (vf->vf2rep_flow[i])
3540 enic_fm_flow_destroy(dev, vf->vf2rep_flow[i], &error);
3542 for (i = 0; i < ARRAY_SIZE(vf->rep2vf_flow); i++) {
3543 if (vf->rep2vf_flow[i])
3544 enic_fm_flow_destroy(dev, vf->rep2vf_flow[i], &error);
3548 static struct enic_flowman *
3549 begin_fm(struct enic *enic)
3551 struct enic_vf_representor *vf;
3552 struct enic_flowman *fm;
3554 /* Representor uses PF flowman */
3555 if (enic_is_vf_rep(enic)) {
3556 vf = VF_ENIC_TO_VF_REP(enic);
3561 /* Save the API caller and lock if representors exist */
3563 if (fm->owner_enic->switchdev_mode)
3564 rte_spinlock_lock(&fm->lock);
3565 fm->user_enic = enic;
3571 end_fm(struct enic_flowman *fm)
3573 fm->user_enic = NULL;
3574 if (fm->owner_enic->switchdev_mode)
3575 rte_spinlock_unlock(&fm->lock);