1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2008-2019 Cisco Systems, Inc. All rights reserved.
8 #include <ethdev_driver.h>
9 #include <rte_flow_driver.h>
10 #include <rte_ether.h>
12 #include <rte_jhash.h>
15 #include <rte_memzone.h>
17 #include "enic_compat.h"
22 #define IP_DEFTTL 64 /* from RFC 1340. */
23 #define IP6_VTC_FLOW 0x60000000
25 /* Highest Item type supported by Flowman */
26 #define FM_MAX_ITEM_TYPE RTE_FLOW_ITEM_TYPE_VXLAN
28 /* Up to 1024 TCAM entries */
29 #define FM_MAX_TCAM_TABLE_SIZE 1024
31 /* Up to 4096 entries per exact match table */
32 #define FM_MAX_EXACT_TABLE_SIZE 4096
34 /* Number of counters to increase on for each increment */
35 #define FM_COUNTERS_EXPAND 100
37 #define FM_INVALID_HANDLE 0
39 /* Low priority used for implicit VF -> representor flow */
40 #define FM_LOWEST_PRIORITY 100000
42 /* High priority used for implicit representor -> VF flow */
43 #define FM_HIGHEST_PRIORITY 0
45 /* Tag used for implicit VF <-> representor flows */
46 #define FM_VF_REP_TAG 1
48 /* Max number of actions supported by VIC is 2K. Make hash table double that. */
49 #define FM_MAX_ACTION_TABLE_SIZE 4096
52 * Flow exact match tables (FET) in the VIC and rte_flow groups.
53 * Use a simple scheme to map groups to tables.
54 * Group 0 uses the single TCAM tables, one for each direction.
55 * Group 1, 2, ... uses its own exact match table.
57 * The TCAM tables are allocated upfront during init.
59 * Exact match tables are allocated on demand. 3 paths that lead allocations.
61 * 1. Add a flow that jumps from group 0 to group N.
63 * If N does not exist, we allocate an exact match table for it, using
64 * a dummy key. A key is required for the table.
66 * 2. Add a flow that uses group N.
68 * If N does not exist, we allocate an exact match table for it, using
69 * the flow's key. Subsequent flows to the same group all should have
72 * Without a jump flow to N, N is not reachable in hardware. No packets
75 * 3. Add a flow to an empty group N.
77 * N has been created via (1) and the dummy key. We free that table, allocate
78 * a new table using the new flow's key. Also re-do the existing jump flow to
79 * point to the new table.
81 #define FM_TCAM_RTE_GROUP 0
84 TAILQ_ENTRY(enic_fm_fet) list;
85 uint32_t group; /* rte_flow group ID */
86 uint64_t handle; /* Exact match table handle from flowman */
89 int ref; /* Reference count via get/put */
90 struct fm_key_template key; /* Key associated with the table */
93 struct enic_fm_counter {
94 SLIST_ENTRY(enic_fm_counter) next;
98 struct enic_fm_action {
101 struct fm_action key;
105 struct enic_fm_flow {
107 uint64_t entry_handle;
108 struct enic_fm_action *action;
109 struct enic_fm_counter *counter;
110 struct enic_fm_fet *fet;
111 /* Auto-added steer action for hairpin flows (e.g. vnic->vnic) */
112 struct enic_fm_flow *hairpin_steer_flow;
115 struct enic_fm_jump_flow {
116 TAILQ_ENTRY(enic_fm_jump_flow) list;
117 struct rte_flow *flow;
119 struct fm_tcam_match_entry match;
120 struct fm_action action;
124 * Flowman uses host memory for commands. This structure is allocated
125 * in DMA-able memory.
127 union enic_flowman_cmd_mem {
128 struct fm_tcam_match_table fm_tcam_match_table;
129 struct fm_exact_match_table fm_exact_match_table;
130 struct fm_tcam_match_entry fm_tcam_match_entry;
131 struct fm_exact_match_entry fm_exact_match_entry;
132 struct fm_action fm_action;
136 * PF has a flowman instance, and VF representors share it with PF.
137 * PF allocates this structure and owns it. VF representors borrow
138 * the PF's structure during API calls (e.g. create, query).
140 struct enic_flowman {
141 struct enic *owner_enic; /* PF */
142 struct enic *user_enic; /* API caller (PF or representor) */
144 * Representors and PF share the same underlying flowman.
145 * Lock API calls to serialize accesses from them. Only used
146 * when VF representors are present.
151 union enic_flowman_cmd_mem *va;
154 /* TCAM tables allocated upfront, used for group 0 */
155 uint64_t ig_tcam_hndl;
156 uint64_t eg_tcam_hndl;
158 SLIST_HEAD(enic_free_counters, enic_fm_counter) counters;
160 uint32_t counters_alloced;
161 /* Exact match tables for groups != 0, dynamically allocated */
162 TAILQ_HEAD(fet_list, enic_fm_fet) fet_list;
164 * Default exact match tables used for jump actions to
165 * non-existent groups.
167 struct enic_fm_fet *default_eg_fet;
168 struct enic_fm_fet *default_ig_fet;
169 /* hash table for Action reuse */
170 struct rte_hash *action_hash;
171 /* Flows that jump to the default table above */
172 TAILQ_HEAD(jump_flow_list, enic_fm_jump_flow) jump_list;
174 * Scratch data used during each invocation of flow_create
177 struct enic_fm_fet *fet;
178 struct fm_tcam_match_entry tcam_entry;
179 struct fm_action action;
180 struct fm_action action_tmp; /* enic_fm_reorder_action_op */
182 /* Tags used for representor flows */
184 /* For auto-added steer action for hairpin */
185 int need_hairpin_steer;
186 uint64_t hairpin_steer_vnic_h;
189 static int enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle);
191 * API functions (create, destroy, validate, flush) call begin_fm()
192 * upon entering to save the caller enic (PF or VF representor) and
193 * lock. Upon exit, they call end_fm() to unlock.
195 static struct enic_flowman *begin_fm(struct enic *enic);
196 static void end_fm(struct enic_flowman *fm);
197 /* Delete internal flows created for representor paths */
198 static void delete_rep_flows(struct enic *enic);
201 * Common arguments passed to copy_item functions. Use this structure
202 * so we can easily add new arguments.
203 * item: Item specification.
204 * fm_tcam_entry: Flowman TCAM match entry.
205 * header_level: 0 for outer header, 1 for inner header.
207 struct copy_item_args {
208 const struct rte_flow_item *item;
209 struct fm_tcam_match_entry *fm_tcam_entry;
210 uint8_t header_level;
213 /* functions for copying items into flowman match */
214 typedef int (enic_copy_item_fn)(struct copy_item_args *arg);
216 /* Info about how to copy items into flowman match */
217 struct enic_fm_items {
218 /* Function for copying and validating an item. */
219 enic_copy_item_fn * const copy_item;
220 /* List of valid previous items. */
221 const enum rte_flow_item_type * const prev_items;
223 * True if it's OK for this item to be the first item. For some NIC
224 * versions, it's invalid to start the stack above layer 3.
226 const uint8_t valid_start_item;
229 static enic_copy_item_fn enic_fm_copy_item_eth;
230 static enic_copy_item_fn enic_fm_copy_item_ipv4;
231 static enic_copy_item_fn enic_fm_copy_item_ipv6;
232 static enic_copy_item_fn enic_fm_copy_item_raw;
233 static enic_copy_item_fn enic_fm_copy_item_sctp;
234 static enic_copy_item_fn enic_fm_copy_item_tcp;
235 static enic_copy_item_fn enic_fm_copy_item_udp;
236 static enic_copy_item_fn enic_fm_copy_item_vlan;
237 static enic_copy_item_fn enic_fm_copy_item_vxlan;
239 /* Ingress actions */
240 static const enum rte_flow_action_type enic_fm_supported_ig_actions[] = {
241 RTE_FLOW_ACTION_TYPE_COUNT,
242 RTE_FLOW_ACTION_TYPE_DROP,
243 RTE_FLOW_ACTION_TYPE_FLAG,
244 RTE_FLOW_ACTION_TYPE_JUMP,
245 RTE_FLOW_ACTION_TYPE_MARK,
246 RTE_FLOW_ACTION_TYPE_OF_POP_VLAN,
247 RTE_FLOW_ACTION_TYPE_PORT_ID,
248 RTE_FLOW_ACTION_TYPE_PASSTHRU,
249 RTE_FLOW_ACTION_TYPE_QUEUE,
250 RTE_FLOW_ACTION_TYPE_RSS,
251 RTE_FLOW_ACTION_TYPE_VOID,
252 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
253 RTE_FLOW_ACTION_TYPE_VXLAN_DECAP,
254 RTE_FLOW_ACTION_TYPE_END, /* END must be the last entry */
258 static const enum rte_flow_action_type enic_fm_supported_eg_actions[] = {
259 RTE_FLOW_ACTION_TYPE_COUNT,
260 RTE_FLOW_ACTION_TYPE_DROP,
261 RTE_FLOW_ACTION_TYPE_JUMP,
262 RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN,
263 RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP,
264 RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID,
265 RTE_FLOW_ACTION_TYPE_PORT_ID,
266 RTE_FLOW_ACTION_TYPE_PASSTHRU,
267 RTE_FLOW_ACTION_TYPE_VOID,
268 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
269 RTE_FLOW_ACTION_TYPE_END,
272 static const struct enic_fm_items enic_fm_items[] = {
273 [RTE_FLOW_ITEM_TYPE_RAW] = {
274 .copy_item = enic_fm_copy_item_raw,
275 .valid_start_item = 0,
276 .prev_items = (const enum rte_flow_item_type[]) {
277 RTE_FLOW_ITEM_TYPE_UDP,
278 RTE_FLOW_ITEM_TYPE_END,
281 [RTE_FLOW_ITEM_TYPE_ETH] = {
282 .copy_item = enic_fm_copy_item_eth,
283 .valid_start_item = 1,
284 .prev_items = (const enum rte_flow_item_type[]) {
285 RTE_FLOW_ITEM_TYPE_END,
288 [RTE_FLOW_ITEM_TYPE_VLAN] = {
289 .copy_item = enic_fm_copy_item_vlan,
290 .valid_start_item = 1,
291 .prev_items = (const enum rte_flow_item_type[]) {
292 RTE_FLOW_ITEM_TYPE_ETH,
293 RTE_FLOW_ITEM_TYPE_END,
296 [RTE_FLOW_ITEM_TYPE_IPV4] = {
297 .copy_item = enic_fm_copy_item_ipv4,
298 .valid_start_item = 1,
299 .prev_items = (const enum rte_flow_item_type[]) {
300 RTE_FLOW_ITEM_TYPE_ETH,
301 RTE_FLOW_ITEM_TYPE_VLAN,
302 RTE_FLOW_ITEM_TYPE_END,
305 [RTE_FLOW_ITEM_TYPE_IPV6] = {
306 .copy_item = enic_fm_copy_item_ipv6,
307 .valid_start_item = 1,
308 .prev_items = (const enum rte_flow_item_type[]) {
309 RTE_FLOW_ITEM_TYPE_ETH,
310 RTE_FLOW_ITEM_TYPE_VLAN,
311 RTE_FLOW_ITEM_TYPE_END,
314 [RTE_FLOW_ITEM_TYPE_UDP] = {
315 .copy_item = enic_fm_copy_item_udp,
316 .valid_start_item = 1,
317 .prev_items = (const enum rte_flow_item_type[]) {
318 RTE_FLOW_ITEM_TYPE_IPV4,
319 RTE_FLOW_ITEM_TYPE_IPV6,
320 RTE_FLOW_ITEM_TYPE_END,
323 [RTE_FLOW_ITEM_TYPE_TCP] = {
324 .copy_item = enic_fm_copy_item_tcp,
325 .valid_start_item = 1,
326 .prev_items = (const enum rte_flow_item_type[]) {
327 RTE_FLOW_ITEM_TYPE_IPV4,
328 RTE_FLOW_ITEM_TYPE_IPV6,
329 RTE_FLOW_ITEM_TYPE_END,
332 [RTE_FLOW_ITEM_TYPE_SCTP] = {
333 .copy_item = enic_fm_copy_item_sctp,
334 .valid_start_item = 0,
335 .prev_items = (const enum rte_flow_item_type[]) {
336 RTE_FLOW_ITEM_TYPE_IPV4,
337 RTE_FLOW_ITEM_TYPE_IPV6,
338 RTE_FLOW_ITEM_TYPE_END,
341 [RTE_FLOW_ITEM_TYPE_VXLAN] = {
342 .copy_item = enic_fm_copy_item_vxlan,
343 .valid_start_item = 1,
344 .prev_items = (const enum rte_flow_item_type[]) {
345 RTE_FLOW_ITEM_TYPE_UDP,
346 RTE_FLOW_ITEM_TYPE_END,
352 enic_fm_copy_item_eth(struct copy_item_args *arg)
354 const struct rte_flow_item *item = arg->item;
355 const struct rte_flow_item_eth *spec = item->spec;
356 const struct rte_flow_item_eth *mask = item->mask;
357 const uint8_t lvl = arg->header_level;
358 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
359 struct fm_header_set *fm_data, *fm_mask;
361 ENICPMD_FUNC_TRACE();
362 /* Match all if no spec */
366 mask = &rte_flow_item_eth_mask;
367 fm_data = &entry->ftm_data.fk_hdrset[lvl];
368 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
369 fm_data->fk_header_select |= FKH_ETHER;
370 fm_mask->fk_header_select |= FKH_ETHER;
371 memcpy(&fm_data->l2.eth, spec, sizeof(struct rte_ether_hdr));
372 memcpy(&fm_mask->l2.eth, mask, sizeof(struct rte_ether_hdr));
377 enic_fm_copy_item_vlan(struct copy_item_args *arg)
379 const struct rte_flow_item *item = arg->item;
380 const struct rte_flow_item_vlan *spec = item->spec;
381 const struct rte_flow_item_vlan *mask = item->mask;
382 const uint8_t lvl = arg->header_level;
383 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
384 struct fm_header_set *fm_data, *fm_mask;
385 struct rte_ether_hdr *eth_mask;
386 struct rte_ether_hdr *eth_val;
389 ENICPMD_FUNC_TRACE();
390 fm_data = &entry->ftm_data.fk_hdrset[lvl];
391 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
392 /* Outer and inner packet vlans need different flags */
393 meta = FKM_VLAN_PRES;
396 fm_data->fk_metadata |= meta;
397 fm_mask->fk_metadata |= meta;
399 /* Match all if no spec */
403 mask = &rte_flow_item_vlan_mask;
405 eth_mask = (void *)&fm_mask->l2.eth;
406 eth_val = (void *)&fm_data->l2.eth;
409 * Outer TPID cannot be matched. If inner_type is 0, use what is
412 if (eth_mask->ether_type && mask->inner_type)
416 * When packet matching, the VIC always compares vlan-stripped
417 * L2, regardless of vlan stripping settings. So, the inner type
418 * from vlan becomes the ether type of the eth header.
420 if (mask->inner_type) {
421 eth_mask->ether_type = mask->inner_type;
422 eth_val->ether_type = spec->inner_type;
424 fm_data->fk_header_select |= FKH_ETHER | FKH_QTAG;
425 fm_mask->fk_header_select |= FKH_ETHER | FKH_QTAG;
426 fm_data->fk_vlan = rte_be_to_cpu_16(spec->tci);
427 fm_mask->fk_vlan = rte_be_to_cpu_16(mask->tci);
432 enic_fm_copy_item_ipv4(struct copy_item_args *arg)
434 const struct rte_flow_item *item = arg->item;
435 const struct rte_flow_item_ipv4 *spec = item->spec;
436 const struct rte_flow_item_ipv4 *mask = item->mask;
437 const uint8_t lvl = arg->header_level;
438 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
439 struct fm_header_set *fm_data, *fm_mask;
441 ENICPMD_FUNC_TRACE();
442 fm_data = &entry->ftm_data.fk_hdrset[lvl];
443 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
444 fm_data->fk_metadata |= FKM_IPV4;
445 fm_mask->fk_metadata |= FKM_IPV4;
450 mask = &rte_flow_item_ipv4_mask;
452 fm_data->fk_header_select |= FKH_IPV4;
453 fm_mask->fk_header_select |= FKH_IPV4;
454 memcpy(&fm_data->l3.ip4, spec, sizeof(*spec));
455 memcpy(&fm_mask->l3.ip4, mask, sizeof(*mask));
460 enic_fm_copy_item_ipv6(struct copy_item_args *arg)
462 const struct rte_flow_item *item = arg->item;
463 const struct rte_flow_item_ipv6 *spec = item->spec;
464 const struct rte_flow_item_ipv6 *mask = item->mask;
465 const uint8_t lvl = arg->header_level;
466 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
467 struct fm_header_set *fm_data, *fm_mask;
469 ENICPMD_FUNC_TRACE();
470 fm_data = &entry->ftm_data.fk_hdrset[lvl];
471 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
472 fm_data->fk_metadata |= FKM_IPV6;
473 fm_mask->fk_metadata |= FKM_IPV6;
478 mask = &rte_flow_item_ipv6_mask;
480 fm_data->fk_header_select |= FKH_IPV6;
481 fm_mask->fk_header_select |= FKH_IPV6;
482 memcpy(&fm_data->l3.ip6, spec, sizeof(struct rte_ipv6_hdr));
483 memcpy(&fm_mask->l3.ip6, mask, sizeof(struct rte_ipv6_hdr));
488 enic_fm_copy_item_udp(struct copy_item_args *arg)
490 const struct rte_flow_item *item = arg->item;
491 const struct rte_flow_item_udp *spec = item->spec;
492 const struct rte_flow_item_udp *mask = item->mask;
493 const uint8_t lvl = arg->header_level;
494 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
495 struct fm_header_set *fm_data, *fm_mask;
497 ENICPMD_FUNC_TRACE();
498 fm_data = &entry->ftm_data.fk_hdrset[lvl];
499 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
500 fm_data->fk_metadata |= FKM_UDP;
501 fm_mask->fk_metadata |= FKM_UDP;
506 mask = &rte_flow_item_udp_mask;
508 fm_data->fk_header_select |= FKH_UDP;
509 fm_mask->fk_header_select |= FKH_UDP;
510 memcpy(&fm_data->l4.udp, spec, sizeof(*spec));
511 memcpy(&fm_mask->l4.udp, mask, sizeof(*mask));
516 enic_fm_copy_item_tcp(struct copy_item_args *arg)
518 const struct rte_flow_item *item = arg->item;
519 const struct rte_flow_item_tcp *spec = item->spec;
520 const struct rte_flow_item_tcp *mask = item->mask;
521 const uint8_t lvl = arg->header_level;
522 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
523 struct fm_header_set *fm_data, *fm_mask;
525 ENICPMD_FUNC_TRACE();
526 fm_data = &entry->ftm_data.fk_hdrset[lvl];
527 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
528 fm_data->fk_metadata |= FKM_TCP;
529 fm_mask->fk_metadata |= FKM_TCP;
534 mask = &rte_flow_item_tcp_mask;
536 fm_data->fk_header_select |= FKH_TCP;
537 fm_mask->fk_header_select |= FKH_TCP;
538 memcpy(&fm_data->l4.tcp, spec, sizeof(*spec));
539 memcpy(&fm_mask->l4.tcp, mask, sizeof(*mask));
544 enic_fm_copy_item_sctp(struct copy_item_args *arg)
546 const struct rte_flow_item *item = arg->item;
547 const struct rte_flow_item_sctp *spec = item->spec;
548 const struct rte_flow_item_sctp *mask = item->mask;
549 const uint8_t lvl = arg->header_level;
550 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
551 struct fm_header_set *fm_data, *fm_mask;
552 uint8_t *ip_proto_mask = NULL;
553 uint8_t *ip_proto = NULL;
556 ENICPMD_FUNC_TRACE();
557 fm_data = &entry->ftm_data.fk_hdrset[lvl];
558 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
560 * The NIC filter API has no flags for "match sctp", so explicitly
561 * set the protocol number in the IP pattern.
563 if (fm_data->fk_metadata & FKM_IPV4) {
564 struct rte_ipv4_hdr *ip;
565 ip = (struct rte_ipv4_hdr *)&fm_mask->l3.ip4;
566 ip_proto_mask = &ip->next_proto_id;
567 ip = (struct rte_ipv4_hdr *)&fm_data->l3.ip4;
568 ip_proto = &ip->next_proto_id;
570 } else if (fm_data->fk_metadata & FKM_IPV6) {
571 struct rte_ipv6_hdr *ip;
572 ip = (struct rte_ipv6_hdr *)&fm_mask->l3.ip6;
573 ip_proto_mask = &ip->proto;
574 ip = (struct rte_ipv6_hdr *)&fm_data->l3.ip6;
575 ip_proto = &ip->proto;
578 /* Need IPv4/IPv6 pattern first */
581 *ip_proto = IPPROTO_SCTP;
582 *ip_proto_mask = 0xff;
583 fm_data->fk_header_select |= l3_fkh;
584 fm_mask->fk_header_select |= l3_fkh;
589 mask = &rte_flow_item_sctp_mask;
591 fm_data->fk_header_select |= FKH_L4RAW;
592 fm_mask->fk_header_select |= FKH_L4RAW;
593 memcpy(fm_data->l4.rawdata, spec, sizeof(*spec));
594 memcpy(fm_mask->l4.rawdata, mask, sizeof(*mask));
599 enic_fm_copy_item_vxlan(struct copy_item_args *arg)
601 const struct rte_flow_item *item = arg->item;
602 const struct rte_flow_item_vxlan *spec = item->spec;
603 const struct rte_flow_item_vxlan *mask = item->mask;
604 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
605 struct fm_header_set *fm_data, *fm_mask;
607 ENICPMD_FUNC_TRACE();
608 /* Only 2 header levels (outer and inner) allowed */
609 if (arg->header_level > 0)
612 fm_data = &entry->ftm_data.fk_hdrset[0];
613 fm_mask = &entry->ftm_mask.fk_hdrset[0];
614 fm_data->fk_metadata |= FKM_VXLAN;
615 fm_mask->fk_metadata |= FKM_VXLAN;
616 /* items from here on out are inner header items */
617 arg->header_level = 1;
619 /* Match all if no spec */
623 mask = &rte_flow_item_vxlan_mask;
625 fm_data->fk_header_select |= FKH_VXLAN;
626 fm_mask->fk_header_select |= FKH_VXLAN;
627 memcpy(&fm_data->vxlan, spec, sizeof(*spec));
628 memcpy(&fm_mask->vxlan, mask, sizeof(*mask));
633 * Currently, raw pattern match is very limited. It is intended for matching
634 * UDP tunnel header (e.g. vxlan or geneve).
637 enic_fm_copy_item_raw(struct copy_item_args *arg)
639 const struct rte_flow_item *item = arg->item;
640 const struct rte_flow_item_raw *spec = item->spec;
641 const struct rte_flow_item_raw *mask = item->mask;
642 const uint8_t lvl = arg->header_level;
643 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
644 struct fm_header_set *fm_data, *fm_mask;
646 ENICPMD_FUNC_TRACE();
647 /* Cannot be used for inner packet */
650 /* Need both spec and mask */
653 /* Only supports relative with offset 0 */
654 if (!spec->relative || spec->offset != 0 || spec->search ||
657 /* Need non-null pattern that fits within the NIC's filter pattern */
658 if (spec->length == 0 ||
659 spec->length + sizeof(struct rte_udp_hdr) > FM_LAYER_SIZE ||
660 !spec->pattern || !mask->pattern)
663 * Mask fields, including length, are often set to zero. Assume that
664 * means "same as spec" to avoid breaking existing apps. If length
665 * is not zero, then it should be >= spec length.
667 * No more pattern follows this, so append to the L4 layer instead of
668 * L5 to work with both recent and older VICs.
670 if (mask->length != 0 && mask->length < spec->length)
673 fm_data = &entry->ftm_data.fk_hdrset[lvl];
674 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
675 fm_data->fk_header_select |= FKH_L4RAW;
676 fm_mask->fk_header_select |= FKH_L4RAW;
677 fm_data->fk_header_select &= ~FKH_UDP;
678 fm_mask->fk_header_select &= ~FKH_UDP;
679 memcpy(fm_data->l4.rawdata + sizeof(struct rte_udp_hdr),
680 spec->pattern, spec->length);
681 memcpy(fm_mask->l4.rawdata + sizeof(struct rte_udp_hdr),
682 mask->pattern, spec->length);
687 flowman_cmd(struct enic_flowman *fm, uint64_t *args, int nargs)
689 return vnic_dev_flowman_cmd(fm->owner_enic->vdev, args, nargs);
693 enic_fet_alloc(struct enic_flowman *fm, uint8_t ingress,
694 struct fm_key_template *key, int entries,
695 struct enic_fm_fet **fet_out)
697 struct fm_exact_match_table *cmd;
698 struct fm_header_set *hdr;
699 struct enic_fm_fet *fet;
703 ENICPMD_FUNC_TRACE();
704 fet = calloc(1, sizeof(struct enic_fm_fet));
707 cmd = &fm->cmd.va->fm_exact_match_table;
708 memset(cmd, 0, sizeof(*cmd));
709 cmd->fet_direction = ingress ? FM_INGRESS : FM_EGRESS;
710 cmd->fet_stage = FM_STAGE_LAST;
711 cmd->fet_max_entries = entries ? entries : FM_MAX_EXACT_TABLE_SIZE;
713 hdr = &cmd->fet_key.fk_hdrset[0];
714 memset(hdr, 0, sizeof(*hdr));
715 hdr->fk_header_select = FKH_IPV4 | FKH_UDP;
716 hdr->l3.ip4.fk_saddr = 0xFFFFFFFF;
717 hdr->l3.ip4.fk_daddr = 0xFFFFFFFF;
718 hdr->l4.udp.fk_source = 0xFFFF;
719 hdr->l4.udp.fk_dest = 0xFFFF;
720 fet->default_key = 1;
722 memcpy(&cmd->fet_key, key, sizeof(*key));
723 memcpy(&fet->key, key, sizeof(*key));
724 fet->default_key = 0;
726 cmd->fet_key.fk_packet_tag = 1;
728 args[0] = FM_EXACT_TABLE_ALLOC;
729 args[1] = fm->cmd.pa;
730 ret = flowman_cmd(fm, args, 2);
732 ENICPMD_LOG(ERR, "cannot alloc exact match table: rc=%d", ret);
736 fet->handle = args[0];
737 fet->ingress = ingress;
738 ENICPMD_LOG(DEBUG, "allocated exact match table: handle=0x%" PRIx64,
745 enic_fet_free(struct enic_flowman *fm, struct enic_fm_fet *fet)
747 ENICPMD_FUNC_TRACE();
748 enic_fm_tbl_free(fm, fet->handle);
749 if (!fet->default_key)
750 TAILQ_REMOVE(&fm->fet_list, fet, list);
755 * Get the exact match table for the given combination of
756 * <group, ingress, key>. Allocate one on the fly as necessary.
759 enic_fet_get(struct enic_flowman *fm,
762 struct fm_key_template *key,
763 struct enic_fm_fet **fet_out,
764 struct rte_flow_error *error)
766 struct enic_fm_fet *fet;
768 ENICPMD_FUNC_TRACE();
769 /* See if we already have this table open */
770 TAILQ_FOREACH(fet, &fm->fet_list, list) {
771 if (fet->group == group && fet->ingress == ingress)
775 /* Jumping to a non-existing group? Use the default table */
777 fet = ingress ? fm->default_ig_fet : fm->default_eg_fet;
778 } else if (enic_fet_alloc(fm, ingress, key, 0, &fet)) {
779 return rte_flow_error_set(error, EINVAL,
780 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
781 NULL, "enic: cannot get exact match table");
784 /* Default table is never on the open table list */
785 if (!fet->default_key)
786 TAILQ_INSERT_HEAD(&fm->fet_list, fet, list);
790 ENICPMD_LOG(DEBUG, "fet_get: %s %s group=%u ref=%u",
791 fet->default_key ? "default" : "",
792 fet->ingress ? "ingress" : "egress",
793 fet->group, fet->ref);
798 enic_fet_put(struct enic_flowman *fm, struct enic_fm_fet *fet)
800 ENICPMD_FUNC_TRACE();
801 RTE_ASSERT(fet->ref > 0);
803 ENICPMD_LOG(DEBUG, "fet_put: %s %s group=%u ref=%u",
804 fet->default_key ? "default" : "",
805 fet->ingress ? "ingress" : "egress",
806 fet->group, fet->ref);
808 enic_fet_free(fm, fet);
811 /* Return 1 if current item is valid on top of the previous one. */
813 fm_item_stacking_valid(enum rte_flow_item_type prev_item,
814 const struct enic_fm_items *item_info,
815 uint8_t is_first_item)
817 enum rte_flow_item_type const *allowed_items = item_info->prev_items;
819 ENICPMD_FUNC_TRACE();
820 for (; *allowed_items != RTE_FLOW_ITEM_TYPE_END; allowed_items++) {
821 if (prev_item == *allowed_items)
825 /* This is the first item in the stack. Check if that's cool */
826 if (is_first_item && item_info->valid_start_item)
832 * Build the flow manager match entry structure from the provided pattern.
833 * The pattern is validated as the items are copied.
836 enic_fm_copy_entry(struct enic_flowman *fm,
837 const struct rte_flow_item pattern[],
838 struct rte_flow_error *error)
840 const struct enic_fm_items *item_info;
841 enum rte_flow_item_type prev_item;
842 const struct rte_flow_item *item;
843 struct copy_item_args args;
844 uint8_t prev_header_level;
845 uint8_t is_first_item;
848 ENICPMD_FUNC_TRACE();
851 prev_item = RTE_FLOW_ITEM_TYPE_END;
853 args.fm_tcam_entry = &fm->tcam_entry;
854 args.header_level = 0;
855 prev_header_level = 0;
856 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
858 * Get info about how to validate and copy the item. If NULL
859 * is returned the nic does not support the item.
861 if (item->type == RTE_FLOW_ITEM_TYPE_VOID)
864 item_info = &enic_fm_items[item->type];
866 if (item->type > FM_MAX_ITEM_TYPE ||
867 item_info->copy_item == NULL) {
868 return rte_flow_error_set(error, ENOTSUP,
869 RTE_FLOW_ERROR_TYPE_ITEM,
870 NULL, "enic: unsupported item");
873 /* check to see if item stacking is valid */
874 if (!fm_item_stacking_valid(prev_item, item_info,
879 ret = item_info->copy_item(&args);
881 goto item_not_supported;
882 /* Going from outer to inner? Treat it as a new packet start */
883 if (prev_header_level != args.header_level) {
884 prev_item = RTE_FLOW_ITEM_TYPE_END;
887 prev_item = item->type;
890 prev_header_level = args.header_level;
895 return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,
896 NULL, "enic: unsupported item type");
899 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
900 item, "enic: unsupported item stack");
904 flow_item_skip_void(const struct rte_flow_item **item)
907 if ((*item)->type != RTE_FLOW_ITEM_TYPE_VOID)
912 append_template(void **template, uint8_t *off, const void *data, int len)
914 memcpy(*template, data, len);
915 *template = (char *)*template + len;
920 enic_fm_append_action_op(struct enic_flowman *fm,
921 struct fm_action_op *fm_op,
922 struct rte_flow_error *error)
926 count = fm->action_op_count;
927 ENICPMD_LOG(DEBUG, "append action op: idx=%d op=%u",
928 count, fm_op->fa_op);
929 if (count == FM_ACTION_OP_MAX) {
930 return rte_flow_error_set(error, EINVAL,
931 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
932 "too many action operations");
934 fm->action.fma_action_ops[count] = *fm_op;
935 fm->action_op_count = count + 1;
939 static struct fm_action_op *
940 find_prev_action_op(struct enic_flowman *fm, uint32_t opcode)
942 struct fm_action_op *op;
945 for (i = 0; i < fm->action_op_count; i++) {
946 op = &fm->action.fma_action_ops[i];
947 if (op->fa_op == opcode)
953 /* NIC requires that 1st steer appear before decap.
954 * Correct example: steer, decap, steer, steer, ...
957 enic_fm_reorder_action_op(struct enic_flowman *fm)
959 struct fm_action_op *op, *steer, *decap;
960 struct fm_action_op tmp_op;
962 ENICPMD_FUNC_TRACE();
963 /* Find 1st steer and decap */
964 op = fm->action.fma_action_ops;
967 while (op->fa_op != FMOP_END) {
968 if (!decap && (op->fa_op == FMOP_DECAP_NOSTRIP ||
969 op->fa_op == FMOP_DECAP_STRIP))
971 else if (!steer && op->fa_op == FMOP_RQ_STEER)
975 /* If decap is before steer, swap */
976 if (steer && decap && decap < steer) {
977 op = fm->action.fma_action_ops;
978 ENICPMD_LOG(DEBUG, "swap decap %ld <-> steer %ld",
979 (long)(decap - op), (long)(steer - op));
986 /* VXLAN decap is done via flowman compound action */
988 enic_fm_copy_vxlan_decap(struct enic_flowman *fm,
989 struct fm_tcam_match_entry *fmt,
990 const struct rte_flow_action *action,
991 struct rte_flow_error *error)
993 struct fm_header_set *fm_data;
994 struct fm_action_op fm_op;
996 ENICPMD_FUNC_TRACE();
997 fm_data = &fmt->ftm_data.fk_hdrset[0];
998 if (!(fm_data->fk_metadata & FKM_VXLAN)) {
999 return rte_flow_error_set(error, EINVAL,
1000 RTE_FLOW_ERROR_TYPE_ACTION, action,
1001 "vxlan-decap: vxlan must be in pattern");
1004 memset(&fm_op, 0, sizeof(fm_op));
1005 fm_op.fa_op = FMOP_DECAP_NOSTRIP;
1006 return enic_fm_append_action_op(fm, &fm_op, error);
1009 /* Generate a reasonable source port number */
1013 /* Min/max below are the default values in OVS-DPDK and Linux */
1014 uint16_t p = rte_rand();
1015 p = RTE_MAX(p, 32768);
1016 p = RTE_MIN(p, 61000);
1017 return rte_cpu_to_be_16(p);
1020 /* VXLAN encap is done via flowman compound action */
1022 enic_fm_copy_vxlan_encap(struct enic_flowman *fm,
1023 const struct rte_flow_item *item,
1024 struct rte_flow_error *error)
1026 struct fm_action_op fm_op;
1027 struct rte_ether_hdr *eth;
1028 struct rte_udp_hdr *udp;
1029 uint16_t *ethertype;
1033 ENICPMD_FUNC_TRACE();
1034 memset(&fm_op, 0, sizeof(fm_op));
1035 fm_op.fa_op = FMOP_ENCAP;
1036 template = fm->action.fma_data;
1039 * Copy flow items to the flowman template starting L2.
1040 * L2 must be ethernet.
1042 flow_item_skip_void(&item);
1043 if (item->type != RTE_FLOW_ITEM_TYPE_ETH)
1044 return rte_flow_error_set(error, EINVAL,
1045 RTE_FLOW_ERROR_TYPE_ITEM, item,
1046 "vxlan-encap: first item should be ethernet");
1047 eth = (struct rte_ether_hdr *)template;
1048 ethertype = ð->ether_type;
1049 append_template(&template, &off, item->spec,
1050 sizeof(struct rte_ether_hdr));
1052 flow_item_skip_void(&item);
1054 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
1055 const struct rte_flow_item_vlan *spec;
1057 ENICPMD_LOG(DEBUG, "vxlan-encap: vlan");
1059 fm_op.encap.outer_vlan = rte_be_to_cpu_16(spec->tci);
1061 flow_item_skip_void(&item);
1063 /* L3 must be IPv4, IPv6 */
1064 switch (item->type) {
1065 case RTE_FLOW_ITEM_TYPE_IPV4:
1067 struct rte_ipv4_hdr *ip4;
1069 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv4");
1070 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
1071 ip4 = (struct rte_ipv4_hdr *)template;
1073 * Offset of IPv4 length field and its initial value
1074 * (IP + UDP + VXLAN) are specified in the action. The NIC
1075 * will add inner packet length.
1077 fm_op.encap.len1_offset = off +
1078 offsetof(struct rte_ipv4_hdr, total_length);
1079 fm_op.encap.len1_delta = sizeof(struct rte_ipv4_hdr) +
1080 sizeof(struct rte_udp_hdr) +
1081 sizeof(struct rte_vxlan_hdr);
1082 append_template(&template, &off, item->spec,
1083 sizeof(struct rte_ipv4_hdr));
1084 ip4->version_ihl = RTE_IPV4_VHL_DEF;
1085 if (ip4->time_to_live == 0)
1086 ip4->time_to_live = IP_DEFTTL;
1087 ip4->next_proto_id = IPPROTO_UDP;
1090 case RTE_FLOW_ITEM_TYPE_IPV6:
1092 struct rte_ipv6_hdr *ip6;
1094 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv6");
1095 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
1096 ip6 = (struct rte_ipv6_hdr *)template;
1097 fm_op.encap.len1_offset = off +
1098 offsetof(struct rte_ipv6_hdr, payload_len);
1099 fm_op.encap.len1_delta = sizeof(struct rte_udp_hdr) +
1100 sizeof(struct rte_vxlan_hdr);
1101 append_template(&template, &off, item->spec,
1102 sizeof(struct rte_ipv6_hdr));
1103 ip6->vtc_flow |= rte_cpu_to_be_32(IP6_VTC_FLOW);
1104 if (ip6->hop_limits == 0)
1105 ip6->hop_limits = IP_DEFTTL;
1106 ip6->proto = IPPROTO_UDP;
1110 return rte_flow_error_set(error,
1111 EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1112 "vxlan-encap: L3 must be IPv4/IPv6");
1115 flow_item_skip_void(&item);
1118 if (item->type != RTE_FLOW_ITEM_TYPE_UDP)
1119 return rte_flow_error_set(error, EINVAL,
1120 RTE_FLOW_ERROR_TYPE_ITEM, item,
1121 "vxlan-encap: UDP must follow IPv4/IPv6");
1122 /* UDP length = UDP + VXLAN. NIC will add inner packet length. */
1123 fm_op.encap.len2_offset =
1124 off + offsetof(struct rte_udp_hdr, dgram_len);
1125 fm_op.encap.len2_delta =
1126 sizeof(struct rte_udp_hdr) + sizeof(struct rte_vxlan_hdr);
1127 udp = (struct rte_udp_hdr *)template;
1128 append_template(&template, &off, item->spec,
1129 sizeof(struct rte_udp_hdr));
1131 * Firmware does not hash/fill source port yet. Generate a
1132 * random port, as there is *usually* one rte_flow for the
1133 * given inner packet stream (i.e. a single stream has one
1136 if (udp->src_port == 0)
1137 udp->src_port = gen_src_port();
1139 flow_item_skip_void(&item);
1142 if (item->type != RTE_FLOW_ITEM_TYPE_VXLAN)
1143 return rte_flow_error_set(error,
1144 EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1145 "vxlan-encap: VXLAN must follow UDP");
1146 append_template(&template, &off, item->spec,
1147 sizeof(struct rte_flow_item_vxlan));
1150 * Fill in the rest of the action structure.
1151 * Indicate that we want to encap with vxlan at packet start.
1153 fm_op.encap.template_offset = 0;
1154 fm_op.encap.template_len = off;
1155 return enic_fm_append_action_op(fm, &fm_op, error);
1159 enic_fm_find_vnic(struct enic *enic, const struct rte_pci_addr *addr,
1166 ENICPMD_FUNC_TRACE();
1167 ENICPMD_LOG(DEBUG, "bdf=%x:%x:%x", addr->bus, addr->devid,
1169 bdf = addr->bus << 8 | addr->devid << 3 | addr->function;
1170 args[0] = FM_VNIC_FIND;
1172 rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
1174 /* Expected to fail if BDF is not on the adapter */
1175 ENICPMD_LOG(DEBUG, "cannot find vnic handle: rc=%d", rc);
1179 ENICPMD_LOG(DEBUG, "found vnic: handle=0x%" PRIx64, *handle);
1184 * Egress: target port should be either PF uplink or VF.
1186 * 1. VF egress -> PF uplink
1187 * PF may be this VF's PF, or another PF, as long as they are on the same VIC.
1188 * 2. VF egress -> VF
1191 * 1. PF egress -> VF
1192 * App should be using representor to pass packets to VF
1195 vf_egress_port_id_action(struct enic_flowman *fm,
1196 struct rte_eth_dev *dst_dev,
1197 uint64_t dst_vnic_h,
1198 struct fm_action_op *fm_op,
1199 struct rte_flow_error *error)
1201 struct enic *src_enic, *dst_enic;
1202 struct enic_vf_representor *vf;
1206 ENICPMD_FUNC_TRACE();
1207 src_enic = fm->user_enic;
1208 dst_enic = pmd_priv(dst_dev);
1209 if (!(src_enic->rte_dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)) {
1210 return rte_flow_error_set(error, EINVAL,
1211 RTE_FLOW_ERROR_TYPE_ACTION,
1212 NULL, "source port is not VF representor");
1215 /* VF -> PF uplink. dst is not VF representor */
1216 if (!(dst_dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)) {
1217 /* PF is the VF's PF? Then nothing to do */
1218 vf = VF_ENIC_TO_VF_REP(src_enic);
1219 if (vf->pf == dst_enic) {
1220 ENICPMD_LOG(DEBUG, "destination port is VF's PF");
1223 /* If not, steer to the remote PF's uplink */
1224 uif = dst_enic->fm_vnic_uif;
1225 ENICPMD_LOG(DEBUG, "steer to uplink %u", uif);
1226 memset(fm_op, 0, sizeof(*fm_op));
1227 fm_op->fa_op = FMOP_SET_EGPORT;
1228 fm_op->set_egport.egport = uif;
1229 ret = enic_fm_append_action_op(fm, fm_op, error);
1233 /* VF -> VF loopback. Hairpin and steer to vnic */
1234 memset(fm_op, 0, sizeof(*fm_op));
1235 fm_op->fa_op = FMOP_EG_HAIRPIN;
1236 ret = enic_fm_append_action_op(fm, fm_op, error);
1239 ENICPMD_LOG(DEBUG, "egress hairpin");
1240 fm->hairpin_steer_vnic_h = dst_vnic_h;
1241 fm->need_hairpin_steer = 1;
1246 enic_fm_check_transfer_dst(struct enic *enic, uint16_t dst_port_id,
1247 struct rte_eth_dev **dst_dev,
1248 struct rte_flow_error *error)
1250 struct rte_eth_dev *dev;
1252 ENICPMD_LOG(DEBUG, "port id %u", dst_port_id);
1253 if (!rte_eth_dev_is_valid_port(dst_port_id)) {
1254 return rte_flow_error_set(error, EINVAL,
1255 RTE_FLOW_ERROR_TYPE_ACTION,
1256 NULL, "invalid port_id");
1258 dev = &rte_eth_devices[dst_port_id];
1259 if (!dev_is_enic(dev)) {
1260 return rte_flow_error_set(error, EINVAL,
1261 RTE_FLOW_ERROR_TYPE_ACTION,
1262 NULL, "port_id is not enic");
1264 if (enic->switch_domain_id != pmd_priv(dev)->switch_domain_id) {
1265 return rte_flow_error_set(error, EINVAL,
1266 RTE_FLOW_ERROR_TYPE_ACTION,
1267 NULL, "destination and source ports are not in the same switch domain");
1274 /* Translate flow actions to flowman TCAM entry actions */
1276 enic_fm_copy_action(struct enic_flowman *fm,
1277 const struct rte_flow_action actions[],
1279 struct rte_flow_error *error)
1290 struct fm_tcam_match_entry *fmt;
1291 struct fm_action_op fm_op;
1292 bool need_ovlan_action;
1301 ENICPMD_FUNC_TRACE();
1302 fmt = &fm->tcam_entry;
1303 need_ovlan_action = false;
1307 enic = fm->user_enic;
1309 vnic_h = enic->fm_vnic_handle;
1311 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1312 switch (actions->type) {
1313 case RTE_FLOW_ACTION_TYPE_VOID:
1315 case RTE_FLOW_ACTION_TYPE_PASSTHRU: {
1316 if (overlap & PASSTHRU)
1318 overlap |= PASSTHRU;
1321 case RTE_FLOW_ACTION_TYPE_JUMP: {
1322 const struct rte_flow_action_jump *jump =
1324 struct enic_fm_fet *fet;
1328 ret = enic_fet_get(fm, jump->group, ingress, NULL,
1333 memset(&fm_op, 0, sizeof(fm_op));
1334 fm_op.fa_op = FMOP_EXACT_MATCH;
1335 fm_op.exact.handle = fet->handle;
1337 ret = enic_fm_append_action_op(fm, &fm_op, error);
1342 case RTE_FLOW_ACTION_TYPE_MARK: {
1343 const struct rte_flow_action_mark *mark =
1346 if (enic->use_noscatter_vec_rx_handler)
1348 if (mark->id >= ENIC_MAGIC_FILTER_ID - 1)
1349 return rte_flow_error_set(error, EINVAL,
1350 RTE_FLOW_ERROR_TYPE_ACTION,
1351 NULL, "invalid mark id");
1352 memset(&fm_op, 0, sizeof(fm_op));
1353 fm_op.fa_op = FMOP_MARK;
1354 fm_op.mark.mark = mark->id + 1;
1355 ret = enic_fm_append_action_op(fm, &fm_op, error);
1360 case RTE_FLOW_ACTION_TYPE_FLAG: {
1361 if (enic->use_noscatter_vec_rx_handler)
1363 /* ENIC_MAGIC_FILTER_ID is reserved for flagging */
1364 memset(&fm_op, 0, sizeof(fm_op));
1365 fm_op.fa_op = FMOP_MARK;
1366 fm_op.mark.mark = ENIC_MAGIC_FILTER_ID;
1367 ret = enic_fm_append_action_op(fm, &fm_op, error);
1372 case RTE_FLOW_ACTION_TYPE_QUEUE: {
1373 const struct rte_flow_action_queue *queue =
1377 * If fate other than QUEUE or RSS, fail. Multiple
1378 * rss and queue actions are ok.
1380 if ((overlap & FATE) && first_rq)
1384 memset(&fm_op, 0, sizeof(fm_op));
1385 fm_op.fa_op = FMOP_RQ_STEER;
1386 fm_op.rq_steer.rq_index =
1387 enic_rte_rq_idx_to_sop_idx(queue->index);
1388 fm_op.rq_steer.rq_count = 1;
1389 fm_op.rq_steer.vnic_handle = vnic_h;
1390 ret = enic_fm_append_action_op(fm, &fm_op, error);
1393 ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1394 fm_op.rq_steer.rq_index);
1398 case RTE_FLOW_ACTION_TYPE_DROP: {
1402 memset(&fm_op, 0, sizeof(fm_op));
1403 fm_op.fa_op = FMOP_DROP;
1404 ret = enic_fm_append_action_op(fm, &fm_op, error);
1407 ENICPMD_LOG(DEBUG, "create DROP action");
1410 case RTE_FLOW_ACTION_TYPE_COUNT: {
1411 if (overlap & COUNT)
1414 /* Count is associated with entry not action on VIC. */
1415 fmt->ftm_flags |= FMEF_COUNTER;
1418 case RTE_FLOW_ACTION_TYPE_RSS: {
1419 const struct rte_flow_action_rss *rss = actions->conf;
1424 * If fate other than QUEUE or RSS, fail. Multiple
1425 * rss and queue actions are ok.
1427 if ((overlap & FATE) && first_rq)
1433 * Hardware only supports RSS actions on outer level
1434 * with default type and function. Queues must be
1437 allow = rss->func == RTE_ETH_HASH_FUNCTION_DEFAULT &&
1438 rss->level == 0 && (rss->types == 0 ||
1439 rss->types == enic->rss_hf) &&
1440 rss->queue_num <= enic->rq_count &&
1441 rss->queue[rss->queue_num - 1] < enic->rq_count;
1444 /* Identity queue map needs to be sequential */
1445 for (i = 1; i < rss->queue_num; i++)
1446 allow = allow && (rss->queue[i] ==
1447 rss->queue[i - 1] + 1);
1451 memset(&fm_op, 0, sizeof(fm_op));
1452 fm_op.fa_op = FMOP_RQ_STEER;
1453 fm_op.rq_steer.rq_index =
1454 enic_rte_rq_idx_to_sop_idx(rss->queue[0]);
1455 fm_op.rq_steer.rq_count = rss->queue_num;
1456 fm_op.rq_steer.vnic_handle = vnic_h;
1457 ret = enic_fm_append_action_op(fm, &fm_op, error);
1460 ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1461 fm_op.rq_steer.rq_index);
1465 case RTE_FLOW_ACTION_TYPE_PORT_ID: {
1466 const struct rte_flow_action_port_id *port;
1467 struct rte_eth_dev *dev = NULL;
1469 if (!ingress && (overlap & PORT_ID)) {
1470 ENICPMD_LOG(DEBUG, "cannot have multiple egress PORT_ID actions");
1473 port = actions->conf;
1474 if (port->original) {
1475 vnic_h = enic->fm_vnic_handle; /* This port */
1478 ret = enic_fm_check_transfer_dst(enic, port->id, &dev,
1482 vnic_h = pmd_priv(dev)->fm_vnic_handle;
1485 * Ingress. Nothing more to do. We add an implicit
1486 * steer at the end if needed.
1491 ret = vf_egress_port_id_action(fm, dev, vnic_h, &fm_op,
1497 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: {
1498 if (overlap & DECAP)
1502 ret = enic_fm_copy_vxlan_decap(fm, fmt, actions,
1508 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: {
1509 const struct rte_flow_action_vxlan_encap *encap;
1511 encap = actions->conf;
1512 if (overlap & ENCAP)
1515 ret = enic_fm_copy_vxlan_encap(fm, encap->definition,
1521 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: {
1522 struct fm_action_op *decap;
1525 * If decap-nostrip appears before pop vlan, this pop
1526 * applies to the inner packet vlan. Turn it into
1529 decap = find_prev_action_op(fm, FMOP_DECAP_NOSTRIP);
1531 ENICPMD_LOG(DEBUG, "pop-vlan inner: decap-nostrip => decap-strip");
1532 decap->fa_op = FMOP_DECAP_STRIP;
1535 memset(&fm_op, 0, sizeof(fm_op));
1536 fm_op.fa_op = FMOP_POP_VLAN;
1537 ret = enic_fm_append_action_op(fm, &fm_op, error);
1542 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: {
1543 const struct rte_flow_action_of_push_vlan *vlan;
1545 if (overlap & PASSTHRU)
1547 vlan = actions->conf;
1548 if (vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN)) {
1549 return rte_flow_error_set(error, EINVAL,
1550 RTE_FLOW_ERROR_TYPE_ACTION,
1551 NULL, "unexpected push_vlan ethertype");
1553 overlap |= PUSH_VLAN;
1554 need_ovlan_action = true;
1557 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: {
1558 const struct rte_flow_action_of_set_vlan_pcp *pcp;
1560 pcp = actions->conf;
1561 if (pcp->vlan_pcp > 7) {
1562 return rte_flow_error_set(error, EINVAL,
1563 RTE_FLOW_ERROR_TYPE_ACTION,
1564 NULL, "invalid vlan_pcp");
1566 need_ovlan_action = true;
1567 ovlan |= ((uint16_t)pcp->vlan_pcp) << 13;
1570 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: {
1571 const struct rte_flow_action_of_set_vlan_vid *vid;
1573 vid = actions->conf;
1574 need_ovlan_action = true;
1575 ovlan |= rte_be_to_cpu_16(vid->vlan_vid);
1578 case RTE_FLOW_ACTION_TYPE_PORT_REPRESENTOR: {
1579 const struct rte_flow_action_ethdev *ethdev;
1580 struct rte_eth_dev *dev = NULL;
1582 ethdev = actions->conf;
1583 ret = enic_fm_check_transfer_dst(enic, ethdev->port_id,
1587 vnic_h = pmd_priv(dev)->fm_vnic_handle;
1590 * Action PORT_REPRESENTOR implies ingress destination.
1591 * Noting to do. We add an implicit stree at the
1597 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: {
1598 const struct rte_flow_action_ethdev *ethdev;
1599 struct rte_eth_dev *dev = NULL;
1601 if (overlap & PORT_ID) {
1602 ENICPMD_LOG(DEBUG, "cannot have multiple egress PORT_ID actions");
1605 ethdev = actions->conf;
1606 ret = enic_fm_check_transfer_dst(enic, ethdev->port_id,
1610 vnic_h = pmd_priv(dev)->fm_vnic_handle;
1612 /* Action REPRESENTED_PORT: always egress destination */
1614 ret = vf_egress_port_id_action(fm, dev, vnic_h, &fm_op,
1625 if (!(overlap & (FATE | PASSTHRU | COUNT | PORT_ID)))
1627 /* Egress from VF: need implicit WQ match */
1628 if (enic_is_vf_rep(enic) && !ingress) {
1629 fmt->ftm_data.fk_wq_id = 0;
1630 fmt->ftm_mask.fk_wq_id = 0xffff;
1631 fmt->ftm_data.fk_wq_vnic = enic->fm_vnic_handle;
1632 ENICPMD_LOG(DEBUG, "add implicit wq id match for vf %d",
1633 VF_ENIC_TO_VF_REP(enic)->vf_id);
1635 if (need_ovlan_action) {
1636 memset(&fm_op, 0, sizeof(fm_op));
1637 fm_op.fa_op = FMOP_SET_OVLAN;
1638 fm_op.ovlan.vlan = ovlan;
1639 ret = enic_fm_append_action_op(fm, &fm_op, error);
1643 /* Add steer op for PORT_ID without QUEUE */
1644 if ((overlap & PORT_ID) && !steer && ingress) {
1645 memset(&fm_op, 0, sizeof(fm_op));
1646 /* Always to queue 0 for now as generic RSS is not available */
1647 fm_op.fa_op = FMOP_RQ_STEER;
1648 fm_op.rq_steer.rq_index = 0;
1649 fm_op.rq_steer.vnic_handle = vnic_h;
1650 ret = enic_fm_append_action_op(fm, &fm_op, error);
1653 ENICPMD_LOG(DEBUG, "add implicit steer op");
1655 /* Add required END */
1656 memset(&fm_op, 0, sizeof(fm_op));
1657 fm_op.fa_op = FMOP_END;
1658 ret = enic_fm_append_action_op(fm, &fm_op, error);
1661 enic_fm_reorder_action_op(fm);
1665 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
1666 NULL, "enic: unsupported action");
1669 /** Check if the action is supported */
1671 enic_fm_match_action(const struct rte_flow_action *action,
1672 const enum rte_flow_action_type *supported_actions)
1674 for (; *supported_actions != RTE_FLOW_ACTION_TYPE_END;
1675 supported_actions++) {
1676 if (action->type == *supported_actions)
1682 /* Debug function to dump internal NIC action structure. */
1684 enic_fm_dump_tcam_actions(const struct fm_action *fm_action)
1686 /* Manually keep in sync with FMOP commands */
1687 const char *fmop_str[FMOP_OP_MAX] = {
1689 [FMOP_DROP] = "drop",
1690 [FMOP_RQ_STEER] = "steer",
1691 [FMOP_EXACT_MATCH] = "exmatch",
1692 [FMOP_MARK] = "mark",
1693 [FMOP_EXT_MARK] = "ext_mark",
1695 [FMOP_EG_HAIRPIN] = "eg_hairpin",
1696 [FMOP_IG_HAIRPIN] = "ig_hairpin",
1697 [FMOP_ENCAP_IVLAN] = "encap_ivlan",
1698 [FMOP_ENCAP_NOIVLAN] = "encap_noivlan",
1699 [FMOP_ENCAP] = "encap",
1700 [FMOP_SET_OVLAN] = "set_ovlan",
1701 [FMOP_DECAP_NOSTRIP] = "decap_nostrip",
1702 [FMOP_DECAP_STRIP] = "decap_strip",
1703 [FMOP_POP_VLAN] = "pop_vlan",
1704 [FMOP_SET_EGPORT] = "set_egport",
1705 [FMOP_RQ_STEER_ONLY] = "rq_steer_only",
1706 [FMOP_SET_ENCAP_VLAN] = "set_encap_vlan",
1707 [FMOP_EMIT] = "emit",
1708 [FMOP_MODIFY] = "modify",
1710 const struct fm_action_op *op = &fm_action->fma_action_ops[0];
1711 char buf[128], *bp = buf;
1716 buf_len = sizeof(buf);
1717 for (i = 0; i < FM_ACTION_OP_MAX; i++) {
1718 if (op->fa_op == FMOP_END)
1720 if (op->fa_op >= FMOP_OP_MAX)
1723 op_str = fmop_str[op->fa_op];
1724 n = snprintf(bp, buf_len, "%s,", op_str);
1725 if (n > 0 && n < buf_len) {
1731 /* Remove trailing comma */
1734 ENICPMD_LOG(DEBUG, " Acions: %s", buf);
1738 bits_to_str(uint32_t bits, const char *strings[], int max,
1739 char *buf, int buf_len)
1741 int i, n = 0, len = 0;
1743 for (i = 0; i < max; i++) {
1744 if (bits & (1 << i)) {
1745 n = snprintf(buf, buf_len, "%s,", strings[i]);
1746 if (n > 0 && n < buf_len) {
1753 /* Remove trailing comma */
1761 /* Debug function to dump internal NIC filter structure. */
1763 __enic_fm_dump_tcam_match(const struct fm_header_set *fk_hdrset, char *buf,
1766 /* Manually keep in sync with FKM_BITS */
1767 const char *fm_fkm_str[FKM_BIT_COUNT] = {
1768 [FKM_QTAG_BIT] = "qtag",
1769 [FKM_CMD_BIT] = "cmd",
1770 [FKM_IPV4_BIT] = "ip4",
1771 [FKM_IPV6_BIT] = "ip6",
1772 [FKM_ROCE_BIT] = "roce",
1773 [FKM_UDP_BIT] = "udp",
1774 [FKM_TCP_BIT] = "tcp",
1775 [FKM_TCPORUDP_BIT] = "tcpportudp",
1776 [FKM_IPFRAG_BIT] = "ipfrag",
1777 [FKM_NVGRE_BIT] = "nvgre",
1778 [FKM_VXLAN_BIT] = "vxlan",
1779 [FKM_GENEVE_BIT] = "geneve",
1780 [FKM_NSH_BIT] = "nsh",
1781 [FKM_ROCEV2_BIT] = "rocev2",
1782 [FKM_VLAN_PRES_BIT] = "vlan_pres",
1783 [FKM_IPOK_BIT] = "ipok",
1784 [FKM_L4OK_BIT] = "l4ok",
1785 [FKM_ROCEOK_BIT] = "roceok",
1786 [FKM_FCSOK_BIT] = "fcsok",
1787 [FKM_EG_SPAN_BIT] = "eg_span",
1788 [FKM_IG_SPAN_BIT] = "ig_span",
1789 [FKM_EG_HAIRPINNED_BIT] = "eg_hairpinned",
1791 /* Manually keep in sync with FKH_BITS */
1792 const char *fm_fkh_str[FKH_BIT_COUNT] = {
1793 [FKH_ETHER_BIT] = "eth",
1794 [FKH_QTAG_BIT] = "qtag",
1795 [FKH_L2RAW_BIT] = "l2raw",
1796 [FKH_IPV4_BIT] = "ip4",
1797 [FKH_IPV6_BIT] = "ip6",
1798 [FKH_L3RAW_BIT] = "l3raw",
1799 [FKH_UDP_BIT] = "udp",
1800 [FKH_TCP_BIT] = "tcp",
1801 [FKH_ICMP_BIT] = "icmp",
1802 [FKH_VXLAN_BIT] = "vxlan",
1803 [FKH_L4RAW_BIT] = "l4raw",
1805 uint32_t fkh_bits = fk_hdrset->fk_header_select;
1806 uint32_t fkm_bits = fk_hdrset->fk_metadata;
1809 if (!fkm_bits && !fkh_bits)
1811 n = snprintf(buf, buf_len, "metadata(");
1812 if (n > 0 && n < buf_len) {
1816 n = bits_to_str(fkm_bits, fm_fkm_str, FKM_BIT_COUNT, buf, buf_len);
1817 if (n > 0 && n < buf_len) {
1821 n = snprintf(buf, buf_len, ") valid hdr fields(");
1822 if (n > 0 && n < buf_len) {
1826 n = bits_to_str(fkh_bits, fm_fkh_str, FKH_BIT_COUNT, buf, buf_len);
1827 if (n > 0 && n < buf_len) {
1831 snprintf(buf, buf_len, ")");
1835 enic_fm_dump_tcam_match(const struct fm_tcam_match_entry *match,
1840 memset(buf, 0, sizeof(buf));
1841 __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[0],
1843 ENICPMD_LOG(DEBUG, " TCAM %s Outer: %s %scounter position %u",
1844 (ingress) ? "IG" : "EG", buf,
1845 (match->ftm_flags & FMEF_COUNTER) ? "" : "no ",
1846 match->ftm_position);
1847 memset(buf, 0, sizeof(buf));
1848 __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[1],
1851 ENICPMD_LOG(DEBUG, " Inner: %s", buf);
1854 /* Debug function to dump internal NIC flow structures. */
1856 enic_fm_dump_tcam_entry(const struct fm_tcam_match_entry *fm_match,
1857 const struct fm_action *fm_action,
1860 if (!rte_log_can_log(enic_pmd_logtype, RTE_LOG_DEBUG))
1862 enic_fm_dump_tcam_match(fm_match, ingress);
1863 enic_fm_dump_tcam_actions(fm_action);
1867 enic_fm_flow_parse(struct enic_flowman *fm,
1868 const struct rte_flow_attr *attrs,
1869 const struct rte_flow_item pattern[],
1870 const struct rte_flow_action actions[],
1871 struct rte_flow_error *error)
1873 const struct rte_flow_action *action;
1875 static const enum rte_flow_action_type *sa;
1877 ENICPMD_FUNC_TRACE();
1880 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM,
1881 NULL, "no pattern specified");
1886 rte_flow_error_set(error, EINVAL,
1887 RTE_FLOW_ERROR_TYPE_ACTION_NUM,
1888 NULL, "no action specified");
1893 if (attrs->group != FM_TCAM_RTE_GROUP && attrs->priority) {
1894 rte_flow_error_set(error, ENOTSUP,
1895 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1897 "priorities are not supported for non-default (0) groups");
1899 } else if (!fm->owner_enic->switchdev_mode && attrs->transfer) {
1900 rte_flow_error_set(error, ENOTSUP,
1901 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1903 "transfer is not supported");
1905 } else if (attrs->ingress && attrs->egress) {
1906 rte_flow_error_set(error, ENOTSUP,
1907 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1909 "bidirectional rules not supported");
1914 rte_flow_error_set(error, EINVAL,
1915 RTE_FLOW_ERROR_TYPE_ATTR,
1916 NULL, "no attribute specified");
1920 /* Verify Actions. */
1921 sa = (attrs->ingress) ? enic_fm_supported_ig_actions :
1922 enic_fm_supported_eg_actions;
1923 for (action = &actions[0]; action->type != RTE_FLOW_ACTION_TYPE_END;
1925 if (action->type == RTE_FLOW_ACTION_TYPE_VOID)
1927 else if (!enic_fm_match_action(action, sa))
1930 if (action->type != RTE_FLOW_ACTION_TYPE_END) {
1931 rte_flow_error_set(error, EPERM, RTE_FLOW_ERROR_TYPE_ACTION,
1932 action, "invalid action");
1935 ret = enic_fm_copy_entry(fm, pattern, error);
1938 ret = enic_fm_copy_action(fm, actions, attrs->ingress, error);
1943 enic_fm_counter_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
1945 if (!fm_flow->counter_valid)
1947 SLIST_INSERT_HEAD(&fm->counters, fm_flow->counter, next);
1948 fm_flow->counter_valid = false;
1952 enic_fm_more_counters(struct enic_flowman *fm)
1954 struct enic_fm_counter *new_stack;
1955 struct enic_fm_counter *ctrs;
1959 ENICPMD_FUNC_TRACE();
1960 new_stack = rte_realloc(fm->counter_stack, (fm->counters_alloced +
1961 FM_COUNTERS_EXPAND) *
1962 sizeof(struct enic_fm_counter), 0);
1963 if (new_stack == NULL) {
1964 ENICPMD_LOG(ERR, "cannot alloc counter memory");
1967 fm->counter_stack = new_stack;
1969 args[0] = FM_COUNTER_BRK;
1970 args[1] = fm->counters_alloced + FM_COUNTERS_EXPAND;
1971 rc = flowman_cmd(fm, args, 2);
1973 ENICPMD_LOG(ERR, "cannot alloc counters rc=%d", rc);
1976 ctrs = (struct enic_fm_counter *)fm->counter_stack +
1977 fm->counters_alloced;
1978 for (i = 0; i < FM_COUNTERS_EXPAND; i++, ctrs++) {
1979 ctrs->handle = fm->counters_alloced + i;
1980 SLIST_INSERT_HEAD(&fm->counters, ctrs, next);
1982 fm->counters_alloced += FM_COUNTERS_EXPAND;
1983 ENICPMD_LOG(DEBUG, "%u counters allocated, total: %u",
1984 FM_COUNTERS_EXPAND, fm->counters_alloced);
1989 enic_fm_counter_zero(struct enic_flowman *fm, struct enic_fm_counter *c)
1994 ENICPMD_FUNC_TRACE();
1995 args[0] = FM_COUNTER_QUERY;
1996 args[1] = c->handle;
1997 args[2] = 1; /* clear */
1998 ret = flowman_cmd(fm, args, 3);
2000 ENICPMD_LOG(ERR, "counter init: rc=%d handle=0x%x",
2008 enic_fm_counter_alloc(struct enic_flowman *fm, struct rte_flow_error *error,
2009 struct enic_fm_counter **ctr)
2011 struct enic_fm_counter *c;
2014 ENICPMD_FUNC_TRACE();
2016 if (SLIST_EMPTY(&fm->counters)) {
2017 ret = enic_fm_more_counters(fm);
2019 return rte_flow_error_set(error, -ret,
2020 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2021 NULL, "enic: out of counters");
2023 c = SLIST_FIRST(&fm->counters);
2024 SLIST_REMOVE_HEAD(&fm->counters, next);
2030 enic_fm_action_free(struct enic_flowman *fm, struct enic_fm_action *ah)
2035 ENICPMD_FUNC_TRACE();
2036 RTE_ASSERT(ah->ref > 0);
2039 args[0] = FM_ACTION_FREE;
2040 args[1] = ah->handle;
2041 ret = flowman_cmd(fm, args, 2);
2043 /* This is a "should never happen" error. */
2044 ENICPMD_LOG(ERR, "freeing action rc=%d handle=0x%"
2045 PRIx64, ret, ah->handle);
2046 rte_hash_del_key(fm->action_hash, (const void *)&ah->key);
2053 enic_fm_entry_free(struct enic_flowman *fm, uint64_t handle)
2058 ENICPMD_FUNC_TRACE();
2059 args[0] = FM_MATCH_ENTRY_REMOVE;
2061 rc = flowman_cmd(fm, args, 2);
2063 ENICPMD_LOG(ERR, "cannot free match entry: rc=%d"
2064 " handle=0x%" PRIx64, rc, handle);
2068 static struct enic_fm_jump_flow *
2069 find_jump_flow(struct enic_flowman *fm, uint32_t group)
2071 struct enic_fm_jump_flow *j;
2073 ENICPMD_FUNC_TRACE();
2074 TAILQ_FOREACH(j, &fm->jump_list, list) {
2075 if (j->group == group)
2082 remove_jump_flow(struct enic_flowman *fm, struct rte_flow *flow)
2084 struct enic_fm_jump_flow *j;
2086 ENICPMD_FUNC_TRACE();
2087 TAILQ_FOREACH(j, &fm->jump_list, list) {
2088 if (j->flow == flow) {
2089 TAILQ_REMOVE(&fm->jump_list, j, list);
2097 save_jump_flow(struct enic_flowman *fm,
2098 struct rte_flow *flow,
2100 struct fm_tcam_match_entry *match,
2101 struct fm_action *action)
2103 struct enic_fm_jump_flow *j;
2105 ENICPMD_FUNC_TRACE();
2106 j = calloc(1, sizeof(struct enic_fm_jump_flow));
2112 j->action = *action;
2113 TAILQ_INSERT_HEAD(&fm->jump_list, j, list);
2114 ENICPMD_LOG(DEBUG, "saved jump flow: flow=%p group=%u", flow, group);
2119 __enic_fm_flow_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
2121 if (fm_flow->entry_handle != FM_INVALID_HANDLE) {
2122 enic_fm_entry_free(fm, fm_flow->entry_handle);
2123 fm_flow->entry_handle = FM_INVALID_HANDLE;
2125 if (fm_flow->action != NULL) {
2126 enic_fm_action_free(fm, fm_flow->action);
2127 fm_flow->action = NULL;
2129 enic_fm_counter_free(fm, fm_flow);
2131 enic_fet_put(fm, fm_flow->fet);
2132 fm_flow->fet = NULL;
2137 enic_fm_flow_free(struct enic_flowman *fm, struct rte_flow *flow)
2139 struct enic_fm_flow *steer = flow->fm->hairpin_steer_flow;
2141 if (flow->fm->fet && flow->fm->fet->default_key)
2142 remove_jump_flow(fm, flow);
2143 __enic_fm_flow_free(fm, flow->fm);
2145 __enic_fm_flow_free(fm, steer);
2153 enic_fm_add_tcam_entry(struct enic_flowman *fm,
2154 struct fm_tcam_match_entry *match_in,
2155 uint64_t *entry_handle,
2157 struct rte_flow_error *error)
2159 struct fm_tcam_match_entry *ftm;
2163 ENICPMD_FUNC_TRACE();
2164 /* Copy entry to the command buffer */
2165 ftm = &fm->cmd.va->fm_tcam_match_entry;
2166 memcpy(ftm, match_in, sizeof(*ftm));
2167 /* Add TCAM entry */
2168 args[0] = FM_TCAM_ENTRY_INSTALL;
2169 args[1] = ingress ? fm->ig_tcam_hndl : fm->eg_tcam_hndl;
2170 args[2] = fm->cmd.pa;
2171 ret = flowman_cmd(fm, args, 3);
2173 ENICPMD_LOG(ERR, "cannot add %s TCAM entry: rc=%d",
2174 ingress ? "ingress" : "egress", ret);
2175 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2176 NULL, "enic: devcmd(tcam-entry-install)");
2179 ENICPMD_LOG(DEBUG, "installed %s TCAM entry: handle=0x%" PRIx64,
2180 ingress ? "ingress" : "egress", (uint64_t)args[0]);
2181 *entry_handle = args[0];
2186 enic_fm_add_exact_entry(struct enic_flowman *fm,
2187 struct fm_tcam_match_entry *match_in,
2188 uint64_t *entry_handle,
2189 struct enic_fm_fet *fet,
2190 struct rte_flow_error *error)
2192 struct fm_exact_match_entry *fem;
2196 ENICPMD_FUNC_TRACE();
2197 /* The new entry must have the table's key */
2198 if (memcmp(fet->key.fk_hdrset, match_in->ftm_mask.fk_hdrset,
2199 sizeof(struct fm_header_set) * FM_HDRSET_MAX)) {
2200 return rte_flow_error_set(error, EINVAL,
2201 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2202 "enic: key does not match group's key");
2205 /* Copy entry to the command buffer */
2206 fem = &fm->cmd.va->fm_exact_match_entry;
2208 * Translate TCAM entry to exact entry. As is only need to drop
2209 * position and mask. The mask is part of the exact match table.
2210 * Position (aka priority) is not supported in the exact match table.
2212 fem->fem_data = match_in->ftm_data;
2213 fem->fem_flags = match_in->ftm_flags;
2214 fem->fem_action = match_in->ftm_action;
2215 fem->fem_counter = match_in->ftm_counter;
2217 /* Add exact entry */
2218 args[0] = FM_EXACT_ENTRY_INSTALL;
2219 args[1] = fet->handle;
2220 args[2] = fm->cmd.pa;
2221 ret = flowman_cmd(fm, args, 3);
2223 ENICPMD_LOG(ERR, "cannot add %s exact entry: group=%u",
2224 fet->ingress ? "ingress" : "egress", fet->group);
2225 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2226 NULL, "enic: devcmd(exact-entry-install)");
2229 ENICPMD_LOG(DEBUG, "installed %s exact entry: group=%u"
2230 " handle=0x%" PRIx64,
2231 fet->ingress ? "ingress" : "egress", fet->group,
2233 *entry_handle = args[0];
2238 enic_action_handle_get(struct enic_flowman *fm, struct fm_action *action_in,
2239 struct rte_flow_error *error,
2240 struct enic_fm_action **ah_o)
2242 struct enic_fm_action *ah;
2243 struct fm_action *fma;
2247 ret = rte_hash_lookup_data(fm->action_hash, action_in,
2249 if (ret < 0 && ret != -ENOENT)
2250 return rte_flow_error_set(error, -ret,
2251 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2252 NULL, "enic: rte_hash_lookup(aciton)");
2254 if (ret == -ENOENT) {
2255 /* Allocate a new action on the NIC. */
2256 fma = &fm->cmd.va->fm_action;
2257 memcpy(fma, action_in, sizeof(*fma));
2259 ah = calloc(1, sizeof(*ah));
2260 memcpy(&ah->key, action_in, sizeof(struct fm_action));
2262 return rte_flow_error_set(error, ENOMEM,
2263 RTE_FLOW_ERROR_TYPE_HANDLE,
2264 NULL, "enic: calloc(fm-action)");
2265 args[0] = FM_ACTION_ALLOC;
2266 args[1] = fm->cmd.pa;
2267 ret = flowman_cmd(fm, args, 2);
2269 rte_flow_error_set(error, -ret,
2270 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2271 NULL, "enic: devcmd(action-alloc)");
2274 ah->handle = args[0];
2275 ret = rte_hash_add_key_data(fm->action_hash,
2276 (const void *)action_in,
2279 rte_flow_error_set(error, -ret,
2280 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2282 "enic: rte_hash_add_key_data(actn)");
2283 goto error_with_action_handle;
2285 ENICPMD_LOG(DEBUG, "action allocated: handle=0x%" PRIx64,
2289 /* Action handle struct is valid, increment reference count. */
2293 error_with_action_handle:
2294 args[0] = FM_ACTION_FREE;
2295 args[1] = ah->handle;
2296 ret = flowman_cmd(fm, args, 2);
2298 rte_flow_error_set(error, -ret,
2299 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2300 NULL, "enic: devcmd(action-free)");
2306 /* Push match-action to the NIC. */
2308 __enic_fm_flow_add_entry(struct enic_flowman *fm,
2309 struct enic_fm_flow *fm_flow,
2310 struct fm_tcam_match_entry *match_in,
2311 struct fm_action *action_in,
2314 struct rte_flow_error *error)
2316 struct enic_fm_counter *ctr;
2317 struct enic_fm_action *ah = NULL;
2321 ENICPMD_FUNC_TRACE();
2323 /* Get or create an aciton handle. */
2324 ret = enic_action_handle_get(fm, action_in, error, &ah);
2327 match_in->ftm_action = ah->handle;
2328 fm_flow->action = ah;
2330 /* Allocate counter if requested. */
2331 if (match_in->ftm_flags & FMEF_COUNTER) {
2332 ret = enic_fm_counter_alloc(fm, error, &ctr);
2333 if (ret) /* error has been filled in */
2335 fm_flow->counter_valid = true;
2336 fm_flow->counter = ctr;
2337 match_in->ftm_counter = ctr->handle;
2341 * Get the group's table (either TCAM or exact match table) and
2342 * add entry to it. If we use the exact match table, the handler
2343 * will translate the TCAM entry (match_in) to the appropriate
2344 * exact match entry and use that instead.
2346 entry_h = FM_INVALID_HANDLE;
2347 if (group == FM_TCAM_RTE_GROUP) {
2348 ret = enic_fm_add_tcam_entry(fm, match_in, &entry_h, ingress,
2352 /* Jump action might have a ref to fet */
2353 fm_flow->fet = fm->fet;
2356 struct enic_fm_fet *fet = NULL;
2358 ret = enic_fet_get(fm, group, ingress,
2359 &match_in->ftm_mask, &fet, error);
2363 ret = enic_fm_add_exact_entry(fm, match_in, &entry_h, fet,
2368 /* Clear counter after adding entry, as it requires in-use counter */
2369 if (fm_flow->counter_valid) {
2370 ret = enic_fm_counter_zero(fm, fm_flow->counter);
2374 fm_flow->entry_handle = entry_h;
2378 /* Push match-action to the NIC. */
2379 static struct rte_flow *
2380 enic_fm_flow_add_entry(struct enic_flowman *fm,
2381 struct fm_tcam_match_entry *match_in,
2382 struct fm_action *action_in,
2383 const struct rte_flow_attr *attrs,
2384 struct rte_flow_error *error)
2386 struct enic_fm_flow *fm_flow;
2387 struct rte_flow *flow;
2389 ENICPMD_FUNC_TRACE();
2390 match_in->ftm_position = attrs->priority;
2391 enic_fm_dump_tcam_entry(match_in, action_in, attrs->ingress);
2392 flow = calloc(1, sizeof(*flow));
2393 fm_flow = calloc(1, sizeof(*fm_flow));
2394 if (flow == NULL || fm_flow == NULL) {
2395 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
2396 NULL, "enic: cannot allocate rte_flow");
2402 fm_flow->action = NULL;
2403 fm_flow->entry_handle = FM_INVALID_HANDLE;
2404 if (__enic_fm_flow_add_entry(fm, fm_flow, match_in, action_in,
2405 attrs->group, attrs->ingress, error)) {
2406 enic_fm_flow_free(fm, flow);
2413 convert_jump_flows(struct enic_flowman *fm, struct enic_fm_fet *fet,
2414 struct rte_flow_error *error)
2416 struct enic_fm_flow *fm_flow;
2417 struct enic_fm_jump_flow *j;
2418 struct fm_action *fma;
2421 ENICPMD_FUNC_TRACE();
2423 * Find the saved flows that should jump to the new table (fet).
2424 * Then delete the old TCAM entry that jumps to the default table,
2425 * and add a new one that jumps to the new table.
2428 j = find_jump_flow(fm, group);
2430 ENICPMD_LOG(DEBUG, "convert jump flow: flow=%p group=%u",
2432 /* Delete old entry */
2433 fm_flow = j->flow->fm;
2434 __enic_fm_flow_free(fm, fm_flow);
2438 fma->fma_action_ops[0].exact.handle = fet->handle;
2439 if (__enic_fm_flow_add_entry(fm, fm_flow, &j->match, fma,
2440 FM_TCAM_RTE_GROUP, fet->ingress, error)) {
2441 /* Cannot roll back changes at the moment */
2442 ENICPMD_LOG(ERR, "cannot convert jump flow: flow=%p",
2447 ENICPMD_LOG(DEBUG, "convert ok: group=%u ref=%u",
2448 fet->group, fet->ref);
2451 TAILQ_REMOVE(&fm->jump_list, j, list);
2453 j = find_jump_flow(fm, group);
2458 add_hairpin_steer(struct enic_flowman *fm, struct rte_flow *flow,
2459 struct rte_flow_error *error)
2461 struct fm_tcam_match_entry *fm_tcam_entry;
2462 struct enic_fm_flow *fm_flow;
2463 struct fm_action *fm_action;
2464 struct fm_action_op fm_op;
2467 ENICPMD_FUNC_TRACE();
2468 fm_flow = calloc(1, sizeof(*fm_flow));
2469 if (fm_flow == NULL) {
2470 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
2471 NULL, "enic: cannot allocate rte_flow");
2474 /* Original egress hairpin flow */
2475 fm_tcam_entry = &fm->tcam_entry;
2476 fm_action = &fm->action;
2477 /* Use the match pattern of the egress flow as is, without counters */
2478 fm_tcam_entry->ftm_flags &= ~FMEF_COUNTER;
2479 /* The only action is steer to vnic */
2480 fm->action_op_count = 0;
2481 memset(fm_action, 0, sizeof(*fm_action));
2482 memset(&fm_op, 0, sizeof(fm_op));
2483 /* Always to queue 0 for now */
2484 fm_op.fa_op = FMOP_RQ_STEER;
2485 fm_op.rq_steer.rq_index = 0;
2486 fm_op.rq_steer.vnic_handle = fm->hairpin_steer_vnic_h;
2487 ret = enic_fm_append_action_op(fm, &fm_op, error);
2489 goto error_with_flow;
2490 ENICPMD_LOG(DEBUG, "add steer op");
2491 /* Add required END */
2492 memset(&fm_op, 0, sizeof(fm_op));
2493 fm_op.fa_op = FMOP_END;
2494 ret = enic_fm_append_action_op(fm, &fm_op, error);
2496 goto error_with_flow;
2497 /* Add the ingress flow */
2498 fm_flow->action = NULL;
2499 fm_flow->entry_handle = FM_INVALID_HANDLE;
2500 ret = __enic_fm_flow_add_entry(fm, fm_flow, fm_tcam_entry, fm_action,
2501 FM_TCAM_RTE_GROUP, 1 /* ingress */, error);
2503 ENICPMD_LOG(ERR, "cannot add hairpin-steer flow");
2504 goto error_with_flow;
2506 /* The new flow is now the egress flow's paired flow */
2507 flow->fm->hairpin_steer_flow = fm_flow;
2516 enic_fm_open_scratch(struct enic_flowman *fm)
2518 fm->action_op_count = 0;
2520 fm->need_hairpin_steer = 0;
2521 fm->hairpin_steer_vnic_h = 0;
2522 memset(&fm->tcam_entry, 0, sizeof(fm->tcam_entry));
2523 memset(&fm->action, 0, sizeof(fm->action));
2527 enic_fm_close_scratch(struct enic_flowman *fm)
2530 enic_fet_put(fm, fm->fet);
2533 fm->action_op_count = 0;
2537 enic_fm_flow_validate(struct rte_eth_dev *dev,
2538 const struct rte_flow_attr *attrs,
2539 const struct rte_flow_item pattern[],
2540 const struct rte_flow_action actions[],
2541 struct rte_flow_error *error)
2543 struct fm_tcam_match_entry *fm_tcam_entry;
2544 struct fm_action *fm_action;
2545 struct enic_flowman *fm;
2548 ENICPMD_FUNC_TRACE();
2549 fm = begin_fm(pmd_priv(dev));
2552 enic_fm_open_scratch(fm);
2553 ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2555 fm_tcam_entry = &fm->tcam_entry;
2556 fm_action = &fm->action;
2557 enic_fm_dump_tcam_entry(fm_tcam_entry, fm_action,
2560 enic_fm_close_scratch(fm);
2566 enic_fm_flow_query_count(struct rte_eth_dev *dev,
2567 struct rte_flow *flow, void *data,
2568 struct rte_flow_error *error)
2570 struct rte_flow_query_count *query;
2571 struct enic_fm_flow *fm_flow;
2572 struct enic_flowman *fm;
2576 ENICPMD_FUNC_TRACE();
2577 fm = begin_fm(pmd_priv(dev));
2580 if (!fm_flow->counter_valid) {
2581 rc = rte_flow_error_set(error, ENOTSUP,
2582 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2583 "enic: flow does not have counter");
2587 args[0] = FM_COUNTER_QUERY;
2588 args[1] = fm_flow->counter->handle;
2589 args[2] = query->reset;
2590 rc = flowman_cmd(fm, args, 3);
2592 ENICPMD_LOG(ERR, "cannot query counter: rc=%d handle=0x%x",
2593 rc, fm_flow->counter->handle);
2596 query->hits_set = 1;
2597 query->hits = args[0];
2598 query->bytes_set = 1;
2599 query->bytes = args[1];
2607 enic_fm_flow_query(struct rte_eth_dev *dev,
2608 struct rte_flow *flow,
2609 const struct rte_flow_action *actions,
2611 struct rte_flow_error *error)
2615 ENICPMD_FUNC_TRACE();
2616 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2617 switch (actions->type) {
2618 case RTE_FLOW_ACTION_TYPE_VOID:
2620 case RTE_FLOW_ACTION_TYPE_COUNT:
2621 ret = enic_fm_flow_query_count(dev, flow, data, error);
2624 return rte_flow_error_set(error, ENOTSUP,
2625 RTE_FLOW_ERROR_TYPE_ACTION,
2627 "action not supported");
2635 static struct rte_flow *
2636 enic_fm_flow_create(struct rte_eth_dev *dev,
2637 const struct rte_flow_attr *attrs,
2638 const struct rte_flow_item pattern[],
2639 const struct rte_flow_action actions[],
2640 struct rte_flow_error *error)
2642 struct fm_tcam_match_entry *fm_tcam_entry;
2643 struct fm_action *fm_action;
2644 struct enic_flowman *fm;
2645 struct enic_fm_fet *fet;
2646 struct rte_flow *flow;
2650 ENICPMD_FUNC_TRACE();
2651 enic = pmd_priv(dev);
2652 fm = begin_fm(enic);
2654 rte_flow_error_set(error, ENOTSUP,
2655 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2656 "flowman is not initialized");
2659 enic_fm_open_scratch(fm);
2661 ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2663 goto error_with_scratch;
2664 fm_tcam_entry = &fm->tcam_entry;
2665 fm_action = &fm->action;
2666 flow = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2669 /* Add ingress rule that pairs with hairpin rule */
2670 if (fm->need_hairpin_steer) {
2671 ret = add_hairpin_steer(fm, flow, error);
2673 enic_fm_flow_free(fm, flow);
2675 goto error_with_scratch;
2678 LIST_INSERT_HEAD(&enic->flows, flow, next);
2679 fet = flow->fm->fet;
2680 if (fet && fet->default_key) {
2682 * Jump to non-existent group? Save the relevant info
2683 * so we can convert this flow when that group
2686 save_jump_flow(fm, flow, fet->group,
2687 fm_tcam_entry, fm_action);
2688 } else if (fet && fet->ref == 1) {
2690 * A new table is created. Convert the saved flows
2691 * that should jump to this group.
2693 convert_jump_flows(fm, fet, error);
2698 enic_fm_close_scratch(fm);
2704 enic_fm_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
2705 __rte_unused struct rte_flow_error *error)
2707 struct enic *enic = pmd_priv(dev);
2708 struct enic_flowman *fm;
2710 ENICPMD_FUNC_TRACE();
2711 fm = begin_fm(enic);
2714 LIST_REMOVE(flow, next);
2715 enic_fm_flow_free(fm, flow);
2721 enic_fm_flow_flush(struct rte_eth_dev *dev,
2722 __rte_unused struct rte_flow_error *error)
2724 LIST_HEAD(enic_flows, rte_flow) internal;
2725 struct enic_fm_flow *fm_flow;
2726 struct enic_flowman *fm;
2727 struct rte_flow *flow;
2728 struct enic *enic = pmd_priv(dev);
2730 ENICPMD_FUNC_TRACE();
2732 fm = begin_fm(enic);
2735 /* Destroy all non-internal flows */
2736 LIST_INIT(&internal);
2737 while (!LIST_EMPTY(&enic->flows)) {
2738 flow = LIST_FIRST(&enic->flows);
2740 LIST_REMOVE(flow, next);
2741 if (flow->internal) {
2742 LIST_INSERT_HEAD(&internal, flow, next);
2746 * If tables are null, then vNIC is closing, and the firmware
2747 * has already cleaned up flowman state. So do not try to free
2748 * resources, as it only causes errors.
2750 if (fm->ig_tcam_hndl == FM_INVALID_HANDLE) {
2751 fm_flow->entry_handle = FM_INVALID_HANDLE;
2752 fm_flow->action = NULL;
2753 fm_flow->fet = NULL;
2755 enic_fm_flow_free(fm, flow);
2757 while (!LIST_EMPTY(&internal)) {
2758 flow = LIST_FIRST(&internal);
2759 LIST_REMOVE(flow, next);
2760 LIST_INSERT_HEAD(&enic->flows, flow, next);
2767 enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle)
2772 args[0] = FM_MATCH_TABLE_FREE;
2774 rc = flowman_cmd(fm, args, 2);
2776 ENICPMD_LOG(ERR, "cannot free table: rc=%d handle=0x%" PRIx64,
2782 enic_fm_tcam_tbl_alloc(struct enic_flowman *fm, uint32_t direction,
2783 uint32_t max_entries, uint64_t *handle)
2785 struct fm_tcam_match_table *tcam_tbl;
2789 ENICPMD_FUNC_TRACE();
2790 tcam_tbl = &fm->cmd.va->fm_tcam_match_table;
2791 tcam_tbl->ftt_direction = direction;
2792 tcam_tbl->ftt_stage = FM_STAGE_LAST;
2793 tcam_tbl->ftt_max_entries = max_entries;
2794 args[0] = FM_TCAM_TABLE_ALLOC;
2795 args[1] = fm->cmd.pa;
2796 rc = flowman_cmd(fm, args, 2);
2798 ENICPMD_LOG(ERR, "cannot alloc %s TCAM table: rc=%d",
2799 (direction == FM_INGRESS) ? "IG" : "EG", rc);
2803 ENICPMD_LOG(DEBUG, "%s TCAM table allocated, handle=0x%" PRIx64,
2804 (direction == FM_INGRESS) ? "IG" : "EG", *handle);
2809 enic_fm_init_actions(struct enic_flowman *fm)
2811 struct rte_hash *a_hash;
2812 char name[RTE_HASH_NAMESIZE];
2813 struct rte_hash_parameters params = {
2814 .entries = FM_MAX_ACTION_TABLE_SIZE,
2815 .key_len = sizeof(struct fm_action),
2816 .hash_func = rte_jhash,
2817 .hash_func_init_val = 0,
2818 .socket_id = rte_socket_id(),
2821 ENICPMD_FUNC_TRACE();
2822 snprintf((char *)name, sizeof(name), "fm-ah-%s",
2823 fm->owner_enic->bdf_name);
2826 a_hash = rte_hash_create(¶ms);
2829 fm->action_hash = a_hash;
2834 enic_fm_init_counters(struct enic_flowman *fm)
2836 ENICPMD_FUNC_TRACE();
2837 SLIST_INIT(&fm->counters);
2838 return enic_fm_more_counters(fm);
2842 enic_fm_free_all_counters(struct enic_flowman *fm)
2847 args[0] = FM_COUNTER_BRK;
2849 rc = flowman_cmd(fm, args, 2);
2851 ENICPMD_LOG(ERR, "cannot free counters: rc=%d", rc);
2852 rte_free(fm->counter_stack);
2856 enic_fm_alloc_tcam_tables(struct enic_flowman *fm)
2860 ENICPMD_FUNC_TRACE();
2861 rc = enic_fm_tcam_tbl_alloc(fm, FM_INGRESS, FM_MAX_TCAM_TABLE_SIZE,
2865 rc = enic_fm_tcam_tbl_alloc(fm, FM_EGRESS, FM_MAX_TCAM_TABLE_SIZE,
2871 enic_fm_free_tcam_tables(struct enic_flowman *fm)
2873 ENICPMD_FUNC_TRACE();
2874 if (fm->ig_tcam_hndl) {
2875 ENICPMD_LOG(DEBUG, "free IG TCAM table handle=0x%" PRIx64,
2877 enic_fm_tbl_free(fm, fm->ig_tcam_hndl);
2878 fm->ig_tcam_hndl = FM_INVALID_HANDLE;
2880 if (fm->eg_tcam_hndl) {
2881 ENICPMD_LOG(DEBUG, "free EG TCAM table handle=0x%" PRIx64,
2883 enic_fm_tbl_free(fm, fm->eg_tcam_hndl);
2884 fm->eg_tcam_hndl = FM_INVALID_HANDLE;
2889 enic_fm_init(struct enic *enic)
2891 const struct rte_pci_addr *addr;
2892 struct enic_flowman *fm;
2893 uint8_t name[RTE_MEMZONE_NAMESIZE];
2896 if (enic->flow_filter_mode != FILTER_FLOWMAN)
2898 ENICPMD_FUNC_TRACE();
2899 /* Get vnic handle and save for port-id action */
2900 if (enic_is_vf_rep(enic))
2901 addr = &VF_ENIC_TO_VF_REP(enic)->bdf;
2903 addr = &RTE_ETH_DEV_TO_PCI(enic->rte_dev)->addr;
2904 rc = enic_fm_find_vnic(enic, addr, &enic->fm_vnic_handle);
2906 ENICPMD_LOG(ERR, "cannot find vnic handle for %x:%x:%x",
2907 addr->bus, addr->devid, addr->function);
2910 /* Save UIF for egport action */
2911 enic->fm_vnic_uif = vnic_dev_uif(enic->vdev);
2912 ENICPMD_LOG(DEBUG, "uif %u", enic->fm_vnic_uif);
2913 /* Nothing else to do for representor. It will share the PF flowman */
2914 if (enic_is_vf_rep(enic))
2916 fm = calloc(1, sizeof(*fm));
2918 ENICPMD_LOG(ERR, "cannot alloc flowman struct");
2921 fm->owner_enic = enic;
2922 rte_spinlock_init(&fm->lock);
2923 TAILQ_INIT(&fm->fet_list);
2924 TAILQ_INIT(&fm->jump_list);
2925 /* Allocate host memory for flowman commands */
2926 snprintf((char *)name, sizeof(name), "fm-cmd-%s", enic->bdf_name);
2927 fm->cmd.va = enic_alloc_consistent(enic,
2928 sizeof(union enic_flowman_cmd_mem), &fm->cmd.pa, name);
2930 ENICPMD_LOG(ERR, "cannot allocate flowman command memory");
2934 /* Allocate TCAM tables upfront as they are the main tables */
2935 rc = enic_fm_alloc_tcam_tables(fm);
2937 ENICPMD_LOG(ERR, "cannot alloc TCAM tables");
2940 /* Then a number of counters */
2941 rc = enic_fm_init_counters(fm);
2943 ENICPMD_LOG(ERR, "cannot alloc counters");
2946 /* set up action handle hash */
2947 rc = enic_fm_init_actions(fm);
2949 ENICPMD_LOG(ERR, "cannot create action hash, error:%d", rc);
2950 goto error_counters;
2953 * One default exact match table for each direction. We hold onto
2956 rc = enic_fet_alloc(fm, 1, NULL, 128, &fm->default_ig_fet);
2958 ENICPMD_LOG(ERR, "cannot alloc default IG exact match table");
2961 fm->default_ig_fet->ref = 1;
2962 rc = enic_fet_alloc(fm, 0, NULL, 128, &fm->default_eg_fet);
2964 ENICPMD_LOG(ERR, "cannot alloc default EG exact match table");
2967 fm->default_eg_fet->ref = 1;
2968 fm->vf_rep_tag = FM_VF_REP_TAG;
2973 enic_fet_free(fm, fm->default_ig_fet);
2975 rte_hash_free(fm->action_hash);
2977 enic_fm_free_all_counters(fm);
2979 enic_fm_free_tcam_tables(fm);
2981 enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
2982 fm->cmd.va, fm->cmd.pa);
2989 enic_fm_destroy(struct enic *enic)
2991 struct enic_flowman *fm;
2992 struct enic_fm_fet *fet;
2994 ENICPMD_FUNC_TRACE();
2995 if (enic_is_vf_rep(enic)) {
2996 delete_rep_flows(enic);
2999 if (enic->fm == NULL)
3002 enic_fm_flow_flush(enic->rte_dev, NULL);
3003 enic_fet_free(fm, fm->default_eg_fet);
3004 enic_fet_free(fm, fm->default_ig_fet);
3005 /* Free all exact match tables still open */
3006 while (!TAILQ_EMPTY(&fm->fet_list)) {
3007 fet = TAILQ_FIRST(&fm->fet_list);
3008 enic_fet_free(fm, fet);
3010 enic_fm_free_tcam_tables(fm);
3011 enic_fm_free_all_counters(fm);
3012 rte_hash_free(fm->action_hash);
3013 enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
3014 fm->cmd.va, fm->cmd.pa);
3021 enic_fm_allocate_switch_domain(struct enic *pf)
3023 const struct rte_pci_addr *cur_a, *prev_a;
3024 struct rte_eth_dev *dev;
3025 struct enic *cur, *prev;
3031 ENICPMD_FUNC_TRACE();
3032 if (enic_is_vf_rep(pf))
3035 cur_a = &RTE_ETH_DEV_TO_PCI(cur->rte_dev)->addr;
3036 /* Go through ports and find another PF that is on the same adapter */
3037 RTE_ETH_FOREACH_DEV(pid) {
3038 dev = &rte_eth_devices[pid];
3039 if (!dev_is_enic(dev))
3041 if (dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)
3043 if (dev == cur->rte_dev)
3045 /* dev is another PF. Is it on the same adapter? */
3046 prev = pmd_priv(dev);
3047 prev_a = &RTE_ETH_DEV_TO_PCI(dev)->addr;
3048 if (!enic_fm_find_vnic(cur, prev_a, &vnic_h)) {
3049 ENICPMD_LOG(DEBUG, "Port %u (PF BDF %x:%x:%x) and port %u (PF BDF %x:%x:%x domain %u) are on the same VIC",
3050 cur->rte_dev->data->port_id,
3051 cur_a->bus, cur_a->devid, cur_a->function,
3053 prev_a->bus, prev_a->devid, prev_a->function,
3054 prev->switch_domain_id);
3055 cur->switch_domain_id = prev->switch_domain_id;
3059 ret = rte_eth_switch_domain_alloc(&domain_id);
3061 ENICPMD_LOG(WARNING, "failed to allocate switch domain for device %d",
3064 cur->switch_domain_id = domain_id;
3065 ENICPMD_LOG(DEBUG, "Port %u (PF BDF %x:%x:%x) is the 1st PF on the VIC. Allocated switch domain id %u",
3066 cur->rte_dev->data->port_id,
3067 cur_a->bus, cur_a->devid, cur_a->function,
3072 const struct rte_flow_ops enic_fm_flow_ops = {
3073 .validate = enic_fm_flow_validate,
3074 .create = enic_fm_flow_create,
3075 .destroy = enic_fm_flow_destroy,
3076 .flush = enic_fm_flow_flush,
3077 .query = enic_fm_flow_query,
3080 /* Add a high priority flow that loops representor packets to VF */
3082 enic_fm_add_rep2vf_flow(struct enic_vf_representor *vf)
3084 struct fm_tcam_match_entry *fm_tcam_entry;
3085 struct rte_flow *flow0, *flow1;
3086 struct fm_action *fm_action;
3087 struct rte_flow_error error;
3088 struct rte_flow_attr attrs;
3089 struct fm_action_op fm_op;
3090 struct enic_flowman *fm;
3096 tag = fm->vf_rep_tag;
3097 enic_fm_open_scratch(fm);
3098 fm_tcam_entry = &fm->tcam_entry;
3099 fm_action = &fm->action;
3100 /* Egress rule: match WQ ID and tag+hairpin */
3101 fm_tcam_entry->ftm_data.fk_wq_id = vf->pf_wq_idx;
3102 fm_tcam_entry->ftm_mask.fk_wq_id = 0xffff;
3103 fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3104 memset(&fm_op, 0, sizeof(fm_op));
3105 fm_op.fa_op = FMOP_TAG;
3106 fm_op.tag.tag = tag;
3107 enic_fm_append_action_op(fm, &fm_op, &error);
3108 memset(&fm_op, 0, sizeof(fm_op));
3109 fm_op.fa_op = FMOP_EG_HAIRPIN;
3110 enic_fm_append_action_op(fm, &fm_op, &error);
3111 memset(&fm_op, 0, sizeof(fm_op));
3112 fm_op.fa_op = FMOP_END;
3113 enic_fm_append_action_op(fm, &fm_op, &error);
3117 attrs.priority = FM_HIGHEST_PRIORITY;
3118 flow0 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3120 enic_fm_close_scratch(fm);
3121 if (flow0 == NULL) {
3122 ENICPMD_LOG(ERR, "Cannot create flow 0 for representor->VF");
3125 LIST_INSERT_HEAD(&pf->flows, flow0, next);
3126 /* Make this flow internal, so the user app cannot delete it */
3127 flow0->internal = 1;
3128 ENICPMD_LOG(DEBUG, "representor->VF %d flow created: wq %d -> tag %d hairpin",
3129 vf->vf_id, vf->pf_wq_idx, tag);
3131 /* Ingress: steer hairpinned to VF RQ 0 */
3132 enic_fm_open_scratch(fm);
3133 fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3134 fm_tcam_entry->ftm_data.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3135 fm_tcam_entry->ftm_mask.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3136 fm_tcam_entry->ftm_data.fk_packet_tag = tag;
3137 fm_tcam_entry->ftm_mask.fk_packet_tag = 0xff;
3138 memset(&fm_op, 0, sizeof(fm_op));
3139 fm_op.fa_op = FMOP_RQ_STEER;
3140 fm_op.rq_steer.rq_index = 0;
3141 fm_op.rq_steer.vnic_handle = vf->enic.fm_vnic_handle;
3142 enic_fm_append_action_op(fm, &fm_op, &error);
3143 memset(&fm_op, 0, sizeof(fm_op));
3144 fm_op.fa_op = FMOP_END;
3145 enic_fm_append_action_op(fm, &fm_op, &error);
3149 attrs.priority = FM_HIGHEST_PRIORITY;
3150 flow1 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3152 enic_fm_close_scratch(fm);
3153 if (flow1 == NULL) {
3154 ENICPMD_LOG(ERR, "Cannot create flow 1 for representor->VF");
3155 enic_fm_flow_destroy(pf->rte_dev, flow0, &error);
3158 LIST_INSERT_HEAD(&pf->flows, flow1, next);
3159 flow1->internal = 1;
3160 ENICPMD_LOG(DEBUG, "representor->VF %d flow created: tag %d hairpinned -> VF RQ %d",
3161 vf->vf_id, tag, fm_op.rq_steer.rq_index);
3162 vf->rep2vf_flow[0] = flow0;
3163 vf->rep2vf_flow[1] = flow1;
3164 /* Done with this tag, use a different one next time */
3170 * Add a low priority flow that matches all packets from VF and loops them
3171 * back to the representor.
3174 enic_fm_add_vf2rep_flow(struct enic_vf_representor *vf)
3176 struct fm_tcam_match_entry *fm_tcam_entry;
3177 struct rte_flow *flow0, *flow1;
3178 struct fm_action *fm_action;
3179 struct rte_flow_error error;
3180 struct rte_flow_attr attrs;
3181 struct fm_action_op fm_op;
3182 struct enic_flowman *fm;
3188 tag = fm->vf_rep_tag;
3189 enic_fm_open_scratch(fm);
3190 fm_tcam_entry = &fm->tcam_entry;
3191 fm_action = &fm->action;
3192 /* Egress rule: match-any and tag+hairpin */
3193 fm_tcam_entry->ftm_data.fk_wq_id = 0;
3194 fm_tcam_entry->ftm_mask.fk_wq_id = 0xffff;
3195 fm_tcam_entry->ftm_data.fk_wq_vnic = vf->enic.fm_vnic_handle;
3196 fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3197 memset(&fm_op, 0, sizeof(fm_op));
3198 fm_op.fa_op = FMOP_TAG;
3199 fm_op.tag.tag = tag;
3200 enic_fm_append_action_op(fm, &fm_op, &error);
3201 memset(&fm_op, 0, sizeof(fm_op));
3202 fm_op.fa_op = FMOP_EG_HAIRPIN;
3203 enic_fm_append_action_op(fm, &fm_op, &error);
3204 memset(&fm_op, 0, sizeof(fm_op));
3205 fm_op.fa_op = FMOP_END;
3206 enic_fm_append_action_op(fm, &fm_op, &error);
3210 attrs.priority = FM_LOWEST_PRIORITY;
3211 flow0 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3213 enic_fm_close_scratch(fm);
3214 if (flow0 == NULL) {
3215 ENICPMD_LOG(ERR, "Cannot create flow 0 for VF->representor");
3218 LIST_INSERT_HEAD(&pf->flows, flow0, next);
3219 /* Make this flow internal, so the user app cannot delete it */
3220 flow0->internal = 1;
3221 ENICPMD_LOG(DEBUG, "VF %d->representor flow created: wq %d (low prio) -> tag %d hairpin",
3222 vf->vf_id, fm_tcam_entry->ftm_data.fk_wq_id, tag);
3224 /* Ingress: steer hairpinned to VF rep RQ */
3225 enic_fm_open_scratch(fm);
3226 fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3227 fm_tcam_entry->ftm_data.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3228 fm_tcam_entry->ftm_mask.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3229 fm_tcam_entry->ftm_data.fk_packet_tag = tag;
3230 fm_tcam_entry->ftm_mask.fk_packet_tag = 0xff;
3231 memset(&fm_op, 0, sizeof(fm_op));
3232 fm_op.fa_op = FMOP_RQ_STEER;
3233 fm_op.rq_steer.rq_index = vf->pf_rq_sop_idx;
3234 fm_op.rq_steer.vnic_handle = pf->fm_vnic_handle;
3235 enic_fm_append_action_op(fm, &fm_op, &error);
3236 memset(&fm_op, 0, sizeof(fm_op));
3237 fm_op.fa_op = FMOP_END;
3238 enic_fm_append_action_op(fm, &fm_op, &error);
3242 attrs.priority = FM_HIGHEST_PRIORITY;
3243 flow1 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3245 enic_fm_close_scratch(fm);
3246 if (flow1 == NULL) {
3247 ENICPMD_LOG(ERR, "Cannot create flow 1 for VF->representor");
3248 enic_fm_flow_destroy(pf->rte_dev, flow0, &error);
3251 LIST_INSERT_HEAD(&pf->flows, flow1, next);
3252 flow1->internal = 1;
3253 ENICPMD_LOG(DEBUG, "VF %d->representor flow created: tag %d hairpinned -> PF RQ %d",
3254 vf->vf_id, tag, vf->pf_rq_sop_idx);
3255 vf->vf2rep_flow[0] = flow0;
3256 vf->vf2rep_flow[1] = flow1;
3257 /* Done with this tag, use a different one next time */
3262 /* Destroy representor flows created by enic_fm_add_{rep2vf,vf2rep}_flow */
3264 delete_rep_flows(struct enic *enic)
3266 struct enic_vf_representor *vf;
3267 struct rte_flow_error error;
3268 struct rte_eth_dev *dev;
3271 RTE_ASSERT(enic_is_vf_rep(enic));
3272 vf = VF_ENIC_TO_VF_REP(enic);
3273 dev = vf->pf->rte_dev;
3274 for (i = 0; i < ARRAY_SIZE(vf->vf2rep_flow); i++) {
3275 if (vf->vf2rep_flow[i])
3276 enic_fm_flow_destroy(dev, vf->vf2rep_flow[i], &error);
3278 for (i = 0; i < ARRAY_SIZE(vf->rep2vf_flow); i++) {
3279 if (vf->rep2vf_flow[i])
3280 enic_fm_flow_destroy(dev, vf->rep2vf_flow[i], &error);
3284 static struct enic_flowman *
3285 begin_fm(struct enic *enic)
3287 struct enic_vf_representor *vf;
3288 struct enic_flowman *fm;
3290 /* Representor uses PF flowman */
3291 if (enic_is_vf_rep(enic)) {
3292 vf = VF_ENIC_TO_VF_REP(enic);
3297 /* Save the API caller and lock if representors exist */
3299 if (fm->owner_enic->switchdev_mode)
3300 rte_spinlock_lock(&fm->lock);
3301 fm->user_enic = enic;
3307 end_fm(struct enic_flowman *fm)
3309 fm->user_enic = NULL;
3310 if (fm->owner_enic->switchdev_mode)
3311 rte_spinlock_unlock(&fm->lock);