1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2008-2019 Cisco Systems, Inc. All rights reserved.
8 #include <ethdev_driver.h>
9 #include <rte_flow_driver.h>
10 #include <rte_ether.h>
11 #include <rte_geneve.h>
13 #include <rte_jhash.h>
16 #include <rte_memzone.h>
18 #include "enic_compat.h"
23 #define IP_DEFTTL 64 /* from RFC 1340. */
24 #define IP6_VTC_FLOW 0x60000000
26 /* Up to 1024 TCAM entries */
27 #define FM_MAX_TCAM_TABLE_SIZE 1024
29 /* Up to 4096 entries per exact match table */
30 #define FM_MAX_EXACT_TABLE_SIZE 4096
32 /* Number of counters to increase on for each increment */
33 #define FM_COUNTERS_EXPAND 100
35 #define FM_INVALID_HANDLE 0
37 /* Low priority used for implicit VF -> representor flow */
38 #define FM_LOWEST_PRIORITY 100000
40 /* High priority used for implicit representor -> VF flow */
41 #define FM_HIGHEST_PRIORITY 0
43 /* Tag used for implicit VF <-> representor flows */
44 #define FM_VF_REP_TAG 1
46 /* Max number of actions supported by VIC is 2K. Make hash table double that. */
47 #define FM_MAX_ACTION_TABLE_SIZE 4096
50 * Flow exact match tables (FET) in the VIC and rte_flow groups.
51 * Use a simple scheme to map groups to tables.
52 * Group 0 uses the single TCAM tables, one for each direction.
53 * Group 1, 2, ... uses its own exact match table.
55 * The TCAM tables are allocated upfront during init.
57 * Exact match tables are allocated on demand. 3 paths that lead allocations.
59 * 1. Add a flow that jumps from group 0 to group N.
61 * If N does not exist, we allocate an exact match table for it, using
62 * a dummy key. A key is required for the table.
64 * 2. Add a flow that uses group N.
66 * If N does not exist, we allocate an exact match table for it, using
67 * the flow's key. Subsequent flows to the same group all should have
70 * Without a jump flow to N, N is not reachable in hardware. No packets
73 * 3. Add a flow to an empty group N.
75 * N has been created via (1) and the dummy key. We free that table, allocate
76 * a new table using the new flow's key. Also re-do the existing jump flow to
77 * point to the new table.
79 #define FM_TCAM_RTE_GROUP 0
82 TAILQ_ENTRY(enic_fm_fet) list;
83 uint32_t group; /* rte_flow group ID */
84 uint64_t handle; /* Exact match table handle from flowman */
87 int ref; /* Reference count via get/put */
88 struct fm_key_template key; /* Key associated with the table */
91 struct enic_fm_counter {
92 SLIST_ENTRY(enic_fm_counter) next;
96 struct enic_fm_action {
103 struct enic_fm_flow {
105 uint64_t entry_handle;
106 struct enic_fm_action *action;
107 struct enic_fm_counter *counter;
108 struct enic_fm_fet *fet;
109 /* Auto-added steer action for hairpin flows (e.g. vnic->vnic) */
110 struct enic_fm_flow *hairpin_steer_flow;
113 struct enic_fm_jump_flow {
114 TAILQ_ENTRY(enic_fm_jump_flow) list;
115 struct rte_flow *flow;
117 struct fm_tcam_match_entry match;
118 struct fm_action action;
122 * Flowman uses host memory for commands. This structure is allocated
123 * in DMA-able memory.
125 union enic_flowman_cmd_mem {
126 struct fm_tcam_match_table fm_tcam_match_table;
127 struct fm_exact_match_table fm_exact_match_table;
128 struct fm_tcam_match_entry fm_tcam_match_entry;
129 struct fm_exact_match_entry fm_exact_match_entry;
130 struct fm_action fm_action;
134 * PF has a flowman instance, and VF representors share it with PF.
135 * PF allocates this structure and owns it. VF representors borrow
136 * the PF's structure during API calls (e.g. create, query).
138 struct enic_flowman {
139 struct enic *owner_enic; /* PF */
140 struct enic *user_enic; /* API caller (PF or representor) */
142 * Representors and PF share the same underlying flowman.
143 * Lock API calls to serialize accesses from them. Only used
144 * when VF representors are present.
149 union enic_flowman_cmd_mem *va;
152 /* TCAM tables allocated upfront, used for group 0 */
153 uint64_t ig_tcam_hndl;
154 uint64_t eg_tcam_hndl;
156 SLIST_HEAD(enic_free_counters, enic_fm_counter) counters;
158 uint32_t counters_alloced;
159 /* Exact match tables for groups != 0, dynamically allocated */
160 TAILQ_HEAD(fet_list, enic_fm_fet) fet_list;
162 * Default exact match tables used for jump actions to
163 * non-existent groups.
165 struct enic_fm_fet *default_eg_fet;
166 struct enic_fm_fet *default_ig_fet;
167 /* hash table for Action reuse */
168 struct rte_hash *action_hash;
169 /* Flows that jump to the default table above */
170 TAILQ_HEAD(jump_flow_list, enic_fm_jump_flow) jump_list;
172 * Scratch data used during each invocation of flow_create
175 struct enic_fm_fet *fet;
176 struct fm_tcam_match_entry tcam_entry;
177 struct fm_action action;
178 struct fm_action action_tmp; /* enic_fm_reorder_action_op */
180 /* Tags used for representor flows */
182 /* For auto-added steer action for hairpin */
183 int need_hairpin_steer;
184 uint64_t hairpin_steer_vnic_h;
187 static int enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle);
189 * API functions (create, destroy, validate, flush) call begin_fm()
190 * upon entering to save the caller enic (PF or VF representor) and
191 * lock. Upon exit, they call end_fm() to unlock.
193 static struct enic_flowman *begin_fm(struct enic *enic);
194 static void end_fm(struct enic_flowman *fm);
195 /* Delete internal flows created for representor paths */
196 static void delete_rep_flows(struct enic *enic);
199 * Common arguments passed to copy_item functions. Use this structure
200 * so we can easily add new arguments.
201 * item: Item specification.
202 * fm_tcam_entry: Flowman TCAM match entry.
203 * header_level: 0 for outer header, 1 for inner header.
205 struct copy_item_args {
206 const struct rte_flow_item *item;
207 struct fm_tcam_match_entry *fm_tcam_entry;
208 uint8_t header_level;
209 struct rte_flow_error *error;
212 /* functions for copying items into flowman match */
213 typedef int (enic_copy_item_fn)(struct copy_item_args *arg);
215 /* Info about how to copy items into flowman match */
216 struct enic_fm_items {
217 /* Function for copying and validating an item. */
218 enic_copy_item_fn * const copy_item;
219 /* List of valid previous items. */
220 const enum rte_flow_item_type * const prev_items;
222 * True if it's OK for this item to be the first item. For some NIC
223 * versions, it's invalid to start the stack above layer 3.
225 const uint8_t valid_start_item;
228 static enic_copy_item_fn enic_fm_copy_item_eth;
229 static enic_copy_item_fn enic_fm_copy_item_ipv4;
230 static enic_copy_item_fn enic_fm_copy_item_ipv6;
231 static enic_copy_item_fn enic_fm_copy_item_raw;
232 static enic_copy_item_fn enic_fm_copy_item_sctp;
233 static enic_copy_item_fn enic_fm_copy_item_tcp;
234 static enic_copy_item_fn enic_fm_copy_item_udp;
235 static enic_copy_item_fn enic_fm_copy_item_vlan;
236 static enic_copy_item_fn enic_fm_copy_item_vxlan;
237 static enic_copy_item_fn enic_fm_copy_item_gtp;
238 static enic_copy_item_fn enic_fm_copy_item_geneve;
239 static enic_copy_item_fn enic_fm_copy_item_geneve_opt;
240 static enic_copy_item_fn enic_fm_copy_item_ecpri;
242 /* Ingress actions */
243 static const enum rte_flow_action_type enic_fm_supported_ig_actions[] = {
244 RTE_FLOW_ACTION_TYPE_COUNT,
245 RTE_FLOW_ACTION_TYPE_DROP,
246 RTE_FLOW_ACTION_TYPE_FLAG,
247 RTE_FLOW_ACTION_TYPE_JUMP,
248 RTE_FLOW_ACTION_TYPE_MARK,
249 RTE_FLOW_ACTION_TYPE_OF_POP_VLAN,
250 RTE_FLOW_ACTION_TYPE_PORT_ID,
251 RTE_FLOW_ACTION_TYPE_PASSTHRU,
252 RTE_FLOW_ACTION_TYPE_QUEUE,
253 RTE_FLOW_ACTION_TYPE_RSS,
254 RTE_FLOW_ACTION_TYPE_VOID,
255 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
256 RTE_FLOW_ACTION_TYPE_VXLAN_DECAP,
257 RTE_FLOW_ACTION_TYPE_END, /* END must be the last entry */
261 static const enum rte_flow_action_type enic_fm_supported_eg_actions[] = {
262 RTE_FLOW_ACTION_TYPE_COUNT,
263 RTE_FLOW_ACTION_TYPE_DROP,
264 RTE_FLOW_ACTION_TYPE_JUMP,
265 RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN,
266 RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP,
267 RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID,
268 RTE_FLOW_ACTION_TYPE_PORT_ID,
269 RTE_FLOW_ACTION_TYPE_PASSTHRU,
270 RTE_FLOW_ACTION_TYPE_VOID,
271 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
272 RTE_FLOW_ACTION_TYPE_END,
275 static const struct enic_fm_items enic_fm_items[] = {
276 [RTE_FLOW_ITEM_TYPE_RAW] = {
277 .copy_item = enic_fm_copy_item_raw,
278 .valid_start_item = 0,
279 .prev_items = (const enum rte_flow_item_type[]) {
280 RTE_FLOW_ITEM_TYPE_UDP,
281 RTE_FLOW_ITEM_TYPE_END,
284 [RTE_FLOW_ITEM_TYPE_ETH] = {
285 .copy_item = enic_fm_copy_item_eth,
286 .valid_start_item = 1,
287 .prev_items = (const enum rte_flow_item_type[]) {
288 RTE_FLOW_ITEM_TYPE_END,
291 [RTE_FLOW_ITEM_TYPE_VLAN] = {
292 .copy_item = enic_fm_copy_item_vlan,
293 .valid_start_item = 1,
294 .prev_items = (const enum rte_flow_item_type[]) {
295 RTE_FLOW_ITEM_TYPE_ETH,
296 RTE_FLOW_ITEM_TYPE_END,
299 [RTE_FLOW_ITEM_TYPE_IPV4] = {
300 .copy_item = enic_fm_copy_item_ipv4,
301 .valid_start_item = 1,
302 .prev_items = (const enum rte_flow_item_type[]) {
303 RTE_FLOW_ITEM_TYPE_ETH,
304 RTE_FLOW_ITEM_TYPE_VLAN,
305 RTE_FLOW_ITEM_TYPE_END,
308 [RTE_FLOW_ITEM_TYPE_IPV6] = {
309 .copy_item = enic_fm_copy_item_ipv6,
310 .valid_start_item = 1,
311 .prev_items = (const enum rte_flow_item_type[]) {
312 RTE_FLOW_ITEM_TYPE_ETH,
313 RTE_FLOW_ITEM_TYPE_VLAN,
314 RTE_FLOW_ITEM_TYPE_END,
317 [RTE_FLOW_ITEM_TYPE_UDP] = {
318 .copy_item = enic_fm_copy_item_udp,
319 .valid_start_item = 1,
320 .prev_items = (const enum rte_flow_item_type[]) {
321 RTE_FLOW_ITEM_TYPE_IPV4,
322 RTE_FLOW_ITEM_TYPE_IPV6,
323 RTE_FLOW_ITEM_TYPE_END,
326 [RTE_FLOW_ITEM_TYPE_TCP] = {
327 .copy_item = enic_fm_copy_item_tcp,
328 .valid_start_item = 1,
329 .prev_items = (const enum rte_flow_item_type[]) {
330 RTE_FLOW_ITEM_TYPE_IPV4,
331 RTE_FLOW_ITEM_TYPE_IPV6,
332 RTE_FLOW_ITEM_TYPE_END,
335 [RTE_FLOW_ITEM_TYPE_SCTP] = {
336 .copy_item = enic_fm_copy_item_sctp,
337 .valid_start_item = 0,
338 .prev_items = (const enum rte_flow_item_type[]) {
339 RTE_FLOW_ITEM_TYPE_IPV4,
340 RTE_FLOW_ITEM_TYPE_IPV6,
341 RTE_FLOW_ITEM_TYPE_END,
344 [RTE_FLOW_ITEM_TYPE_VXLAN] = {
345 .copy_item = enic_fm_copy_item_vxlan,
346 .valid_start_item = 1,
347 .prev_items = (const enum rte_flow_item_type[]) {
348 RTE_FLOW_ITEM_TYPE_UDP,
349 RTE_FLOW_ITEM_TYPE_END,
352 [RTE_FLOW_ITEM_TYPE_GTP] = {
353 .copy_item = enic_fm_copy_item_gtp,
354 .valid_start_item = 0,
355 .prev_items = (const enum rte_flow_item_type[]) {
356 RTE_FLOW_ITEM_TYPE_UDP,
357 RTE_FLOW_ITEM_TYPE_END,
360 [RTE_FLOW_ITEM_TYPE_GTPC] = {
361 .copy_item = enic_fm_copy_item_gtp,
362 .valid_start_item = 1,
363 .prev_items = (const enum rte_flow_item_type[]) {
364 RTE_FLOW_ITEM_TYPE_UDP,
365 RTE_FLOW_ITEM_TYPE_END,
368 [RTE_FLOW_ITEM_TYPE_GTPU] = {
369 .copy_item = enic_fm_copy_item_gtp,
370 .valid_start_item = 1,
371 .prev_items = (const enum rte_flow_item_type[]) {
372 RTE_FLOW_ITEM_TYPE_UDP,
373 RTE_FLOW_ITEM_TYPE_END,
376 [RTE_FLOW_ITEM_TYPE_GENEVE] = {
377 .copy_item = enic_fm_copy_item_geneve,
378 .valid_start_item = 1,
379 .prev_items = (const enum rte_flow_item_type[]) {
380 RTE_FLOW_ITEM_TYPE_ETH,
381 RTE_FLOW_ITEM_TYPE_IPV4,
382 RTE_FLOW_ITEM_TYPE_IPV6,
383 RTE_FLOW_ITEM_TYPE_UDP,
384 RTE_FLOW_ITEM_TYPE_END,
387 [RTE_FLOW_ITEM_TYPE_GENEVE_OPT] = {
388 .copy_item = enic_fm_copy_item_geneve_opt,
389 .valid_start_item = 1,
390 /* Can match at most 1 option */
391 .prev_items = (const enum rte_flow_item_type[]) {
392 RTE_FLOW_ITEM_TYPE_GENEVE,
393 RTE_FLOW_ITEM_TYPE_END,
396 [RTE_FLOW_ITEM_TYPE_ECPRI] = {
397 .copy_item = enic_fm_copy_item_ecpri,
398 .valid_start_item = 1,
399 .prev_items = (const enum rte_flow_item_type[]) {
400 RTE_FLOW_ITEM_TYPE_ETH,
401 RTE_FLOW_ITEM_TYPE_UDP,
402 RTE_FLOW_ITEM_TYPE_END,
408 enic_fm_copy_item_eth(struct copy_item_args *arg)
410 const struct rte_flow_item *item = arg->item;
411 const struct rte_flow_item_eth *spec = item->spec;
412 const struct rte_flow_item_eth *mask = item->mask;
413 const uint8_t lvl = arg->header_level;
414 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
415 struct fm_header_set *fm_data, *fm_mask;
417 ENICPMD_FUNC_TRACE();
418 /* Match all if no spec */
422 mask = &rte_flow_item_eth_mask;
423 fm_data = &entry->ftm_data.fk_hdrset[lvl];
424 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
425 fm_data->fk_header_select |= FKH_ETHER;
426 fm_mask->fk_header_select |= FKH_ETHER;
427 memcpy(&fm_data->l2.eth, spec, sizeof(struct rte_ether_hdr));
428 memcpy(&fm_mask->l2.eth, mask, sizeof(struct rte_ether_hdr));
433 enic_fm_copy_item_vlan(struct copy_item_args *arg)
435 const struct rte_flow_item *item = arg->item;
436 const struct rte_flow_item_vlan *spec = item->spec;
437 const struct rte_flow_item_vlan *mask = item->mask;
438 const uint8_t lvl = arg->header_level;
439 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
440 struct fm_header_set *fm_data, *fm_mask;
441 struct rte_ether_hdr *eth_mask;
442 struct rte_ether_hdr *eth_val;
445 ENICPMD_FUNC_TRACE();
446 fm_data = &entry->ftm_data.fk_hdrset[lvl];
447 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
448 /* Outer and inner packet vlans need different flags */
449 meta = FKM_VLAN_PRES;
452 fm_data->fk_metadata |= meta;
453 fm_mask->fk_metadata |= meta;
455 /* Match all if no spec */
459 mask = &rte_flow_item_vlan_mask;
461 eth_mask = (void *)&fm_mask->l2.eth;
462 eth_val = (void *)&fm_data->l2.eth;
465 * Outer TPID cannot be matched. If inner_type is 0, use what is
468 if (eth_mask->ether_type && mask->inner_type)
472 * When packet matching, the VIC always compares vlan-stripped
473 * L2, regardless of vlan stripping settings. So, the inner type
474 * from vlan becomes the ether type of the eth header.
476 if (mask->inner_type) {
477 eth_mask->ether_type = mask->inner_type;
478 eth_val->ether_type = spec->inner_type;
480 fm_data->fk_header_select |= FKH_ETHER | FKH_QTAG;
481 fm_mask->fk_header_select |= FKH_ETHER | FKH_QTAG;
482 fm_data->fk_vlan = rte_be_to_cpu_16(spec->tci);
483 fm_mask->fk_vlan = rte_be_to_cpu_16(mask->tci);
488 enic_fm_copy_item_ipv4(struct copy_item_args *arg)
490 const struct rte_flow_item *item = arg->item;
491 const struct rte_flow_item_ipv4 *spec = item->spec;
492 const struct rte_flow_item_ipv4 *mask = item->mask;
493 const uint8_t lvl = arg->header_level;
494 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
495 struct fm_header_set *fm_data, *fm_mask;
497 ENICPMD_FUNC_TRACE();
498 fm_data = &entry->ftm_data.fk_hdrset[lvl];
499 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
500 fm_data->fk_metadata |= FKM_IPV4;
501 fm_mask->fk_metadata |= FKM_IPV4;
506 mask = &rte_flow_item_ipv4_mask;
508 fm_data->fk_header_select |= FKH_IPV4;
509 fm_mask->fk_header_select |= FKH_IPV4;
510 memcpy(&fm_data->l3.ip4, spec, sizeof(*spec));
511 memcpy(&fm_mask->l3.ip4, mask, sizeof(*mask));
516 enic_fm_copy_item_ipv6(struct copy_item_args *arg)
518 const struct rte_flow_item *item = arg->item;
519 const struct rte_flow_item_ipv6 *spec = item->spec;
520 const struct rte_flow_item_ipv6 *mask = item->mask;
521 const uint8_t lvl = arg->header_level;
522 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
523 struct fm_header_set *fm_data, *fm_mask;
525 ENICPMD_FUNC_TRACE();
526 fm_data = &entry->ftm_data.fk_hdrset[lvl];
527 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
528 fm_data->fk_metadata |= FKM_IPV6;
529 fm_mask->fk_metadata |= FKM_IPV6;
534 mask = &rte_flow_item_ipv6_mask;
536 fm_data->fk_header_select |= FKH_IPV6;
537 fm_mask->fk_header_select |= FKH_IPV6;
538 memcpy(&fm_data->l3.ip6, spec, sizeof(struct rte_ipv6_hdr));
539 memcpy(&fm_mask->l3.ip6, mask, sizeof(struct rte_ipv6_hdr));
544 enic_fm_copy_item_udp(struct copy_item_args *arg)
546 const struct rte_flow_item *item = arg->item;
547 const struct rte_flow_item_udp *spec = item->spec;
548 const struct rte_flow_item_udp *mask = item->mask;
549 const uint8_t lvl = arg->header_level;
550 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
551 struct fm_header_set *fm_data, *fm_mask;
553 ENICPMD_FUNC_TRACE();
554 fm_data = &entry->ftm_data.fk_hdrset[lvl];
555 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
556 fm_data->fk_metadata |= FKM_UDP;
557 fm_mask->fk_metadata |= FKM_UDP;
562 mask = &rte_flow_item_udp_mask;
564 fm_data->fk_header_select |= FKH_UDP;
565 fm_mask->fk_header_select |= FKH_UDP;
566 memcpy(&fm_data->l4.udp, spec, sizeof(*spec));
567 memcpy(&fm_mask->l4.udp, mask, sizeof(*mask));
572 enic_fm_copy_item_tcp(struct copy_item_args *arg)
574 const struct rte_flow_item *item = arg->item;
575 const struct rte_flow_item_tcp *spec = item->spec;
576 const struct rte_flow_item_tcp *mask = item->mask;
577 const uint8_t lvl = arg->header_level;
578 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
579 struct fm_header_set *fm_data, *fm_mask;
581 ENICPMD_FUNC_TRACE();
582 fm_data = &entry->ftm_data.fk_hdrset[lvl];
583 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
584 fm_data->fk_metadata |= FKM_TCP;
585 fm_mask->fk_metadata |= FKM_TCP;
590 mask = &rte_flow_item_tcp_mask;
592 fm_data->fk_header_select |= FKH_TCP;
593 fm_mask->fk_header_select |= FKH_TCP;
594 memcpy(&fm_data->l4.tcp, spec, sizeof(*spec));
595 memcpy(&fm_mask->l4.tcp, mask, sizeof(*mask));
600 enic_fm_copy_item_sctp(struct copy_item_args *arg)
602 const struct rte_flow_item *item = arg->item;
603 const struct rte_flow_item_sctp *spec = item->spec;
604 const struct rte_flow_item_sctp *mask = item->mask;
605 const uint8_t lvl = arg->header_level;
606 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
607 struct fm_header_set *fm_data, *fm_mask;
608 uint8_t *ip_proto_mask = NULL;
609 uint8_t *ip_proto = NULL;
612 ENICPMD_FUNC_TRACE();
613 fm_data = &entry->ftm_data.fk_hdrset[lvl];
614 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
616 * The NIC filter API has no flags for "match sctp", so explicitly
617 * set the protocol number in the IP pattern.
619 if (fm_data->fk_metadata & FKM_IPV4) {
620 struct rte_ipv4_hdr *ip;
621 ip = (struct rte_ipv4_hdr *)&fm_mask->l3.ip4;
622 ip_proto_mask = &ip->next_proto_id;
623 ip = (struct rte_ipv4_hdr *)&fm_data->l3.ip4;
624 ip_proto = &ip->next_proto_id;
626 } else if (fm_data->fk_metadata & FKM_IPV6) {
627 struct rte_ipv6_hdr *ip;
628 ip = (struct rte_ipv6_hdr *)&fm_mask->l3.ip6;
629 ip_proto_mask = &ip->proto;
630 ip = (struct rte_ipv6_hdr *)&fm_data->l3.ip6;
631 ip_proto = &ip->proto;
634 /* Need IPv4/IPv6 pattern first */
637 *ip_proto = IPPROTO_SCTP;
638 *ip_proto_mask = 0xff;
639 fm_data->fk_header_select |= l3_fkh;
640 fm_mask->fk_header_select |= l3_fkh;
645 mask = &rte_flow_item_sctp_mask;
647 fm_data->fk_header_select |= FKH_L4RAW;
648 fm_mask->fk_header_select |= FKH_L4RAW;
649 memcpy(fm_data->l4.rawdata, spec, sizeof(*spec));
650 memcpy(fm_mask->l4.rawdata, mask, sizeof(*mask));
655 enic_fm_copy_item_vxlan(struct copy_item_args *arg)
657 const struct rte_flow_item *item = arg->item;
658 const struct rte_flow_item_vxlan *spec = item->spec;
659 const struct rte_flow_item_vxlan *mask = item->mask;
660 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
661 struct fm_header_set *fm_data, *fm_mask;
663 ENICPMD_FUNC_TRACE();
664 /* Only 2 header levels (outer and inner) allowed */
665 if (arg->header_level > 0)
668 fm_data = &entry->ftm_data.fk_hdrset[0];
669 fm_mask = &entry->ftm_mask.fk_hdrset[0];
670 fm_data->fk_metadata |= FKM_VXLAN;
671 fm_mask->fk_metadata |= FKM_VXLAN;
672 /* items from here on out are inner header items */
673 arg->header_level = 1;
675 /* Match all if no spec */
679 mask = &rte_flow_item_vxlan_mask;
681 fm_data->fk_header_select |= FKH_VXLAN;
682 fm_mask->fk_header_select |= FKH_VXLAN;
683 memcpy(&fm_data->vxlan, spec, sizeof(*spec));
684 memcpy(&fm_mask->vxlan, mask, sizeof(*mask));
689 enic_fm_copy_item_gtp(struct copy_item_args *arg)
691 const struct rte_flow_item *item = arg->item;
692 const struct rte_flow_item_gtp *spec = item->spec;
693 const struct rte_flow_item_gtp *mask = item->mask;
694 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
695 struct fm_header_set *fm_data, *fm_mask;
697 uint16_t udp_gtp_uc_port_be = 0;
699 ENICPMD_FUNC_TRACE();
700 /* Only 2 header levels (outer and inner) allowed */
701 if (arg->header_level > 0)
704 fm_data = &entry->ftm_data.fk_hdrset[0];
705 fm_mask = &entry->ftm_mask.fk_hdrset[0];
707 switch (item->type) {
708 case RTE_FLOW_ITEM_TYPE_GTP:
710 /* For vanilla GTP, the UDP destination port must be specified
711 * but value of the port is not enforced here.
713 if (!(fm_data->fk_metadata & FKM_UDP) ||
714 !(fm_data->fk_header_select & FKH_UDP) ||
715 fm_data->l4.udp.fk_dest == 0)
717 if (!(fm_mask->fk_metadata & FKM_UDP) ||
718 !(fm_mask->fk_header_select & FKH_UDP) ||
719 fm_mask->l4.udp.fk_dest != 0xFFFF)
723 case RTE_FLOW_ITEM_TYPE_GTPC:
725 udp_gtp_uc_port_be = rte_cpu_to_be_16(RTE_GTPC_UDP_PORT);
728 case RTE_FLOW_ITEM_TYPE_GTPU:
730 udp_gtp_uc_port_be = rte_cpu_to_be_16(RTE_GTPU_UDP_PORT);
737 /* The GTP-C or GTP-U UDP destination port must be matched. */
738 if (udp_gtp_uc_port_be) {
739 if (fm_data->fk_metadata & FKM_UDP &&
740 fm_data->fk_header_select & FKH_UDP &&
741 fm_data->l4.udp.fk_dest != udp_gtp_uc_port_be)
743 if (fm_mask->fk_metadata & FKM_UDP &&
744 fm_mask->fk_header_select & FKH_UDP &&
745 fm_mask->l4.udp.fk_dest != 0xFFFF)
748 /* In any case, add match for GTP-C GTP-U UDP dst port */
749 fm_data->fk_metadata |= FKM_UDP;
750 fm_data->fk_header_select |= FKH_UDP;
751 fm_data->l4.udp.fk_dest = udp_gtp_uc_port_be;
752 fm_mask->fk_metadata |= FKM_UDP;
753 fm_mask->fk_header_select |= FKH_UDP;
754 fm_mask->l4.udp.fk_dest = 0xFFFF;
757 /* NIC does not support GTP tunnels. No Items are allowed after this.
758 * This prevents the specification of further items.
760 arg->header_level = 0;
762 /* Match all if no spec */
766 mask = &rte_flow_item_gtp_mask;
769 * Use the raw L4 buffer to match GTP as fm_header_set does not have
770 * GTP header. UDP dst port must be specific. Using the raw buffer
771 * does not affect such UDP item, since we skip UDP in the raw buffer.
773 fm_data->fk_header_select |= FKH_L4RAW;
774 fm_mask->fk_header_select |= FKH_L4RAW;
775 off = sizeof(fm_data->l4.udp);
776 memcpy(&fm_data->l4.rawdata[off], spec, sizeof(*spec));
777 memcpy(&fm_mask->l4.rawdata[off], mask, sizeof(*mask));
782 enic_fm_copy_item_geneve(struct copy_item_args *arg)
784 const struct rte_flow_item *item = arg->item;
785 const struct rte_flow_item_geneve *spec = item->spec;
786 const struct rte_flow_item_geneve *mask = item->mask;
787 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
788 struct fm_header_set *fm_data, *fm_mask;
791 ENICPMD_FUNC_TRACE();
792 /* Only 2 header levels (outer and inner) allowed */
793 if (arg->header_level > 0)
796 fm_data = &entry->ftm_data.fk_hdrset[0];
797 fm_mask = &entry->ftm_mask.fk_hdrset[0];
798 fm_data->fk_metadata |= FKM_GENEVE;
799 fm_mask->fk_metadata |= FKM_GENEVE;
800 /* items from here on out are inner header items, except options */
801 arg->header_level = 1;
803 /* Match all if no spec */
807 mask = &rte_flow_item_geneve_mask;
810 * Use the raw L4 buffer to match geneve as fm_header_set does
811 * not have geneve header. A UDP item may precede the geneve
812 * item. Using the raw buffer does not affect such UDP item,
813 * since we skip UDP in the raw buffer.
815 fm_data->fk_header_select |= FKH_L4RAW;
816 fm_mask->fk_header_select |= FKH_L4RAW;
817 off = sizeof(fm_data->l4.udp);
818 memcpy(&fm_data->l4.rawdata[off], spec, sizeof(struct rte_geneve_hdr));
819 memcpy(&fm_mask->l4.rawdata[off], mask, sizeof(struct rte_geneve_hdr));
824 enic_fm_copy_item_geneve_opt(struct copy_item_args *arg)
826 const struct rte_flow_item *item = arg->item;
827 const struct rte_flow_item_geneve_opt *spec = item->spec;
828 const struct rte_flow_item_geneve_opt *mask = item->mask;
829 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
830 struct fm_header_set *fm_data, *fm_mask;
831 struct rte_geneve_hdr *geneve;
834 ENICPMD_FUNC_TRACE();
835 fm_data = &entry->ftm_data.fk_hdrset[0];
836 fm_mask = &entry->ftm_mask.fk_hdrset[0];
837 /* Match all if no spec */
841 mask = &rte_flow_item_geneve_opt_mask;
843 if (spec->option_len > 0 &&
844 (spec->data == NULL || mask->data == NULL)) {
845 return rte_flow_error_set(arg->error, EINVAL,
846 RTE_FLOW_ERROR_TYPE_ITEM,
847 NULL, "enic: geneve_opt unexpected null data");
850 * Geneve item must already be in the raw buffer. Append the
851 * option pattern to it. There are two limitations.
852 * (1) Can match only the 1st option, the first one following Geneve
853 * (2) Geneve header must specify option length, as HW does not
854 * have "has Geneve option" flag.
856 RTE_ASSERT((fm_data->fk_header_select & FKH_L4RAW) != 0);
857 RTE_ASSERT((fm_mask->fk_header_select & FKH_L4RAW) != 0);
858 off = sizeof(fm_data->l4.udp);
859 geneve = (struct rte_geneve_hdr *)&fm_data->l4.rawdata[off];
860 if (geneve->opt_len == 0) {
861 return rte_flow_error_set(arg->error, EINVAL,
862 RTE_FLOW_ERROR_TYPE_ITEM,
863 NULL, "enic: geneve_opt requires non-zero geneve option length");
865 geneve = (struct rte_geneve_hdr *)&fm_mask->l4.rawdata[off];
866 if (geneve->opt_len == 0) {
867 return rte_flow_error_set(arg->error, EINVAL,
868 RTE_FLOW_ERROR_TYPE_ITEM,
869 NULL, "enic: geneve_opt requires non-zero geneve option length mask");
871 off = sizeof(fm_data->l4.udp) + sizeof(struct rte_geneve_hdr);
872 if (off + (spec->option_len + 1) * 4 > FM_LAYER_SIZE) {
873 return rte_flow_error_set(arg->error, EINVAL,
874 RTE_FLOW_ERROR_TYPE_ITEM,
875 NULL, "enic: geneve_opt too large");
877 /* Copy option header */
878 memcpy(&fm_data->l4.rawdata[off], spec, 4);
879 memcpy(&fm_mask->l4.rawdata[off], mask, 4);
880 /* Copy option data */
881 if (spec->option_len > 0) {
883 len = spec->option_len * 4;
884 memcpy(&fm_data->l4.rawdata[off], spec->data, len);
885 memcpy(&fm_mask->l4.rawdata[off], mask->data, len);
890 /* Match eCPRI combined message header */
892 enic_fm_copy_item_ecpri(struct copy_item_args *arg)
894 const struct rte_flow_item *item = arg->item;
895 const struct rte_flow_item_ecpri *spec = item->spec;
896 const struct rte_flow_item_ecpri *mask = item->mask;
897 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
898 struct fm_header_set *fm_data, *fm_mask;
899 uint8_t *fm_data_to, *fm_mask_to;
901 ENICPMD_FUNC_TRACE();
903 /* Tunneling not supported- only matching on inner eCPRI fields. */
904 if (arg->header_level > 0)
907 /* Need both spec and mask */
911 fm_data = &entry->ftm_data.fk_hdrset[0];
912 fm_mask = &entry->ftm_mask.fk_hdrset[0];
914 /* eCPRI can only follow L2/VLAN layer if ethernet type is 0xAEFE. */
915 if (!(fm_data->fk_metadata & FKM_UDP) &&
916 (fm_mask->l2.eth.fk_ethtype != UINT16_MAX ||
917 rte_cpu_to_be_16(fm_data->l2.eth.fk_ethtype) !=
918 RTE_ETHER_TYPE_ECPRI))
921 if (fm_data->fk_metadata & FKM_UDP) {
923 fm_data->fk_header_select |= FKH_L4RAW;
924 fm_mask->fk_header_select |= FKH_L4RAW;
925 fm_data_to = &fm_data->l4.rawdata[sizeof(fm_data->l4.udp)];
926 fm_mask_to = &fm_mask->l4.rawdata[sizeof(fm_data->l4.udp)];
928 /* eCPRI directly after Etherent header */
929 fm_data->fk_header_select |= FKH_L3RAW;
930 fm_mask->fk_header_select |= FKH_L3RAW;
931 fm_data_to = &fm_data->l3.rawdata[0];
932 fm_mask_to = &fm_mask->l3.rawdata[0];
936 * Use the raw L3 or L4 buffer to match eCPRI since fm_header_set does
937 * not have eCPRI header. Only 1st message header of PDU can be matched.
940 memcpy(fm_data_to, spec, sizeof(*spec));
941 memcpy(fm_mask_to, mask, sizeof(*mask));
946 * Currently, raw pattern match is very limited. It is intended for matching
947 * UDP tunnel header (e.g. vxlan or geneve).
950 enic_fm_copy_item_raw(struct copy_item_args *arg)
952 const struct rte_flow_item *item = arg->item;
953 const struct rte_flow_item_raw *spec = item->spec;
954 const struct rte_flow_item_raw *mask = item->mask;
955 const uint8_t lvl = arg->header_level;
956 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
957 struct fm_header_set *fm_data, *fm_mask;
959 ENICPMD_FUNC_TRACE();
960 /* Cannot be used for inner packet */
963 /* Need both spec and mask */
966 /* Only supports relative with offset 0 */
967 if (!spec->relative || spec->offset != 0 || spec->search ||
970 /* Need non-null pattern that fits within the NIC's filter pattern */
971 if (spec->length == 0 ||
972 spec->length + sizeof(struct rte_udp_hdr) > FM_LAYER_SIZE ||
973 !spec->pattern || !mask->pattern)
976 * Mask fields, including length, are often set to zero. Assume that
977 * means "same as spec" to avoid breaking existing apps. If length
978 * is not zero, then it should be >= spec length.
980 * No more pattern follows this, so append to the L4 layer instead of
981 * L5 to work with both recent and older VICs.
983 if (mask->length != 0 && mask->length < spec->length)
986 fm_data = &entry->ftm_data.fk_hdrset[lvl];
987 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
988 fm_data->fk_header_select |= FKH_L4RAW;
989 fm_mask->fk_header_select |= FKH_L4RAW;
990 fm_data->fk_header_select &= ~FKH_UDP;
991 fm_mask->fk_header_select &= ~FKH_UDP;
992 memcpy(fm_data->l4.rawdata + sizeof(struct rte_udp_hdr),
993 spec->pattern, spec->length);
994 memcpy(fm_mask->l4.rawdata + sizeof(struct rte_udp_hdr),
995 mask->pattern, spec->length);
1000 flowman_cmd(struct enic_flowman *fm, uint64_t *args, int nargs)
1002 return vnic_dev_flowman_cmd(fm->owner_enic->vdev, args, nargs);
1006 enic_fet_alloc(struct enic_flowman *fm, uint8_t ingress,
1007 struct fm_key_template *key, int entries,
1008 struct enic_fm_fet **fet_out)
1010 struct fm_exact_match_table *cmd;
1011 struct fm_header_set *hdr;
1012 struct enic_fm_fet *fet;
1016 ENICPMD_FUNC_TRACE();
1017 fet = calloc(1, sizeof(struct enic_fm_fet));
1020 cmd = &fm->cmd.va->fm_exact_match_table;
1021 memset(cmd, 0, sizeof(*cmd));
1022 cmd->fet_direction = ingress ? FM_INGRESS : FM_EGRESS;
1023 cmd->fet_stage = FM_STAGE_LAST;
1024 cmd->fet_max_entries = entries ? entries : FM_MAX_EXACT_TABLE_SIZE;
1026 hdr = &cmd->fet_key.fk_hdrset[0];
1027 memset(hdr, 0, sizeof(*hdr));
1028 hdr->fk_header_select = FKH_IPV4 | FKH_UDP;
1029 hdr->l3.ip4.fk_saddr = 0xFFFFFFFF;
1030 hdr->l3.ip4.fk_daddr = 0xFFFFFFFF;
1031 hdr->l4.udp.fk_source = 0xFFFF;
1032 hdr->l4.udp.fk_dest = 0xFFFF;
1033 fet->default_key = 1;
1035 memcpy(&cmd->fet_key, key, sizeof(*key));
1036 memcpy(&fet->key, key, sizeof(*key));
1037 fet->default_key = 0;
1039 cmd->fet_key.fk_packet_tag = 1;
1041 args[0] = FM_EXACT_TABLE_ALLOC;
1042 args[1] = fm->cmd.pa;
1043 ret = flowman_cmd(fm, args, 2);
1045 ENICPMD_LOG(ERR, "cannot alloc exact match table: rc=%d", ret);
1049 fet->handle = args[0];
1050 fet->ingress = ingress;
1051 ENICPMD_LOG(DEBUG, "allocated exact match table: handle=0x%" PRIx64,
1058 enic_fet_free(struct enic_flowman *fm, struct enic_fm_fet *fet)
1060 ENICPMD_FUNC_TRACE();
1061 enic_fm_tbl_free(fm, fet->handle);
1062 if (!fet->default_key)
1063 TAILQ_REMOVE(&fm->fet_list, fet, list);
1068 * Get the exact match table for the given combination of
1069 * <group, ingress, key>. Allocate one on the fly as necessary.
1072 enic_fet_get(struct enic_flowman *fm,
1075 struct fm_key_template *key,
1076 struct enic_fm_fet **fet_out,
1077 struct rte_flow_error *error)
1079 struct enic_fm_fet *fet;
1081 ENICPMD_FUNC_TRACE();
1082 /* See if we already have this table open */
1083 TAILQ_FOREACH(fet, &fm->fet_list, list) {
1084 if (fet->group == group && fet->ingress == ingress)
1088 /* Jumping to a non-existing group? Use the default table */
1090 fet = ingress ? fm->default_ig_fet : fm->default_eg_fet;
1091 } else if (enic_fet_alloc(fm, ingress, key, 0, &fet)) {
1092 return rte_flow_error_set(error, EINVAL,
1093 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1094 NULL, "enic: cannot get exact match table");
1097 /* Default table is never on the open table list */
1098 if (!fet->default_key)
1099 TAILQ_INSERT_HEAD(&fm->fet_list, fet, list);
1103 ENICPMD_LOG(DEBUG, "fet_get: %s %s group=%u ref=%u",
1104 fet->default_key ? "default" : "",
1105 fet->ingress ? "ingress" : "egress",
1106 fet->group, fet->ref);
1111 enic_fet_put(struct enic_flowman *fm, struct enic_fm_fet *fet)
1113 ENICPMD_FUNC_TRACE();
1114 RTE_ASSERT(fet->ref > 0);
1116 ENICPMD_LOG(DEBUG, "fet_put: %s %s group=%u ref=%u",
1117 fet->default_key ? "default" : "",
1118 fet->ingress ? "ingress" : "egress",
1119 fet->group, fet->ref);
1121 enic_fet_free(fm, fet);
1124 /* Return 1 if current item is valid on top of the previous one. */
1126 fm_item_stacking_valid(enum rte_flow_item_type prev_item,
1127 const struct enic_fm_items *item_info,
1128 uint8_t is_first_item)
1130 enum rte_flow_item_type const *allowed_items = item_info->prev_items;
1132 ENICPMD_FUNC_TRACE();
1133 for (; *allowed_items != RTE_FLOW_ITEM_TYPE_END; allowed_items++) {
1134 if (prev_item == *allowed_items)
1138 /* This is the first item in the stack. Check if that's cool */
1139 if (is_first_item && item_info->valid_start_item)
1145 * Build the flow manager match entry structure from the provided pattern.
1146 * The pattern is validated as the items are copied.
1149 enic_fm_copy_entry(struct enic_flowman *fm,
1150 const struct rte_flow_item pattern[],
1151 struct rte_flow_error *error)
1153 const struct enic_fm_items *item_info;
1154 enum rte_flow_item_type prev_item;
1155 const struct rte_flow_item *item;
1156 struct copy_item_args args;
1157 uint8_t prev_header_level;
1158 uint8_t is_first_item;
1161 ENICPMD_FUNC_TRACE();
1164 prev_item = RTE_FLOW_ITEM_TYPE_END;
1166 args.fm_tcam_entry = &fm->tcam_entry;
1167 args.header_level = 0;
1168 prev_header_level = 0;
1169 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
1171 * Get info about how to validate and copy the item. If NULL
1172 * is returned the nic does not support the item.
1174 if (item->type == RTE_FLOW_ITEM_TYPE_VOID)
1177 item_info = &enic_fm_items[item->type];
1179 if (item->type >= RTE_DIM(enic_fm_items) ||
1180 item_info->copy_item == NULL) {
1181 return rte_flow_error_set(error, ENOTSUP,
1182 RTE_FLOW_ERROR_TYPE_ITEM,
1183 NULL, "enic: unsupported item");
1186 * Check vNIC feature dependencies. Geneve item needs
1187 * Geneve offload feature
1189 if (item->type == RTE_FLOW_ITEM_TYPE_GENEVE &&
1190 !fm->user_enic->geneve) {
1191 return rte_flow_error_set(error, ENOTSUP,
1192 RTE_FLOW_ERROR_TYPE_ITEM,
1193 NULL, "enic: geneve not supported");
1195 /* check to see if item stacking is valid */
1196 if (!fm_item_stacking_valid(prev_item, item_info,
1198 goto stacking_error;
1203 error->type = RTE_FLOW_ERROR_TYPE_NONE;
1204 ret = item_info->copy_item(&args);
1206 /* If copy_item set the error, return that */
1207 if (error->type != RTE_FLOW_ERROR_TYPE_NONE)
1209 goto item_not_supported;
1211 /* Going from outer to inner? Treat it as a new packet start */
1212 if (prev_header_level != args.header_level) {
1213 prev_item = RTE_FLOW_ITEM_TYPE_END;
1216 prev_item = item->type;
1219 prev_header_level = args.header_level;
1224 return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,
1225 NULL, "enic: unsupported item type");
1228 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
1229 item, "enic: unsupported item stack");
1233 flow_item_skip_void(const struct rte_flow_item **item)
1235 for ( ; ; (*item)++)
1236 if ((*item)->type != RTE_FLOW_ITEM_TYPE_VOID)
1241 append_template(void **template, uint8_t *off, const void *data, int len)
1243 memcpy(*template, data, len);
1244 *template = (char *)*template + len;
1249 enic_fm_append_action_op(struct enic_flowman *fm,
1250 struct fm_action_op *fm_op,
1251 struct rte_flow_error *error)
1255 count = fm->action_op_count;
1256 ENICPMD_LOG(DEBUG, "append action op: idx=%d op=%u",
1257 count, fm_op->fa_op);
1258 if (count == FM_ACTION_OP_MAX) {
1259 return rte_flow_error_set(error, EINVAL,
1260 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1261 "too many action operations");
1263 fm->action.fma_action_ops[count] = *fm_op;
1264 fm->action_op_count = count + 1;
1268 static struct fm_action_op *
1269 find_prev_action_op(struct enic_flowman *fm, uint32_t opcode)
1271 struct fm_action_op *op;
1274 for (i = 0; i < fm->action_op_count; i++) {
1275 op = &fm->action.fma_action_ops[i];
1276 if (op->fa_op == opcode)
1282 /* NIC requires that 1st steer appear before decap.
1283 * Correct example: steer, decap, steer, steer, ...
1286 enic_fm_reorder_action_op(struct enic_flowman *fm)
1288 struct fm_action_op *op, *steer, *decap;
1289 struct fm_action_op tmp_op;
1291 ENICPMD_FUNC_TRACE();
1292 /* Find 1st steer and decap */
1293 op = fm->action.fma_action_ops;
1296 while (op->fa_op != FMOP_END) {
1297 if (!decap && (op->fa_op == FMOP_DECAP_NOSTRIP ||
1298 op->fa_op == FMOP_DECAP_STRIP))
1300 else if (!steer && op->fa_op == FMOP_RQ_STEER)
1304 /* If decap is before steer, swap */
1305 if (steer && decap && decap < steer) {
1306 op = fm->action.fma_action_ops;
1307 ENICPMD_LOG(DEBUG, "swap decap %ld <-> steer %ld",
1308 (long)(decap - op), (long)(steer - op));
1315 /* VXLAN decap is done via flowman compound action */
1317 enic_fm_copy_vxlan_decap(struct enic_flowman *fm,
1318 struct fm_tcam_match_entry *fmt,
1319 const struct rte_flow_action *action,
1320 struct rte_flow_error *error)
1322 struct fm_header_set *fm_data;
1323 struct fm_action_op fm_op;
1325 ENICPMD_FUNC_TRACE();
1326 fm_data = &fmt->ftm_data.fk_hdrset[0];
1327 if (!(fm_data->fk_metadata & FKM_VXLAN)) {
1328 return rte_flow_error_set(error, EINVAL,
1329 RTE_FLOW_ERROR_TYPE_ACTION, action,
1330 "vxlan-decap: vxlan must be in pattern");
1333 memset(&fm_op, 0, sizeof(fm_op));
1334 fm_op.fa_op = FMOP_DECAP_NOSTRIP;
1335 return enic_fm_append_action_op(fm, &fm_op, error);
1338 /* Generate a reasonable source port number */
1342 /* Min/max below are the default values in OVS-DPDK and Linux */
1343 uint16_t p = rte_rand();
1344 p = RTE_MAX(p, 32768);
1345 p = RTE_MIN(p, 61000);
1346 return rte_cpu_to_be_16(p);
1349 /* VXLAN encap is done via flowman compound action */
1351 enic_fm_copy_vxlan_encap(struct enic_flowman *fm,
1352 const struct rte_flow_item *item,
1353 struct rte_flow_error *error)
1355 struct fm_action_op fm_op;
1356 struct rte_ether_hdr *eth;
1357 struct rte_udp_hdr *udp;
1358 uint16_t *ethertype;
1362 ENICPMD_FUNC_TRACE();
1363 memset(&fm_op, 0, sizeof(fm_op));
1364 fm_op.fa_op = FMOP_ENCAP;
1365 template = fm->action.fma_data;
1368 * Copy flow items to the flowman template starting L2.
1369 * L2 must be ethernet.
1371 flow_item_skip_void(&item);
1372 if (item->type != RTE_FLOW_ITEM_TYPE_ETH)
1373 return rte_flow_error_set(error, EINVAL,
1374 RTE_FLOW_ERROR_TYPE_ITEM, item,
1375 "vxlan-encap: first item should be ethernet");
1376 eth = (struct rte_ether_hdr *)template;
1377 ethertype = ð->ether_type;
1378 append_template(&template, &off, item->spec,
1379 sizeof(struct rte_ether_hdr));
1381 flow_item_skip_void(&item);
1383 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
1384 const struct rte_flow_item_vlan *spec;
1386 ENICPMD_LOG(DEBUG, "vxlan-encap: vlan");
1388 fm_op.encap.outer_vlan = rte_be_to_cpu_16(spec->tci);
1390 flow_item_skip_void(&item);
1392 /* L3 must be IPv4, IPv6 */
1393 switch (item->type) {
1394 case RTE_FLOW_ITEM_TYPE_IPV4:
1396 struct rte_ipv4_hdr *ip4;
1398 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv4");
1399 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
1400 ip4 = (struct rte_ipv4_hdr *)template;
1402 * Offset of IPv4 length field and its initial value
1403 * (IP + UDP + VXLAN) are specified in the action. The NIC
1404 * will add inner packet length.
1406 fm_op.encap.len1_offset = off +
1407 offsetof(struct rte_ipv4_hdr, total_length);
1408 fm_op.encap.len1_delta = sizeof(struct rte_ipv4_hdr) +
1409 sizeof(struct rte_udp_hdr) +
1410 sizeof(struct rte_vxlan_hdr);
1411 append_template(&template, &off, item->spec,
1412 sizeof(struct rte_ipv4_hdr));
1413 ip4->version_ihl = RTE_IPV4_VHL_DEF;
1414 if (ip4->time_to_live == 0)
1415 ip4->time_to_live = IP_DEFTTL;
1416 ip4->next_proto_id = IPPROTO_UDP;
1419 case RTE_FLOW_ITEM_TYPE_IPV6:
1421 struct rte_ipv6_hdr *ip6;
1423 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv6");
1424 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
1425 ip6 = (struct rte_ipv6_hdr *)template;
1426 fm_op.encap.len1_offset = off +
1427 offsetof(struct rte_ipv6_hdr, payload_len);
1428 fm_op.encap.len1_delta = sizeof(struct rte_udp_hdr) +
1429 sizeof(struct rte_vxlan_hdr);
1430 append_template(&template, &off, item->spec,
1431 sizeof(struct rte_ipv6_hdr));
1432 ip6->vtc_flow |= rte_cpu_to_be_32(IP6_VTC_FLOW);
1433 if (ip6->hop_limits == 0)
1434 ip6->hop_limits = IP_DEFTTL;
1435 ip6->proto = IPPROTO_UDP;
1439 return rte_flow_error_set(error,
1440 EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1441 "vxlan-encap: L3 must be IPv4/IPv6");
1444 flow_item_skip_void(&item);
1447 if (item->type != RTE_FLOW_ITEM_TYPE_UDP)
1448 return rte_flow_error_set(error, EINVAL,
1449 RTE_FLOW_ERROR_TYPE_ITEM, item,
1450 "vxlan-encap: UDP must follow IPv4/IPv6");
1451 /* UDP length = UDP + VXLAN. NIC will add inner packet length. */
1452 fm_op.encap.len2_offset =
1453 off + offsetof(struct rte_udp_hdr, dgram_len);
1454 fm_op.encap.len2_delta =
1455 sizeof(struct rte_udp_hdr) + sizeof(struct rte_vxlan_hdr);
1456 udp = (struct rte_udp_hdr *)template;
1457 append_template(&template, &off, item->spec,
1458 sizeof(struct rte_udp_hdr));
1460 * Firmware does not hash/fill source port yet. Generate a
1461 * random port, as there is *usually* one rte_flow for the
1462 * given inner packet stream (i.e. a single stream has one
1465 if (udp->src_port == 0)
1466 udp->src_port = gen_src_port();
1468 flow_item_skip_void(&item);
1471 if (item->type != RTE_FLOW_ITEM_TYPE_VXLAN)
1472 return rte_flow_error_set(error,
1473 EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1474 "vxlan-encap: VXLAN must follow UDP");
1475 append_template(&template, &off, item->spec,
1476 sizeof(struct rte_flow_item_vxlan));
1479 * Fill in the rest of the action structure.
1480 * Indicate that we want to encap with vxlan at packet start.
1482 fm_op.encap.template_offset = 0;
1483 fm_op.encap.template_len = off;
1484 return enic_fm_append_action_op(fm, &fm_op, error);
1488 enic_fm_find_vnic(struct enic *enic, const struct rte_pci_addr *addr,
1495 ENICPMD_FUNC_TRACE();
1496 ENICPMD_LOG(DEBUG, "bdf=%x:%x:%x", addr->bus, addr->devid,
1498 bdf = addr->bus << 8 | addr->devid << 3 | addr->function;
1499 args[0] = FM_VNIC_FIND;
1501 rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
1503 /* Expected to fail if BDF is not on the adapter */
1504 ENICPMD_LOG(DEBUG, "cannot find vnic handle: rc=%d", rc);
1508 ENICPMD_LOG(DEBUG, "found vnic: handle=0x%" PRIx64, *handle);
1513 * Egress: target port should be either PF uplink or VF.
1515 * 1. VF egress -> PF uplink
1516 * PF may be this VF's PF, or another PF, as long as they are on the same VIC.
1517 * 2. VF egress -> VF
1520 * 1. PF egress -> VF
1521 * App should be using representor to pass packets to VF
1524 vf_egress_port_id_action(struct enic_flowman *fm,
1525 struct rte_eth_dev *dst_dev,
1526 uint64_t dst_vnic_h,
1527 struct fm_action_op *fm_op,
1528 struct rte_flow_error *error)
1530 struct enic *src_enic, *dst_enic;
1531 struct enic_vf_representor *vf;
1535 ENICPMD_FUNC_TRACE();
1536 src_enic = fm->user_enic;
1537 dst_enic = pmd_priv(dst_dev);
1538 if (!(src_enic->rte_dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)) {
1539 return rte_flow_error_set(error, EINVAL,
1540 RTE_FLOW_ERROR_TYPE_ACTION,
1541 NULL, "source port is not VF representor");
1544 /* VF -> PF uplink. dst is not VF representor */
1545 if (!(dst_dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)) {
1546 /* PF is the VF's PF? Then nothing to do */
1547 vf = VF_ENIC_TO_VF_REP(src_enic);
1548 if (vf->pf == dst_enic) {
1549 ENICPMD_LOG(DEBUG, "destination port is VF's PF");
1552 /* If not, steer to the remote PF's uplink */
1553 uif = dst_enic->fm_vnic_uif;
1554 ENICPMD_LOG(DEBUG, "steer to uplink %u", uif);
1555 memset(fm_op, 0, sizeof(*fm_op));
1556 fm_op->fa_op = FMOP_SET_EGPORT;
1557 fm_op->set_egport.egport = uif;
1558 ret = enic_fm_append_action_op(fm, fm_op, error);
1562 /* VF -> VF loopback. Hairpin and steer to vnic */
1563 memset(fm_op, 0, sizeof(*fm_op));
1564 fm_op->fa_op = FMOP_EG_HAIRPIN;
1565 ret = enic_fm_append_action_op(fm, fm_op, error);
1568 ENICPMD_LOG(DEBUG, "egress hairpin");
1569 fm->hairpin_steer_vnic_h = dst_vnic_h;
1570 fm->need_hairpin_steer = 1;
1575 enic_fm_check_transfer_dst(struct enic *enic, uint16_t dst_port_id,
1576 struct rte_eth_dev **dst_dev,
1577 struct rte_flow_error *error)
1579 struct rte_eth_dev *dev;
1581 ENICPMD_LOG(DEBUG, "port id %u", dst_port_id);
1582 if (!rte_eth_dev_is_valid_port(dst_port_id)) {
1583 return rte_flow_error_set(error, EINVAL,
1584 RTE_FLOW_ERROR_TYPE_ACTION,
1585 NULL, "invalid port_id");
1587 dev = &rte_eth_devices[dst_port_id];
1588 if (!dev_is_enic(dev)) {
1589 return rte_flow_error_set(error, EINVAL,
1590 RTE_FLOW_ERROR_TYPE_ACTION,
1591 NULL, "port_id is not enic");
1593 if (enic->switch_domain_id != pmd_priv(dev)->switch_domain_id) {
1594 return rte_flow_error_set(error, EINVAL,
1595 RTE_FLOW_ERROR_TYPE_ACTION,
1596 NULL, "destination and source ports are not in the same switch domain");
1603 /* Translate flow actions to flowman TCAM entry actions */
1605 enic_fm_copy_action(struct enic_flowman *fm,
1606 const struct rte_flow_action actions[],
1608 struct rte_flow_error *error)
1619 struct fm_tcam_match_entry *fmt;
1620 struct fm_action_op fm_op;
1621 bool need_ovlan_action;
1630 ENICPMD_FUNC_TRACE();
1631 fmt = &fm->tcam_entry;
1632 need_ovlan_action = false;
1636 enic = fm->user_enic;
1638 vnic_h = enic->fm_vnic_handle;
1640 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1641 switch (actions->type) {
1642 case RTE_FLOW_ACTION_TYPE_VOID:
1644 case RTE_FLOW_ACTION_TYPE_PASSTHRU: {
1645 if (overlap & PASSTHRU)
1647 overlap |= PASSTHRU;
1650 case RTE_FLOW_ACTION_TYPE_JUMP: {
1651 const struct rte_flow_action_jump *jump =
1653 struct enic_fm_fet *fet;
1657 ret = enic_fet_get(fm, jump->group, ingress, NULL,
1662 memset(&fm_op, 0, sizeof(fm_op));
1663 fm_op.fa_op = FMOP_EXACT_MATCH;
1664 fm_op.exact.handle = fet->handle;
1666 ret = enic_fm_append_action_op(fm, &fm_op, error);
1671 case RTE_FLOW_ACTION_TYPE_MARK: {
1672 const struct rte_flow_action_mark *mark =
1675 if (enic->use_noscatter_vec_rx_handler)
1677 if (mark->id >= ENIC_MAGIC_FILTER_ID - 1)
1678 return rte_flow_error_set(error, EINVAL,
1679 RTE_FLOW_ERROR_TYPE_ACTION,
1680 NULL, "invalid mark id");
1681 memset(&fm_op, 0, sizeof(fm_op));
1682 fm_op.fa_op = FMOP_MARK;
1683 fm_op.mark.mark = mark->id + 1;
1684 ret = enic_fm_append_action_op(fm, &fm_op, error);
1689 case RTE_FLOW_ACTION_TYPE_FLAG: {
1690 if (enic->use_noscatter_vec_rx_handler)
1692 /* ENIC_MAGIC_FILTER_ID is reserved for flagging */
1693 memset(&fm_op, 0, sizeof(fm_op));
1694 fm_op.fa_op = FMOP_MARK;
1695 fm_op.mark.mark = ENIC_MAGIC_FILTER_ID;
1696 ret = enic_fm_append_action_op(fm, &fm_op, error);
1701 case RTE_FLOW_ACTION_TYPE_QUEUE: {
1702 const struct rte_flow_action_queue *queue =
1706 * If fate other than QUEUE or RSS, fail. Multiple
1707 * rss and queue actions are ok.
1709 if ((overlap & FATE) && first_rq)
1713 memset(&fm_op, 0, sizeof(fm_op));
1714 fm_op.fa_op = FMOP_RQ_STEER;
1715 fm_op.rq_steer.rq_index =
1716 enic_rte_rq_idx_to_sop_idx(queue->index);
1717 fm_op.rq_steer.rq_count = 1;
1718 fm_op.rq_steer.vnic_handle = vnic_h;
1719 ret = enic_fm_append_action_op(fm, &fm_op, error);
1722 ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1723 fm_op.rq_steer.rq_index);
1727 case RTE_FLOW_ACTION_TYPE_DROP: {
1731 memset(&fm_op, 0, sizeof(fm_op));
1732 fm_op.fa_op = FMOP_DROP;
1733 ret = enic_fm_append_action_op(fm, &fm_op, error);
1736 ENICPMD_LOG(DEBUG, "create DROP action");
1739 case RTE_FLOW_ACTION_TYPE_COUNT: {
1740 if (overlap & COUNT)
1743 /* Count is associated with entry not action on VIC. */
1744 fmt->ftm_flags |= FMEF_COUNTER;
1747 case RTE_FLOW_ACTION_TYPE_RSS: {
1748 const struct rte_flow_action_rss *rss = actions->conf;
1753 * If fate other than QUEUE or RSS, fail. Multiple
1754 * rss and queue actions are ok.
1756 if ((overlap & FATE) && first_rq)
1762 * Hardware only supports RSS actions on outer level
1763 * with default type and function. Queues must be
1766 allow = rss->func == RTE_ETH_HASH_FUNCTION_DEFAULT &&
1767 rss->level == 0 && (rss->types == 0 ||
1768 rss->types == enic->rss_hf) &&
1769 rss->queue_num <= enic->rq_count &&
1770 rss->queue[rss->queue_num - 1] < enic->rq_count;
1773 /* Identity queue map needs to be sequential */
1774 for (i = 1; i < rss->queue_num; i++)
1775 allow = allow && (rss->queue[i] ==
1776 rss->queue[i - 1] + 1);
1780 memset(&fm_op, 0, sizeof(fm_op));
1781 fm_op.fa_op = FMOP_RQ_STEER;
1782 fm_op.rq_steer.rq_index =
1783 enic_rte_rq_idx_to_sop_idx(rss->queue[0]);
1784 fm_op.rq_steer.rq_count = rss->queue_num;
1785 fm_op.rq_steer.vnic_handle = vnic_h;
1786 ret = enic_fm_append_action_op(fm, &fm_op, error);
1789 ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1790 fm_op.rq_steer.rq_index);
1794 case RTE_FLOW_ACTION_TYPE_PORT_ID: {
1795 const struct rte_flow_action_port_id *port;
1796 struct rte_eth_dev *dev = NULL;
1798 if (!ingress && (overlap & PORT_ID)) {
1799 ENICPMD_LOG(DEBUG, "cannot have multiple egress PORT_ID actions");
1802 port = actions->conf;
1803 if (port->original) {
1804 vnic_h = enic->fm_vnic_handle; /* This port */
1807 ret = enic_fm_check_transfer_dst(enic, port->id, &dev,
1811 vnic_h = pmd_priv(dev)->fm_vnic_handle;
1814 * Ingress. Nothing more to do. We add an implicit
1815 * steer at the end if needed.
1820 ret = vf_egress_port_id_action(fm, dev, vnic_h, &fm_op,
1826 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: {
1827 if (overlap & DECAP)
1831 ret = enic_fm_copy_vxlan_decap(fm, fmt, actions,
1837 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: {
1838 const struct rte_flow_action_vxlan_encap *encap;
1840 encap = actions->conf;
1841 if (overlap & ENCAP)
1844 ret = enic_fm_copy_vxlan_encap(fm, encap->definition,
1850 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: {
1851 struct fm_action_op *decap;
1854 * If decap-nostrip appears before pop vlan, this pop
1855 * applies to the inner packet vlan. Turn it into
1858 decap = find_prev_action_op(fm, FMOP_DECAP_NOSTRIP);
1860 ENICPMD_LOG(DEBUG, "pop-vlan inner: decap-nostrip => decap-strip");
1861 decap->fa_op = FMOP_DECAP_STRIP;
1864 memset(&fm_op, 0, sizeof(fm_op));
1865 fm_op.fa_op = FMOP_POP_VLAN;
1866 ret = enic_fm_append_action_op(fm, &fm_op, error);
1871 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: {
1872 const struct rte_flow_action_of_push_vlan *vlan;
1874 if (overlap & PASSTHRU)
1876 vlan = actions->conf;
1877 if (vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN)) {
1878 return rte_flow_error_set(error, EINVAL,
1879 RTE_FLOW_ERROR_TYPE_ACTION,
1880 NULL, "unexpected push_vlan ethertype");
1882 overlap |= PUSH_VLAN;
1883 need_ovlan_action = true;
1886 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: {
1887 const struct rte_flow_action_of_set_vlan_pcp *pcp;
1889 pcp = actions->conf;
1890 if (pcp->vlan_pcp > 7) {
1891 return rte_flow_error_set(error, EINVAL,
1892 RTE_FLOW_ERROR_TYPE_ACTION,
1893 NULL, "invalid vlan_pcp");
1895 need_ovlan_action = true;
1896 ovlan |= ((uint16_t)pcp->vlan_pcp) << 13;
1899 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: {
1900 const struct rte_flow_action_of_set_vlan_vid *vid;
1902 vid = actions->conf;
1903 need_ovlan_action = true;
1904 ovlan |= rte_be_to_cpu_16(vid->vlan_vid);
1907 case RTE_FLOW_ACTION_TYPE_PORT_REPRESENTOR: {
1908 const struct rte_flow_action_ethdev *ethdev;
1909 struct rte_eth_dev *dev = NULL;
1911 ethdev = actions->conf;
1912 ret = enic_fm_check_transfer_dst(enic, ethdev->port_id,
1916 vnic_h = pmd_priv(dev)->fm_vnic_handle;
1919 * Action PORT_REPRESENTOR implies ingress destination.
1920 * Noting to do. We add an implicit stree at the
1926 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: {
1927 const struct rte_flow_action_ethdev *ethdev;
1928 struct rte_eth_dev *dev = NULL;
1930 if (overlap & PORT_ID) {
1931 ENICPMD_LOG(DEBUG, "cannot have multiple egress PORT_ID actions");
1934 ethdev = actions->conf;
1935 ret = enic_fm_check_transfer_dst(enic, ethdev->port_id,
1939 vnic_h = pmd_priv(dev)->fm_vnic_handle;
1941 /* Action REPRESENTED_PORT: always egress destination */
1943 ret = vf_egress_port_id_action(fm, dev, vnic_h, &fm_op,
1954 if (!(overlap & (FATE | PASSTHRU | COUNT | PORT_ID)))
1956 /* Egress from VF: need implicit WQ match */
1957 if (enic_is_vf_rep(enic) && !ingress) {
1958 fmt->ftm_data.fk_wq_id = 0;
1959 fmt->ftm_mask.fk_wq_id = 0xffff;
1960 fmt->ftm_data.fk_wq_vnic = enic->fm_vnic_handle;
1961 ENICPMD_LOG(DEBUG, "add implicit wq id match for vf %d",
1962 VF_ENIC_TO_VF_REP(enic)->vf_id);
1964 if (need_ovlan_action) {
1965 memset(&fm_op, 0, sizeof(fm_op));
1966 fm_op.fa_op = FMOP_SET_OVLAN;
1967 fm_op.ovlan.vlan = ovlan;
1968 ret = enic_fm_append_action_op(fm, &fm_op, error);
1972 /* Add steer op for PORT_ID without QUEUE */
1973 if ((overlap & PORT_ID) && !steer && ingress) {
1974 memset(&fm_op, 0, sizeof(fm_op));
1975 /* Always to queue 0 for now as generic RSS is not available */
1976 fm_op.fa_op = FMOP_RQ_STEER;
1977 fm_op.rq_steer.rq_index = 0;
1978 fm_op.rq_steer.vnic_handle = vnic_h;
1979 ret = enic_fm_append_action_op(fm, &fm_op, error);
1982 ENICPMD_LOG(DEBUG, "add implicit steer op");
1984 /* Add required END */
1985 memset(&fm_op, 0, sizeof(fm_op));
1986 fm_op.fa_op = FMOP_END;
1987 ret = enic_fm_append_action_op(fm, &fm_op, error);
1990 enic_fm_reorder_action_op(fm);
1994 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
1995 NULL, "enic: unsupported action");
1998 /** Check if the action is supported */
2000 enic_fm_match_action(const struct rte_flow_action *action,
2001 const enum rte_flow_action_type *supported_actions)
2003 for (; *supported_actions != RTE_FLOW_ACTION_TYPE_END;
2004 supported_actions++) {
2005 if (action->type == *supported_actions)
2011 /* Debug function to dump internal NIC action structure. */
2013 enic_fm_dump_tcam_actions(const struct fm_action *fm_action)
2015 /* Manually keep in sync with FMOP commands */
2016 const char *fmop_str[FMOP_OP_MAX] = {
2018 [FMOP_DROP] = "drop",
2019 [FMOP_RQ_STEER] = "steer",
2020 [FMOP_EXACT_MATCH] = "exmatch",
2021 [FMOP_MARK] = "mark",
2022 [FMOP_EXT_MARK] = "ext_mark",
2024 [FMOP_EG_HAIRPIN] = "eg_hairpin",
2025 [FMOP_IG_HAIRPIN] = "ig_hairpin",
2026 [FMOP_ENCAP_IVLAN] = "encap_ivlan",
2027 [FMOP_ENCAP_NOIVLAN] = "encap_noivlan",
2028 [FMOP_ENCAP] = "encap",
2029 [FMOP_SET_OVLAN] = "set_ovlan",
2030 [FMOP_DECAP_NOSTRIP] = "decap_nostrip",
2031 [FMOP_DECAP_STRIP] = "decap_strip",
2032 [FMOP_POP_VLAN] = "pop_vlan",
2033 [FMOP_SET_EGPORT] = "set_egport",
2034 [FMOP_RQ_STEER_ONLY] = "rq_steer_only",
2035 [FMOP_SET_ENCAP_VLAN] = "set_encap_vlan",
2036 [FMOP_EMIT] = "emit",
2037 [FMOP_MODIFY] = "modify",
2039 const struct fm_action_op *op = &fm_action->fma_action_ops[0];
2040 char buf[128], *bp = buf;
2045 buf_len = sizeof(buf);
2046 for (i = 0; i < FM_ACTION_OP_MAX; i++) {
2047 if (op->fa_op == FMOP_END)
2049 if (op->fa_op >= FMOP_OP_MAX)
2052 op_str = fmop_str[op->fa_op];
2053 n = snprintf(bp, buf_len, "%s,", op_str);
2054 if (n > 0 && n < buf_len) {
2060 /* Remove trailing comma */
2063 ENICPMD_LOG(DEBUG, " Actions: %s", buf);
2067 bits_to_str(uint32_t bits, const char *strings[], int max,
2068 char *buf, int buf_len)
2070 int i, n = 0, len = 0;
2072 for (i = 0; i < max; i++) {
2073 if (bits & (1 << i)) {
2074 n = snprintf(buf, buf_len, "%s,", strings[i]);
2075 if (n > 0 && n < buf_len) {
2082 /* Remove trailing comma */
2090 /* Debug function to dump internal NIC filter structure. */
2092 __enic_fm_dump_tcam_match(const struct fm_header_set *fk_hdrset, char *buf,
2095 /* Manually keep in sync with FKM_BITS */
2096 const char *fm_fkm_str[FKM_BIT_COUNT] = {
2097 [FKM_QTAG_BIT] = "qtag",
2098 [FKM_CMD_BIT] = "cmd",
2099 [FKM_IPV4_BIT] = "ip4",
2100 [FKM_IPV6_BIT] = "ip6",
2101 [FKM_ROCE_BIT] = "roce",
2102 [FKM_UDP_BIT] = "udp",
2103 [FKM_TCP_BIT] = "tcp",
2104 [FKM_TCPORUDP_BIT] = "tcpportudp",
2105 [FKM_IPFRAG_BIT] = "ipfrag",
2106 [FKM_NVGRE_BIT] = "nvgre",
2107 [FKM_VXLAN_BIT] = "vxlan",
2108 [FKM_GENEVE_BIT] = "geneve",
2109 [FKM_NSH_BIT] = "nsh",
2110 [FKM_ROCEV2_BIT] = "rocev2",
2111 [FKM_VLAN_PRES_BIT] = "vlan_pres",
2112 [FKM_IPOK_BIT] = "ipok",
2113 [FKM_L4OK_BIT] = "l4ok",
2114 [FKM_ROCEOK_BIT] = "roceok",
2115 [FKM_FCSOK_BIT] = "fcsok",
2116 [FKM_EG_SPAN_BIT] = "eg_span",
2117 [FKM_IG_SPAN_BIT] = "ig_span",
2118 [FKM_EG_HAIRPINNED_BIT] = "eg_hairpinned",
2120 /* Manually keep in sync with FKH_BITS */
2121 const char *fm_fkh_str[FKH_BIT_COUNT] = {
2122 [FKH_ETHER_BIT] = "eth",
2123 [FKH_QTAG_BIT] = "qtag",
2124 [FKH_L2RAW_BIT] = "l2raw",
2125 [FKH_IPV4_BIT] = "ip4",
2126 [FKH_IPV6_BIT] = "ip6",
2127 [FKH_L3RAW_BIT] = "l3raw",
2128 [FKH_UDP_BIT] = "udp",
2129 [FKH_TCP_BIT] = "tcp",
2130 [FKH_ICMP_BIT] = "icmp",
2131 [FKH_VXLAN_BIT] = "vxlan",
2132 [FKH_L4RAW_BIT] = "l4raw",
2134 uint32_t fkh_bits = fk_hdrset->fk_header_select;
2135 uint32_t fkm_bits = fk_hdrset->fk_metadata;
2138 if (!fkm_bits && !fkh_bits)
2140 n = snprintf(buf, buf_len, "metadata(");
2141 if (n > 0 && n < buf_len) {
2145 n = bits_to_str(fkm_bits, fm_fkm_str, FKM_BIT_COUNT, buf, buf_len);
2146 if (n > 0 && n < buf_len) {
2150 n = snprintf(buf, buf_len, ") valid hdr fields(");
2151 if (n > 0 && n < buf_len) {
2155 n = bits_to_str(fkh_bits, fm_fkh_str, FKH_BIT_COUNT, buf, buf_len);
2156 if (n > 0 && n < buf_len) {
2160 snprintf(buf, buf_len, ")");
2164 enic_fm_dump_tcam_match(const struct fm_tcam_match_entry *match,
2169 memset(buf, 0, sizeof(buf));
2170 __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[0],
2172 ENICPMD_LOG(DEBUG, " TCAM %s Outer: %s %scounter position %u",
2173 (ingress) ? "IG" : "EG", buf,
2174 (match->ftm_flags & FMEF_COUNTER) ? "" : "no ",
2175 match->ftm_position);
2176 memset(buf, 0, sizeof(buf));
2177 __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[1],
2180 ENICPMD_LOG(DEBUG, " Inner: %s", buf);
2183 /* Debug function to dump internal NIC flow structures. */
2185 enic_fm_dump_tcam_entry(const struct fm_tcam_match_entry *fm_match,
2186 const struct fm_action *fm_action,
2189 if (!rte_log_can_log(enic_pmd_logtype, RTE_LOG_DEBUG))
2191 enic_fm_dump_tcam_match(fm_match, ingress);
2192 enic_fm_dump_tcam_actions(fm_action);
2196 enic_fm_flow_parse(struct enic_flowman *fm,
2197 const struct rte_flow_attr *attrs,
2198 const struct rte_flow_item pattern[],
2199 const struct rte_flow_action actions[],
2200 struct rte_flow_error *error)
2202 const struct rte_flow_action *action;
2204 static const enum rte_flow_action_type *sa;
2206 ENICPMD_FUNC_TRACE();
2209 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM,
2210 NULL, "no pattern specified");
2215 rte_flow_error_set(error, EINVAL,
2216 RTE_FLOW_ERROR_TYPE_ACTION_NUM,
2217 NULL, "no action specified");
2222 if (attrs->group != FM_TCAM_RTE_GROUP && attrs->priority) {
2223 rte_flow_error_set(error, ENOTSUP,
2224 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
2226 "priorities are not supported for non-default (0) groups");
2228 } else if (!fm->owner_enic->switchdev_mode && attrs->transfer) {
2229 rte_flow_error_set(error, ENOTSUP,
2230 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
2232 "transfer is not supported");
2234 } else if (attrs->ingress && attrs->egress) {
2235 rte_flow_error_set(error, ENOTSUP,
2236 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
2238 "bidirectional rules not supported");
2243 rte_flow_error_set(error, EINVAL,
2244 RTE_FLOW_ERROR_TYPE_ATTR,
2245 NULL, "no attribute specified");
2249 /* Verify Actions. */
2250 sa = (attrs->ingress) ? enic_fm_supported_ig_actions :
2251 enic_fm_supported_eg_actions;
2252 for (action = &actions[0]; action->type != RTE_FLOW_ACTION_TYPE_END;
2254 if (action->type == RTE_FLOW_ACTION_TYPE_VOID)
2256 else if (!enic_fm_match_action(action, sa))
2259 if (action->type != RTE_FLOW_ACTION_TYPE_END) {
2260 rte_flow_error_set(error, EPERM, RTE_FLOW_ERROR_TYPE_ACTION,
2261 action, "invalid action");
2264 ret = enic_fm_copy_entry(fm, pattern, error);
2267 ret = enic_fm_copy_action(fm, actions, attrs->ingress, error);
2272 enic_fm_counter_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
2274 if (!fm_flow->counter_valid)
2276 SLIST_INSERT_HEAD(&fm->counters, fm_flow->counter, next);
2277 fm_flow->counter_valid = false;
2281 enic_fm_more_counters(struct enic_flowman *fm)
2283 struct enic_fm_counter *new_stack;
2284 struct enic_fm_counter *ctrs;
2288 ENICPMD_FUNC_TRACE();
2289 new_stack = rte_realloc(fm->counter_stack, (fm->counters_alloced +
2290 FM_COUNTERS_EXPAND) *
2291 sizeof(struct enic_fm_counter), 0);
2292 if (new_stack == NULL) {
2293 ENICPMD_LOG(ERR, "cannot alloc counter memory");
2296 fm->counter_stack = new_stack;
2298 args[0] = FM_COUNTER_BRK;
2299 args[1] = fm->counters_alloced + FM_COUNTERS_EXPAND;
2300 rc = flowman_cmd(fm, args, 2);
2302 ENICPMD_LOG(ERR, "cannot alloc counters rc=%d", rc);
2305 ctrs = (struct enic_fm_counter *)fm->counter_stack +
2306 fm->counters_alloced;
2307 for (i = 0; i < FM_COUNTERS_EXPAND; i++, ctrs++) {
2308 ctrs->handle = fm->counters_alloced + i;
2309 SLIST_INSERT_HEAD(&fm->counters, ctrs, next);
2311 fm->counters_alloced += FM_COUNTERS_EXPAND;
2312 ENICPMD_LOG(DEBUG, "%u counters allocated, total: %u",
2313 FM_COUNTERS_EXPAND, fm->counters_alloced);
2318 enic_fm_counter_zero(struct enic_flowman *fm, struct enic_fm_counter *c)
2323 ENICPMD_FUNC_TRACE();
2324 args[0] = FM_COUNTER_QUERY;
2325 args[1] = c->handle;
2326 args[2] = 1; /* clear */
2327 ret = flowman_cmd(fm, args, 3);
2329 ENICPMD_LOG(ERR, "counter init: rc=%d handle=0x%x",
2337 enic_fm_counter_alloc(struct enic_flowman *fm, struct rte_flow_error *error,
2338 struct enic_fm_counter **ctr)
2340 struct enic_fm_counter *c;
2343 ENICPMD_FUNC_TRACE();
2345 if (SLIST_EMPTY(&fm->counters)) {
2346 ret = enic_fm_more_counters(fm);
2348 return rte_flow_error_set(error, -ret,
2349 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2350 NULL, "enic: out of counters");
2352 c = SLIST_FIRST(&fm->counters);
2353 SLIST_REMOVE_HEAD(&fm->counters, next);
2359 enic_fm_action_free(struct enic_flowman *fm, struct enic_fm_action *ah)
2364 ENICPMD_FUNC_TRACE();
2365 RTE_ASSERT(ah->ref > 0);
2368 args[0] = FM_ACTION_FREE;
2369 args[1] = ah->handle;
2370 ret = flowman_cmd(fm, args, 2);
2372 /* This is a "should never happen" error. */
2373 ENICPMD_LOG(ERR, "freeing action rc=%d handle=0x%"
2374 PRIx64, ret, ah->handle);
2375 rte_hash_del_key(fm->action_hash, (const void *)&ah->key);
2382 enic_fm_entry_free(struct enic_flowman *fm, uint64_t handle)
2387 ENICPMD_FUNC_TRACE();
2388 args[0] = FM_MATCH_ENTRY_REMOVE;
2390 rc = flowman_cmd(fm, args, 2);
2392 ENICPMD_LOG(ERR, "cannot free match entry: rc=%d"
2393 " handle=0x%" PRIx64, rc, handle);
2397 static struct enic_fm_jump_flow *
2398 find_jump_flow(struct enic_flowman *fm, uint32_t group)
2400 struct enic_fm_jump_flow *j;
2402 ENICPMD_FUNC_TRACE();
2403 TAILQ_FOREACH(j, &fm->jump_list, list) {
2404 if (j->group == group)
2411 remove_jump_flow(struct enic_flowman *fm, struct rte_flow *flow)
2413 struct enic_fm_jump_flow *j;
2415 ENICPMD_FUNC_TRACE();
2416 TAILQ_FOREACH(j, &fm->jump_list, list) {
2417 if (j->flow == flow) {
2418 TAILQ_REMOVE(&fm->jump_list, j, list);
2426 save_jump_flow(struct enic_flowman *fm,
2427 struct rte_flow *flow,
2429 struct fm_tcam_match_entry *match,
2430 struct fm_action *action)
2432 struct enic_fm_jump_flow *j;
2434 ENICPMD_FUNC_TRACE();
2435 j = calloc(1, sizeof(struct enic_fm_jump_flow));
2441 j->action = *action;
2442 TAILQ_INSERT_HEAD(&fm->jump_list, j, list);
2443 ENICPMD_LOG(DEBUG, "saved jump flow: flow=%p group=%u", flow, group);
2448 __enic_fm_flow_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
2450 if (fm_flow->entry_handle != FM_INVALID_HANDLE) {
2451 enic_fm_entry_free(fm, fm_flow->entry_handle);
2452 fm_flow->entry_handle = FM_INVALID_HANDLE;
2454 if (fm_flow->action != NULL) {
2455 enic_fm_action_free(fm, fm_flow->action);
2456 fm_flow->action = NULL;
2458 enic_fm_counter_free(fm, fm_flow);
2460 enic_fet_put(fm, fm_flow->fet);
2461 fm_flow->fet = NULL;
2466 enic_fm_flow_free(struct enic_flowman *fm, struct rte_flow *flow)
2468 struct enic_fm_flow *steer = flow->fm->hairpin_steer_flow;
2470 if (flow->fm->fet && flow->fm->fet->default_key)
2471 remove_jump_flow(fm, flow);
2472 __enic_fm_flow_free(fm, flow->fm);
2474 __enic_fm_flow_free(fm, steer);
2482 enic_fm_add_tcam_entry(struct enic_flowman *fm,
2483 struct fm_tcam_match_entry *match_in,
2484 uint64_t *entry_handle,
2486 struct rte_flow_error *error)
2488 struct fm_tcam_match_entry *ftm;
2492 ENICPMD_FUNC_TRACE();
2493 /* Copy entry to the command buffer */
2494 ftm = &fm->cmd.va->fm_tcam_match_entry;
2495 memcpy(ftm, match_in, sizeof(*ftm));
2496 /* Add TCAM entry */
2497 args[0] = FM_TCAM_ENTRY_INSTALL;
2498 args[1] = ingress ? fm->ig_tcam_hndl : fm->eg_tcam_hndl;
2499 args[2] = fm->cmd.pa;
2500 ret = flowman_cmd(fm, args, 3);
2502 ENICPMD_LOG(ERR, "cannot add %s TCAM entry: rc=%d",
2503 ingress ? "ingress" : "egress", ret);
2504 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2505 NULL, "enic: devcmd(tcam-entry-install)");
2508 ENICPMD_LOG(DEBUG, "installed %s TCAM entry: handle=0x%" PRIx64,
2509 ingress ? "ingress" : "egress", (uint64_t)args[0]);
2510 *entry_handle = args[0];
2515 enic_fm_add_exact_entry(struct enic_flowman *fm,
2516 struct fm_tcam_match_entry *match_in,
2517 uint64_t *entry_handle,
2518 struct enic_fm_fet *fet,
2519 struct rte_flow_error *error)
2521 struct fm_exact_match_entry *fem;
2525 ENICPMD_FUNC_TRACE();
2526 /* The new entry must have the table's key */
2527 if (memcmp(fet->key.fk_hdrset, match_in->ftm_mask.fk_hdrset,
2528 sizeof(struct fm_header_set) * FM_HDRSET_MAX)) {
2529 return rte_flow_error_set(error, EINVAL,
2530 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2531 "enic: key does not match group's key");
2534 /* Copy entry to the command buffer */
2535 fem = &fm->cmd.va->fm_exact_match_entry;
2537 * Translate TCAM entry to exact entry. As is only need to drop
2538 * position and mask. The mask is part of the exact match table.
2539 * Position (aka priority) is not supported in the exact match table.
2541 fem->fem_data = match_in->ftm_data;
2542 fem->fem_flags = match_in->ftm_flags;
2543 fem->fem_action = match_in->ftm_action;
2544 fem->fem_counter = match_in->ftm_counter;
2546 /* Add exact entry */
2547 args[0] = FM_EXACT_ENTRY_INSTALL;
2548 args[1] = fet->handle;
2549 args[2] = fm->cmd.pa;
2550 ret = flowman_cmd(fm, args, 3);
2552 ENICPMD_LOG(ERR, "cannot add %s exact entry: group=%u",
2553 fet->ingress ? "ingress" : "egress", fet->group);
2554 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2555 NULL, "enic: devcmd(exact-entry-install)");
2558 ENICPMD_LOG(DEBUG, "installed %s exact entry: group=%u"
2559 " handle=0x%" PRIx64,
2560 fet->ingress ? "ingress" : "egress", fet->group,
2562 *entry_handle = args[0];
2567 enic_action_handle_get(struct enic_flowman *fm, struct fm_action *action_in,
2568 struct rte_flow_error *error,
2569 struct enic_fm_action **ah_o)
2571 struct enic_fm_action *ah;
2572 struct fm_action *fma;
2576 ret = rte_hash_lookup_data(fm->action_hash, action_in,
2578 if (ret < 0 && ret != -ENOENT)
2579 return rte_flow_error_set(error, -ret,
2580 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2581 NULL, "enic: rte_hash_lookup(action)");
2583 if (ret == -ENOENT) {
2584 /* Allocate a new action on the NIC. */
2585 fma = &fm->cmd.va->fm_action;
2586 memcpy(fma, action_in, sizeof(*fma));
2588 ah = calloc(1, sizeof(*ah));
2590 return rte_flow_error_set(error, ENOMEM,
2591 RTE_FLOW_ERROR_TYPE_HANDLE,
2592 NULL, "enic: calloc(fm-action)");
2593 memcpy(&ah->key, action_in, sizeof(struct fm_action));
2594 args[0] = FM_ACTION_ALLOC;
2595 args[1] = fm->cmd.pa;
2596 ret = flowman_cmd(fm, args, 2);
2598 rte_flow_error_set(error, -ret,
2599 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2600 NULL, "enic: devcmd(action-alloc)");
2603 ah->handle = args[0];
2604 ret = rte_hash_add_key_data(fm->action_hash,
2605 (const void *)action_in,
2608 rte_flow_error_set(error, -ret,
2609 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2611 "enic: rte_hash_add_key_data(actn)");
2612 goto error_with_action_handle;
2614 ENICPMD_LOG(DEBUG, "action allocated: handle=0x%" PRIx64,
2618 /* Action handle struct is valid, increment reference count. */
2622 error_with_action_handle:
2623 args[0] = FM_ACTION_FREE;
2624 args[1] = ah->handle;
2625 ret = flowman_cmd(fm, args, 2);
2627 rte_flow_error_set(error, -ret,
2628 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2629 NULL, "enic: devcmd(action-free)");
2635 /* Push match-action to the NIC. */
2637 __enic_fm_flow_add_entry(struct enic_flowman *fm,
2638 struct enic_fm_flow *fm_flow,
2639 struct fm_tcam_match_entry *match_in,
2640 struct fm_action *action_in,
2643 struct rte_flow_error *error)
2645 struct enic_fm_counter *ctr;
2646 struct enic_fm_action *ah = NULL;
2650 ENICPMD_FUNC_TRACE();
2652 /* Get or create an action handle. */
2653 ret = enic_action_handle_get(fm, action_in, error, &ah);
2656 match_in->ftm_action = ah->handle;
2657 fm_flow->action = ah;
2659 /* Allocate counter if requested. */
2660 if (match_in->ftm_flags & FMEF_COUNTER) {
2661 ret = enic_fm_counter_alloc(fm, error, &ctr);
2662 if (ret) /* error has been filled in */
2664 fm_flow->counter_valid = true;
2665 fm_flow->counter = ctr;
2666 match_in->ftm_counter = ctr->handle;
2670 * Get the group's table (either TCAM or exact match table) and
2671 * add entry to it. If we use the exact match table, the handler
2672 * will translate the TCAM entry (match_in) to the appropriate
2673 * exact match entry and use that instead.
2675 entry_h = FM_INVALID_HANDLE;
2676 if (group == FM_TCAM_RTE_GROUP) {
2677 ret = enic_fm_add_tcam_entry(fm, match_in, &entry_h, ingress,
2681 /* Jump action might have a ref to fet */
2682 fm_flow->fet = fm->fet;
2685 struct enic_fm_fet *fet = NULL;
2687 ret = enic_fet_get(fm, group, ingress,
2688 &match_in->ftm_mask, &fet, error);
2692 ret = enic_fm_add_exact_entry(fm, match_in, &entry_h, fet,
2697 /* Clear counter after adding entry, as it requires in-use counter */
2698 if (fm_flow->counter_valid) {
2699 ret = enic_fm_counter_zero(fm, fm_flow->counter);
2703 fm_flow->entry_handle = entry_h;
2707 /* Push match-action to the NIC. */
2708 static struct rte_flow *
2709 enic_fm_flow_add_entry(struct enic_flowman *fm,
2710 struct fm_tcam_match_entry *match_in,
2711 struct fm_action *action_in,
2712 const struct rte_flow_attr *attrs,
2713 struct rte_flow_error *error)
2715 struct enic_fm_flow *fm_flow;
2716 struct rte_flow *flow;
2718 ENICPMD_FUNC_TRACE();
2719 match_in->ftm_position = attrs->priority;
2720 enic_fm_dump_tcam_entry(match_in, action_in, attrs->ingress);
2721 flow = calloc(1, sizeof(*flow));
2722 fm_flow = calloc(1, sizeof(*fm_flow));
2723 if (flow == NULL || fm_flow == NULL) {
2724 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
2725 NULL, "enic: cannot allocate rte_flow");
2731 fm_flow->action = NULL;
2732 fm_flow->entry_handle = FM_INVALID_HANDLE;
2733 if (__enic_fm_flow_add_entry(fm, fm_flow, match_in, action_in,
2734 attrs->group, attrs->ingress, error)) {
2735 enic_fm_flow_free(fm, flow);
2742 convert_jump_flows(struct enic_flowman *fm, struct enic_fm_fet *fet,
2743 struct rte_flow_error *error)
2745 struct enic_fm_flow *fm_flow;
2746 struct enic_fm_jump_flow *j;
2747 struct fm_action *fma;
2750 ENICPMD_FUNC_TRACE();
2752 * Find the saved flows that should jump to the new table (fet).
2753 * Then delete the old TCAM entry that jumps to the default table,
2754 * and add a new one that jumps to the new table.
2757 j = find_jump_flow(fm, group);
2759 ENICPMD_LOG(DEBUG, "convert jump flow: flow=%p group=%u",
2761 /* Delete old entry */
2762 fm_flow = j->flow->fm;
2763 __enic_fm_flow_free(fm, fm_flow);
2767 fma->fma_action_ops[0].exact.handle = fet->handle;
2768 if (__enic_fm_flow_add_entry(fm, fm_flow, &j->match, fma,
2769 FM_TCAM_RTE_GROUP, fet->ingress, error)) {
2770 /* Cannot roll back changes at the moment */
2771 ENICPMD_LOG(ERR, "cannot convert jump flow: flow=%p",
2776 ENICPMD_LOG(DEBUG, "convert ok: group=%u ref=%u",
2777 fet->group, fet->ref);
2780 TAILQ_REMOVE(&fm->jump_list, j, list);
2782 j = find_jump_flow(fm, group);
2787 add_hairpin_steer(struct enic_flowman *fm, struct rte_flow *flow,
2788 struct rte_flow_error *error)
2790 struct fm_tcam_match_entry *fm_tcam_entry;
2791 struct enic_fm_flow *fm_flow;
2792 struct fm_action *fm_action;
2793 struct fm_action_op fm_op;
2796 ENICPMD_FUNC_TRACE();
2797 fm_flow = calloc(1, sizeof(*fm_flow));
2798 if (fm_flow == NULL) {
2799 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
2800 NULL, "enic: cannot allocate rte_flow");
2803 /* Original egress hairpin flow */
2804 fm_tcam_entry = &fm->tcam_entry;
2805 fm_action = &fm->action;
2806 /* Use the match pattern of the egress flow as is, without counters */
2807 fm_tcam_entry->ftm_flags &= ~FMEF_COUNTER;
2808 /* The only action is steer to vnic */
2809 fm->action_op_count = 0;
2810 memset(fm_action, 0, sizeof(*fm_action));
2811 memset(&fm_op, 0, sizeof(fm_op));
2812 /* Always to queue 0 for now */
2813 fm_op.fa_op = FMOP_RQ_STEER;
2814 fm_op.rq_steer.rq_index = 0;
2815 fm_op.rq_steer.vnic_handle = fm->hairpin_steer_vnic_h;
2816 ret = enic_fm_append_action_op(fm, &fm_op, error);
2818 goto error_with_flow;
2819 ENICPMD_LOG(DEBUG, "add steer op");
2820 /* Add required END */
2821 memset(&fm_op, 0, sizeof(fm_op));
2822 fm_op.fa_op = FMOP_END;
2823 ret = enic_fm_append_action_op(fm, &fm_op, error);
2825 goto error_with_flow;
2826 /* Add the ingress flow */
2827 fm_flow->action = NULL;
2828 fm_flow->entry_handle = FM_INVALID_HANDLE;
2829 ret = __enic_fm_flow_add_entry(fm, fm_flow, fm_tcam_entry, fm_action,
2830 FM_TCAM_RTE_GROUP, 1 /* ingress */, error);
2832 ENICPMD_LOG(ERR, "cannot add hairpin-steer flow");
2833 goto error_with_flow;
2835 /* The new flow is now the egress flow's paired flow */
2836 flow->fm->hairpin_steer_flow = fm_flow;
2845 enic_fm_open_scratch(struct enic_flowman *fm)
2847 fm->action_op_count = 0;
2849 fm->need_hairpin_steer = 0;
2850 fm->hairpin_steer_vnic_h = 0;
2851 memset(&fm->tcam_entry, 0, sizeof(fm->tcam_entry));
2852 memset(&fm->action, 0, sizeof(fm->action));
2856 enic_fm_close_scratch(struct enic_flowman *fm)
2859 enic_fet_put(fm, fm->fet);
2862 fm->action_op_count = 0;
2866 enic_fm_flow_validate(struct rte_eth_dev *dev,
2867 const struct rte_flow_attr *attrs,
2868 const struct rte_flow_item pattern[],
2869 const struct rte_flow_action actions[],
2870 struct rte_flow_error *error)
2872 struct fm_tcam_match_entry *fm_tcam_entry;
2873 struct fm_action *fm_action;
2874 struct enic_flowman *fm;
2877 ENICPMD_FUNC_TRACE();
2878 fm = begin_fm(pmd_priv(dev));
2881 enic_fm_open_scratch(fm);
2882 ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2884 fm_tcam_entry = &fm->tcam_entry;
2885 fm_action = &fm->action;
2886 enic_fm_dump_tcam_entry(fm_tcam_entry, fm_action,
2889 enic_fm_close_scratch(fm);
2895 enic_fm_flow_query_count(struct rte_eth_dev *dev,
2896 struct rte_flow *flow, void *data,
2897 struct rte_flow_error *error)
2899 struct rte_flow_query_count *query;
2900 struct enic_fm_flow *fm_flow;
2901 struct enic_flowman *fm;
2905 ENICPMD_FUNC_TRACE();
2906 fm = begin_fm(pmd_priv(dev));
2909 if (!fm_flow->counter_valid) {
2910 rc = rte_flow_error_set(error, ENOTSUP,
2911 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2912 "enic: flow does not have counter");
2916 args[0] = FM_COUNTER_QUERY;
2917 args[1] = fm_flow->counter->handle;
2918 args[2] = query->reset;
2919 rc = flowman_cmd(fm, args, 3);
2921 ENICPMD_LOG(ERR, "cannot query counter: rc=%d handle=0x%x",
2922 rc, fm_flow->counter->handle);
2925 query->hits_set = 1;
2926 query->hits = args[0];
2927 query->bytes_set = 1;
2928 query->bytes = args[1];
2936 enic_fm_flow_query(struct rte_eth_dev *dev,
2937 struct rte_flow *flow,
2938 const struct rte_flow_action *actions,
2940 struct rte_flow_error *error)
2944 ENICPMD_FUNC_TRACE();
2945 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2946 switch (actions->type) {
2947 case RTE_FLOW_ACTION_TYPE_VOID:
2949 case RTE_FLOW_ACTION_TYPE_COUNT:
2950 ret = enic_fm_flow_query_count(dev, flow, data, error);
2953 return rte_flow_error_set(error, ENOTSUP,
2954 RTE_FLOW_ERROR_TYPE_ACTION,
2956 "action not supported");
2964 static struct rte_flow *
2965 enic_fm_flow_create(struct rte_eth_dev *dev,
2966 const struct rte_flow_attr *attrs,
2967 const struct rte_flow_item pattern[],
2968 const struct rte_flow_action actions[],
2969 struct rte_flow_error *error)
2971 struct fm_tcam_match_entry *fm_tcam_entry;
2972 struct fm_action *fm_action;
2973 struct enic_flowman *fm;
2974 struct enic_fm_fet *fet;
2975 struct rte_flow *flow;
2979 ENICPMD_FUNC_TRACE();
2980 enic = pmd_priv(dev);
2981 fm = begin_fm(enic);
2983 rte_flow_error_set(error, ENOTSUP,
2984 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2985 "flowman is not initialized");
2988 enic_fm_open_scratch(fm);
2990 ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2992 goto error_with_scratch;
2993 fm_tcam_entry = &fm->tcam_entry;
2994 fm_action = &fm->action;
2995 flow = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2998 /* Add ingress rule that pairs with hairpin rule */
2999 if (fm->need_hairpin_steer) {
3000 ret = add_hairpin_steer(fm, flow, error);
3002 enic_fm_flow_free(fm, flow);
3004 goto error_with_scratch;
3007 LIST_INSERT_HEAD(&enic->flows, flow, next);
3008 fet = flow->fm->fet;
3009 if (fet && fet->default_key) {
3011 * Jump to non-existent group? Save the relevant info
3012 * so we can convert this flow when that group
3015 save_jump_flow(fm, flow, fet->group,
3016 fm_tcam_entry, fm_action);
3017 } else if (fet && fet->ref == 1) {
3019 * A new table is created. Convert the saved flows
3020 * that should jump to this group.
3022 convert_jump_flows(fm, fet, error);
3027 enic_fm_close_scratch(fm);
3033 enic_fm_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
3034 __rte_unused struct rte_flow_error *error)
3036 struct enic *enic = pmd_priv(dev);
3037 struct enic_flowman *fm;
3039 ENICPMD_FUNC_TRACE();
3040 fm = begin_fm(enic);
3043 LIST_REMOVE(flow, next);
3044 enic_fm_flow_free(fm, flow);
3050 enic_fm_flow_flush(struct rte_eth_dev *dev,
3051 __rte_unused struct rte_flow_error *error)
3053 LIST_HEAD(enic_flows, rte_flow) internal;
3054 struct enic_fm_flow *fm_flow;
3055 struct enic_flowman *fm;
3056 struct rte_flow *flow;
3057 struct enic *enic = pmd_priv(dev);
3059 ENICPMD_FUNC_TRACE();
3061 fm = begin_fm(enic);
3064 /* Destroy all non-internal flows */
3065 LIST_INIT(&internal);
3066 while (!LIST_EMPTY(&enic->flows)) {
3067 flow = LIST_FIRST(&enic->flows);
3069 LIST_REMOVE(flow, next);
3070 if (flow->internal) {
3071 LIST_INSERT_HEAD(&internal, flow, next);
3075 * If tables are null, then vNIC is closing, and the firmware
3076 * has already cleaned up flowman state. So do not try to free
3077 * resources, as it only causes errors.
3079 if (fm->ig_tcam_hndl == FM_INVALID_HANDLE) {
3080 fm_flow->entry_handle = FM_INVALID_HANDLE;
3081 fm_flow->action = NULL;
3082 fm_flow->fet = NULL;
3084 enic_fm_flow_free(fm, flow);
3086 while (!LIST_EMPTY(&internal)) {
3087 flow = LIST_FIRST(&internal);
3088 LIST_REMOVE(flow, next);
3089 LIST_INSERT_HEAD(&enic->flows, flow, next);
3096 enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle)
3101 args[0] = FM_MATCH_TABLE_FREE;
3103 rc = flowman_cmd(fm, args, 2);
3105 ENICPMD_LOG(ERR, "cannot free table: rc=%d handle=0x%" PRIx64,
3111 enic_fm_tcam_tbl_alloc(struct enic_flowman *fm, uint32_t direction,
3112 uint32_t max_entries, uint64_t *handle)
3114 struct fm_tcam_match_table *tcam_tbl;
3118 ENICPMD_FUNC_TRACE();
3119 tcam_tbl = &fm->cmd.va->fm_tcam_match_table;
3120 tcam_tbl->ftt_direction = direction;
3121 tcam_tbl->ftt_stage = FM_STAGE_LAST;
3122 tcam_tbl->ftt_max_entries = max_entries;
3123 args[0] = FM_TCAM_TABLE_ALLOC;
3124 args[1] = fm->cmd.pa;
3125 rc = flowman_cmd(fm, args, 2);
3127 ENICPMD_LOG(ERR, "cannot alloc %s TCAM table: rc=%d",
3128 (direction == FM_INGRESS) ? "IG" : "EG", rc);
3132 ENICPMD_LOG(DEBUG, "%s TCAM table allocated, handle=0x%" PRIx64,
3133 (direction == FM_INGRESS) ? "IG" : "EG", *handle);
3138 enic_fm_init_actions(struct enic_flowman *fm)
3140 struct rte_hash *a_hash;
3141 char name[RTE_HASH_NAMESIZE];
3142 struct rte_hash_parameters params = {
3143 .entries = FM_MAX_ACTION_TABLE_SIZE,
3144 .key_len = sizeof(struct fm_action),
3145 .hash_func = rte_jhash,
3146 .hash_func_init_val = 0,
3147 .socket_id = rte_socket_id(),
3150 ENICPMD_FUNC_TRACE();
3151 snprintf((char *)name, sizeof(name), "fm-ah-%s",
3152 fm->owner_enic->bdf_name);
3155 a_hash = rte_hash_create(¶ms);
3158 fm->action_hash = a_hash;
3163 enic_fm_init_counters(struct enic_flowman *fm)
3165 ENICPMD_FUNC_TRACE();
3166 SLIST_INIT(&fm->counters);
3167 return enic_fm_more_counters(fm);
3171 enic_fm_free_all_counters(struct enic_flowman *fm)
3176 args[0] = FM_COUNTER_BRK;
3178 rc = flowman_cmd(fm, args, 2);
3180 ENICPMD_LOG(ERR, "cannot free counters: rc=%d", rc);
3181 rte_free(fm->counter_stack);
3185 enic_fm_alloc_tcam_tables(struct enic_flowman *fm)
3189 ENICPMD_FUNC_TRACE();
3190 rc = enic_fm_tcam_tbl_alloc(fm, FM_INGRESS, FM_MAX_TCAM_TABLE_SIZE,
3194 rc = enic_fm_tcam_tbl_alloc(fm, FM_EGRESS, FM_MAX_TCAM_TABLE_SIZE,
3200 enic_fm_free_tcam_tables(struct enic_flowman *fm)
3202 ENICPMD_FUNC_TRACE();
3203 if (fm->ig_tcam_hndl) {
3204 ENICPMD_LOG(DEBUG, "free IG TCAM table handle=0x%" PRIx64,
3206 enic_fm_tbl_free(fm, fm->ig_tcam_hndl);
3207 fm->ig_tcam_hndl = FM_INVALID_HANDLE;
3209 if (fm->eg_tcam_hndl) {
3210 ENICPMD_LOG(DEBUG, "free EG TCAM table handle=0x%" PRIx64,
3212 enic_fm_tbl_free(fm, fm->eg_tcam_hndl);
3213 fm->eg_tcam_hndl = FM_INVALID_HANDLE;
3218 enic_fm_init(struct enic *enic)
3220 const struct rte_pci_addr *addr;
3221 struct enic_flowman *fm;
3222 uint8_t name[RTE_MEMZONE_NAMESIZE];
3225 if (enic->flow_filter_mode != FILTER_FLOWMAN)
3227 ENICPMD_FUNC_TRACE();
3228 /* Get vnic handle and save for port-id action */
3229 if (enic_is_vf_rep(enic))
3230 addr = &VF_ENIC_TO_VF_REP(enic)->bdf;
3232 addr = &RTE_ETH_DEV_TO_PCI(enic->rte_dev)->addr;
3233 rc = enic_fm_find_vnic(enic, addr, &enic->fm_vnic_handle);
3235 ENICPMD_LOG(ERR, "cannot find vnic handle for %x:%x:%x",
3236 addr->bus, addr->devid, addr->function);
3239 /* Save UIF for egport action */
3240 enic->fm_vnic_uif = vnic_dev_uif(enic->vdev);
3241 ENICPMD_LOG(DEBUG, "uif %u", enic->fm_vnic_uif);
3242 /* Nothing else to do for representor. It will share the PF flowman */
3243 if (enic_is_vf_rep(enic))
3245 fm = calloc(1, sizeof(*fm));
3247 ENICPMD_LOG(ERR, "cannot alloc flowman struct");
3250 fm->owner_enic = enic;
3251 rte_spinlock_init(&fm->lock);
3252 TAILQ_INIT(&fm->fet_list);
3253 TAILQ_INIT(&fm->jump_list);
3254 /* Allocate host memory for flowman commands */
3255 snprintf((char *)name, sizeof(name), "fm-cmd-%s", enic->bdf_name);
3256 fm->cmd.va = enic_alloc_consistent(enic,
3257 sizeof(union enic_flowman_cmd_mem), &fm->cmd.pa, name);
3259 ENICPMD_LOG(ERR, "cannot allocate flowman command memory");
3263 /* Allocate TCAM tables upfront as they are the main tables */
3264 rc = enic_fm_alloc_tcam_tables(fm);
3266 ENICPMD_LOG(ERR, "cannot alloc TCAM tables");
3269 /* Then a number of counters */
3270 rc = enic_fm_init_counters(fm);
3272 ENICPMD_LOG(ERR, "cannot alloc counters");
3275 /* set up action handle hash */
3276 rc = enic_fm_init_actions(fm);
3278 ENICPMD_LOG(ERR, "cannot create action hash, error:%d", rc);
3279 goto error_counters;
3282 * One default exact match table for each direction. We hold onto
3285 rc = enic_fet_alloc(fm, 1, NULL, 128, &fm->default_ig_fet);
3287 ENICPMD_LOG(ERR, "cannot alloc default IG exact match table");
3290 fm->default_ig_fet->ref = 1;
3291 rc = enic_fet_alloc(fm, 0, NULL, 128, &fm->default_eg_fet);
3293 ENICPMD_LOG(ERR, "cannot alloc default EG exact match table");
3296 fm->default_eg_fet->ref = 1;
3297 fm->vf_rep_tag = FM_VF_REP_TAG;
3302 enic_fet_free(fm, fm->default_ig_fet);
3304 rte_hash_free(fm->action_hash);
3306 enic_fm_free_all_counters(fm);
3308 enic_fm_free_tcam_tables(fm);
3310 enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
3311 fm->cmd.va, fm->cmd.pa);
3318 enic_fm_destroy(struct enic *enic)
3320 struct enic_flowman *fm;
3321 struct enic_fm_fet *fet;
3323 ENICPMD_FUNC_TRACE();
3324 if (enic_is_vf_rep(enic)) {
3325 delete_rep_flows(enic);
3328 if (enic->fm == NULL)
3331 enic_fm_flow_flush(enic->rte_dev, NULL);
3332 enic_fet_free(fm, fm->default_eg_fet);
3333 enic_fet_free(fm, fm->default_ig_fet);
3334 /* Free all exact match tables still open */
3335 while (!TAILQ_EMPTY(&fm->fet_list)) {
3336 fet = TAILQ_FIRST(&fm->fet_list);
3337 enic_fet_free(fm, fet);
3339 enic_fm_free_tcam_tables(fm);
3340 enic_fm_free_all_counters(fm);
3341 rte_hash_free(fm->action_hash);
3342 enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
3343 fm->cmd.va, fm->cmd.pa);
3350 enic_fm_allocate_switch_domain(struct enic *pf)
3352 const struct rte_pci_addr *cur_a, *prev_a;
3353 struct rte_eth_dev *dev;
3354 struct enic *cur, *prev;
3360 ENICPMD_FUNC_TRACE();
3361 if (enic_is_vf_rep(pf))
3364 cur_a = &RTE_ETH_DEV_TO_PCI(cur->rte_dev)->addr;
3365 /* Go through ports and find another PF that is on the same adapter */
3366 RTE_ETH_FOREACH_DEV(pid) {
3367 dev = &rte_eth_devices[pid];
3368 if (!dev_is_enic(dev))
3370 if (dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)
3372 if (dev == cur->rte_dev)
3374 /* dev is another PF. Is it on the same adapter? */
3375 prev = pmd_priv(dev);
3376 prev_a = &RTE_ETH_DEV_TO_PCI(dev)->addr;
3377 if (!enic_fm_find_vnic(cur, prev_a, &vnic_h)) {
3378 ENICPMD_LOG(DEBUG, "Port %u (PF BDF %x:%x:%x) and port %u (PF BDF %x:%x:%x domain %u) are on the same VIC",
3379 cur->rte_dev->data->port_id,
3380 cur_a->bus, cur_a->devid, cur_a->function,
3382 prev_a->bus, prev_a->devid, prev_a->function,
3383 prev->switch_domain_id);
3384 cur->switch_domain_id = prev->switch_domain_id;
3388 ret = rte_eth_switch_domain_alloc(&domain_id);
3390 ENICPMD_LOG(WARNING, "failed to allocate switch domain for device %d",
3393 cur->switch_domain_id = domain_id;
3394 ENICPMD_LOG(DEBUG, "Port %u (PF BDF %x:%x:%x) is the 1st PF on the VIC. Allocated switch domain id %u",
3395 cur->rte_dev->data->port_id,
3396 cur_a->bus, cur_a->devid, cur_a->function,
3401 const struct rte_flow_ops enic_fm_flow_ops = {
3402 .validate = enic_fm_flow_validate,
3403 .create = enic_fm_flow_create,
3404 .destroy = enic_fm_flow_destroy,
3405 .flush = enic_fm_flow_flush,
3406 .query = enic_fm_flow_query,
3409 /* Add a high priority flow that loops representor packets to VF */
3411 enic_fm_add_rep2vf_flow(struct enic_vf_representor *vf)
3413 struct fm_tcam_match_entry *fm_tcam_entry;
3414 struct rte_flow *flow0, *flow1;
3415 struct fm_action *fm_action;
3416 struct rte_flow_error error;
3417 struct rte_flow_attr attrs;
3418 struct fm_action_op fm_op;
3419 struct enic_flowman *fm;
3425 tag = fm->vf_rep_tag;
3426 enic_fm_open_scratch(fm);
3427 fm_tcam_entry = &fm->tcam_entry;
3428 fm_action = &fm->action;
3429 /* Egress rule: match WQ ID and tag+hairpin */
3430 fm_tcam_entry->ftm_data.fk_wq_id = vf->pf_wq_idx;
3431 fm_tcam_entry->ftm_mask.fk_wq_id = 0xffff;
3432 fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3433 memset(&fm_op, 0, sizeof(fm_op));
3434 fm_op.fa_op = FMOP_TAG;
3435 fm_op.tag.tag = tag;
3436 enic_fm_append_action_op(fm, &fm_op, &error);
3437 memset(&fm_op, 0, sizeof(fm_op));
3438 fm_op.fa_op = FMOP_EG_HAIRPIN;
3439 enic_fm_append_action_op(fm, &fm_op, &error);
3440 memset(&fm_op, 0, sizeof(fm_op));
3441 fm_op.fa_op = FMOP_END;
3442 enic_fm_append_action_op(fm, &fm_op, &error);
3446 attrs.priority = FM_HIGHEST_PRIORITY;
3447 flow0 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3449 enic_fm_close_scratch(fm);
3450 if (flow0 == NULL) {
3451 ENICPMD_LOG(ERR, "Cannot create flow 0 for representor->VF");
3454 LIST_INSERT_HEAD(&pf->flows, flow0, next);
3455 /* Make this flow internal, so the user app cannot delete it */
3456 flow0->internal = 1;
3457 ENICPMD_LOG(DEBUG, "representor->VF %d flow created: wq %d -> tag %d hairpin",
3458 vf->vf_id, vf->pf_wq_idx, tag);
3460 /* Ingress: steer hairpinned to VF RQ 0 */
3461 enic_fm_open_scratch(fm);
3462 fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3463 fm_tcam_entry->ftm_data.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3464 fm_tcam_entry->ftm_mask.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3465 fm_tcam_entry->ftm_data.fk_packet_tag = tag;
3466 fm_tcam_entry->ftm_mask.fk_packet_tag = 0xff;
3467 memset(&fm_op, 0, sizeof(fm_op));
3468 fm_op.fa_op = FMOP_RQ_STEER;
3469 fm_op.rq_steer.rq_index = 0;
3470 fm_op.rq_steer.vnic_handle = vf->enic.fm_vnic_handle;
3471 enic_fm_append_action_op(fm, &fm_op, &error);
3472 memset(&fm_op, 0, sizeof(fm_op));
3473 fm_op.fa_op = FMOP_END;
3474 enic_fm_append_action_op(fm, &fm_op, &error);
3478 attrs.priority = FM_HIGHEST_PRIORITY;
3479 flow1 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3481 enic_fm_close_scratch(fm);
3482 if (flow1 == NULL) {
3483 ENICPMD_LOG(ERR, "Cannot create flow 1 for representor->VF");
3484 enic_fm_flow_destroy(pf->rte_dev, flow0, &error);
3487 LIST_INSERT_HEAD(&pf->flows, flow1, next);
3488 flow1->internal = 1;
3489 ENICPMD_LOG(DEBUG, "representor->VF %d flow created: tag %d hairpinned -> VF RQ %d",
3490 vf->vf_id, tag, fm_op.rq_steer.rq_index);
3491 vf->rep2vf_flow[0] = flow0;
3492 vf->rep2vf_flow[1] = flow1;
3493 /* Done with this tag, use a different one next time */
3499 * Add a low priority flow that matches all packets from VF and loops them
3500 * back to the representor.
3503 enic_fm_add_vf2rep_flow(struct enic_vf_representor *vf)
3505 struct fm_tcam_match_entry *fm_tcam_entry;
3506 struct rte_flow *flow0, *flow1;
3507 struct fm_action *fm_action;
3508 struct rte_flow_error error;
3509 struct rte_flow_attr attrs;
3510 struct fm_action_op fm_op;
3511 struct enic_flowman *fm;
3517 tag = fm->vf_rep_tag;
3518 enic_fm_open_scratch(fm);
3519 fm_tcam_entry = &fm->tcam_entry;
3520 fm_action = &fm->action;
3521 /* Egress rule: match-any and tag+hairpin */
3522 fm_tcam_entry->ftm_data.fk_wq_id = 0;
3523 fm_tcam_entry->ftm_mask.fk_wq_id = 0xffff;
3524 fm_tcam_entry->ftm_data.fk_wq_vnic = vf->enic.fm_vnic_handle;
3525 fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3526 memset(&fm_op, 0, sizeof(fm_op));
3527 fm_op.fa_op = FMOP_TAG;
3528 fm_op.tag.tag = tag;
3529 enic_fm_append_action_op(fm, &fm_op, &error);
3530 memset(&fm_op, 0, sizeof(fm_op));
3531 fm_op.fa_op = FMOP_EG_HAIRPIN;
3532 enic_fm_append_action_op(fm, &fm_op, &error);
3533 memset(&fm_op, 0, sizeof(fm_op));
3534 fm_op.fa_op = FMOP_END;
3535 enic_fm_append_action_op(fm, &fm_op, &error);
3539 attrs.priority = FM_LOWEST_PRIORITY;
3540 flow0 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3542 enic_fm_close_scratch(fm);
3543 if (flow0 == NULL) {
3544 ENICPMD_LOG(ERR, "Cannot create flow 0 for VF->representor");
3547 LIST_INSERT_HEAD(&pf->flows, flow0, next);
3548 /* Make this flow internal, so the user app cannot delete it */
3549 flow0->internal = 1;
3550 ENICPMD_LOG(DEBUG, "VF %d->representor flow created: wq %d (low prio) -> tag %d hairpin",
3551 vf->vf_id, fm_tcam_entry->ftm_data.fk_wq_id, tag);
3553 /* Ingress: steer hairpinned to VF rep RQ */
3554 enic_fm_open_scratch(fm);
3555 fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3556 fm_tcam_entry->ftm_data.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3557 fm_tcam_entry->ftm_mask.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3558 fm_tcam_entry->ftm_data.fk_packet_tag = tag;
3559 fm_tcam_entry->ftm_mask.fk_packet_tag = 0xff;
3560 memset(&fm_op, 0, sizeof(fm_op));
3561 fm_op.fa_op = FMOP_RQ_STEER;
3562 fm_op.rq_steer.rq_index = vf->pf_rq_sop_idx;
3563 fm_op.rq_steer.vnic_handle = pf->fm_vnic_handle;
3564 enic_fm_append_action_op(fm, &fm_op, &error);
3565 memset(&fm_op, 0, sizeof(fm_op));
3566 fm_op.fa_op = FMOP_END;
3567 enic_fm_append_action_op(fm, &fm_op, &error);
3571 attrs.priority = FM_HIGHEST_PRIORITY;
3572 flow1 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3574 enic_fm_close_scratch(fm);
3575 if (flow1 == NULL) {
3576 ENICPMD_LOG(ERR, "Cannot create flow 1 for VF->representor");
3577 enic_fm_flow_destroy(pf->rte_dev, flow0, &error);
3580 LIST_INSERT_HEAD(&pf->flows, flow1, next);
3581 flow1->internal = 1;
3582 ENICPMD_LOG(DEBUG, "VF %d->representor flow created: tag %d hairpinned -> PF RQ %d",
3583 vf->vf_id, tag, vf->pf_rq_sop_idx);
3584 vf->vf2rep_flow[0] = flow0;
3585 vf->vf2rep_flow[1] = flow1;
3586 /* Done with this tag, use a different one next time */
3591 /* Destroy representor flows created by enic_fm_add_{rep2vf,vf2rep}_flow */
3593 delete_rep_flows(struct enic *enic)
3595 struct enic_vf_representor *vf;
3596 struct rte_flow_error error;
3597 struct rte_eth_dev *dev;
3600 RTE_ASSERT(enic_is_vf_rep(enic));
3601 vf = VF_ENIC_TO_VF_REP(enic);
3602 dev = vf->pf->rte_dev;
3603 for (i = 0; i < ARRAY_SIZE(vf->vf2rep_flow); i++) {
3604 if (vf->vf2rep_flow[i])
3605 enic_fm_flow_destroy(dev, vf->vf2rep_flow[i], &error);
3607 for (i = 0; i < ARRAY_SIZE(vf->rep2vf_flow); i++) {
3608 if (vf->rep2vf_flow[i])
3609 enic_fm_flow_destroy(dev, vf->rep2vf_flow[i], &error);
3613 static struct enic_flowman *
3614 begin_fm(struct enic *enic)
3616 struct enic_vf_representor *vf;
3617 struct enic_flowman *fm;
3619 /* Representor uses PF flowman */
3620 if (enic_is_vf_rep(enic)) {
3621 vf = VF_ENIC_TO_VF_REP(enic);
3626 /* Save the API caller and lock if representors exist */
3628 if (fm->owner_enic->switchdev_mode)
3629 rte_spinlock_lock(&fm->lock);
3630 fm->user_enic = enic;
3636 end_fm(struct enic_flowman *fm)
3638 fm->user_enic = NULL;
3639 if (fm->owner_enic->switchdev_mode)
3640 rte_spinlock_unlock(&fm->lock);