1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2008-2019 Cisco Systems, Inc. All rights reserved.
8 #include <ethdev_driver.h>
9 #include <rte_flow_driver.h>
10 #include <rte_ether.h>
12 #include <rte_jhash.h>
15 #include <rte_memzone.h>
17 #include "enic_compat.h"
22 #define IP_DEFTTL 64 /* from RFC 1340. */
23 #define IP6_VTC_FLOW 0x60000000
25 /* Up to 1024 TCAM entries */
26 #define FM_MAX_TCAM_TABLE_SIZE 1024
28 /* Up to 4096 entries per exact match table */
29 #define FM_MAX_EXACT_TABLE_SIZE 4096
31 /* Number of counters to increase on for each increment */
32 #define FM_COUNTERS_EXPAND 100
34 #define FM_INVALID_HANDLE 0
36 /* Low priority used for implicit VF -> representor flow */
37 #define FM_LOWEST_PRIORITY 100000
39 /* High priority used for implicit representor -> VF flow */
40 #define FM_HIGHEST_PRIORITY 0
42 /* Tag used for implicit VF <-> representor flows */
43 #define FM_VF_REP_TAG 1
45 /* Max number of actions supported by VIC is 2K. Make hash table double that. */
46 #define FM_MAX_ACTION_TABLE_SIZE 4096
49 * Flow exact match tables (FET) in the VIC and rte_flow groups.
50 * Use a simple scheme to map groups to tables.
51 * Group 0 uses the single TCAM tables, one for each direction.
52 * Group 1, 2, ... uses its own exact match table.
54 * The TCAM tables are allocated upfront during init.
56 * Exact match tables are allocated on demand. 3 paths that lead allocations.
58 * 1. Add a flow that jumps from group 0 to group N.
60 * If N does not exist, we allocate an exact match table for it, using
61 * a dummy key. A key is required for the table.
63 * 2. Add a flow that uses group N.
65 * If N does not exist, we allocate an exact match table for it, using
66 * the flow's key. Subsequent flows to the same group all should have
69 * Without a jump flow to N, N is not reachable in hardware. No packets
72 * 3. Add a flow to an empty group N.
74 * N has been created via (1) and the dummy key. We free that table, allocate
75 * a new table using the new flow's key. Also re-do the existing jump flow to
76 * point to the new table.
78 #define FM_TCAM_RTE_GROUP 0
81 TAILQ_ENTRY(enic_fm_fet) list;
82 uint32_t group; /* rte_flow group ID */
83 uint64_t handle; /* Exact match table handle from flowman */
86 int ref; /* Reference count via get/put */
87 struct fm_key_template key; /* Key associated with the table */
90 struct enic_fm_counter {
91 SLIST_ENTRY(enic_fm_counter) next;
95 struct enic_fm_action {
102 struct enic_fm_flow {
104 uint64_t entry_handle;
105 struct enic_fm_action *action;
106 struct enic_fm_counter *counter;
107 struct enic_fm_fet *fet;
108 /* Auto-added steer action for hairpin flows (e.g. vnic->vnic) */
109 struct enic_fm_flow *hairpin_steer_flow;
112 struct enic_fm_jump_flow {
113 TAILQ_ENTRY(enic_fm_jump_flow) list;
114 struct rte_flow *flow;
116 struct fm_tcam_match_entry match;
117 struct fm_action action;
121 * Flowman uses host memory for commands. This structure is allocated
122 * in DMA-able memory.
124 union enic_flowman_cmd_mem {
125 struct fm_tcam_match_table fm_tcam_match_table;
126 struct fm_exact_match_table fm_exact_match_table;
127 struct fm_tcam_match_entry fm_tcam_match_entry;
128 struct fm_exact_match_entry fm_exact_match_entry;
129 struct fm_action fm_action;
133 * PF has a flowman instance, and VF representors share it with PF.
134 * PF allocates this structure and owns it. VF representors borrow
135 * the PF's structure during API calls (e.g. create, query).
137 struct enic_flowman {
138 struct enic *owner_enic; /* PF */
139 struct enic *user_enic; /* API caller (PF or representor) */
141 * Representors and PF share the same underlying flowman.
142 * Lock API calls to serialize accesses from them. Only used
143 * when VF representors are present.
148 union enic_flowman_cmd_mem *va;
151 /* TCAM tables allocated upfront, used for group 0 */
152 uint64_t ig_tcam_hndl;
153 uint64_t eg_tcam_hndl;
155 SLIST_HEAD(enic_free_counters, enic_fm_counter) counters;
157 uint32_t counters_alloced;
158 /* Exact match tables for groups != 0, dynamically allocated */
159 TAILQ_HEAD(fet_list, enic_fm_fet) fet_list;
161 * Default exact match tables used for jump actions to
162 * non-existent groups.
164 struct enic_fm_fet *default_eg_fet;
165 struct enic_fm_fet *default_ig_fet;
166 /* hash table for Action reuse */
167 struct rte_hash *action_hash;
168 /* Flows that jump to the default table above */
169 TAILQ_HEAD(jump_flow_list, enic_fm_jump_flow) jump_list;
171 * Scratch data used during each invocation of flow_create
174 struct enic_fm_fet *fet;
175 struct fm_tcam_match_entry tcam_entry;
176 struct fm_action action;
177 struct fm_action action_tmp; /* enic_fm_reorder_action_op */
179 /* Tags used for representor flows */
181 /* For auto-added steer action for hairpin */
182 int need_hairpin_steer;
183 uint64_t hairpin_steer_vnic_h;
186 static int enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle);
188 * API functions (create, destroy, validate, flush) call begin_fm()
189 * upon entering to save the caller enic (PF or VF representor) and
190 * lock. Upon exit, they call end_fm() to unlock.
192 static struct enic_flowman *begin_fm(struct enic *enic);
193 static void end_fm(struct enic_flowman *fm);
194 /* Delete internal flows created for representor paths */
195 static void delete_rep_flows(struct enic *enic);
198 * Common arguments passed to copy_item functions. Use this structure
199 * so we can easily add new arguments.
200 * item: Item specification.
201 * fm_tcam_entry: Flowman TCAM match entry.
202 * header_level: 0 for outer header, 1 for inner header.
204 struct copy_item_args {
205 const struct rte_flow_item *item;
206 struct fm_tcam_match_entry *fm_tcam_entry;
207 uint8_t header_level;
210 /* functions for copying items into flowman match */
211 typedef int (enic_copy_item_fn)(struct copy_item_args *arg);
213 /* Info about how to copy items into flowman match */
214 struct enic_fm_items {
215 /* Function for copying and validating an item. */
216 enic_copy_item_fn * const copy_item;
217 /* List of valid previous items. */
218 const enum rte_flow_item_type * const prev_items;
220 * True if it's OK for this item to be the first item. For some NIC
221 * versions, it's invalid to start the stack above layer 3.
223 const uint8_t valid_start_item;
226 static enic_copy_item_fn enic_fm_copy_item_eth;
227 static enic_copy_item_fn enic_fm_copy_item_ipv4;
228 static enic_copy_item_fn enic_fm_copy_item_ipv6;
229 static enic_copy_item_fn enic_fm_copy_item_raw;
230 static enic_copy_item_fn enic_fm_copy_item_sctp;
231 static enic_copy_item_fn enic_fm_copy_item_tcp;
232 static enic_copy_item_fn enic_fm_copy_item_udp;
233 static enic_copy_item_fn enic_fm_copy_item_vlan;
234 static enic_copy_item_fn enic_fm_copy_item_vxlan;
235 static enic_copy_item_fn enic_fm_copy_item_gtp;
237 /* Ingress actions */
238 static const enum rte_flow_action_type enic_fm_supported_ig_actions[] = {
239 RTE_FLOW_ACTION_TYPE_COUNT,
240 RTE_FLOW_ACTION_TYPE_DROP,
241 RTE_FLOW_ACTION_TYPE_FLAG,
242 RTE_FLOW_ACTION_TYPE_JUMP,
243 RTE_FLOW_ACTION_TYPE_MARK,
244 RTE_FLOW_ACTION_TYPE_OF_POP_VLAN,
245 RTE_FLOW_ACTION_TYPE_PORT_ID,
246 RTE_FLOW_ACTION_TYPE_PASSTHRU,
247 RTE_FLOW_ACTION_TYPE_QUEUE,
248 RTE_FLOW_ACTION_TYPE_RSS,
249 RTE_FLOW_ACTION_TYPE_VOID,
250 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
251 RTE_FLOW_ACTION_TYPE_VXLAN_DECAP,
252 RTE_FLOW_ACTION_TYPE_END, /* END must be the last entry */
256 static const enum rte_flow_action_type enic_fm_supported_eg_actions[] = {
257 RTE_FLOW_ACTION_TYPE_COUNT,
258 RTE_FLOW_ACTION_TYPE_DROP,
259 RTE_FLOW_ACTION_TYPE_JUMP,
260 RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN,
261 RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP,
262 RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID,
263 RTE_FLOW_ACTION_TYPE_PORT_ID,
264 RTE_FLOW_ACTION_TYPE_PASSTHRU,
265 RTE_FLOW_ACTION_TYPE_VOID,
266 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
267 RTE_FLOW_ACTION_TYPE_END,
270 static const struct enic_fm_items enic_fm_items[] = {
271 [RTE_FLOW_ITEM_TYPE_RAW] = {
272 .copy_item = enic_fm_copy_item_raw,
273 .valid_start_item = 0,
274 .prev_items = (const enum rte_flow_item_type[]) {
275 RTE_FLOW_ITEM_TYPE_UDP,
276 RTE_FLOW_ITEM_TYPE_END,
279 [RTE_FLOW_ITEM_TYPE_ETH] = {
280 .copy_item = enic_fm_copy_item_eth,
281 .valid_start_item = 1,
282 .prev_items = (const enum rte_flow_item_type[]) {
283 RTE_FLOW_ITEM_TYPE_END,
286 [RTE_FLOW_ITEM_TYPE_VLAN] = {
287 .copy_item = enic_fm_copy_item_vlan,
288 .valid_start_item = 1,
289 .prev_items = (const enum rte_flow_item_type[]) {
290 RTE_FLOW_ITEM_TYPE_ETH,
291 RTE_FLOW_ITEM_TYPE_END,
294 [RTE_FLOW_ITEM_TYPE_IPV4] = {
295 .copy_item = enic_fm_copy_item_ipv4,
296 .valid_start_item = 1,
297 .prev_items = (const enum rte_flow_item_type[]) {
298 RTE_FLOW_ITEM_TYPE_ETH,
299 RTE_FLOW_ITEM_TYPE_VLAN,
300 RTE_FLOW_ITEM_TYPE_END,
303 [RTE_FLOW_ITEM_TYPE_IPV6] = {
304 .copy_item = enic_fm_copy_item_ipv6,
305 .valid_start_item = 1,
306 .prev_items = (const enum rte_flow_item_type[]) {
307 RTE_FLOW_ITEM_TYPE_ETH,
308 RTE_FLOW_ITEM_TYPE_VLAN,
309 RTE_FLOW_ITEM_TYPE_END,
312 [RTE_FLOW_ITEM_TYPE_UDP] = {
313 .copy_item = enic_fm_copy_item_udp,
314 .valid_start_item = 1,
315 .prev_items = (const enum rte_flow_item_type[]) {
316 RTE_FLOW_ITEM_TYPE_IPV4,
317 RTE_FLOW_ITEM_TYPE_IPV6,
318 RTE_FLOW_ITEM_TYPE_END,
321 [RTE_FLOW_ITEM_TYPE_TCP] = {
322 .copy_item = enic_fm_copy_item_tcp,
323 .valid_start_item = 1,
324 .prev_items = (const enum rte_flow_item_type[]) {
325 RTE_FLOW_ITEM_TYPE_IPV4,
326 RTE_FLOW_ITEM_TYPE_IPV6,
327 RTE_FLOW_ITEM_TYPE_END,
330 [RTE_FLOW_ITEM_TYPE_SCTP] = {
331 .copy_item = enic_fm_copy_item_sctp,
332 .valid_start_item = 0,
333 .prev_items = (const enum rte_flow_item_type[]) {
334 RTE_FLOW_ITEM_TYPE_IPV4,
335 RTE_FLOW_ITEM_TYPE_IPV6,
336 RTE_FLOW_ITEM_TYPE_END,
339 [RTE_FLOW_ITEM_TYPE_VXLAN] = {
340 .copy_item = enic_fm_copy_item_vxlan,
341 .valid_start_item = 1,
342 .prev_items = (const enum rte_flow_item_type[]) {
343 RTE_FLOW_ITEM_TYPE_UDP,
344 RTE_FLOW_ITEM_TYPE_END,
347 [RTE_FLOW_ITEM_TYPE_GTP] = {
348 .copy_item = enic_fm_copy_item_gtp,
349 .valid_start_item = 0,
350 .prev_items = (const enum rte_flow_item_type[]) {
351 RTE_FLOW_ITEM_TYPE_UDP,
352 RTE_FLOW_ITEM_TYPE_END,
355 [RTE_FLOW_ITEM_TYPE_GTPC] = {
356 .copy_item = enic_fm_copy_item_gtp,
357 .valid_start_item = 1,
358 .prev_items = (const enum rte_flow_item_type[]) {
359 RTE_FLOW_ITEM_TYPE_UDP,
360 RTE_FLOW_ITEM_TYPE_END,
363 [RTE_FLOW_ITEM_TYPE_GTPU] = {
364 .copy_item = enic_fm_copy_item_gtp,
365 .valid_start_item = 1,
366 .prev_items = (const enum rte_flow_item_type[]) {
367 RTE_FLOW_ITEM_TYPE_UDP,
368 RTE_FLOW_ITEM_TYPE_END,
374 enic_fm_copy_item_eth(struct copy_item_args *arg)
376 const struct rte_flow_item *item = arg->item;
377 const struct rte_flow_item_eth *spec = item->spec;
378 const struct rte_flow_item_eth *mask = item->mask;
379 const uint8_t lvl = arg->header_level;
380 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
381 struct fm_header_set *fm_data, *fm_mask;
383 ENICPMD_FUNC_TRACE();
384 /* Match all if no spec */
388 mask = &rte_flow_item_eth_mask;
389 fm_data = &entry->ftm_data.fk_hdrset[lvl];
390 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
391 fm_data->fk_header_select |= FKH_ETHER;
392 fm_mask->fk_header_select |= FKH_ETHER;
393 memcpy(&fm_data->l2.eth, spec, sizeof(struct rte_ether_hdr));
394 memcpy(&fm_mask->l2.eth, mask, sizeof(struct rte_ether_hdr));
399 enic_fm_copy_item_vlan(struct copy_item_args *arg)
401 const struct rte_flow_item *item = arg->item;
402 const struct rte_flow_item_vlan *spec = item->spec;
403 const struct rte_flow_item_vlan *mask = item->mask;
404 const uint8_t lvl = arg->header_level;
405 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
406 struct fm_header_set *fm_data, *fm_mask;
407 struct rte_ether_hdr *eth_mask;
408 struct rte_ether_hdr *eth_val;
411 ENICPMD_FUNC_TRACE();
412 fm_data = &entry->ftm_data.fk_hdrset[lvl];
413 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
414 /* Outer and inner packet vlans need different flags */
415 meta = FKM_VLAN_PRES;
418 fm_data->fk_metadata |= meta;
419 fm_mask->fk_metadata |= meta;
421 /* Match all if no spec */
425 mask = &rte_flow_item_vlan_mask;
427 eth_mask = (void *)&fm_mask->l2.eth;
428 eth_val = (void *)&fm_data->l2.eth;
431 * Outer TPID cannot be matched. If inner_type is 0, use what is
434 if (eth_mask->ether_type && mask->inner_type)
438 * When packet matching, the VIC always compares vlan-stripped
439 * L2, regardless of vlan stripping settings. So, the inner type
440 * from vlan becomes the ether type of the eth header.
442 if (mask->inner_type) {
443 eth_mask->ether_type = mask->inner_type;
444 eth_val->ether_type = spec->inner_type;
446 fm_data->fk_header_select |= FKH_ETHER | FKH_QTAG;
447 fm_mask->fk_header_select |= FKH_ETHER | FKH_QTAG;
448 fm_data->fk_vlan = rte_be_to_cpu_16(spec->tci);
449 fm_mask->fk_vlan = rte_be_to_cpu_16(mask->tci);
454 enic_fm_copy_item_ipv4(struct copy_item_args *arg)
456 const struct rte_flow_item *item = arg->item;
457 const struct rte_flow_item_ipv4 *spec = item->spec;
458 const struct rte_flow_item_ipv4 *mask = item->mask;
459 const uint8_t lvl = arg->header_level;
460 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
461 struct fm_header_set *fm_data, *fm_mask;
463 ENICPMD_FUNC_TRACE();
464 fm_data = &entry->ftm_data.fk_hdrset[lvl];
465 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
466 fm_data->fk_metadata |= FKM_IPV4;
467 fm_mask->fk_metadata |= FKM_IPV4;
472 mask = &rte_flow_item_ipv4_mask;
474 fm_data->fk_header_select |= FKH_IPV4;
475 fm_mask->fk_header_select |= FKH_IPV4;
476 memcpy(&fm_data->l3.ip4, spec, sizeof(*spec));
477 memcpy(&fm_mask->l3.ip4, mask, sizeof(*mask));
482 enic_fm_copy_item_ipv6(struct copy_item_args *arg)
484 const struct rte_flow_item *item = arg->item;
485 const struct rte_flow_item_ipv6 *spec = item->spec;
486 const struct rte_flow_item_ipv6 *mask = item->mask;
487 const uint8_t lvl = arg->header_level;
488 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
489 struct fm_header_set *fm_data, *fm_mask;
491 ENICPMD_FUNC_TRACE();
492 fm_data = &entry->ftm_data.fk_hdrset[lvl];
493 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
494 fm_data->fk_metadata |= FKM_IPV6;
495 fm_mask->fk_metadata |= FKM_IPV6;
500 mask = &rte_flow_item_ipv6_mask;
502 fm_data->fk_header_select |= FKH_IPV6;
503 fm_mask->fk_header_select |= FKH_IPV6;
504 memcpy(&fm_data->l3.ip6, spec, sizeof(struct rte_ipv6_hdr));
505 memcpy(&fm_mask->l3.ip6, mask, sizeof(struct rte_ipv6_hdr));
510 enic_fm_copy_item_udp(struct copy_item_args *arg)
512 const struct rte_flow_item *item = arg->item;
513 const struct rte_flow_item_udp *spec = item->spec;
514 const struct rte_flow_item_udp *mask = item->mask;
515 const uint8_t lvl = arg->header_level;
516 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
517 struct fm_header_set *fm_data, *fm_mask;
519 ENICPMD_FUNC_TRACE();
520 fm_data = &entry->ftm_data.fk_hdrset[lvl];
521 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
522 fm_data->fk_metadata |= FKM_UDP;
523 fm_mask->fk_metadata |= FKM_UDP;
528 mask = &rte_flow_item_udp_mask;
530 fm_data->fk_header_select |= FKH_UDP;
531 fm_mask->fk_header_select |= FKH_UDP;
532 memcpy(&fm_data->l4.udp, spec, sizeof(*spec));
533 memcpy(&fm_mask->l4.udp, mask, sizeof(*mask));
538 enic_fm_copy_item_tcp(struct copy_item_args *arg)
540 const struct rte_flow_item *item = arg->item;
541 const struct rte_flow_item_tcp *spec = item->spec;
542 const struct rte_flow_item_tcp *mask = item->mask;
543 const uint8_t lvl = arg->header_level;
544 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
545 struct fm_header_set *fm_data, *fm_mask;
547 ENICPMD_FUNC_TRACE();
548 fm_data = &entry->ftm_data.fk_hdrset[lvl];
549 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
550 fm_data->fk_metadata |= FKM_TCP;
551 fm_mask->fk_metadata |= FKM_TCP;
556 mask = &rte_flow_item_tcp_mask;
558 fm_data->fk_header_select |= FKH_TCP;
559 fm_mask->fk_header_select |= FKH_TCP;
560 memcpy(&fm_data->l4.tcp, spec, sizeof(*spec));
561 memcpy(&fm_mask->l4.tcp, mask, sizeof(*mask));
566 enic_fm_copy_item_sctp(struct copy_item_args *arg)
568 const struct rte_flow_item *item = arg->item;
569 const struct rte_flow_item_sctp *spec = item->spec;
570 const struct rte_flow_item_sctp *mask = item->mask;
571 const uint8_t lvl = arg->header_level;
572 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
573 struct fm_header_set *fm_data, *fm_mask;
574 uint8_t *ip_proto_mask = NULL;
575 uint8_t *ip_proto = NULL;
578 ENICPMD_FUNC_TRACE();
579 fm_data = &entry->ftm_data.fk_hdrset[lvl];
580 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
582 * The NIC filter API has no flags for "match sctp", so explicitly
583 * set the protocol number in the IP pattern.
585 if (fm_data->fk_metadata & FKM_IPV4) {
586 struct rte_ipv4_hdr *ip;
587 ip = (struct rte_ipv4_hdr *)&fm_mask->l3.ip4;
588 ip_proto_mask = &ip->next_proto_id;
589 ip = (struct rte_ipv4_hdr *)&fm_data->l3.ip4;
590 ip_proto = &ip->next_proto_id;
592 } else if (fm_data->fk_metadata & FKM_IPV6) {
593 struct rte_ipv6_hdr *ip;
594 ip = (struct rte_ipv6_hdr *)&fm_mask->l3.ip6;
595 ip_proto_mask = &ip->proto;
596 ip = (struct rte_ipv6_hdr *)&fm_data->l3.ip6;
597 ip_proto = &ip->proto;
600 /* Need IPv4/IPv6 pattern first */
603 *ip_proto = IPPROTO_SCTP;
604 *ip_proto_mask = 0xff;
605 fm_data->fk_header_select |= l3_fkh;
606 fm_mask->fk_header_select |= l3_fkh;
611 mask = &rte_flow_item_sctp_mask;
613 fm_data->fk_header_select |= FKH_L4RAW;
614 fm_mask->fk_header_select |= FKH_L4RAW;
615 memcpy(fm_data->l4.rawdata, spec, sizeof(*spec));
616 memcpy(fm_mask->l4.rawdata, mask, sizeof(*mask));
621 enic_fm_copy_item_vxlan(struct copy_item_args *arg)
623 const struct rte_flow_item *item = arg->item;
624 const struct rte_flow_item_vxlan *spec = item->spec;
625 const struct rte_flow_item_vxlan *mask = item->mask;
626 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
627 struct fm_header_set *fm_data, *fm_mask;
629 ENICPMD_FUNC_TRACE();
630 /* Only 2 header levels (outer and inner) allowed */
631 if (arg->header_level > 0)
634 fm_data = &entry->ftm_data.fk_hdrset[0];
635 fm_mask = &entry->ftm_mask.fk_hdrset[0];
636 fm_data->fk_metadata |= FKM_VXLAN;
637 fm_mask->fk_metadata |= FKM_VXLAN;
638 /* items from here on out are inner header items */
639 arg->header_level = 1;
641 /* Match all if no spec */
645 mask = &rte_flow_item_vxlan_mask;
647 fm_data->fk_header_select |= FKH_VXLAN;
648 fm_mask->fk_header_select |= FKH_VXLAN;
649 memcpy(&fm_data->vxlan, spec, sizeof(*spec));
650 memcpy(&fm_mask->vxlan, mask, sizeof(*mask));
655 enic_fm_copy_item_gtp(struct copy_item_args *arg)
657 const struct rte_flow_item *item = arg->item;
658 const struct rte_flow_item_gtp *spec = item->spec;
659 const struct rte_flow_item_gtp *mask = item->mask;
660 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
661 struct fm_header_set *fm_data, *fm_mask;
663 uint16_t udp_gtp_uc_port_be = 0;
665 ENICPMD_FUNC_TRACE();
666 /* Only 2 header levels (outer and inner) allowed */
667 if (arg->header_level > 0)
670 fm_data = &entry->ftm_data.fk_hdrset[0];
671 fm_mask = &entry->ftm_mask.fk_hdrset[0];
673 switch (item->type) {
674 case RTE_FLOW_ITEM_TYPE_GTP:
676 /* For vanilla GTP, the UDP destination port must be specified
677 * but value of the port is not enforced here.
679 if (!(fm_data->fk_metadata & FKM_UDP) ||
680 !(fm_data->fk_header_select & FKH_UDP) ||
681 fm_data->l4.udp.fk_dest == 0)
683 if (!(fm_mask->fk_metadata & FKM_UDP) ||
684 !(fm_mask->fk_header_select & FKH_UDP) ||
685 fm_mask->l4.udp.fk_dest != 0xFFFF)
689 case RTE_FLOW_ITEM_TYPE_GTPC:
691 udp_gtp_uc_port_be = rte_cpu_to_be_16(RTE_GTPC_UDP_PORT);
694 case RTE_FLOW_ITEM_TYPE_GTPU:
696 udp_gtp_uc_port_be = rte_cpu_to_be_16(RTE_GTPU_UDP_PORT);
703 /* The GTP-C or GTP-U UDP destination port must be matched. */
704 if (udp_gtp_uc_port_be) {
705 if (fm_data->fk_metadata & FKM_UDP &&
706 fm_data->fk_header_select & FKH_UDP &&
707 fm_data->l4.udp.fk_dest != udp_gtp_uc_port_be)
709 if (fm_mask->fk_metadata & FKM_UDP &&
710 fm_mask->fk_header_select & FKH_UDP &&
711 fm_mask->l4.udp.fk_dest != 0xFFFF)
714 /* In any case, add match for GTP-C GTP-U UDP dst port */
715 fm_data->fk_metadata |= FKM_UDP;
716 fm_data->fk_header_select |= FKH_UDP;
717 fm_data->l4.udp.fk_dest = udp_gtp_uc_port_be;
718 fm_mask->fk_metadata |= FKM_UDP;
719 fm_mask->fk_header_select |= FKH_UDP;
720 fm_mask->l4.udp.fk_dest = 0xFFFF;
723 /* NIC does not support GTP tunnels. No Items are allowed after this.
724 * This prevents the specificaiton of further items.
726 arg->header_level = 0;
728 /* Match all if no spec */
732 mask = &rte_flow_item_gtp_mask;
735 * Use the raw L4 buffer to match GTP as fm_header_set does not have
736 * GTP header. UDP dst port must be specifiec. Using the raw buffer
737 * does not affect such UDP item, since we skip UDP in the raw buffer.
739 fm_data->fk_header_select |= FKH_L4RAW;
740 fm_mask->fk_header_select |= FKH_L4RAW;
741 off = sizeof(fm_data->l4.udp);
742 memcpy(&fm_data->l4.rawdata[off], spec, sizeof(*spec));
743 memcpy(&fm_mask->l4.rawdata[off], mask, sizeof(*mask));
748 * Currently, raw pattern match is very limited. It is intended for matching
749 * UDP tunnel header (e.g. vxlan or geneve).
752 enic_fm_copy_item_raw(struct copy_item_args *arg)
754 const struct rte_flow_item *item = arg->item;
755 const struct rte_flow_item_raw *spec = item->spec;
756 const struct rte_flow_item_raw *mask = item->mask;
757 const uint8_t lvl = arg->header_level;
758 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
759 struct fm_header_set *fm_data, *fm_mask;
761 ENICPMD_FUNC_TRACE();
762 /* Cannot be used for inner packet */
765 /* Need both spec and mask */
768 /* Only supports relative with offset 0 */
769 if (!spec->relative || spec->offset != 0 || spec->search ||
772 /* Need non-null pattern that fits within the NIC's filter pattern */
773 if (spec->length == 0 ||
774 spec->length + sizeof(struct rte_udp_hdr) > FM_LAYER_SIZE ||
775 !spec->pattern || !mask->pattern)
778 * Mask fields, including length, are often set to zero. Assume that
779 * means "same as spec" to avoid breaking existing apps. If length
780 * is not zero, then it should be >= spec length.
782 * No more pattern follows this, so append to the L4 layer instead of
783 * L5 to work with both recent and older VICs.
785 if (mask->length != 0 && mask->length < spec->length)
788 fm_data = &entry->ftm_data.fk_hdrset[lvl];
789 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
790 fm_data->fk_header_select |= FKH_L4RAW;
791 fm_mask->fk_header_select |= FKH_L4RAW;
792 fm_data->fk_header_select &= ~FKH_UDP;
793 fm_mask->fk_header_select &= ~FKH_UDP;
794 memcpy(fm_data->l4.rawdata + sizeof(struct rte_udp_hdr),
795 spec->pattern, spec->length);
796 memcpy(fm_mask->l4.rawdata + sizeof(struct rte_udp_hdr),
797 mask->pattern, spec->length);
802 flowman_cmd(struct enic_flowman *fm, uint64_t *args, int nargs)
804 return vnic_dev_flowman_cmd(fm->owner_enic->vdev, args, nargs);
808 enic_fet_alloc(struct enic_flowman *fm, uint8_t ingress,
809 struct fm_key_template *key, int entries,
810 struct enic_fm_fet **fet_out)
812 struct fm_exact_match_table *cmd;
813 struct fm_header_set *hdr;
814 struct enic_fm_fet *fet;
818 ENICPMD_FUNC_TRACE();
819 fet = calloc(1, sizeof(struct enic_fm_fet));
822 cmd = &fm->cmd.va->fm_exact_match_table;
823 memset(cmd, 0, sizeof(*cmd));
824 cmd->fet_direction = ingress ? FM_INGRESS : FM_EGRESS;
825 cmd->fet_stage = FM_STAGE_LAST;
826 cmd->fet_max_entries = entries ? entries : FM_MAX_EXACT_TABLE_SIZE;
828 hdr = &cmd->fet_key.fk_hdrset[0];
829 memset(hdr, 0, sizeof(*hdr));
830 hdr->fk_header_select = FKH_IPV4 | FKH_UDP;
831 hdr->l3.ip4.fk_saddr = 0xFFFFFFFF;
832 hdr->l3.ip4.fk_daddr = 0xFFFFFFFF;
833 hdr->l4.udp.fk_source = 0xFFFF;
834 hdr->l4.udp.fk_dest = 0xFFFF;
835 fet->default_key = 1;
837 memcpy(&cmd->fet_key, key, sizeof(*key));
838 memcpy(&fet->key, key, sizeof(*key));
839 fet->default_key = 0;
841 cmd->fet_key.fk_packet_tag = 1;
843 args[0] = FM_EXACT_TABLE_ALLOC;
844 args[1] = fm->cmd.pa;
845 ret = flowman_cmd(fm, args, 2);
847 ENICPMD_LOG(ERR, "cannot alloc exact match table: rc=%d", ret);
851 fet->handle = args[0];
852 fet->ingress = ingress;
853 ENICPMD_LOG(DEBUG, "allocated exact match table: handle=0x%" PRIx64,
860 enic_fet_free(struct enic_flowman *fm, struct enic_fm_fet *fet)
862 ENICPMD_FUNC_TRACE();
863 enic_fm_tbl_free(fm, fet->handle);
864 if (!fet->default_key)
865 TAILQ_REMOVE(&fm->fet_list, fet, list);
870 * Get the exact match table for the given combination of
871 * <group, ingress, key>. Allocate one on the fly as necessary.
874 enic_fet_get(struct enic_flowman *fm,
877 struct fm_key_template *key,
878 struct enic_fm_fet **fet_out,
879 struct rte_flow_error *error)
881 struct enic_fm_fet *fet;
883 ENICPMD_FUNC_TRACE();
884 /* See if we already have this table open */
885 TAILQ_FOREACH(fet, &fm->fet_list, list) {
886 if (fet->group == group && fet->ingress == ingress)
890 /* Jumping to a non-existing group? Use the default table */
892 fet = ingress ? fm->default_ig_fet : fm->default_eg_fet;
893 } else if (enic_fet_alloc(fm, ingress, key, 0, &fet)) {
894 return rte_flow_error_set(error, EINVAL,
895 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
896 NULL, "enic: cannot get exact match table");
899 /* Default table is never on the open table list */
900 if (!fet->default_key)
901 TAILQ_INSERT_HEAD(&fm->fet_list, fet, list);
905 ENICPMD_LOG(DEBUG, "fet_get: %s %s group=%u ref=%u",
906 fet->default_key ? "default" : "",
907 fet->ingress ? "ingress" : "egress",
908 fet->group, fet->ref);
913 enic_fet_put(struct enic_flowman *fm, struct enic_fm_fet *fet)
915 ENICPMD_FUNC_TRACE();
916 RTE_ASSERT(fet->ref > 0);
918 ENICPMD_LOG(DEBUG, "fet_put: %s %s group=%u ref=%u",
919 fet->default_key ? "default" : "",
920 fet->ingress ? "ingress" : "egress",
921 fet->group, fet->ref);
923 enic_fet_free(fm, fet);
926 /* Return 1 if current item is valid on top of the previous one. */
928 fm_item_stacking_valid(enum rte_flow_item_type prev_item,
929 const struct enic_fm_items *item_info,
930 uint8_t is_first_item)
932 enum rte_flow_item_type const *allowed_items = item_info->prev_items;
934 ENICPMD_FUNC_TRACE();
935 for (; *allowed_items != RTE_FLOW_ITEM_TYPE_END; allowed_items++) {
936 if (prev_item == *allowed_items)
940 /* This is the first item in the stack. Check if that's cool */
941 if (is_first_item && item_info->valid_start_item)
947 * Build the flow manager match entry structure from the provided pattern.
948 * The pattern is validated as the items are copied.
951 enic_fm_copy_entry(struct enic_flowman *fm,
952 const struct rte_flow_item pattern[],
953 struct rte_flow_error *error)
955 const struct enic_fm_items *item_info;
956 enum rte_flow_item_type prev_item;
957 const struct rte_flow_item *item;
958 struct copy_item_args args;
959 uint8_t prev_header_level;
960 uint8_t is_first_item;
963 ENICPMD_FUNC_TRACE();
966 prev_item = RTE_FLOW_ITEM_TYPE_END;
968 args.fm_tcam_entry = &fm->tcam_entry;
969 args.header_level = 0;
970 prev_header_level = 0;
971 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
973 * Get info about how to validate and copy the item. If NULL
974 * is returned the nic does not support the item.
976 if (item->type == RTE_FLOW_ITEM_TYPE_VOID)
979 item_info = &enic_fm_items[item->type];
981 if (item->type >= RTE_DIM(enic_fm_items) ||
982 item_info->copy_item == NULL) {
983 return rte_flow_error_set(error, ENOTSUP,
984 RTE_FLOW_ERROR_TYPE_ITEM,
985 NULL, "enic: unsupported item");
988 /* check to see if item stacking is valid */
989 if (!fm_item_stacking_valid(prev_item, item_info,
994 ret = item_info->copy_item(&args);
996 goto item_not_supported;
997 /* Going from outer to inner? Treat it as a new packet start */
998 if (prev_header_level != args.header_level) {
999 prev_item = RTE_FLOW_ITEM_TYPE_END;
1002 prev_item = item->type;
1005 prev_header_level = args.header_level;
1010 return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,
1011 NULL, "enic: unsupported item type");
1014 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
1015 item, "enic: unsupported item stack");
1019 flow_item_skip_void(const struct rte_flow_item **item)
1021 for ( ; ; (*item)++)
1022 if ((*item)->type != RTE_FLOW_ITEM_TYPE_VOID)
1027 append_template(void **template, uint8_t *off, const void *data, int len)
1029 memcpy(*template, data, len);
1030 *template = (char *)*template + len;
1035 enic_fm_append_action_op(struct enic_flowman *fm,
1036 struct fm_action_op *fm_op,
1037 struct rte_flow_error *error)
1041 count = fm->action_op_count;
1042 ENICPMD_LOG(DEBUG, "append action op: idx=%d op=%u",
1043 count, fm_op->fa_op);
1044 if (count == FM_ACTION_OP_MAX) {
1045 return rte_flow_error_set(error, EINVAL,
1046 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1047 "too many action operations");
1049 fm->action.fma_action_ops[count] = *fm_op;
1050 fm->action_op_count = count + 1;
1054 static struct fm_action_op *
1055 find_prev_action_op(struct enic_flowman *fm, uint32_t opcode)
1057 struct fm_action_op *op;
1060 for (i = 0; i < fm->action_op_count; i++) {
1061 op = &fm->action.fma_action_ops[i];
1062 if (op->fa_op == opcode)
1068 /* NIC requires that 1st steer appear before decap.
1069 * Correct example: steer, decap, steer, steer, ...
1072 enic_fm_reorder_action_op(struct enic_flowman *fm)
1074 struct fm_action_op *op, *steer, *decap;
1075 struct fm_action_op tmp_op;
1077 ENICPMD_FUNC_TRACE();
1078 /* Find 1st steer and decap */
1079 op = fm->action.fma_action_ops;
1082 while (op->fa_op != FMOP_END) {
1083 if (!decap && (op->fa_op == FMOP_DECAP_NOSTRIP ||
1084 op->fa_op == FMOP_DECAP_STRIP))
1086 else if (!steer && op->fa_op == FMOP_RQ_STEER)
1090 /* If decap is before steer, swap */
1091 if (steer && decap && decap < steer) {
1092 op = fm->action.fma_action_ops;
1093 ENICPMD_LOG(DEBUG, "swap decap %ld <-> steer %ld",
1094 (long)(decap - op), (long)(steer - op));
1101 /* VXLAN decap is done via flowman compound action */
1103 enic_fm_copy_vxlan_decap(struct enic_flowman *fm,
1104 struct fm_tcam_match_entry *fmt,
1105 const struct rte_flow_action *action,
1106 struct rte_flow_error *error)
1108 struct fm_header_set *fm_data;
1109 struct fm_action_op fm_op;
1111 ENICPMD_FUNC_TRACE();
1112 fm_data = &fmt->ftm_data.fk_hdrset[0];
1113 if (!(fm_data->fk_metadata & FKM_VXLAN)) {
1114 return rte_flow_error_set(error, EINVAL,
1115 RTE_FLOW_ERROR_TYPE_ACTION, action,
1116 "vxlan-decap: vxlan must be in pattern");
1119 memset(&fm_op, 0, sizeof(fm_op));
1120 fm_op.fa_op = FMOP_DECAP_NOSTRIP;
1121 return enic_fm_append_action_op(fm, &fm_op, error);
1124 /* Generate a reasonable source port number */
1128 /* Min/max below are the default values in OVS-DPDK and Linux */
1129 uint16_t p = rte_rand();
1130 p = RTE_MAX(p, 32768);
1131 p = RTE_MIN(p, 61000);
1132 return rte_cpu_to_be_16(p);
1135 /* VXLAN encap is done via flowman compound action */
1137 enic_fm_copy_vxlan_encap(struct enic_flowman *fm,
1138 const struct rte_flow_item *item,
1139 struct rte_flow_error *error)
1141 struct fm_action_op fm_op;
1142 struct rte_ether_hdr *eth;
1143 struct rte_udp_hdr *udp;
1144 uint16_t *ethertype;
1148 ENICPMD_FUNC_TRACE();
1149 memset(&fm_op, 0, sizeof(fm_op));
1150 fm_op.fa_op = FMOP_ENCAP;
1151 template = fm->action.fma_data;
1154 * Copy flow items to the flowman template starting L2.
1155 * L2 must be ethernet.
1157 flow_item_skip_void(&item);
1158 if (item->type != RTE_FLOW_ITEM_TYPE_ETH)
1159 return rte_flow_error_set(error, EINVAL,
1160 RTE_FLOW_ERROR_TYPE_ITEM, item,
1161 "vxlan-encap: first item should be ethernet");
1162 eth = (struct rte_ether_hdr *)template;
1163 ethertype = ð->ether_type;
1164 append_template(&template, &off, item->spec,
1165 sizeof(struct rte_ether_hdr));
1167 flow_item_skip_void(&item);
1169 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
1170 const struct rte_flow_item_vlan *spec;
1172 ENICPMD_LOG(DEBUG, "vxlan-encap: vlan");
1174 fm_op.encap.outer_vlan = rte_be_to_cpu_16(spec->tci);
1176 flow_item_skip_void(&item);
1178 /* L3 must be IPv4, IPv6 */
1179 switch (item->type) {
1180 case RTE_FLOW_ITEM_TYPE_IPV4:
1182 struct rte_ipv4_hdr *ip4;
1184 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv4");
1185 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
1186 ip4 = (struct rte_ipv4_hdr *)template;
1188 * Offset of IPv4 length field and its initial value
1189 * (IP + UDP + VXLAN) are specified in the action. The NIC
1190 * will add inner packet length.
1192 fm_op.encap.len1_offset = off +
1193 offsetof(struct rte_ipv4_hdr, total_length);
1194 fm_op.encap.len1_delta = sizeof(struct rte_ipv4_hdr) +
1195 sizeof(struct rte_udp_hdr) +
1196 sizeof(struct rte_vxlan_hdr);
1197 append_template(&template, &off, item->spec,
1198 sizeof(struct rte_ipv4_hdr));
1199 ip4->version_ihl = RTE_IPV4_VHL_DEF;
1200 if (ip4->time_to_live == 0)
1201 ip4->time_to_live = IP_DEFTTL;
1202 ip4->next_proto_id = IPPROTO_UDP;
1205 case RTE_FLOW_ITEM_TYPE_IPV6:
1207 struct rte_ipv6_hdr *ip6;
1209 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv6");
1210 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
1211 ip6 = (struct rte_ipv6_hdr *)template;
1212 fm_op.encap.len1_offset = off +
1213 offsetof(struct rte_ipv6_hdr, payload_len);
1214 fm_op.encap.len1_delta = sizeof(struct rte_udp_hdr) +
1215 sizeof(struct rte_vxlan_hdr);
1216 append_template(&template, &off, item->spec,
1217 sizeof(struct rte_ipv6_hdr));
1218 ip6->vtc_flow |= rte_cpu_to_be_32(IP6_VTC_FLOW);
1219 if (ip6->hop_limits == 0)
1220 ip6->hop_limits = IP_DEFTTL;
1221 ip6->proto = IPPROTO_UDP;
1225 return rte_flow_error_set(error,
1226 EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1227 "vxlan-encap: L3 must be IPv4/IPv6");
1230 flow_item_skip_void(&item);
1233 if (item->type != RTE_FLOW_ITEM_TYPE_UDP)
1234 return rte_flow_error_set(error, EINVAL,
1235 RTE_FLOW_ERROR_TYPE_ITEM, item,
1236 "vxlan-encap: UDP must follow IPv4/IPv6");
1237 /* UDP length = UDP + VXLAN. NIC will add inner packet length. */
1238 fm_op.encap.len2_offset =
1239 off + offsetof(struct rte_udp_hdr, dgram_len);
1240 fm_op.encap.len2_delta =
1241 sizeof(struct rte_udp_hdr) + sizeof(struct rte_vxlan_hdr);
1242 udp = (struct rte_udp_hdr *)template;
1243 append_template(&template, &off, item->spec,
1244 sizeof(struct rte_udp_hdr));
1246 * Firmware does not hash/fill source port yet. Generate a
1247 * random port, as there is *usually* one rte_flow for the
1248 * given inner packet stream (i.e. a single stream has one
1251 if (udp->src_port == 0)
1252 udp->src_port = gen_src_port();
1254 flow_item_skip_void(&item);
1257 if (item->type != RTE_FLOW_ITEM_TYPE_VXLAN)
1258 return rte_flow_error_set(error,
1259 EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1260 "vxlan-encap: VXLAN must follow UDP");
1261 append_template(&template, &off, item->spec,
1262 sizeof(struct rte_flow_item_vxlan));
1265 * Fill in the rest of the action structure.
1266 * Indicate that we want to encap with vxlan at packet start.
1268 fm_op.encap.template_offset = 0;
1269 fm_op.encap.template_len = off;
1270 return enic_fm_append_action_op(fm, &fm_op, error);
1274 enic_fm_find_vnic(struct enic *enic, const struct rte_pci_addr *addr,
1281 ENICPMD_FUNC_TRACE();
1282 ENICPMD_LOG(DEBUG, "bdf=%x:%x:%x", addr->bus, addr->devid,
1284 bdf = addr->bus << 8 | addr->devid << 3 | addr->function;
1285 args[0] = FM_VNIC_FIND;
1287 rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
1289 /* Expected to fail if BDF is not on the adapter */
1290 ENICPMD_LOG(DEBUG, "cannot find vnic handle: rc=%d", rc);
1294 ENICPMD_LOG(DEBUG, "found vnic: handle=0x%" PRIx64, *handle);
1299 * Egress: target port should be either PF uplink or VF.
1301 * 1. VF egress -> PF uplink
1302 * PF may be this VF's PF, or another PF, as long as they are on the same VIC.
1303 * 2. VF egress -> VF
1306 * 1. PF egress -> VF
1307 * App should be using representor to pass packets to VF
1310 vf_egress_port_id_action(struct enic_flowman *fm,
1311 struct rte_eth_dev *dst_dev,
1312 uint64_t dst_vnic_h,
1313 struct fm_action_op *fm_op,
1314 struct rte_flow_error *error)
1316 struct enic *src_enic, *dst_enic;
1317 struct enic_vf_representor *vf;
1321 ENICPMD_FUNC_TRACE();
1322 src_enic = fm->user_enic;
1323 dst_enic = pmd_priv(dst_dev);
1324 if (!(src_enic->rte_dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)) {
1325 return rte_flow_error_set(error, EINVAL,
1326 RTE_FLOW_ERROR_TYPE_ACTION,
1327 NULL, "source port is not VF representor");
1330 /* VF -> PF uplink. dst is not VF representor */
1331 if (!(dst_dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)) {
1332 /* PF is the VF's PF? Then nothing to do */
1333 vf = VF_ENIC_TO_VF_REP(src_enic);
1334 if (vf->pf == dst_enic) {
1335 ENICPMD_LOG(DEBUG, "destination port is VF's PF");
1338 /* If not, steer to the remote PF's uplink */
1339 uif = dst_enic->fm_vnic_uif;
1340 ENICPMD_LOG(DEBUG, "steer to uplink %u", uif);
1341 memset(fm_op, 0, sizeof(*fm_op));
1342 fm_op->fa_op = FMOP_SET_EGPORT;
1343 fm_op->set_egport.egport = uif;
1344 ret = enic_fm_append_action_op(fm, fm_op, error);
1348 /* VF -> VF loopback. Hairpin and steer to vnic */
1349 memset(fm_op, 0, sizeof(*fm_op));
1350 fm_op->fa_op = FMOP_EG_HAIRPIN;
1351 ret = enic_fm_append_action_op(fm, fm_op, error);
1354 ENICPMD_LOG(DEBUG, "egress hairpin");
1355 fm->hairpin_steer_vnic_h = dst_vnic_h;
1356 fm->need_hairpin_steer = 1;
1361 enic_fm_check_transfer_dst(struct enic *enic, uint16_t dst_port_id,
1362 struct rte_eth_dev **dst_dev,
1363 struct rte_flow_error *error)
1365 struct rte_eth_dev *dev;
1367 ENICPMD_LOG(DEBUG, "port id %u", dst_port_id);
1368 if (!rte_eth_dev_is_valid_port(dst_port_id)) {
1369 return rte_flow_error_set(error, EINVAL,
1370 RTE_FLOW_ERROR_TYPE_ACTION,
1371 NULL, "invalid port_id");
1373 dev = &rte_eth_devices[dst_port_id];
1374 if (!dev_is_enic(dev)) {
1375 return rte_flow_error_set(error, EINVAL,
1376 RTE_FLOW_ERROR_TYPE_ACTION,
1377 NULL, "port_id is not enic");
1379 if (enic->switch_domain_id != pmd_priv(dev)->switch_domain_id) {
1380 return rte_flow_error_set(error, EINVAL,
1381 RTE_FLOW_ERROR_TYPE_ACTION,
1382 NULL, "destination and source ports are not in the same switch domain");
1389 /* Translate flow actions to flowman TCAM entry actions */
1391 enic_fm_copy_action(struct enic_flowman *fm,
1392 const struct rte_flow_action actions[],
1394 struct rte_flow_error *error)
1405 struct fm_tcam_match_entry *fmt;
1406 struct fm_action_op fm_op;
1407 bool need_ovlan_action;
1416 ENICPMD_FUNC_TRACE();
1417 fmt = &fm->tcam_entry;
1418 need_ovlan_action = false;
1422 enic = fm->user_enic;
1424 vnic_h = enic->fm_vnic_handle;
1426 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1427 switch (actions->type) {
1428 case RTE_FLOW_ACTION_TYPE_VOID:
1430 case RTE_FLOW_ACTION_TYPE_PASSTHRU: {
1431 if (overlap & PASSTHRU)
1433 overlap |= PASSTHRU;
1436 case RTE_FLOW_ACTION_TYPE_JUMP: {
1437 const struct rte_flow_action_jump *jump =
1439 struct enic_fm_fet *fet;
1443 ret = enic_fet_get(fm, jump->group, ingress, NULL,
1448 memset(&fm_op, 0, sizeof(fm_op));
1449 fm_op.fa_op = FMOP_EXACT_MATCH;
1450 fm_op.exact.handle = fet->handle;
1452 ret = enic_fm_append_action_op(fm, &fm_op, error);
1457 case RTE_FLOW_ACTION_TYPE_MARK: {
1458 const struct rte_flow_action_mark *mark =
1461 if (enic->use_noscatter_vec_rx_handler)
1463 if (mark->id >= ENIC_MAGIC_FILTER_ID - 1)
1464 return rte_flow_error_set(error, EINVAL,
1465 RTE_FLOW_ERROR_TYPE_ACTION,
1466 NULL, "invalid mark id");
1467 memset(&fm_op, 0, sizeof(fm_op));
1468 fm_op.fa_op = FMOP_MARK;
1469 fm_op.mark.mark = mark->id + 1;
1470 ret = enic_fm_append_action_op(fm, &fm_op, error);
1475 case RTE_FLOW_ACTION_TYPE_FLAG: {
1476 if (enic->use_noscatter_vec_rx_handler)
1478 /* ENIC_MAGIC_FILTER_ID is reserved for flagging */
1479 memset(&fm_op, 0, sizeof(fm_op));
1480 fm_op.fa_op = FMOP_MARK;
1481 fm_op.mark.mark = ENIC_MAGIC_FILTER_ID;
1482 ret = enic_fm_append_action_op(fm, &fm_op, error);
1487 case RTE_FLOW_ACTION_TYPE_QUEUE: {
1488 const struct rte_flow_action_queue *queue =
1492 * If fate other than QUEUE or RSS, fail. Multiple
1493 * rss and queue actions are ok.
1495 if ((overlap & FATE) && first_rq)
1499 memset(&fm_op, 0, sizeof(fm_op));
1500 fm_op.fa_op = FMOP_RQ_STEER;
1501 fm_op.rq_steer.rq_index =
1502 enic_rte_rq_idx_to_sop_idx(queue->index);
1503 fm_op.rq_steer.rq_count = 1;
1504 fm_op.rq_steer.vnic_handle = vnic_h;
1505 ret = enic_fm_append_action_op(fm, &fm_op, error);
1508 ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1509 fm_op.rq_steer.rq_index);
1513 case RTE_FLOW_ACTION_TYPE_DROP: {
1517 memset(&fm_op, 0, sizeof(fm_op));
1518 fm_op.fa_op = FMOP_DROP;
1519 ret = enic_fm_append_action_op(fm, &fm_op, error);
1522 ENICPMD_LOG(DEBUG, "create DROP action");
1525 case RTE_FLOW_ACTION_TYPE_COUNT: {
1526 if (overlap & COUNT)
1529 /* Count is associated with entry not action on VIC. */
1530 fmt->ftm_flags |= FMEF_COUNTER;
1533 case RTE_FLOW_ACTION_TYPE_RSS: {
1534 const struct rte_flow_action_rss *rss = actions->conf;
1539 * If fate other than QUEUE or RSS, fail. Multiple
1540 * rss and queue actions are ok.
1542 if ((overlap & FATE) && first_rq)
1548 * Hardware only supports RSS actions on outer level
1549 * with default type and function. Queues must be
1552 allow = rss->func == RTE_ETH_HASH_FUNCTION_DEFAULT &&
1553 rss->level == 0 && (rss->types == 0 ||
1554 rss->types == enic->rss_hf) &&
1555 rss->queue_num <= enic->rq_count &&
1556 rss->queue[rss->queue_num - 1] < enic->rq_count;
1559 /* Identity queue map needs to be sequential */
1560 for (i = 1; i < rss->queue_num; i++)
1561 allow = allow && (rss->queue[i] ==
1562 rss->queue[i - 1] + 1);
1566 memset(&fm_op, 0, sizeof(fm_op));
1567 fm_op.fa_op = FMOP_RQ_STEER;
1568 fm_op.rq_steer.rq_index =
1569 enic_rte_rq_idx_to_sop_idx(rss->queue[0]);
1570 fm_op.rq_steer.rq_count = rss->queue_num;
1571 fm_op.rq_steer.vnic_handle = vnic_h;
1572 ret = enic_fm_append_action_op(fm, &fm_op, error);
1575 ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1576 fm_op.rq_steer.rq_index);
1580 case RTE_FLOW_ACTION_TYPE_PORT_ID: {
1581 const struct rte_flow_action_port_id *port;
1582 struct rte_eth_dev *dev = NULL;
1584 if (!ingress && (overlap & PORT_ID)) {
1585 ENICPMD_LOG(DEBUG, "cannot have multiple egress PORT_ID actions");
1588 port = actions->conf;
1589 if (port->original) {
1590 vnic_h = enic->fm_vnic_handle; /* This port */
1593 ret = enic_fm_check_transfer_dst(enic, port->id, &dev,
1597 vnic_h = pmd_priv(dev)->fm_vnic_handle;
1600 * Ingress. Nothing more to do. We add an implicit
1601 * steer at the end if needed.
1606 ret = vf_egress_port_id_action(fm, dev, vnic_h, &fm_op,
1612 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: {
1613 if (overlap & DECAP)
1617 ret = enic_fm_copy_vxlan_decap(fm, fmt, actions,
1623 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: {
1624 const struct rte_flow_action_vxlan_encap *encap;
1626 encap = actions->conf;
1627 if (overlap & ENCAP)
1630 ret = enic_fm_copy_vxlan_encap(fm, encap->definition,
1636 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN: {
1637 struct fm_action_op *decap;
1640 * If decap-nostrip appears before pop vlan, this pop
1641 * applies to the inner packet vlan. Turn it into
1644 decap = find_prev_action_op(fm, FMOP_DECAP_NOSTRIP);
1646 ENICPMD_LOG(DEBUG, "pop-vlan inner: decap-nostrip => decap-strip");
1647 decap->fa_op = FMOP_DECAP_STRIP;
1650 memset(&fm_op, 0, sizeof(fm_op));
1651 fm_op.fa_op = FMOP_POP_VLAN;
1652 ret = enic_fm_append_action_op(fm, &fm_op, error);
1657 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: {
1658 const struct rte_flow_action_of_push_vlan *vlan;
1660 if (overlap & PASSTHRU)
1662 vlan = actions->conf;
1663 if (vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN)) {
1664 return rte_flow_error_set(error, EINVAL,
1665 RTE_FLOW_ERROR_TYPE_ACTION,
1666 NULL, "unexpected push_vlan ethertype");
1668 overlap |= PUSH_VLAN;
1669 need_ovlan_action = true;
1672 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP: {
1673 const struct rte_flow_action_of_set_vlan_pcp *pcp;
1675 pcp = actions->conf;
1676 if (pcp->vlan_pcp > 7) {
1677 return rte_flow_error_set(error, EINVAL,
1678 RTE_FLOW_ERROR_TYPE_ACTION,
1679 NULL, "invalid vlan_pcp");
1681 need_ovlan_action = true;
1682 ovlan |= ((uint16_t)pcp->vlan_pcp) << 13;
1685 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: {
1686 const struct rte_flow_action_of_set_vlan_vid *vid;
1688 vid = actions->conf;
1689 need_ovlan_action = true;
1690 ovlan |= rte_be_to_cpu_16(vid->vlan_vid);
1693 case RTE_FLOW_ACTION_TYPE_PORT_REPRESENTOR: {
1694 const struct rte_flow_action_ethdev *ethdev;
1695 struct rte_eth_dev *dev = NULL;
1697 ethdev = actions->conf;
1698 ret = enic_fm_check_transfer_dst(enic, ethdev->port_id,
1702 vnic_h = pmd_priv(dev)->fm_vnic_handle;
1705 * Action PORT_REPRESENTOR implies ingress destination.
1706 * Noting to do. We add an implicit stree at the
1712 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: {
1713 const struct rte_flow_action_ethdev *ethdev;
1714 struct rte_eth_dev *dev = NULL;
1716 if (overlap & PORT_ID) {
1717 ENICPMD_LOG(DEBUG, "cannot have multiple egress PORT_ID actions");
1720 ethdev = actions->conf;
1721 ret = enic_fm_check_transfer_dst(enic, ethdev->port_id,
1725 vnic_h = pmd_priv(dev)->fm_vnic_handle;
1727 /* Action REPRESENTED_PORT: always egress destination */
1729 ret = vf_egress_port_id_action(fm, dev, vnic_h, &fm_op,
1740 if (!(overlap & (FATE | PASSTHRU | COUNT | PORT_ID)))
1742 /* Egress from VF: need implicit WQ match */
1743 if (enic_is_vf_rep(enic) && !ingress) {
1744 fmt->ftm_data.fk_wq_id = 0;
1745 fmt->ftm_mask.fk_wq_id = 0xffff;
1746 fmt->ftm_data.fk_wq_vnic = enic->fm_vnic_handle;
1747 ENICPMD_LOG(DEBUG, "add implicit wq id match for vf %d",
1748 VF_ENIC_TO_VF_REP(enic)->vf_id);
1750 if (need_ovlan_action) {
1751 memset(&fm_op, 0, sizeof(fm_op));
1752 fm_op.fa_op = FMOP_SET_OVLAN;
1753 fm_op.ovlan.vlan = ovlan;
1754 ret = enic_fm_append_action_op(fm, &fm_op, error);
1758 /* Add steer op for PORT_ID without QUEUE */
1759 if ((overlap & PORT_ID) && !steer && ingress) {
1760 memset(&fm_op, 0, sizeof(fm_op));
1761 /* Always to queue 0 for now as generic RSS is not available */
1762 fm_op.fa_op = FMOP_RQ_STEER;
1763 fm_op.rq_steer.rq_index = 0;
1764 fm_op.rq_steer.vnic_handle = vnic_h;
1765 ret = enic_fm_append_action_op(fm, &fm_op, error);
1768 ENICPMD_LOG(DEBUG, "add implicit steer op");
1770 /* Add required END */
1771 memset(&fm_op, 0, sizeof(fm_op));
1772 fm_op.fa_op = FMOP_END;
1773 ret = enic_fm_append_action_op(fm, &fm_op, error);
1776 enic_fm_reorder_action_op(fm);
1780 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
1781 NULL, "enic: unsupported action");
1784 /** Check if the action is supported */
1786 enic_fm_match_action(const struct rte_flow_action *action,
1787 const enum rte_flow_action_type *supported_actions)
1789 for (; *supported_actions != RTE_FLOW_ACTION_TYPE_END;
1790 supported_actions++) {
1791 if (action->type == *supported_actions)
1797 /* Debug function to dump internal NIC action structure. */
1799 enic_fm_dump_tcam_actions(const struct fm_action *fm_action)
1801 /* Manually keep in sync with FMOP commands */
1802 const char *fmop_str[FMOP_OP_MAX] = {
1804 [FMOP_DROP] = "drop",
1805 [FMOP_RQ_STEER] = "steer",
1806 [FMOP_EXACT_MATCH] = "exmatch",
1807 [FMOP_MARK] = "mark",
1808 [FMOP_EXT_MARK] = "ext_mark",
1810 [FMOP_EG_HAIRPIN] = "eg_hairpin",
1811 [FMOP_IG_HAIRPIN] = "ig_hairpin",
1812 [FMOP_ENCAP_IVLAN] = "encap_ivlan",
1813 [FMOP_ENCAP_NOIVLAN] = "encap_noivlan",
1814 [FMOP_ENCAP] = "encap",
1815 [FMOP_SET_OVLAN] = "set_ovlan",
1816 [FMOP_DECAP_NOSTRIP] = "decap_nostrip",
1817 [FMOP_DECAP_STRIP] = "decap_strip",
1818 [FMOP_POP_VLAN] = "pop_vlan",
1819 [FMOP_SET_EGPORT] = "set_egport",
1820 [FMOP_RQ_STEER_ONLY] = "rq_steer_only",
1821 [FMOP_SET_ENCAP_VLAN] = "set_encap_vlan",
1822 [FMOP_EMIT] = "emit",
1823 [FMOP_MODIFY] = "modify",
1825 const struct fm_action_op *op = &fm_action->fma_action_ops[0];
1826 char buf[128], *bp = buf;
1831 buf_len = sizeof(buf);
1832 for (i = 0; i < FM_ACTION_OP_MAX; i++) {
1833 if (op->fa_op == FMOP_END)
1835 if (op->fa_op >= FMOP_OP_MAX)
1838 op_str = fmop_str[op->fa_op];
1839 n = snprintf(bp, buf_len, "%s,", op_str);
1840 if (n > 0 && n < buf_len) {
1846 /* Remove trailing comma */
1849 ENICPMD_LOG(DEBUG, " Acions: %s", buf);
1853 bits_to_str(uint32_t bits, const char *strings[], int max,
1854 char *buf, int buf_len)
1856 int i, n = 0, len = 0;
1858 for (i = 0; i < max; i++) {
1859 if (bits & (1 << i)) {
1860 n = snprintf(buf, buf_len, "%s,", strings[i]);
1861 if (n > 0 && n < buf_len) {
1868 /* Remove trailing comma */
1876 /* Debug function to dump internal NIC filter structure. */
1878 __enic_fm_dump_tcam_match(const struct fm_header_set *fk_hdrset, char *buf,
1881 /* Manually keep in sync with FKM_BITS */
1882 const char *fm_fkm_str[FKM_BIT_COUNT] = {
1883 [FKM_QTAG_BIT] = "qtag",
1884 [FKM_CMD_BIT] = "cmd",
1885 [FKM_IPV4_BIT] = "ip4",
1886 [FKM_IPV6_BIT] = "ip6",
1887 [FKM_ROCE_BIT] = "roce",
1888 [FKM_UDP_BIT] = "udp",
1889 [FKM_TCP_BIT] = "tcp",
1890 [FKM_TCPORUDP_BIT] = "tcpportudp",
1891 [FKM_IPFRAG_BIT] = "ipfrag",
1892 [FKM_NVGRE_BIT] = "nvgre",
1893 [FKM_VXLAN_BIT] = "vxlan",
1894 [FKM_GENEVE_BIT] = "geneve",
1895 [FKM_NSH_BIT] = "nsh",
1896 [FKM_ROCEV2_BIT] = "rocev2",
1897 [FKM_VLAN_PRES_BIT] = "vlan_pres",
1898 [FKM_IPOK_BIT] = "ipok",
1899 [FKM_L4OK_BIT] = "l4ok",
1900 [FKM_ROCEOK_BIT] = "roceok",
1901 [FKM_FCSOK_BIT] = "fcsok",
1902 [FKM_EG_SPAN_BIT] = "eg_span",
1903 [FKM_IG_SPAN_BIT] = "ig_span",
1904 [FKM_EG_HAIRPINNED_BIT] = "eg_hairpinned",
1906 /* Manually keep in sync with FKH_BITS */
1907 const char *fm_fkh_str[FKH_BIT_COUNT] = {
1908 [FKH_ETHER_BIT] = "eth",
1909 [FKH_QTAG_BIT] = "qtag",
1910 [FKH_L2RAW_BIT] = "l2raw",
1911 [FKH_IPV4_BIT] = "ip4",
1912 [FKH_IPV6_BIT] = "ip6",
1913 [FKH_L3RAW_BIT] = "l3raw",
1914 [FKH_UDP_BIT] = "udp",
1915 [FKH_TCP_BIT] = "tcp",
1916 [FKH_ICMP_BIT] = "icmp",
1917 [FKH_VXLAN_BIT] = "vxlan",
1918 [FKH_L4RAW_BIT] = "l4raw",
1920 uint32_t fkh_bits = fk_hdrset->fk_header_select;
1921 uint32_t fkm_bits = fk_hdrset->fk_metadata;
1924 if (!fkm_bits && !fkh_bits)
1926 n = snprintf(buf, buf_len, "metadata(");
1927 if (n > 0 && n < buf_len) {
1931 n = bits_to_str(fkm_bits, fm_fkm_str, FKM_BIT_COUNT, buf, buf_len);
1932 if (n > 0 && n < buf_len) {
1936 n = snprintf(buf, buf_len, ") valid hdr fields(");
1937 if (n > 0 && n < buf_len) {
1941 n = bits_to_str(fkh_bits, fm_fkh_str, FKH_BIT_COUNT, buf, buf_len);
1942 if (n > 0 && n < buf_len) {
1946 snprintf(buf, buf_len, ")");
1950 enic_fm_dump_tcam_match(const struct fm_tcam_match_entry *match,
1955 memset(buf, 0, sizeof(buf));
1956 __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[0],
1958 ENICPMD_LOG(DEBUG, " TCAM %s Outer: %s %scounter position %u",
1959 (ingress) ? "IG" : "EG", buf,
1960 (match->ftm_flags & FMEF_COUNTER) ? "" : "no ",
1961 match->ftm_position);
1962 memset(buf, 0, sizeof(buf));
1963 __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[1],
1966 ENICPMD_LOG(DEBUG, " Inner: %s", buf);
1969 /* Debug function to dump internal NIC flow structures. */
1971 enic_fm_dump_tcam_entry(const struct fm_tcam_match_entry *fm_match,
1972 const struct fm_action *fm_action,
1975 if (!rte_log_can_log(enic_pmd_logtype, RTE_LOG_DEBUG))
1977 enic_fm_dump_tcam_match(fm_match, ingress);
1978 enic_fm_dump_tcam_actions(fm_action);
1982 enic_fm_flow_parse(struct enic_flowman *fm,
1983 const struct rte_flow_attr *attrs,
1984 const struct rte_flow_item pattern[],
1985 const struct rte_flow_action actions[],
1986 struct rte_flow_error *error)
1988 const struct rte_flow_action *action;
1990 static const enum rte_flow_action_type *sa;
1992 ENICPMD_FUNC_TRACE();
1995 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM,
1996 NULL, "no pattern specified");
2001 rte_flow_error_set(error, EINVAL,
2002 RTE_FLOW_ERROR_TYPE_ACTION_NUM,
2003 NULL, "no action specified");
2008 if (attrs->group != FM_TCAM_RTE_GROUP && attrs->priority) {
2009 rte_flow_error_set(error, ENOTSUP,
2010 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
2012 "priorities are not supported for non-default (0) groups");
2014 } else if (!fm->owner_enic->switchdev_mode && attrs->transfer) {
2015 rte_flow_error_set(error, ENOTSUP,
2016 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
2018 "transfer is not supported");
2020 } else if (attrs->ingress && attrs->egress) {
2021 rte_flow_error_set(error, ENOTSUP,
2022 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
2024 "bidirectional rules not supported");
2029 rte_flow_error_set(error, EINVAL,
2030 RTE_FLOW_ERROR_TYPE_ATTR,
2031 NULL, "no attribute specified");
2035 /* Verify Actions. */
2036 sa = (attrs->ingress) ? enic_fm_supported_ig_actions :
2037 enic_fm_supported_eg_actions;
2038 for (action = &actions[0]; action->type != RTE_FLOW_ACTION_TYPE_END;
2040 if (action->type == RTE_FLOW_ACTION_TYPE_VOID)
2042 else if (!enic_fm_match_action(action, sa))
2045 if (action->type != RTE_FLOW_ACTION_TYPE_END) {
2046 rte_flow_error_set(error, EPERM, RTE_FLOW_ERROR_TYPE_ACTION,
2047 action, "invalid action");
2050 ret = enic_fm_copy_entry(fm, pattern, error);
2053 ret = enic_fm_copy_action(fm, actions, attrs->ingress, error);
2058 enic_fm_counter_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
2060 if (!fm_flow->counter_valid)
2062 SLIST_INSERT_HEAD(&fm->counters, fm_flow->counter, next);
2063 fm_flow->counter_valid = false;
2067 enic_fm_more_counters(struct enic_flowman *fm)
2069 struct enic_fm_counter *new_stack;
2070 struct enic_fm_counter *ctrs;
2074 ENICPMD_FUNC_TRACE();
2075 new_stack = rte_realloc(fm->counter_stack, (fm->counters_alloced +
2076 FM_COUNTERS_EXPAND) *
2077 sizeof(struct enic_fm_counter), 0);
2078 if (new_stack == NULL) {
2079 ENICPMD_LOG(ERR, "cannot alloc counter memory");
2082 fm->counter_stack = new_stack;
2084 args[0] = FM_COUNTER_BRK;
2085 args[1] = fm->counters_alloced + FM_COUNTERS_EXPAND;
2086 rc = flowman_cmd(fm, args, 2);
2088 ENICPMD_LOG(ERR, "cannot alloc counters rc=%d", rc);
2091 ctrs = (struct enic_fm_counter *)fm->counter_stack +
2092 fm->counters_alloced;
2093 for (i = 0; i < FM_COUNTERS_EXPAND; i++, ctrs++) {
2094 ctrs->handle = fm->counters_alloced + i;
2095 SLIST_INSERT_HEAD(&fm->counters, ctrs, next);
2097 fm->counters_alloced += FM_COUNTERS_EXPAND;
2098 ENICPMD_LOG(DEBUG, "%u counters allocated, total: %u",
2099 FM_COUNTERS_EXPAND, fm->counters_alloced);
2104 enic_fm_counter_zero(struct enic_flowman *fm, struct enic_fm_counter *c)
2109 ENICPMD_FUNC_TRACE();
2110 args[0] = FM_COUNTER_QUERY;
2111 args[1] = c->handle;
2112 args[2] = 1; /* clear */
2113 ret = flowman_cmd(fm, args, 3);
2115 ENICPMD_LOG(ERR, "counter init: rc=%d handle=0x%x",
2123 enic_fm_counter_alloc(struct enic_flowman *fm, struct rte_flow_error *error,
2124 struct enic_fm_counter **ctr)
2126 struct enic_fm_counter *c;
2129 ENICPMD_FUNC_TRACE();
2131 if (SLIST_EMPTY(&fm->counters)) {
2132 ret = enic_fm_more_counters(fm);
2134 return rte_flow_error_set(error, -ret,
2135 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2136 NULL, "enic: out of counters");
2138 c = SLIST_FIRST(&fm->counters);
2139 SLIST_REMOVE_HEAD(&fm->counters, next);
2145 enic_fm_action_free(struct enic_flowman *fm, struct enic_fm_action *ah)
2150 ENICPMD_FUNC_TRACE();
2151 RTE_ASSERT(ah->ref > 0);
2154 args[0] = FM_ACTION_FREE;
2155 args[1] = ah->handle;
2156 ret = flowman_cmd(fm, args, 2);
2158 /* This is a "should never happen" error. */
2159 ENICPMD_LOG(ERR, "freeing action rc=%d handle=0x%"
2160 PRIx64, ret, ah->handle);
2161 rte_hash_del_key(fm->action_hash, (const void *)&ah->key);
2168 enic_fm_entry_free(struct enic_flowman *fm, uint64_t handle)
2173 ENICPMD_FUNC_TRACE();
2174 args[0] = FM_MATCH_ENTRY_REMOVE;
2176 rc = flowman_cmd(fm, args, 2);
2178 ENICPMD_LOG(ERR, "cannot free match entry: rc=%d"
2179 " handle=0x%" PRIx64, rc, handle);
2183 static struct enic_fm_jump_flow *
2184 find_jump_flow(struct enic_flowman *fm, uint32_t group)
2186 struct enic_fm_jump_flow *j;
2188 ENICPMD_FUNC_TRACE();
2189 TAILQ_FOREACH(j, &fm->jump_list, list) {
2190 if (j->group == group)
2197 remove_jump_flow(struct enic_flowman *fm, struct rte_flow *flow)
2199 struct enic_fm_jump_flow *j;
2201 ENICPMD_FUNC_TRACE();
2202 TAILQ_FOREACH(j, &fm->jump_list, list) {
2203 if (j->flow == flow) {
2204 TAILQ_REMOVE(&fm->jump_list, j, list);
2212 save_jump_flow(struct enic_flowman *fm,
2213 struct rte_flow *flow,
2215 struct fm_tcam_match_entry *match,
2216 struct fm_action *action)
2218 struct enic_fm_jump_flow *j;
2220 ENICPMD_FUNC_TRACE();
2221 j = calloc(1, sizeof(struct enic_fm_jump_flow));
2227 j->action = *action;
2228 TAILQ_INSERT_HEAD(&fm->jump_list, j, list);
2229 ENICPMD_LOG(DEBUG, "saved jump flow: flow=%p group=%u", flow, group);
2234 __enic_fm_flow_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
2236 if (fm_flow->entry_handle != FM_INVALID_HANDLE) {
2237 enic_fm_entry_free(fm, fm_flow->entry_handle);
2238 fm_flow->entry_handle = FM_INVALID_HANDLE;
2240 if (fm_flow->action != NULL) {
2241 enic_fm_action_free(fm, fm_flow->action);
2242 fm_flow->action = NULL;
2244 enic_fm_counter_free(fm, fm_flow);
2246 enic_fet_put(fm, fm_flow->fet);
2247 fm_flow->fet = NULL;
2252 enic_fm_flow_free(struct enic_flowman *fm, struct rte_flow *flow)
2254 struct enic_fm_flow *steer = flow->fm->hairpin_steer_flow;
2256 if (flow->fm->fet && flow->fm->fet->default_key)
2257 remove_jump_flow(fm, flow);
2258 __enic_fm_flow_free(fm, flow->fm);
2260 __enic_fm_flow_free(fm, steer);
2268 enic_fm_add_tcam_entry(struct enic_flowman *fm,
2269 struct fm_tcam_match_entry *match_in,
2270 uint64_t *entry_handle,
2272 struct rte_flow_error *error)
2274 struct fm_tcam_match_entry *ftm;
2278 ENICPMD_FUNC_TRACE();
2279 /* Copy entry to the command buffer */
2280 ftm = &fm->cmd.va->fm_tcam_match_entry;
2281 memcpy(ftm, match_in, sizeof(*ftm));
2282 /* Add TCAM entry */
2283 args[0] = FM_TCAM_ENTRY_INSTALL;
2284 args[1] = ingress ? fm->ig_tcam_hndl : fm->eg_tcam_hndl;
2285 args[2] = fm->cmd.pa;
2286 ret = flowman_cmd(fm, args, 3);
2288 ENICPMD_LOG(ERR, "cannot add %s TCAM entry: rc=%d",
2289 ingress ? "ingress" : "egress", ret);
2290 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2291 NULL, "enic: devcmd(tcam-entry-install)");
2294 ENICPMD_LOG(DEBUG, "installed %s TCAM entry: handle=0x%" PRIx64,
2295 ingress ? "ingress" : "egress", (uint64_t)args[0]);
2296 *entry_handle = args[0];
2301 enic_fm_add_exact_entry(struct enic_flowman *fm,
2302 struct fm_tcam_match_entry *match_in,
2303 uint64_t *entry_handle,
2304 struct enic_fm_fet *fet,
2305 struct rte_flow_error *error)
2307 struct fm_exact_match_entry *fem;
2311 ENICPMD_FUNC_TRACE();
2312 /* The new entry must have the table's key */
2313 if (memcmp(fet->key.fk_hdrset, match_in->ftm_mask.fk_hdrset,
2314 sizeof(struct fm_header_set) * FM_HDRSET_MAX)) {
2315 return rte_flow_error_set(error, EINVAL,
2316 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2317 "enic: key does not match group's key");
2320 /* Copy entry to the command buffer */
2321 fem = &fm->cmd.va->fm_exact_match_entry;
2323 * Translate TCAM entry to exact entry. As is only need to drop
2324 * position and mask. The mask is part of the exact match table.
2325 * Position (aka priority) is not supported in the exact match table.
2327 fem->fem_data = match_in->ftm_data;
2328 fem->fem_flags = match_in->ftm_flags;
2329 fem->fem_action = match_in->ftm_action;
2330 fem->fem_counter = match_in->ftm_counter;
2332 /* Add exact entry */
2333 args[0] = FM_EXACT_ENTRY_INSTALL;
2334 args[1] = fet->handle;
2335 args[2] = fm->cmd.pa;
2336 ret = flowman_cmd(fm, args, 3);
2338 ENICPMD_LOG(ERR, "cannot add %s exact entry: group=%u",
2339 fet->ingress ? "ingress" : "egress", fet->group);
2340 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2341 NULL, "enic: devcmd(exact-entry-install)");
2344 ENICPMD_LOG(DEBUG, "installed %s exact entry: group=%u"
2345 " handle=0x%" PRIx64,
2346 fet->ingress ? "ingress" : "egress", fet->group,
2348 *entry_handle = args[0];
2353 enic_action_handle_get(struct enic_flowman *fm, struct fm_action *action_in,
2354 struct rte_flow_error *error,
2355 struct enic_fm_action **ah_o)
2357 struct enic_fm_action *ah;
2358 struct fm_action *fma;
2362 ret = rte_hash_lookup_data(fm->action_hash, action_in,
2364 if (ret < 0 && ret != -ENOENT)
2365 return rte_flow_error_set(error, -ret,
2366 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2367 NULL, "enic: rte_hash_lookup(aciton)");
2369 if (ret == -ENOENT) {
2370 /* Allocate a new action on the NIC. */
2371 fma = &fm->cmd.va->fm_action;
2372 memcpy(fma, action_in, sizeof(*fma));
2374 ah = calloc(1, sizeof(*ah));
2375 memcpy(&ah->key, action_in, sizeof(struct fm_action));
2377 return rte_flow_error_set(error, ENOMEM,
2378 RTE_FLOW_ERROR_TYPE_HANDLE,
2379 NULL, "enic: calloc(fm-action)");
2380 args[0] = FM_ACTION_ALLOC;
2381 args[1] = fm->cmd.pa;
2382 ret = flowman_cmd(fm, args, 2);
2384 rte_flow_error_set(error, -ret,
2385 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2386 NULL, "enic: devcmd(action-alloc)");
2389 ah->handle = args[0];
2390 ret = rte_hash_add_key_data(fm->action_hash,
2391 (const void *)action_in,
2394 rte_flow_error_set(error, -ret,
2395 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2397 "enic: rte_hash_add_key_data(actn)");
2398 goto error_with_action_handle;
2400 ENICPMD_LOG(DEBUG, "action allocated: handle=0x%" PRIx64,
2404 /* Action handle struct is valid, increment reference count. */
2408 error_with_action_handle:
2409 args[0] = FM_ACTION_FREE;
2410 args[1] = ah->handle;
2411 ret = flowman_cmd(fm, args, 2);
2413 rte_flow_error_set(error, -ret,
2414 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2415 NULL, "enic: devcmd(action-free)");
2421 /* Push match-action to the NIC. */
2423 __enic_fm_flow_add_entry(struct enic_flowman *fm,
2424 struct enic_fm_flow *fm_flow,
2425 struct fm_tcam_match_entry *match_in,
2426 struct fm_action *action_in,
2429 struct rte_flow_error *error)
2431 struct enic_fm_counter *ctr;
2432 struct enic_fm_action *ah = NULL;
2436 ENICPMD_FUNC_TRACE();
2438 /* Get or create an aciton handle. */
2439 ret = enic_action_handle_get(fm, action_in, error, &ah);
2442 match_in->ftm_action = ah->handle;
2443 fm_flow->action = ah;
2445 /* Allocate counter if requested. */
2446 if (match_in->ftm_flags & FMEF_COUNTER) {
2447 ret = enic_fm_counter_alloc(fm, error, &ctr);
2448 if (ret) /* error has been filled in */
2450 fm_flow->counter_valid = true;
2451 fm_flow->counter = ctr;
2452 match_in->ftm_counter = ctr->handle;
2456 * Get the group's table (either TCAM or exact match table) and
2457 * add entry to it. If we use the exact match table, the handler
2458 * will translate the TCAM entry (match_in) to the appropriate
2459 * exact match entry and use that instead.
2461 entry_h = FM_INVALID_HANDLE;
2462 if (group == FM_TCAM_RTE_GROUP) {
2463 ret = enic_fm_add_tcam_entry(fm, match_in, &entry_h, ingress,
2467 /* Jump action might have a ref to fet */
2468 fm_flow->fet = fm->fet;
2471 struct enic_fm_fet *fet = NULL;
2473 ret = enic_fet_get(fm, group, ingress,
2474 &match_in->ftm_mask, &fet, error);
2478 ret = enic_fm_add_exact_entry(fm, match_in, &entry_h, fet,
2483 /* Clear counter after adding entry, as it requires in-use counter */
2484 if (fm_flow->counter_valid) {
2485 ret = enic_fm_counter_zero(fm, fm_flow->counter);
2489 fm_flow->entry_handle = entry_h;
2493 /* Push match-action to the NIC. */
2494 static struct rte_flow *
2495 enic_fm_flow_add_entry(struct enic_flowman *fm,
2496 struct fm_tcam_match_entry *match_in,
2497 struct fm_action *action_in,
2498 const struct rte_flow_attr *attrs,
2499 struct rte_flow_error *error)
2501 struct enic_fm_flow *fm_flow;
2502 struct rte_flow *flow;
2504 ENICPMD_FUNC_TRACE();
2505 match_in->ftm_position = attrs->priority;
2506 enic_fm_dump_tcam_entry(match_in, action_in, attrs->ingress);
2507 flow = calloc(1, sizeof(*flow));
2508 fm_flow = calloc(1, sizeof(*fm_flow));
2509 if (flow == NULL || fm_flow == NULL) {
2510 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
2511 NULL, "enic: cannot allocate rte_flow");
2517 fm_flow->action = NULL;
2518 fm_flow->entry_handle = FM_INVALID_HANDLE;
2519 if (__enic_fm_flow_add_entry(fm, fm_flow, match_in, action_in,
2520 attrs->group, attrs->ingress, error)) {
2521 enic_fm_flow_free(fm, flow);
2528 convert_jump_flows(struct enic_flowman *fm, struct enic_fm_fet *fet,
2529 struct rte_flow_error *error)
2531 struct enic_fm_flow *fm_flow;
2532 struct enic_fm_jump_flow *j;
2533 struct fm_action *fma;
2536 ENICPMD_FUNC_TRACE();
2538 * Find the saved flows that should jump to the new table (fet).
2539 * Then delete the old TCAM entry that jumps to the default table,
2540 * and add a new one that jumps to the new table.
2543 j = find_jump_flow(fm, group);
2545 ENICPMD_LOG(DEBUG, "convert jump flow: flow=%p group=%u",
2547 /* Delete old entry */
2548 fm_flow = j->flow->fm;
2549 __enic_fm_flow_free(fm, fm_flow);
2553 fma->fma_action_ops[0].exact.handle = fet->handle;
2554 if (__enic_fm_flow_add_entry(fm, fm_flow, &j->match, fma,
2555 FM_TCAM_RTE_GROUP, fet->ingress, error)) {
2556 /* Cannot roll back changes at the moment */
2557 ENICPMD_LOG(ERR, "cannot convert jump flow: flow=%p",
2562 ENICPMD_LOG(DEBUG, "convert ok: group=%u ref=%u",
2563 fet->group, fet->ref);
2566 TAILQ_REMOVE(&fm->jump_list, j, list);
2568 j = find_jump_flow(fm, group);
2573 add_hairpin_steer(struct enic_flowman *fm, struct rte_flow *flow,
2574 struct rte_flow_error *error)
2576 struct fm_tcam_match_entry *fm_tcam_entry;
2577 struct enic_fm_flow *fm_flow;
2578 struct fm_action *fm_action;
2579 struct fm_action_op fm_op;
2582 ENICPMD_FUNC_TRACE();
2583 fm_flow = calloc(1, sizeof(*fm_flow));
2584 if (fm_flow == NULL) {
2585 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
2586 NULL, "enic: cannot allocate rte_flow");
2589 /* Original egress hairpin flow */
2590 fm_tcam_entry = &fm->tcam_entry;
2591 fm_action = &fm->action;
2592 /* Use the match pattern of the egress flow as is, without counters */
2593 fm_tcam_entry->ftm_flags &= ~FMEF_COUNTER;
2594 /* The only action is steer to vnic */
2595 fm->action_op_count = 0;
2596 memset(fm_action, 0, sizeof(*fm_action));
2597 memset(&fm_op, 0, sizeof(fm_op));
2598 /* Always to queue 0 for now */
2599 fm_op.fa_op = FMOP_RQ_STEER;
2600 fm_op.rq_steer.rq_index = 0;
2601 fm_op.rq_steer.vnic_handle = fm->hairpin_steer_vnic_h;
2602 ret = enic_fm_append_action_op(fm, &fm_op, error);
2604 goto error_with_flow;
2605 ENICPMD_LOG(DEBUG, "add steer op");
2606 /* Add required END */
2607 memset(&fm_op, 0, sizeof(fm_op));
2608 fm_op.fa_op = FMOP_END;
2609 ret = enic_fm_append_action_op(fm, &fm_op, error);
2611 goto error_with_flow;
2612 /* Add the ingress flow */
2613 fm_flow->action = NULL;
2614 fm_flow->entry_handle = FM_INVALID_HANDLE;
2615 ret = __enic_fm_flow_add_entry(fm, fm_flow, fm_tcam_entry, fm_action,
2616 FM_TCAM_RTE_GROUP, 1 /* ingress */, error);
2618 ENICPMD_LOG(ERR, "cannot add hairpin-steer flow");
2619 goto error_with_flow;
2621 /* The new flow is now the egress flow's paired flow */
2622 flow->fm->hairpin_steer_flow = fm_flow;
2631 enic_fm_open_scratch(struct enic_flowman *fm)
2633 fm->action_op_count = 0;
2635 fm->need_hairpin_steer = 0;
2636 fm->hairpin_steer_vnic_h = 0;
2637 memset(&fm->tcam_entry, 0, sizeof(fm->tcam_entry));
2638 memset(&fm->action, 0, sizeof(fm->action));
2642 enic_fm_close_scratch(struct enic_flowman *fm)
2645 enic_fet_put(fm, fm->fet);
2648 fm->action_op_count = 0;
2652 enic_fm_flow_validate(struct rte_eth_dev *dev,
2653 const struct rte_flow_attr *attrs,
2654 const struct rte_flow_item pattern[],
2655 const struct rte_flow_action actions[],
2656 struct rte_flow_error *error)
2658 struct fm_tcam_match_entry *fm_tcam_entry;
2659 struct fm_action *fm_action;
2660 struct enic_flowman *fm;
2663 ENICPMD_FUNC_TRACE();
2664 fm = begin_fm(pmd_priv(dev));
2667 enic_fm_open_scratch(fm);
2668 ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2670 fm_tcam_entry = &fm->tcam_entry;
2671 fm_action = &fm->action;
2672 enic_fm_dump_tcam_entry(fm_tcam_entry, fm_action,
2675 enic_fm_close_scratch(fm);
2681 enic_fm_flow_query_count(struct rte_eth_dev *dev,
2682 struct rte_flow *flow, void *data,
2683 struct rte_flow_error *error)
2685 struct rte_flow_query_count *query;
2686 struct enic_fm_flow *fm_flow;
2687 struct enic_flowman *fm;
2691 ENICPMD_FUNC_TRACE();
2692 fm = begin_fm(pmd_priv(dev));
2695 if (!fm_flow->counter_valid) {
2696 rc = rte_flow_error_set(error, ENOTSUP,
2697 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2698 "enic: flow does not have counter");
2702 args[0] = FM_COUNTER_QUERY;
2703 args[1] = fm_flow->counter->handle;
2704 args[2] = query->reset;
2705 rc = flowman_cmd(fm, args, 3);
2707 ENICPMD_LOG(ERR, "cannot query counter: rc=%d handle=0x%x",
2708 rc, fm_flow->counter->handle);
2711 query->hits_set = 1;
2712 query->hits = args[0];
2713 query->bytes_set = 1;
2714 query->bytes = args[1];
2722 enic_fm_flow_query(struct rte_eth_dev *dev,
2723 struct rte_flow *flow,
2724 const struct rte_flow_action *actions,
2726 struct rte_flow_error *error)
2730 ENICPMD_FUNC_TRACE();
2731 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2732 switch (actions->type) {
2733 case RTE_FLOW_ACTION_TYPE_VOID:
2735 case RTE_FLOW_ACTION_TYPE_COUNT:
2736 ret = enic_fm_flow_query_count(dev, flow, data, error);
2739 return rte_flow_error_set(error, ENOTSUP,
2740 RTE_FLOW_ERROR_TYPE_ACTION,
2742 "action not supported");
2750 static struct rte_flow *
2751 enic_fm_flow_create(struct rte_eth_dev *dev,
2752 const struct rte_flow_attr *attrs,
2753 const struct rte_flow_item pattern[],
2754 const struct rte_flow_action actions[],
2755 struct rte_flow_error *error)
2757 struct fm_tcam_match_entry *fm_tcam_entry;
2758 struct fm_action *fm_action;
2759 struct enic_flowman *fm;
2760 struct enic_fm_fet *fet;
2761 struct rte_flow *flow;
2765 ENICPMD_FUNC_TRACE();
2766 enic = pmd_priv(dev);
2767 fm = begin_fm(enic);
2769 rte_flow_error_set(error, ENOTSUP,
2770 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2771 "flowman is not initialized");
2774 enic_fm_open_scratch(fm);
2776 ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2778 goto error_with_scratch;
2779 fm_tcam_entry = &fm->tcam_entry;
2780 fm_action = &fm->action;
2781 flow = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2784 /* Add ingress rule that pairs with hairpin rule */
2785 if (fm->need_hairpin_steer) {
2786 ret = add_hairpin_steer(fm, flow, error);
2788 enic_fm_flow_free(fm, flow);
2790 goto error_with_scratch;
2793 LIST_INSERT_HEAD(&enic->flows, flow, next);
2794 fet = flow->fm->fet;
2795 if (fet && fet->default_key) {
2797 * Jump to non-existent group? Save the relevant info
2798 * so we can convert this flow when that group
2801 save_jump_flow(fm, flow, fet->group,
2802 fm_tcam_entry, fm_action);
2803 } else if (fet && fet->ref == 1) {
2805 * A new table is created. Convert the saved flows
2806 * that should jump to this group.
2808 convert_jump_flows(fm, fet, error);
2813 enic_fm_close_scratch(fm);
2819 enic_fm_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
2820 __rte_unused struct rte_flow_error *error)
2822 struct enic *enic = pmd_priv(dev);
2823 struct enic_flowman *fm;
2825 ENICPMD_FUNC_TRACE();
2826 fm = begin_fm(enic);
2829 LIST_REMOVE(flow, next);
2830 enic_fm_flow_free(fm, flow);
2836 enic_fm_flow_flush(struct rte_eth_dev *dev,
2837 __rte_unused struct rte_flow_error *error)
2839 LIST_HEAD(enic_flows, rte_flow) internal;
2840 struct enic_fm_flow *fm_flow;
2841 struct enic_flowman *fm;
2842 struct rte_flow *flow;
2843 struct enic *enic = pmd_priv(dev);
2845 ENICPMD_FUNC_TRACE();
2847 fm = begin_fm(enic);
2850 /* Destroy all non-internal flows */
2851 LIST_INIT(&internal);
2852 while (!LIST_EMPTY(&enic->flows)) {
2853 flow = LIST_FIRST(&enic->flows);
2855 LIST_REMOVE(flow, next);
2856 if (flow->internal) {
2857 LIST_INSERT_HEAD(&internal, flow, next);
2861 * If tables are null, then vNIC is closing, and the firmware
2862 * has already cleaned up flowman state. So do not try to free
2863 * resources, as it only causes errors.
2865 if (fm->ig_tcam_hndl == FM_INVALID_HANDLE) {
2866 fm_flow->entry_handle = FM_INVALID_HANDLE;
2867 fm_flow->action = NULL;
2868 fm_flow->fet = NULL;
2870 enic_fm_flow_free(fm, flow);
2872 while (!LIST_EMPTY(&internal)) {
2873 flow = LIST_FIRST(&internal);
2874 LIST_REMOVE(flow, next);
2875 LIST_INSERT_HEAD(&enic->flows, flow, next);
2882 enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle)
2887 args[0] = FM_MATCH_TABLE_FREE;
2889 rc = flowman_cmd(fm, args, 2);
2891 ENICPMD_LOG(ERR, "cannot free table: rc=%d handle=0x%" PRIx64,
2897 enic_fm_tcam_tbl_alloc(struct enic_flowman *fm, uint32_t direction,
2898 uint32_t max_entries, uint64_t *handle)
2900 struct fm_tcam_match_table *tcam_tbl;
2904 ENICPMD_FUNC_TRACE();
2905 tcam_tbl = &fm->cmd.va->fm_tcam_match_table;
2906 tcam_tbl->ftt_direction = direction;
2907 tcam_tbl->ftt_stage = FM_STAGE_LAST;
2908 tcam_tbl->ftt_max_entries = max_entries;
2909 args[0] = FM_TCAM_TABLE_ALLOC;
2910 args[1] = fm->cmd.pa;
2911 rc = flowman_cmd(fm, args, 2);
2913 ENICPMD_LOG(ERR, "cannot alloc %s TCAM table: rc=%d",
2914 (direction == FM_INGRESS) ? "IG" : "EG", rc);
2918 ENICPMD_LOG(DEBUG, "%s TCAM table allocated, handle=0x%" PRIx64,
2919 (direction == FM_INGRESS) ? "IG" : "EG", *handle);
2924 enic_fm_init_actions(struct enic_flowman *fm)
2926 struct rte_hash *a_hash;
2927 char name[RTE_HASH_NAMESIZE];
2928 struct rte_hash_parameters params = {
2929 .entries = FM_MAX_ACTION_TABLE_SIZE,
2930 .key_len = sizeof(struct fm_action),
2931 .hash_func = rte_jhash,
2932 .hash_func_init_val = 0,
2933 .socket_id = rte_socket_id(),
2936 ENICPMD_FUNC_TRACE();
2937 snprintf((char *)name, sizeof(name), "fm-ah-%s",
2938 fm->owner_enic->bdf_name);
2941 a_hash = rte_hash_create(¶ms);
2944 fm->action_hash = a_hash;
2949 enic_fm_init_counters(struct enic_flowman *fm)
2951 ENICPMD_FUNC_TRACE();
2952 SLIST_INIT(&fm->counters);
2953 return enic_fm_more_counters(fm);
2957 enic_fm_free_all_counters(struct enic_flowman *fm)
2962 args[0] = FM_COUNTER_BRK;
2964 rc = flowman_cmd(fm, args, 2);
2966 ENICPMD_LOG(ERR, "cannot free counters: rc=%d", rc);
2967 rte_free(fm->counter_stack);
2971 enic_fm_alloc_tcam_tables(struct enic_flowman *fm)
2975 ENICPMD_FUNC_TRACE();
2976 rc = enic_fm_tcam_tbl_alloc(fm, FM_INGRESS, FM_MAX_TCAM_TABLE_SIZE,
2980 rc = enic_fm_tcam_tbl_alloc(fm, FM_EGRESS, FM_MAX_TCAM_TABLE_SIZE,
2986 enic_fm_free_tcam_tables(struct enic_flowman *fm)
2988 ENICPMD_FUNC_TRACE();
2989 if (fm->ig_tcam_hndl) {
2990 ENICPMD_LOG(DEBUG, "free IG TCAM table handle=0x%" PRIx64,
2992 enic_fm_tbl_free(fm, fm->ig_tcam_hndl);
2993 fm->ig_tcam_hndl = FM_INVALID_HANDLE;
2995 if (fm->eg_tcam_hndl) {
2996 ENICPMD_LOG(DEBUG, "free EG TCAM table handle=0x%" PRIx64,
2998 enic_fm_tbl_free(fm, fm->eg_tcam_hndl);
2999 fm->eg_tcam_hndl = FM_INVALID_HANDLE;
3004 enic_fm_init(struct enic *enic)
3006 const struct rte_pci_addr *addr;
3007 struct enic_flowman *fm;
3008 uint8_t name[RTE_MEMZONE_NAMESIZE];
3011 if (enic->flow_filter_mode != FILTER_FLOWMAN)
3013 ENICPMD_FUNC_TRACE();
3014 /* Get vnic handle and save for port-id action */
3015 if (enic_is_vf_rep(enic))
3016 addr = &VF_ENIC_TO_VF_REP(enic)->bdf;
3018 addr = &RTE_ETH_DEV_TO_PCI(enic->rte_dev)->addr;
3019 rc = enic_fm_find_vnic(enic, addr, &enic->fm_vnic_handle);
3021 ENICPMD_LOG(ERR, "cannot find vnic handle for %x:%x:%x",
3022 addr->bus, addr->devid, addr->function);
3025 /* Save UIF for egport action */
3026 enic->fm_vnic_uif = vnic_dev_uif(enic->vdev);
3027 ENICPMD_LOG(DEBUG, "uif %u", enic->fm_vnic_uif);
3028 /* Nothing else to do for representor. It will share the PF flowman */
3029 if (enic_is_vf_rep(enic))
3031 fm = calloc(1, sizeof(*fm));
3033 ENICPMD_LOG(ERR, "cannot alloc flowman struct");
3036 fm->owner_enic = enic;
3037 rte_spinlock_init(&fm->lock);
3038 TAILQ_INIT(&fm->fet_list);
3039 TAILQ_INIT(&fm->jump_list);
3040 /* Allocate host memory for flowman commands */
3041 snprintf((char *)name, sizeof(name), "fm-cmd-%s", enic->bdf_name);
3042 fm->cmd.va = enic_alloc_consistent(enic,
3043 sizeof(union enic_flowman_cmd_mem), &fm->cmd.pa, name);
3045 ENICPMD_LOG(ERR, "cannot allocate flowman command memory");
3049 /* Allocate TCAM tables upfront as they are the main tables */
3050 rc = enic_fm_alloc_tcam_tables(fm);
3052 ENICPMD_LOG(ERR, "cannot alloc TCAM tables");
3055 /* Then a number of counters */
3056 rc = enic_fm_init_counters(fm);
3058 ENICPMD_LOG(ERR, "cannot alloc counters");
3061 /* set up action handle hash */
3062 rc = enic_fm_init_actions(fm);
3064 ENICPMD_LOG(ERR, "cannot create action hash, error:%d", rc);
3065 goto error_counters;
3068 * One default exact match table for each direction. We hold onto
3071 rc = enic_fet_alloc(fm, 1, NULL, 128, &fm->default_ig_fet);
3073 ENICPMD_LOG(ERR, "cannot alloc default IG exact match table");
3076 fm->default_ig_fet->ref = 1;
3077 rc = enic_fet_alloc(fm, 0, NULL, 128, &fm->default_eg_fet);
3079 ENICPMD_LOG(ERR, "cannot alloc default EG exact match table");
3082 fm->default_eg_fet->ref = 1;
3083 fm->vf_rep_tag = FM_VF_REP_TAG;
3088 enic_fet_free(fm, fm->default_ig_fet);
3090 rte_hash_free(fm->action_hash);
3092 enic_fm_free_all_counters(fm);
3094 enic_fm_free_tcam_tables(fm);
3096 enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
3097 fm->cmd.va, fm->cmd.pa);
3104 enic_fm_destroy(struct enic *enic)
3106 struct enic_flowman *fm;
3107 struct enic_fm_fet *fet;
3109 ENICPMD_FUNC_TRACE();
3110 if (enic_is_vf_rep(enic)) {
3111 delete_rep_flows(enic);
3114 if (enic->fm == NULL)
3117 enic_fm_flow_flush(enic->rte_dev, NULL);
3118 enic_fet_free(fm, fm->default_eg_fet);
3119 enic_fet_free(fm, fm->default_ig_fet);
3120 /* Free all exact match tables still open */
3121 while (!TAILQ_EMPTY(&fm->fet_list)) {
3122 fet = TAILQ_FIRST(&fm->fet_list);
3123 enic_fet_free(fm, fet);
3125 enic_fm_free_tcam_tables(fm);
3126 enic_fm_free_all_counters(fm);
3127 rte_hash_free(fm->action_hash);
3128 enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
3129 fm->cmd.va, fm->cmd.pa);
3136 enic_fm_allocate_switch_domain(struct enic *pf)
3138 const struct rte_pci_addr *cur_a, *prev_a;
3139 struct rte_eth_dev *dev;
3140 struct enic *cur, *prev;
3146 ENICPMD_FUNC_TRACE();
3147 if (enic_is_vf_rep(pf))
3150 cur_a = &RTE_ETH_DEV_TO_PCI(cur->rte_dev)->addr;
3151 /* Go through ports and find another PF that is on the same adapter */
3152 RTE_ETH_FOREACH_DEV(pid) {
3153 dev = &rte_eth_devices[pid];
3154 if (!dev_is_enic(dev))
3156 if (dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)
3158 if (dev == cur->rte_dev)
3160 /* dev is another PF. Is it on the same adapter? */
3161 prev = pmd_priv(dev);
3162 prev_a = &RTE_ETH_DEV_TO_PCI(dev)->addr;
3163 if (!enic_fm_find_vnic(cur, prev_a, &vnic_h)) {
3164 ENICPMD_LOG(DEBUG, "Port %u (PF BDF %x:%x:%x) and port %u (PF BDF %x:%x:%x domain %u) are on the same VIC",
3165 cur->rte_dev->data->port_id,
3166 cur_a->bus, cur_a->devid, cur_a->function,
3168 prev_a->bus, prev_a->devid, prev_a->function,
3169 prev->switch_domain_id);
3170 cur->switch_domain_id = prev->switch_domain_id;
3174 ret = rte_eth_switch_domain_alloc(&domain_id);
3176 ENICPMD_LOG(WARNING, "failed to allocate switch domain for device %d",
3179 cur->switch_domain_id = domain_id;
3180 ENICPMD_LOG(DEBUG, "Port %u (PF BDF %x:%x:%x) is the 1st PF on the VIC. Allocated switch domain id %u",
3181 cur->rte_dev->data->port_id,
3182 cur_a->bus, cur_a->devid, cur_a->function,
3187 const struct rte_flow_ops enic_fm_flow_ops = {
3188 .validate = enic_fm_flow_validate,
3189 .create = enic_fm_flow_create,
3190 .destroy = enic_fm_flow_destroy,
3191 .flush = enic_fm_flow_flush,
3192 .query = enic_fm_flow_query,
3195 /* Add a high priority flow that loops representor packets to VF */
3197 enic_fm_add_rep2vf_flow(struct enic_vf_representor *vf)
3199 struct fm_tcam_match_entry *fm_tcam_entry;
3200 struct rte_flow *flow0, *flow1;
3201 struct fm_action *fm_action;
3202 struct rte_flow_error error;
3203 struct rte_flow_attr attrs;
3204 struct fm_action_op fm_op;
3205 struct enic_flowman *fm;
3211 tag = fm->vf_rep_tag;
3212 enic_fm_open_scratch(fm);
3213 fm_tcam_entry = &fm->tcam_entry;
3214 fm_action = &fm->action;
3215 /* Egress rule: match WQ ID and tag+hairpin */
3216 fm_tcam_entry->ftm_data.fk_wq_id = vf->pf_wq_idx;
3217 fm_tcam_entry->ftm_mask.fk_wq_id = 0xffff;
3218 fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3219 memset(&fm_op, 0, sizeof(fm_op));
3220 fm_op.fa_op = FMOP_TAG;
3221 fm_op.tag.tag = tag;
3222 enic_fm_append_action_op(fm, &fm_op, &error);
3223 memset(&fm_op, 0, sizeof(fm_op));
3224 fm_op.fa_op = FMOP_EG_HAIRPIN;
3225 enic_fm_append_action_op(fm, &fm_op, &error);
3226 memset(&fm_op, 0, sizeof(fm_op));
3227 fm_op.fa_op = FMOP_END;
3228 enic_fm_append_action_op(fm, &fm_op, &error);
3232 attrs.priority = FM_HIGHEST_PRIORITY;
3233 flow0 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3235 enic_fm_close_scratch(fm);
3236 if (flow0 == NULL) {
3237 ENICPMD_LOG(ERR, "Cannot create flow 0 for representor->VF");
3240 LIST_INSERT_HEAD(&pf->flows, flow0, next);
3241 /* Make this flow internal, so the user app cannot delete it */
3242 flow0->internal = 1;
3243 ENICPMD_LOG(DEBUG, "representor->VF %d flow created: wq %d -> tag %d hairpin",
3244 vf->vf_id, vf->pf_wq_idx, tag);
3246 /* Ingress: steer hairpinned to VF RQ 0 */
3247 enic_fm_open_scratch(fm);
3248 fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3249 fm_tcam_entry->ftm_data.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3250 fm_tcam_entry->ftm_mask.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3251 fm_tcam_entry->ftm_data.fk_packet_tag = tag;
3252 fm_tcam_entry->ftm_mask.fk_packet_tag = 0xff;
3253 memset(&fm_op, 0, sizeof(fm_op));
3254 fm_op.fa_op = FMOP_RQ_STEER;
3255 fm_op.rq_steer.rq_index = 0;
3256 fm_op.rq_steer.vnic_handle = vf->enic.fm_vnic_handle;
3257 enic_fm_append_action_op(fm, &fm_op, &error);
3258 memset(&fm_op, 0, sizeof(fm_op));
3259 fm_op.fa_op = FMOP_END;
3260 enic_fm_append_action_op(fm, &fm_op, &error);
3264 attrs.priority = FM_HIGHEST_PRIORITY;
3265 flow1 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3267 enic_fm_close_scratch(fm);
3268 if (flow1 == NULL) {
3269 ENICPMD_LOG(ERR, "Cannot create flow 1 for representor->VF");
3270 enic_fm_flow_destroy(pf->rte_dev, flow0, &error);
3273 LIST_INSERT_HEAD(&pf->flows, flow1, next);
3274 flow1->internal = 1;
3275 ENICPMD_LOG(DEBUG, "representor->VF %d flow created: tag %d hairpinned -> VF RQ %d",
3276 vf->vf_id, tag, fm_op.rq_steer.rq_index);
3277 vf->rep2vf_flow[0] = flow0;
3278 vf->rep2vf_flow[1] = flow1;
3279 /* Done with this tag, use a different one next time */
3285 * Add a low priority flow that matches all packets from VF and loops them
3286 * back to the representor.
3289 enic_fm_add_vf2rep_flow(struct enic_vf_representor *vf)
3291 struct fm_tcam_match_entry *fm_tcam_entry;
3292 struct rte_flow *flow0, *flow1;
3293 struct fm_action *fm_action;
3294 struct rte_flow_error error;
3295 struct rte_flow_attr attrs;
3296 struct fm_action_op fm_op;
3297 struct enic_flowman *fm;
3303 tag = fm->vf_rep_tag;
3304 enic_fm_open_scratch(fm);
3305 fm_tcam_entry = &fm->tcam_entry;
3306 fm_action = &fm->action;
3307 /* Egress rule: match-any and tag+hairpin */
3308 fm_tcam_entry->ftm_data.fk_wq_id = 0;
3309 fm_tcam_entry->ftm_mask.fk_wq_id = 0xffff;
3310 fm_tcam_entry->ftm_data.fk_wq_vnic = vf->enic.fm_vnic_handle;
3311 fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3312 memset(&fm_op, 0, sizeof(fm_op));
3313 fm_op.fa_op = FMOP_TAG;
3314 fm_op.tag.tag = tag;
3315 enic_fm_append_action_op(fm, &fm_op, &error);
3316 memset(&fm_op, 0, sizeof(fm_op));
3317 fm_op.fa_op = FMOP_EG_HAIRPIN;
3318 enic_fm_append_action_op(fm, &fm_op, &error);
3319 memset(&fm_op, 0, sizeof(fm_op));
3320 fm_op.fa_op = FMOP_END;
3321 enic_fm_append_action_op(fm, &fm_op, &error);
3325 attrs.priority = FM_LOWEST_PRIORITY;
3326 flow0 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3328 enic_fm_close_scratch(fm);
3329 if (flow0 == NULL) {
3330 ENICPMD_LOG(ERR, "Cannot create flow 0 for VF->representor");
3333 LIST_INSERT_HEAD(&pf->flows, flow0, next);
3334 /* Make this flow internal, so the user app cannot delete it */
3335 flow0->internal = 1;
3336 ENICPMD_LOG(DEBUG, "VF %d->representor flow created: wq %d (low prio) -> tag %d hairpin",
3337 vf->vf_id, fm_tcam_entry->ftm_data.fk_wq_id, tag);
3339 /* Ingress: steer hairpinned to VF rep RQ */
3340 enic_fm_open_scratch(fm);
3341 fm_tcam_entry->ftm_flags |= FMEF_COUNTER;
3342 fm_tcam_entry->ftm_data.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3343 fm_tcam_entry->ftm_mask.fk_hdrset[0].fk_metadata |= FKM_EG_HAIRPINNED;
3344 fm_tcam_entry->ftm_data.fk_packet_tag = tag;
3345 fm_tcam_entry->ftm_mask.fk_packet_tag = 0xff;
3346 memset(&fm_op, 0, sizeof(fm_op));
3347 fm_op.fa_op = FMOP_RQ_STEER;
3348 fm_op.rq_steer.rq_index = vf->pf_rq_sop_idx;
3349 fm_op.rq_steer.vnic_handle = pf->fm_vnic_handle;
3350 enic_fm_append_action_op(fm, &fm_op, &error);
3351 memset(&fm_op, 0, sizeof(fm_op));
3352 fm_op.fa_op = FMOP_END;
3353 enic_fm_append_action_op(fm, &fm_op, &error);
3357 attrs.priority = FM_HIGHEST_PRIORITY;
3358 flow1 = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
3360 enic_fm_close_scratch(fm);
3361 if (flow1 == NULL) {
3362 ENICPMD_LOG(ERR, "Cannot create flow 1 for VF->representor");
3363 enic_fm_flow_destroy(pf->rte_dev, flow0, &error);
3366 LIST_INSERT_HEAD(&pf->flows, flow1, next);
3367 flow1->internal = 1;
3368 ENICPMD_LOG(DEBUG, "VF %d->representor flow created: tag %d hairpinned -> PF RQ %d",
3369 vf->vf_id, tag, vf->pf_rq_sop_idx);
3370 vf->vf2rep_flow[0] = flow0;
3371 vf->vf2rep_flow[1] = flow1;
3372 /* Done with this tag, use a different one next time */
3377 /* Destroy representor flows created by enic_fm_add_{rep2vf,vf2rep}_flow */
3379 delete_rep_flows(struct enic *enic)
3381 struct enic_vf_representor *vf;
3382 struct rte_flow_error error;
3383 struct rte_eth_dev *dev;
3386 RTE_ASSERT(enic_is_vf_rep(enic));
3387 vf = VF_ENIC_TO_VF_REP(enic);
3388 dev = vf->pf->rte_dev;
3389 for (i = 0; i < ARRAY_SIZE(vf->vf2rep_flow); i++) {
3390 if (vf->vf2rep_flow[i])
3391 enic_fm_flow_destroy(dev, vf->vf2rep_flow[i], &error);
3393 for (i = 0; i < ARRAY_SIZE(vf->rep2vf_flow); i++) {
3394 if (vf->rep2vf_flow[i])
3395 enic_fm_flow_destroy(dev, vf->rep2vf_flow[i], &error);
3399 static struct enic_flowman *
3400 begin_fm(struct enic *enic)
3402 struct enic_vf_representor *vf;
3403 struct enic_flowman *fm;
3405 /* Representor uses PF flowman */
3406 if (enic_is_vf_rep(enic)) {
3407 vf = VF_ENIC_TO_VF_REP(enic);
3412 /* Save the API caller and lock if representors exist */
3414 if (fm->owner_enic->switchdev_mode)
3415 rte_spinlock_lock(&fm->lock);
3416 fm->user_enic = enic;
3422 end_fm(struct enic_flowman *fm)
3424 fm->user_enic = NULL;
3425 if (fm->owner_enic->switchdev_mode)
3426 rte_spinlock_unlock(&fm->lock);