1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2008-2019 Cisco Systems, Inc. All rights reserved.
8 #include <rte_ethdev_driver.h>
9 #include <rte_flow_driver.h>
10 #include <rte_ether.h>
14 #include "enic_compat.h"
19 #define IP_DEFTTL 64 /* from RFC 1340. */
20 #define IP6_VTC_FLOW 0x60000000
22 /* Highest Item type supported by Flowman */
23 #define FM_MAX_ITEM_TYPE RTE_FLOW_ITEM_TYPE_VXLAN
25 /* Up to 1024 TCAM entries */
26 #define FM_MAX_TCAM_TABLE_SIZE 1024
28 /* Up to 4096 entries per exact match table */
29 #define FM_MAX_EXACT_TABLE_SIZE 4096
31 /* Number of counters to increase on for each increment */
32 #define FM_COUNTERS_EXPAND 100
34 #define FM_INVALID_HANDLE 0
37 * Flow exact match tables (FET) in the VIC and rte_flow groups.
38 * Use a simple scheme to map groups to tables.
39 * Group 0 uses the single TCAM tables, one for each direction.
40 * Group 1, 2, ... uses its own exact match table.
42 * The TCAM tables are allocated upfront during init.
44 * Exact match tables are allocated on demand. 3 paths that lead allocations.
46 * 1. Add a flow that jumps from group 0 to group N.
48 * If N does not exist, we allocate an exact match table for it, using
49 * a dummy key. A key is required for the table.
51 * 2. Add a flow that uses group N.
53 * If N does not exist, we allocate an exact match table for it, using
54 * the flow's key. Subsequent flows to the same group all should have
57 * Without a jump flow to N, N is not reachable in hardware. No packets
60 * 3. Add a flow to an empty group N.
62 * N has been created via (1) and the dummy key. We free that table, allocate
63 * a new table using the new flow's key. Also re-do the existing jump flow to
64 * point to the new table.
66 #define FM_TCAM_RTE_GROUP 0
69 TAILQ_ENTRY(enic_fm_fet) list;
70 uint32_t group; /* rte_flow group ID */
71 uint64_t handle; /* Exact match table handle from flowman */
74 int ref; /* Reference count via get/put */
75 struct fm_key_template key; /* Key associated with the table */
78 struct enic_fm_counter {
79 SLIST_ENTRY(enic_fm_counter) next;
86 uint64_t entry_handle;
87 uint64_t action_handle;
88 struct enic_fm_counter *counter;
89 struct enic_fm_fet *fet;
92 struct enic_fm_jump_flow {
93 TAILQ_ENTRY(enic_fm_jump_flow) list;
94 struct rte_flow *flow;
96 struct fm_tcam_match_entry match;
97 struct fm_action action;
101 * Flowman uses host memory for commands. This structure is allocated
102 * in DMA-able memory.
104 union enic_flowman_cmd_mem {
105 struct fm_tcam_match_table fm_tcam_match_table;
106 struct fm_exact_match_table fm_exact_match_table;
107 struct fm_tcam_match_entry fm_tcam_match_entry;
108 struct fm_exact_match_entry fm_exact_match_entry;
109 struct fm_action fm_action;
112 struct enic_flowman {
116 union enic_flowman_cmd_mem *va;
119 /* TCAM tables allocated upfront, used for group 0 */
120 uint64_t ig_tcam_hndl;
121 uint64_t eg_tcam_hndl;
123 SLIST_HEAD(enic_free_counters, enic_fm_counter) counters;
125 uint32_t counters_alloced;
126 /* Exact match tables for groups != 0, dynamically allocated */
127 TAILQ_HEAD(fet_list, enic_fm_fet) fet_list;
129 * Default exact match tables used for jump actions to
130 * non-existent groups.
132 struct enic_fm_fet *default_eg_fet;
133 struct enic_fm_fet *default_ig_fet;
134 /* Flows that jump to the default table above */
135 TAILQ_HEAD(jump_flow_list, enic_fm_jump_flow) jump_list;
137 * Scratch data used during each invocation of flow_create
140 struct enic_fm_fet *fet;
141 struct fm_tcam_match_entry tcam_entry;
142 struct fm_action action;
143 struct fm_action action_tmp; /* enic_fm_reorder_action_op */
147 static int enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle);
150 * Common arguments passed to copy_item functions. Use this structure
151 * so we can easily add new arguments.
152 * item: Item specification.
153 * fm_tcam_entry: Flowman TCAM match entry.
154 * header_level: 0 for outer header, 1 for inner header.
156 struct copy_item_args {
157 const struct rte_flow_item *item;
158 struct fm_tcam_match_entry *fm_tcam_entry;
159 uint8_t header_level;
162 /* functions for copying items into flowman match */
163 typedef int (enic_copy_item_fn)(struct copy_item_args *arg);
165 /* Info about how to copy items into flowman match */
166 struct enic_fm_items {
167 /* Function for copying and validating an item. */
168 enic_copy_item_fn * const copy_item;
169 /* List of valid previous items. */
170 const enum rte_flow_item_type * const prev_items;
172 * True if it's OK for this item to be the first item. For some NIC
173 * versions, it's invalid to start the stack above layer 3.
175 const uint8_t valid_start_item;
178 static enic_copy_item_fn enic_fm_copy_item_eth;
179 static enic_copy_item_fn enic_fm_copy_item_ipv4;
180 static enic_copy_item_fn enic_fm_copy_item_ipv6;
181 static enic_copy_item_fn enic_fm_copy_item_raw;
182 static enic_copy_item_fn enic_fm_copy_item_sctp;
183 static enic_copy_item_fn enic_fm_copy_item_tcp;
184 static enic_copy_item_fn enic_fm_copy_item_udp;
185 static enic_copy_item_fn enic_fm_copy_item_vlan;
186 static enic_copy_item_fn enic_fm_copy_item_vxlan;
188 /* Ingress actions */
189 static const enum rte_flow_action_type enic_fm_supported_ig_actions[] = {
190 RTE_FLOW_ACTION_TYPE_COUNT,
191 RTE_FLOW_ACTION_TYPE_DROP,
192 RTE_FLOW_ACTION_TYPE_FLAG,
193 RTE_FLOW_ACTION_TYPE_JUMP,
194 RTE_FLOW_ACTION_TYPE_MARK,
195 RTE_FLOW_ACTION_TYPE_PORT_ID,
196 RTE_FLOW_ACTION_TYPE_PASSTHRU,
197 RTE_FLOW_ACTION_TYPE_QUEUE,
198 RTE_FLOW_ACTION_TYPE_RSS,
199 RTE_FLOW_ACTION_TYPE_VOID,
200 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
201 RTE_FLOW_ACTION_TYPE_VXLAN_DECAP,
202 RTE_FLOW_ACTION_TYPE_END, /* END must be the last entry */
206 static const enum rte_flow_action_type enic_fm_supported_eg_actions[] = {
207 RTE_FLOW_ACTION_TYPE_COUNT,
208 RTE_FLOW_ACTION_TYPE_DROP,
209 RTE_FLOW_ACTION_TYPE_JUMP,
210 RTE_FLOW_ACTION_TYPE_PASSTHRU,
211 RTE_FLOW_ACTION_TYPE_VOID,
212 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
213 RTE_FLOW_ACTION_TYPE_END,
216 static const struct enic_fm_items enic_fm_items[] = {
217 [RTE_FLOW_ITEM_TYPE_RAW] = {
218 .copy_item = enic_fm_copy_item_raw,
219 .valid_start_item = 0,
220 .prev_items = (const enum rte_flow_item_type[]) {
221 RTE_FLOW_ITEM_TYPE_UDP,
222 RTE_FLOW_ITEM_TYPE_END,
225 [RTE_FLOW_ITEM_TYPE_ETH] = {
226 .copy_item = enic_fm_copy_item_eth,
227 .valid_start_item = 1,
228 .prev_items = (const enum rte_flow_item_type[]) {
229 RTE_FLOW_ITEM_TYPE_END,
232 [RTE_FLOW_ITEM_TYPE_VLAN] = {
233 .copy_item = enic_fm_copy_item_vlan,
234 .valid_start_item = 1,
235 .prev_items = (const enum rte_flow_item_type[]) {
236 RTE_FLOW_ITEM_TYPE_ETH,
237 RTE_FLOW_ITEM_TYPE_END,
240 [RTE_FLOW_ITEM_TYPE_IPV4] = {
241 .copy_item = enic_fm_copy_item_ipv4,
242 .valid_start_item = 1,
243 .prev_items = (const enum rte_flow_item_type[]) {
244 RTE_FLOW_ITEM_TYPE_ETH,
245 RTE_FLOW_ITEM_TYPE_VLAN,
246 RTE_FLOW_ITEM_TYPE_END,
249 [RTE_FLOW_ITEM_TYPE_IPV6] = {
250 .copy_item = enic_fm_copy_item_ipv6,
251 .valid_start_item = 1,
252 .prev_items = (const enum rte_flow_item_type[]) {
253 RTE_FLOW_ITEM_TYPE_ETH,
254 RTE_FLOW_ITEM_TYPE_VLAN,
255 RTE_FLOW_ITEM_TYPE_END,
258 [RTE_FLOW_ITEM_TYPE_UDP] = {
259 .copy_item = enic_fm_copy_item_udp,
260 .valid_start_item = 1,
261 .prev_items = (const enum rte_flow_item_type[]) {
262 RTE_FLOW_ITEM_TYPE_IPV4,
263 RTE_FLOW_ITEM_TYPE_IPV6,
264 RTE_FLOW_ITEM_TYPE_END,
267 [RTE_FLOW_ITEM_TYPE_TCP] = {
268 .copy_item = enic_fm_copy_item_tcp,
269 .valid_start_item = 1,
270 .prev_items = (const enum rte_flow_item_type[]) {
271 RTE_FLOW_ITEM_TYPE_IPV4,
272 RTE_FLOW_ITEM_TYPE_IPV6,
273 RTE_FLOW_ITEM_TYPE_END,
276 [RTE_FLOW_ITEM_TYPE_SCTP] = {
277 .copy_item = enic_fm_copy_item_sctp,
278 .valid_start_item = 0,
279 .prev_items = (const enum rte_flow_item_type[]) {
280 RTE_FLOW_ITEM_TYPE_IPV4,
281 RTE_FLOW_ITEM_TYPE_IPV6,
282 RTE_FLOW_ITEM_TYPE_END,
285 [RTE_FLOW_ITEM_TYPE_VXLAN] = {
286 .copy_item = enic_fm_copy_item_vxlan,
287 .valid_start_item = 1,
288 .prev_items = (const enum rte_flow_item_type[]) {
289 RTE_FLOW_ITEM_TYPE_UDP,
290 RTE_FLOW_ITEM_TYPE_END,
296 enic_fm_copy_item_eth(struct copy_item_args *arg)
298 const struct rte_flow_item *item = arg->item;
299 const struct rte_flow_item_eth *spec = item->spec;
300 const struct rte_flow_item_eth *mask = item->mask;
301 const uint8_t lvl = arg->header_level;
302 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
303 struct fm_header_set *fm_data, *fm_mask;
305 ENICPMD_FUNC_TRACE();
306 /* Match all if no spec */
310 mask = &rte_flow_item_eth_mask;
311 fm_data = &entry->ftm_data.fk_hdrset[lvl];
312 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
313 fm_data->fk_header_select |= FKH_ETHER;
314 fm_mask->fk_header_select |= FKH_ETHER;
315 memcpy(&fm_data->l2.eth, spec, sizeof(*spec));
316 memcpy(&fm_mask->l2.eth, mask, sizeof(*mask));
321 enic_fm_copy_item_vlan(struct copy_item_args *arg)
323 const struct rte_flow_item *item = arg->item;
324 const struct rte_flow_item_vlan *spec = item->spec;
325 const struct rte_flow_item_vlan *mask = item->mask;
326 const uint8_t lvl = arg->header_level;
327 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
328 struct fm_header_set *fm_data, *fm_mask;
329 struct rte_ether_hdr *eth_mask;
330 struct rte_ether_hdr *eth_val;
333 ENICPMD_FUNC_TRACE();
334 fm_data = &entry->ftm_data.fk_hdrset[lvl];
335 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
336 /* Outer and inner packet vlans need different flags */
337 meta = FKM_VLAN_PRES;
340 fm_data->fk_metadata |= meta;
341 fm_mask->fk_metadata |= meta;
343 /* Match all if no spec */
347 mask = &rte_flow_item_vlan_mask;
349 eth_mask = (void *)&fm_mask->l2.eth;
350 eth_val = (void *)&fm_data->l2.eth;
352 /* Outer TPID cannot be matched */
353 if (eth_mask->ether_type)
357 * When packet matching, the VIC always compares vlan-stripped
358 * L2, regardless of vlan stripping settings. So, the inner type
359 * from vlan becomes the ether type of the eth header.
361 eth_mask->ether_type = mask->inner_type;
362 eth_val->ether_type = spec->inner_type;
363 fm_data->fk_header_select |= FKH_ETHER | FKH_QTAG;
364 fm_mask->fk_header_select |= FKH_ETHER | FKH_QTAG;
365 fm_data->fk_vlan = rte_be_to_cpu_16(spec->tci);
366 fm_mask->fk_vlan = rte_be_to_cpu_16(mask->tci);
371 enic_fm_copy_item_ipv4(struct copy_item_args *arg)
373 const struct rte_flow_item *item = arg->item;
374 const struct rte_flow_item_ipv4 *spec = item->spec;
375 const struct rte_flow_item_ipv4 *mask = item->mask;
376 const uint8_t lvl = arg->header_level;
377 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
378 struct fm_header_set *fm_data, *fm_mask;
380 ENICPMD_FUNC_TRACE();
381 fm_data = &entry->ftm_data.fk_hdrset[lvl];
382 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
383 fm_data->fk_metadata |= FKM_IPV4;
384 fm_mask->fk_metadata |= FKM_IPV4;
389 mask = &rte_flow_item_ipv4_mask;
391 fm_data->fk_header_select |= FKH_IPV4;
392 fm_mask->fk_header_select |= FKH_IPV4;
393 memcpy(&fm_data->l3.ip4, spec, sizeof(*spec));
394 memcpy(&fm_mask->l3.ip4, mask, sizeof(*mask));
399 enic_fm_copy_item_ipv6(struct copy_item_args *arg)
401 const struct rte_flow_item *item = arg->item;
402 const struct rte_flow_item_ipv6 *spec = item->spec;
403 const struct rte_flow_item_ipv6 *mask = item->mask;
404 const uint8_t lvl = arg->header_level;
405 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
406 struct fm_header_set *fm_data, *fm_mask;
408 ENICPMD_FUNC_TRACE();
409 fm_data = &entry->ftm_data.fk_hdrset[lvl];
410 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
411 fm_data->fk_metadata |= FKM_IPV6;
412 fm_mask->fk_metadata |= FKM_IPV6;
417 mask = &rte_flow_item_ipv6_mask;
419 fm_data->fk_header_select |= FKH_IPV6;
420 fm_mask->fk_header_select |= FKH_IPV6;
421 memcpy(&fm_data->l3.ip6, spec, sizeof(*spec));
422 memcpy(&fm_mask->l3.ip6, mask, sizeof(*mask));
427 enic_fm_copy_item_udp(struct copy_item_args *arg)
429 const struct rte_flow_item *item = arg->item;
430 const struct rte_flow_item_udp *spec = item->spec;
431 const struct rte_flow_item_udp *mask = item->mask;
432 const uint8_t lvl = arg->header_level;
433 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
434 struct fm_header_set *fm_data, *fm_mask;
436 ENICPMD_FUNC_TRACE();
437 fm_data = &entry->ftm_data.fk_hdrset[lvl];
438 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
439 fm_data->fk_metadata |= FKM_UDP;
440 fm_mask->fk_metadata |= FKM_UDP;
445 mask = &rte_flow_item_udp_mask;
447 fm_data->fk_header_select |= FKH_UDP;
448 fm_mask->fk_header_select |= FKH_UDP;
449 memcpy(&fm_data->l4.udp, spec, sizeof(*spec));
450 memcpy(&fm_mask->l4.udp, mask, sizeof(*mask));
455 enic_fm_copy_item_tcp(struct copy_item_args *arg)
457 const struct rte_flow_item *item = arg->item;
458 const struct rte_flow_item_tcp *spec = item->spec;
459 const struct rte_flow_item_tcp *mask = item->mask;
460 const uint8_t lvl = arg->header_level;
461 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
462 struct fm_header_set *fm_data, *fm_mask;
464 ENICPMD_FUNC_TRACE();
465 fm_data = &entry->ftm_data.fk_hdrset[lvl];
466 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
467 fm_data->fk_metadata |= FKM_TCP;
468 fm_mask->fk_metadata |= FKM_TCP;
473 mask = &rte_flow_item_tcp_mask;
475 fm_data->fk_header_select |= FKH_TCP;
476 fm_mask->fk_header_select |= FKH_TCP;
477 memcpy(&fm_data->l4.tcp, spec, sizeof(*spec));
478 memcpy(&fm_mask->l4.tcp, mask, sizeof(*mask));
483 enic_fm_copy_item_sctp(struct copy_item_args *arg)
485 const struct rte_flow_item *item = arg->item;
486 const struct rte_flow_item_sctp *spec = item->spec;
487 const struct rte_flow_item_sctp *mask = item->mask;
488 const uint8_t lvl = arg->header_level;
489 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
490 struct fm_header_set *fm_data, *fm_mask;
491 uint8_t *ip_proto_mask = NULL;
492 uint8_t *ip_proto = NULL;
495 ENICPMD_FUNC_TRACE();
496 fm_data = &entry->ftm_data.fk_hdrset[lvl];
497 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
499 * The NIC filter API has no flags for "match sctp", so explicitly
500 * set the protocol number in the IP pattern.
502 if (fm_data->fk_metadata & FKM_IPV4) {
503 struct rte_ipv4_hdr *ip;
504 ip = (struct rte_ipv4_hdr *)&fm_mask->l3.ip4;
505 ip_proto_mask = &ip->next_proto_id;
506 ip = (struct rte_ipv4_hdr *)&fm_data->l3.ip4;
507 ip_proto = &ip->next_proto_id;
509 } else if (fm_data->fk_metadata & FKM_IPV6) {
510 struct rte_ipv6_hdr *ip;
511 ip = (struct rte_ipv6_hdr *)&fm_mask->l3.ip6;
512 ip_proto_mask = &ip->proto;
513 ip = (struct rte_ipv6_hdr *)&fm_data->l3.ip6;
514 ip_proto = &ip->proto;
517 /* Need IPv4/IPv6 pattern first */
520 *ip_proto = IPPROTO_SCTP;
521 *ip_proto_mask = 0xff;
522 fm_data->fk_header_select |= l3_fkh;
523 fm_mask->fk_header_select |= l3_fkh;
528 mask = &rte_flow_item_sctp_mask;
530 fm_data->fk_header_select |= FKH_L4RAW;
531 fm_mask->fk_header_select |= FKH_L4RAW;
532 memcpy(fm_data->l4.rawdata, spec, sizeof(*spec));
533 memcpy(fm_mask->l4.rawdata, mask, sizeof(*mask));
538 enic_fm_copy_item_vxlan(struct copy_item_args *arg)
540 const struct rte_flow_item *item = arg->item;
541 const struct rte_flow_item_vxlan *spec = item->spec;
542 const struct rte_flow_item_vxlan *mask = item->mask;
543 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
544 struct fm_header_set *fm_data, *fm_mask;
546 ENICPMD_FUNC_TRACE();
547 /* Only 2 header levels (outer and inner) allowed */
548 if (arg->header_level > 0)
551 fm_data = &entry->ftm_data.fk_hdrset[0];
552 fm_mask = &entry->ftm_mask.fk_hdrset[0];
553 fm_data->fk_metadata |= FKM_VXLAN;
554 fm_mask->fk_metadata |= FKM_VXLAN;
555 /* items from here on out are inner header items */
556 arg->header_level = 1;
558 /* Match all if no spec */
562 mask = &rte_flow_item_vxlan_mask;
564 fm_data->fk_header_select |= FKH_VXLAN;
565 fm_mask->fk_header_select |= FKH_VXLAN;
566 memcpy(&fm_data->vxlan, spec, sizeof(*spec));
567 memcpy(&fm_mask->vxlan, mask, sizeof(*mask));
572 * Currently, raw pattern match is very limited. It is intended for matching
573 * UDP tunnel header (e.g. vxlan or geneve).
576 enic_fm_copy_item_raw(struct copy_item_args *arg)
578 const struct rte_flow_item *item = arg->item;
579 const struct rte_flow_item_raw *spec = item->spec;
580 const struct rte_flow_item_raw *mask = item->mask;
581 const uint8_t lvl = arg->header_level;
582 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
583 struct fm_header_set *fm_data, *fm_mask;
585 ENICPMD_FUNC_TRACE();
586 /* Cannot be used for inner packet */
589 /* Need both spec and mask */
592 /* Only supports relative with offset 0 */
593 if (!spec->relative || spec->offset != 0 || spec->search ||
596 /* Need non-null pattern that fits within the NIC's filter pattern */
597 if (spec->length == 0 ||
598 spec->length + sizeof(struct rte_udp_hdr) > FM_LAYER_SIZE ||
599 !spec->pattern || !mask->pattern)
602 * Mask fields, including length, are often set to zero. Assume that
603 * means "same as spec" to avoid breaking existing apps. If length
604 * is not zero, then it should be >= spec length.
606 * No more pattern follows this, so append to the L4 layer instead of
607 * L5 to work with both recent and older VICs.
609 if (mask->length != 0 && mask->length < spec->length)
612 fm_data = &entry->ftm_data.fk_hdrset[lvl];
613 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
614 fm_data->fk_header_select |= FKH_L4RAW;
615 fm_mask->fk_header_select |= FKH_L4RAW;
616 fm_data->fk_header_select &= ~FKH_UDP;
617 fm_mask->fk_header_select &= ~FKH_UDP;
618 memcpy(fm_data->l4.rawdata + sizeof(struct rte_udp_hdr),
619 spec->pattern, spec->length);
620 memcpy(fm_mask->l4.rawdata + sizeof(struct rte_udp_hdr),
621 mask->pattern, spec->length);
626 enic_fet_alloc(struct enic_flowman *fm, uint8_t ingress,
627 struct fm_key_template *key, int entries,
628 struct enic_fm_fet **fet_out)
630 struct fm_exact_match_table *cmd;
631 struct fm_header_set *hdr;
632 struct enic_fm_fet *fet;
636 ENICPMD_FUNC_TRACE();
637 fet = calloc(1, sizeof(struct enic_fm_fet));
640 cmd = &fm->cmd.va->fm_exact_match_table;
641 memset(cmd, 0, sizeof(*cmd));
642 cmd->fet_direction = ingress ? FM_INGRESS : FM_EGRESS;
643 cmd->fet_stage = FM_STAGE_LAST;
644 cmd->fet_max_entries = entries ? entries : FM_MAX_EXACT_TABLE_SIZE;
646 hdr = &cmd->fet_key.fk_hdrset[0];
647 memset(hdr, 0, sizeof(*hdr));
648 hdr->fk_header_select = FKH_IPV4 | FKH_UDP;
649 hdr->l3.ip4.fk_saddr = 0xFFFFFFFF;
650 hdr->l3.ip4.fk_daddr = 0xFFFFFFFF;
651 hdr->l4.udp.fk_source = 0xFFFF;
652 hdr->l4.udp.fk_dest = 0xFFFF;
653 fet->default_key = 1;
655 memcpy(&cmd->fet_key, key, sizeof(*key));
656 memcpy(&fet->key, key, sizeof(*key));
657 fet->default_key = 0;
659 cmd->fet_key.fk_packet_tag = 1;
661 args[0] = FM_EXACT_TABLE_ALLOC;
662 args[1] = fm->cmd.pa;
663 ret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
665 ENICPMD_LOG(ERR, "cannot alloc exact match table: rc=%d", ret);
669 fet->handle = args[0];
670 fet->ingress = ingress;
671 ENICPMD_LOG(DEBUG, "allocated exact match table: handle=0x%" PRIx64,
678 enic_fet_free(struct enic_flowman *fm, struct enic_fm_fet *fet)
680 ENICPMD_FUNC_TRACE();
681 enic_fm_tbl_free(fm, fet->handle);
682 if (!fet->default_key)
683 TAILQ_REMOVE(&fm->fet_list, fet, list);
688 * Get the exact match table for the given combination of
689 * <group, ingress, key>. Allocate one on the fly as necessary.
692 enic_fet_get(struct enic_flowman *fm,
695 struct fm_key_template *key,
696 struct enic_fm_fet **fet_out,
697 struct rte_flow_error *error)
699 struct enic_fm_fet *fet;
701 ENICPMD_FUNC_TRACE();
702 /* See if we already have this table open */
703 TAILQ_FOREACH(fet, &fm->fet_list, list) {
704 if (fet->group == group && fet->ingress == ingress)
708 /* Jumping to a non-existing group? Use the default table */
710 fet = ingress ? fm->default_ig_fet : fm->default_eg_fet;
711 } else if (enic_fet_alloc(fm, ingress, key, 0, &fet)) {
712 return rte_flow_error_set(error, EINVAL,
713 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
714 NULL, "enic: cannot get exact match table");
717 /* Default table is never on the open table list */
718 if (!fet->default_key)
719 TAILQ_INSERT_HEAD(&fm->fet_list, fet, list);
723 ENICPMD_LOG(DEBUG, "fet_get: %s %s group=%u ref=%u",
724 fet->default_key ? "default" : "",
725 fet->ingress ? "ingress" : "egress",
726 fet->group, fet->ref);
731 enic_fet_put(struct enic_flowman *fm, struct enic_fm_fet *fet)
733 ENICPMD_FUNC_TRACE();
734 RTE_ASSERT(fet->ref > 0);
736 ENICPMD_LOG(DEBUG, "fet_put: %s %s group=%u ref=%u",
737 fet->default_key ? "default" : "",
738 fet->ingress ? "ingress" : "egress",
739 fet->group, fet->ref);
741 enic_fet_free(fm, fet);
744 /* Return 1 if current item is valid on top of the previous one. */
746 fm_item_stacking_valid(enum rte_flow_item_type prev_item,
747 const struct enic_fm_items *item_info,
748 uint8_t is_first_item)
750 enum rte_flow_item_type const *allowed_items = item_info->prev_items;
752 ENICPMD_FUNC_TRACE();
753 for (; *allowed_items != RTE_FLOW_ITEM_TYPE_END; allowed_items++) {
754 if (prev_item == *allowed_items)
758 /* This is the first item in the stack. Check if that's cool */
759 if (is_first_item && item_info->valid_start_item)
765 * Build the flow manager match entry structure from the provided pattern.
766 * The pattern is validated as the items are copied.
769 enic_fm_copy_entry(struct enic_flowman *fm,
770 const struct rte_flow_item pattern[],
771 struct rte_flow_error *error)
773 const struct enic_fm_items *item_info;
774 enum rte_flow_item_type prev_item;
775 const struct rte_flow_item *item;
776 struct copy_item_args args;
777 uint8_t prev_header_level;
778 uint8_t is_first_item;
781 ENICPMD_FUNC_TRACE();
784 prev_item = RTE_FLOW_ITEM_TYPE_END;
786 args.fm_tcam_entry = &fm->tcam_entry;
787 args.header_level = 0;
788 prev_header_level = 0;
789 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
791 * Get info about how to validate and copy the item. If NULL
792 * is returned the nic does not support the item.
794 if (item->type == RTE_FLOW_ITEM_TYPE_VOID)
797 item_info = &enic_fm_items[item->type];
799 if (item->type > FM_MAX_ITEM_TYPE ||
800 item_info->copy_item == NULL) {
801 return rte_flow_error_set(error, ENOTSUP,
802 RTE_FLOW_ERROR_TYPE_ITEM,
803 NULL, "enic: unsupported item");
806 /* check to see if item stacking is valid */
807 if (!fm_item_stacking_valid(prev_item, item_info,
812 ret = item_info->copy_item(&args);
814 goto item_not_supported;
815 /* Going from outer to inner? Treat it as a new packet start */
816 if (prev_header_level != args.header_level) {
817 prev_item = RTE_FLOW_ITEM_TYPE_END;
820 prev_item = item->type;
823 prev_header_level = args.header_level;
828 return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,
829 NULL, "enic: unsupported item type");
832 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
833 item, "enic: unsupported item stack");
837 flow_item_skip_void(const struct rte_flow_item **item)
840 if ((*item)->type != RTE_FLOW_ITEM_TYPE_VOID)
845 append_template(void **template, uint8_t *off, const void *data, int len)
847 memcpy(*template, data, len);
848 *template = (char *)*template + len;
853 enic_fm_append_action_op(struct enic_flowman *fm,
854 struct fm_action_op *fm_op,
855 struct rte_flow_error *error)
859 count = fm->action_op_count;
860 ENICPMD_LOG(DEBUG, "append action op: idx=%d op=%u",
861 count, fm_op->fa_op);
862 if (count == FM_ACTION_OP_MAX) {
863 return rte_flow_error_set(error, EINVAL,
864 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
865 "too many action operations");
867 fm->action.fma_action_ops[count] = *fm_op;
868 fm->action_op_count = count + 1;
872 /* Steer operations need to appear before other ops */
874 enic_fm_reorder_action_op(struct enic_flowman *fm)
876 struct fm_action_op *dst, *dst_head, *src, *src_head;
878 ENICPMD_FUNC_TRACE();
879 /* Move steer ops to the front. */
880 src = fm->action.fma_action_ops;
882 dst = fm->action_tmp.fma_action_ops;
884 /* Copy steer ops to tmp */
885 while (src->fa_op != FMOP_END) {
886 if (src->fa_op == FMOP_RQ_STEER) {
887 ENICPMD_LOG(DEBUG, "move op: %ld -> dst %ld",
888 (long)(src - src_head),
889 (long)(dst - dst_head));
895 /* Then append non-steer ops */
897 while (src->fa_op != FMOP_END) {
898 if (src->fa_op != FMOP_RQ_STEER) {
899 ENICPMD_LOG(DEBUG, "move op: %ld -> dst %ld",
900 (long)(src - src_head),
901 (long)(dst - dst_head));
909 /* Finally replace the original action with the reordered one */
910 memcpy(fm->action.fma_action_ops, fm->action_tmp.fma_action_ops,
911 sizeof(fm->action.fma_action_ops));
914 /* VXLAN decap is done via flowman compound action */
916 enic_fm_copy_vxlan_decap(struct enic_flowman *fm,
917 struct fm_tcam_match_entry *fmt,
918 const struct rte_flow_action *action,
919 struct rte_flow_error *error)
921 struct fm_header_set *fm_data;
922 struct fm_action_op fm_op;
924 ENICPMD_FUNC_TRACE();
925 fm_data = &fmt->ftm_data.fk_hdrset[0];
926 if (!(fm_data->fk_metadata & FKM_VXLAN)) {
927 return rte_flow_error_set(error, EINVAL,
928 RTE_FLOW_ERROR_TYPE_ACTION, action,
929 "vxlan-decap: vxlan must be in pattern");
932 memset(&fm_op, 0, sizeof(fm_op));
933 fm_op.fa_op = FMOP_DECAP_NOSTRIP;
934 return enic_fm_append_action_op(fm, &fm_op, error);
937 /* VXLAN encap is done via flowman compound action */
939 enic_fm_copy_vxlan_encap(struct enic_flowman *fm,
940 const struct rte_flow_item *item,
941 struct rte_flow_error *error)
943 struct fm_action_op fm_op;
944 struct rte_ether_hdr *eth;
949 ENICPMD_FUNC_TRACE();
950 memset(&fm_op, 0, sizeof(fm_op));
951 fm_op.fa_op = FMOP_ENCAP;
952 template = fm->action.fma_data;
955 * Copy flow items to the flowman template starting L2.
956 * L2 must be ethernet.
958 flow_item_skip_void(&item);
959 if (item->type != RTE_FLOW_ITEM_TYPE_ETH)
960 return rte_flow_error_set(error, EINVAL,
961 RTE_FLOW_ERROR_TYPE_ITEM, item,
962 "vxlan-encap: first item should be ethernet");
963 eth = (struct rte_ether_hdr *)template;
964 ethertype = ð->ether_type;
965 append_template(&template, &off, item->spec,
966 sizeof(struct rte_flow_item_eth));
968 flow_item_skip_void(&item);
970 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
971 const struct rte_flow_item_vlan *spec;
973 ENICPMD_LOG(DEBUG, "vxlan-encap: vlan");
975 fm_op.encap.outer_vlan = rte_be_to_cpu_16(spec->tci);
977 flow_item_skip_void(&item);
979 /* L3 must be IPv4, IPv6 */
980 switch (item->type) {
981 case RTE_FLOW_ITEM_TYPE_IPV4:
983 struct rte_ipv4_hdr *ip4;
985 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv4");
986 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
987 ip4 = (struct rte_ipv4_hdr *)template;
989 * Offset of IPv4 length field and its initial value
990 * (IP + UDP + VXLAN) are specified in the action. The NIC
991 * will add inner packet length.
993 fm_op.encap.len1_offset = off +
994 offsetof(struct rte_ipv4_hdr, total_length);
995 fm_op.encap.len1_delta = sizeof(struct rte_ipv4_hdr) +
996 sizeof(struct rte_udp_hdr) +
997 sizeof(struct rte_vxlan_hdr);
998 append_template(&template, &off, item->spec,
999 sizeof(struct rte_ipv4_hdr));
1000 ip4->version_ihl = RTE_IPV4_VHL_DEF;
1001 if (ip4->time_to_live == 0)
1002 ip4->time_to_live = IP_DEFTTL;
1003 ip4->next_proto_id = IPPROTO_UDP;
1006 case RTE_FLOW_ITEM_TYPE_IPV6:
1008 struct rte_ipv6_hdr *ip6;
1010 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv6");
1011 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
1012 ip6 = (struct rte_ipv6_hdr *)template;
1013 fm_op.encap.len1_offset = off +
1014 offsetof(struct rte_ipv6_hdr, payload_len);
1015 fm_op.encap.len1_delta = sizeof(struct rte_udp_hdr) +
1016 sizeof(struct rte_vxlan_hdr);
1017 append_template(&template, &off, item->spec,
1018 sizeof(struct rte_ipv6_hdr));
1019 ip6->vtc_flow |= rte_cpu_to_be_32(IP6_VTC_FLOW);
1020 if (ip6->hop_limits == 0)
1021 ip6->hop_limits = IP_DEFTTL;
1022 ip6->proto = IPPROTO_UDP;
1026 return rte_flow_error_set(error,
1027 EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1028 "vxlan-encap: L3 must be IPv4/IPv6");
1031 flow_item_skip_void(&item);
1034 if (item->type != RTE_FLOW_ITEM_TYPE_UDP)
1035 return rte_flow_error_set(error, EINVAL,
1036 RTE_FLOW_ERROR_TYPE_ITEM, item,
1037 "vxlan-encap: UDP must follow IPv4/IPv6");
1038 /* UDP length = UDP + VXLAN. NIC will add inner packet length. */
1039 fm_op.encap.len2_offset =
1040 off + offsetof(struct rte_udp_hdr, dgram_len);
1041 fm_op.encap.len2_delta =
1042 sizeof(struct rte_udp_hdr) + sizeof(struct rte_vxlan_hdr);
1043 append_template(&template, &off, item->spec,
1044 sizeof(struct rte_udp_hdr));
1046 flow_item_skip_void(&item);
1049 if (item->type != RTE_FLOW_ITEM_TYPE_VXLAN)
1050 return rte_flow_error_set(error,
1051 EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1052 "vxlan-encap: VXLAN must follow UDP");
1053 append_template(&template, &off, item->spec,
1054 sizeof(struct rte_flow_item_vxlan));
1057 * Fill in the rest of the action structure.
1058 * Indicate that we want to encap with vxlan at packet start.
1060 fm_op.encap.template_offset = 0;
1061 fm_op.encap.template_len = off;
1062 return enic_fm_append_action_op(fm, &fm_op, error);
1066 enic_fm_find_vnic(struct enic *enic, const struct rte_pci_addr *addr,
1073 ENICPMD_FUNC_TRACE();
1074 ENICPMD_LOG(DEBUG, "bdf=%x:%x:%x", addr->bus, addr->devid,
1076 bdf = addr->bus << 8 | addr->devid << 3 | addr->function;
1077 args[0] = FM_VNIC_FIND;
1079 rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
1081 ENICPMD_LOG(ERR, "allocating counters rc=%d", rc);
1085 ENICPMD_LOG(DEBUG, "found vnic: handle=0x%" PRIx64, *handle);
1089 /* Translate flow actions to flowman TCAM entry actions */
1091 enic_fm_copy_action(struct enic_flowman *fm,
1092 const struct rte_flow_action actions[],
1094 struct rte_flow_error *error)
1103 struct fm_tcam_match_entry *fmt;
1104 struct fm_action_op fm_op;
1111 ENICPMD_FUNC_TRACE();
1112 fmt = &fm->tcam_entry;
1116 vnic_h = 0; /* 0 = current vNIC */
1117 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1118 switch (actions->type) {
1119 case RTE_FLOW_ACTION_TYPE_VOID:
1121 case RTE_FLOW_ACTION_TYPE_PASSTHRU: {
1122 if (overlap & PASSTHRU)
1124 overlap |= PASSTHRU;
1127 case RTE_FLOW_ACTION_TYPE_JUMP: {
1128 const struct rte_flow_action_jump *jump =
1130 struct enic_fm_fet *fet;
1134 ret = enic_fet_get(fm, jump->group, ingress, NULL,
1139 memset(&fm_op, 0, sizeof(fm_op));
1140 fm_op.fa_op = FMOP_EXACT_MATCH;
1141 fm_op.exact.handle = fet->handle;
1143 ret = enic_fm_append_action_op(fm, &fm_op, error);
1148 case RTE_FLOW_ACTION_TYPE_MARK: {
1149 const struct rte_flow_action_mark *mark =
1155 if (mark->id >= ENIC_MAGIC_FILTER_ID - 1)
1156 return rte_flow_error_set(error, EINVAL,
1157 RTE_FLOW_ERROR_TYPE_ACTION,
1158 NULL, "invalid mark id");
1159 memset(&fm_op, 0, sizeof(fm_op));
1160 fm_op.fa_op = FMOP_MARK;
1161 fm_op.mark.mark = mark->id + 1;
1162 ret = enic_fm_append_action_op(fm, &fm_op, error);
1167 case RTE_FLOW_ACTION_TYPE_FLAG: {
1171 /* ENIC_MAGIC_FILTER_ID is reserved for flagging */
1172 memset(&fm_op, 0, sizeof(fm_op));
1173 fm_op.fa_op = FMOP_MARK;
1174 fm_op.mark.mark = ENIC_MAGIC_FILTER_ID;
1175 ret = enic_fm_append_action_op(fm, &fm_op, error);
1180 case RTE_FLOW_ACTION_TYPE_QUEUE: {
1181 const struct rte_flow_action_queue *queue =
1185 * If other fate kind is set, fail. Multiple
1186 * queue actions are ok.
1188 if ((overlap & FATE) && first_rq)
1192 memset(&fm_op, 0, sizeof(fm_op));
1193 fm_op.fa_op = FMOP_RQ_STEER;
1194 fm_op.rq_steer.rq_index =
1195 enic_rte_rq_idx_to_sop_idx(queue->index);
1196 fm_op.rq_steer.vnic_handle = vnic_h;
1197 ret = enic_fm_append_action_op(fm, &fm_op, error);
1200 ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1201 fm_op.rq_steer.rq_index);
1204 case RTE_FLOW_ACTION_TYPE_DROP: {
1208 memset(&fm_op, 0, sizeof(fm_op));
1209 fm_op.fa_op = FMOP_DROP;
1210 ret = enic_fm_append_action_op(fm, &fm_op, error);
1213 ENICPMD_LOG(DEBUG, "create DROP action");
1216 case RTE_FLOW_ACTION_TYPE_COUNT: {
1217 if (overlap & COUNT)
1220 /* Count is associated with entry not action on VIC. */
1221 fmt->ftm_flags |= FMEF_COUNTER;
1224 case RTE_FLOW_ACTION_TYPE_RSS: {
1225 const struct rte_flow_action_rss *rss = actions->conf;
1230 * Hardware does not support general RSS actions, but
1231 * we can still support the dummy one that is used to
1232 * "receive normally".
1234 allow = rss->func == RTE_ETH_HASH_FUNCTION_DEFAULT &&
1237 rss->types == enic->rss_hf) &&
1238 rss->queue_num == enic->rq_count &&
1240 /* Identity queue map is ok */
1241 for (i = 0; i < rss->queue_num; i++)
1242 allow = allow && (i == rss->queue[i]);
1247 /* Need MARK or FLAG */
1248 if (!(overlap & MARK))
1253 case RTE_FLOW_ACTION_TYPE_PORT_ID: {
1254 const struct rte_flow_action_port_id *port;
1255 struct rte_pci_device *pdev;
1256 struct rte_eth_dev *dev;
1258 port = actions->conf;
1259 if (port->original) {
1260 vnic_h = 0; /* This port */
1263 ENICPMD_LOG(DEBUG, "port id %u", port->id);
1264 if (!rte_eth_dev_is_valid_port(port->id)) {
1265 return rte_flow_error_set(error, EINVAL,
1266 RTE_FLOW_ERROR_TYPE_ACTION,
1267 NULL, "invalid port_id");
1269 dev = &rte_eth_devices[port->id];
1270 if (!dev_is_enic(dev)) {
1271 return rte_flow_error_set(error, EINVAL,
1272 RTE_FLOW_ERROR_TYPE_ACTION,
1273 NULL, "port_id is not enic");
1275 pdev = RTE_ETH_DEV_TO_PCI(dev);
1276 if (enic_fm_find_vnic(enic, &pdev->addr, &vnic_h)) {
1277 return rte_flow_error_set(error, EINVAL,
1278 RTE_FLOW_ERROR_TYPE_ACTION,
1279 NULL, "port_id is not vnic");
1283 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: {
1284 ret = enic_fm_copy_vxlan_decap(fm, fmt, actions,
1290 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: {
1291 const struct rte_flow_action_vxlan_encap *encap;
1293 encap = actions->conf;
1294 if (overlap & ENCAP)
1297 ret = enic_fm_copy_vxlan_encap(fm, encap->definition,
1308 if (!(overlap & (FATE | PASSTHRU | COUNT)))
1310 memset(&fm_op, 0, sizeof(fm_op));
1311 fm_op.fa_op = FMOP_END;
1312 ret = enic_fm_append_action_op(fm, &fm_op, error);
1315 enic_fm_reorder_action_op(fm);
1319 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
1320 NULL, "enic: unsupported action");
1323 /** Check if the action is supported */
1325 enic_fm_match_action(const struct rte_flow_action *action,
1326 const enum rte_flow_action_type *supported_actions)
1328 for (; *supported_actions != RTE_FLOW_ACTION_TYPE_END;
1329 supported_actions++) {
1330 if (action->type == *supported_actions)
1336 /* Debug function to dump internal NIC action structure. */
1338 enic_fm_dump_tcam_actions(const struct fm_action *fm_action)
1340 /* Manually keep in sync with FMOP commands */
1341 const char *fmop_str[FMOP_OP_MAX] = {
1343 [FMOP_DROP] = "drop",
1344 [FMOP_RQ_STEER] = "steer",
1345 [FMOP_EXACT_MATCH] = "exmatch",
1346 [FMOP_MARK] = "mark",
1347 [FMOP_EXT_MARK] = "ext_mark",
1349 [FMOP_EG_HAIRPIN] = "eg_hairpin",
1350 [FMOP_IG_HAIRPIN] = "ig_hairpin",
1351 [FMOP_ENCAP_IVLAN] = "encap_ivlan",
1352 [FMOP_ENCAP_NOIVLAN] = "encap_noivlan",
1353 [FMOP_ENCAP] = "encap",
1354 [FMOP_SET_OVLAN] = "set_ovlan",
1355 [FMOP_DECAP_NOSTRIP] = "decap_nostrip",
1357 const struct fm_action_op *op = &fm_action->fma_action_ops[0];
1358 char buf[128], *bp = buf;
1363 buf_len = sizeof(buf);
1364 for (i = 0; i < FM_ACTION_OP_MAX; i++) {
1365 if (op->fa_op == FMOP_END)
1367 if (op->fa_op >= FMOP_OP_MAX)
1370 op_str = fmop_str[op->fa_op];
1371 n = snprintf(bp, buf_len, "%s,", op_str);
1372 if (n > 0 && n < buf_len) {
1378 /* Remove trailing comma */
1381 ENICPMD_LOG(DEBUG, " Acions: %s", buf);
1385 bits_to_str(uint32_t bits, const char *strings[], int max,
1386 char *buf, int buf_len)
1388 int i, n = 0, len = 0;
1390 for (i = 0; i < max; i++) {
1391 if (bits & (1 << i)) {
1392 n = snprintf(buf, buf_len, "%s,", strings[i]);
1393 if (n > 0 && n < buf_len) {
1400 /* Remove trailing comma */
1408 /* Debug function to dump internal NIC filter structure. */
1410 __enic_fm_dump_tcam_match(const struct fm_header_set *fk_hdrset, char *buf,
1413 /* Manually keep in sync with FKM_BITS */
1414 const char *fm_fkm_str[FKM_BIT_COUNT] = {
1415 [FKM_QTAG_BIT] = "qtag",
1416 [FKM_CMD_BIT] = "cmd",
1417 [FKM_IPV4_BIT] = "ip4",
1418 [FKM_IPV6_BIT] = "ip6",
1419 [FKM_ROCE_BIT] = "roce",
1420 [FKM_UDP_BIT] = "udp",
1421 [FKM_TCP_BIT] = "tcp",
1422 [FKM_TCPORUDP_BIT] = "tcpportudp",
1423 [FKM_IPFRAG_BIT] = "ipfrag",
1424 [FKM_NVGRE_BIT] = "nvgre",
1425 [FKM_VXLAN_BIT] = "vxlan",
1426 [FKM_GENEVE_BIT] = "geneve",
1427 [FKM_NSH_BIT] = "nsh",
1428 [FKM_ROCEV2_BIT] = "rocev2",
1429 [FKM_VLAN_PRES_BIT] = "vlan_pres",
1430 [FKM_IPOK_BIT] = "ipok",
1431 [FKM_L4OK_BIT] = "l4ok",
1432 [FKM_ROCEOK_BIT] = "roceok",
1433 [FKM_FCSOK_BIT] = "fcsok",
1434 [FKM_EG_SPAN_BIT] = "eg_span",
1435 [FKM_IG_SPAN_BIT] = "ig_span",
1436 [FKM_EG_HAIRPINNED_BIT] = "eg_hairpinned",
1438 /* Manually keep in sync with FKH_BITS */
1439 const char *fm_fkh_str[FKH_BIT_COUNT] = {
1440 [FKH_ETHER_BIT] = "eth",
1441 [FKH_QTAG_BIT] = "qtag",
1442 [FKH_L2RAW_BIT] = "l2raw",
1443 [FKH_IPV4_BIT] = "ip4",
1444 [FKH_IPV6_BIT] = "ip6",
1445 [FKH_L3RAW_BIT] = "l3raw",
1446 [FKH_UDP_BIT] = "udp",
1447 [FKH_TCP_BIT] = "tcp",
1448 [FKH_ICMP_BIT] = "icmp",
1449 [FKH_VXLAN_BIT] = "vxlan",
1450 [FKH_L4RAW_BIT] = "l4raw",
1452 uint32_t fkh_bits = fk_hdrset->fk_header_select;
1453 uint32_t fkm_bits = fk_hdrset->fk_metadata;
1456 if (!fkm_bits && !fkh_bits)
1458 n = snprintf(buf, buf_len, "metadata(");
1459 if (n > 0 && n < buf_len) {
1463 n = bits_to_str(fkm_bits, fm_fkm_str, FKM_BIT_COUNT, buf, buf_len);
1464 if (n > 0 && n < buf_len) {
1468 n = snprintf(buf, buf_len, ") valid hdr fields(");
1469 if (n > 0 && n < buf_len) {
1473 n = bits_to_str(fkh_bits, fm_fkh_str, FKH_BIT_COUNT, buf, buf_len);
1474 if (n > 0 && n < buf_len) {
1478 snprintf(buf, buf_len, ")");
1482 enic_fm_dump_tcam_match(const struct fm_tcam_match_entry *match,
1487 memset(buf, 0, sizeof(buf));
1488 __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[0],
1490 ENICPMD_LOG(DEBUG, " TCAM %s Outer: %s %scounter",
1491 (ingress) ? "IG" : "EG", buf,
1492 (match->ftm_flags & FMEF_COUNTER) ? "" : "no ");
1493 memset(buf, 0, sizeof(buf));
1494 __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[1],
1497 ENICPMD_LOG(DEBUG, " Inner: %s", buf);
1500 /* Debug function to dump internal NIC flow structures. */
1502 enic_fm_dump_tcam_entry(const struct fm_tcam_match_entry *fm_match,
1503 const struct fm_action *fm_action,
1506 if (rte_log_get_level(enic_pmd_logtype) < (int)RTE_LOG_DEBUG)
1508 enic_fm_dump_tcam_match(fm_match, ingress);
1509 enic_fm_dump_tcam_actions(fm_action);
1513 enic_fm_flow_parse(struct enic_flowman *fm,
1514 const struct rte_flow_attr *attrs,
1515 const struct rte_flow_item pattern[],
1516 const struct rte_flow_action actions[],
1517 struct rte_flow_error *error)
1519 const struct rte_flow_action *action;
1521 static const enum rte_flow_action_type *sa;
1523 ENICPMD_FUNC_TRACE();
1526 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM,
1527 NULL, "no pattern specified");
1532 rte_flow_error_set(error, EINVAL,
1533 RTE_FLOW_ERROR_TYPE_ACTION_NUM,
1534 NULL, "no action specified");
1539 if (attrs->priority) {
1540 rte_flow_error_set(error, ENOTSUP,
1541 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1543 "priorities are not supported");
1545 } else if (attrs->transfer) {
1546 rte_flow_error_set(error, ENOTSUP,
1547 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1549 "transfer is not supported");
1551 } else if (attrs->ingress && attrs->egress) {
1552 rte_flow_error_set(error, ENOTSUP,
1553 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1555 "bidirectional rules not supported");
1560 rte_flow_error_set(error, EINVAL,
1561 RTE_FLOW_ERROR_TYPE_ATTR,
1562 NULL, "no attribute specified");
1566 /* Verify Actions. */
1567 sa = (attrs->ingress) ? enic_fm_supported_ig_actions :
1568 enic_fm_supported_eg_actions;
1569 for (action = &actions[0]; action->type != RTE_FLOW_ACTION_TYPE_END;
1571 if (action->type == RTE_FLOW_ACTION_TYPE_VOID)
1573 else if (!enic_fm_match_action(action, sa))
1576 if (action->type != RTE_FLOW_ACTION_TYPE_END) {
1577 rte_flow_error_set(error, EPERM, RTE_FLOW_ERROR_TYPE_ACTION,
1578 action, "invalid action");
1581 ret = enic_fm_copy_entry(fm, pattern, error);
1584 ret = enic_fm_copy_action(fm, actions, attrs->ingress, error);
1589 enic_fm_counter_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
1591 if (!fm_flow->counter_valid)
1593 SLIST_INSERT_HEAD(&fm->counters, fm_flow->counter, next);
1594 fm_flow->counter_valid = false;
1598 enic_fm_more_counters(struct enic_flowman *fm)
1600 struct enic_fm_counter *new_stack;
1601 struct enic_fm_counter *ctrs;
1606 ENICPMD_FUNC_TRACE();
1608 new_stack = rte_realloc(fm->counter_stack, (fm->counters_alloced +
1609 FM_COUNTERS_EXPAND) *
1610 sizeof(struct enic_fm_counter), 0);
1611 if (new_stack == NULL) {
1612 ENICPMD_LOG(ERR, "cannot alloc counter memory");
1615 fm->counter_stack = new_stack;
1617 args[0] = FM_COUNTER_BRK;
1618 args[1] = fm->counters_alloced + FM_COUNTERS_EXPAND;
1619 rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
1621 ENICPMD_LOG(ERR, "cannot alloc counters rc=%d", rc);
1624 ctrs = (struct enic_fm_counter *)fm->counter_stack +
1625 fm->counters_alloced;
1626 for (i = 0; i < FM_COUNTERS_EXPAND; i++, ctrs++) {
1627 ctrs->handle = fm->counters_alloced + i;
1628 SLIST_INSERT_HEAD(&fm->counters, ctrs, next);
1630 fm->counters_alloced += FM_COUNTERS_EXPAND;
1631 ENICPMD_LOG(DEBUG, "%u counters allocated, total: %u",
1632 FM_COUNTERS_EXPAND, fm->counters_alloced);
1637 enic_fm_counter_zero(struct enic_flowman *fm, struct enic_fm_counter *c)
1643 ENICPMD_FUNC_TRACE();
1645 args[0] = FM_COUNTER_QUERY;
1646 args[1] = c->handle;
1647 args[2] = 1; /* clear */
1648 ret = vnic_dev_flowman_cmd(enic->vdev, args, 3);
1650 ENICPMD_LOG(ERR, "counter init: rc=%d handle=0x%x",
1658 enic_fm_counter_alloc(struct enic_flowman *fm, struct rte_flow_error *error,
1659 struct enic_fm_counter **ctr)
1661 struct enic_fm_counter *c;
1664 ENICPMD_FUNC_TRACE();
1666 if (SLIST_EMPTY(&fm->counters)) {
1667 ret = enic_fm_more_counters(fm);
1669 return rte_flow_error_set(error, -ret,
1670 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1671 NULL, "enic: out of counters");
1673 c = SLIST_FIRST(&fm->counters);
1674 SLIST_REMOVE_HEAD(&fm->counters, next);
1680 enic_fm_action_free(struct enic_flowman *fm, uint64_t handle)
1685 ENICPMD_FUNC_TRACE();
1686 args[0] = FM_ACTION_FREE;
1688 rc = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
1690 ENICPMD_LOG(ERR, "cannot free action: rc=%d handle=0x%" PRIx64,
1696 enic_fm_entry_free(struct enic_flowman *fm, uint64_t handle)
1701 ENICPMD_FUNC_TRACE();
1702 args[0] = FM_MATCH_ENTRY_REMOVE;
1704 rc = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
1706 ENICPMD_LOG(ERR, "cannot free match entry: rc=%d"
1707 " handle=0x%" PRIx64, rc, handle);
1711 static struct enic_fm_jump_flow *
1712 find_jump_flow(struct enic_flowman *fm, uint32_t group)
1714 struct enic_fm_jump_flow *j;
1716 ENICPMD_FUNC_TRACE();
1717 TAILQ_FOREACH(j, &fm->jump_list, list) {
1718 if (j->group == group)
1725 remove_jump_flow(struct enic_flowman *fm, struct rte_flow *flow)
1727 struct enic_fm_jump_flow *j;
1729 ENICPMD_FUNC_TRACE();
1730 TAILQ_FOREACH(j, &fm->jump_list, list) {
1731 if (j->flow == flow) {
1732 TAILQ_REMOVE(&fm->jump_list, j, list);
1740 save_jump_flow(struct enic_flowman *fm,
1741 struct rte_flow *flow,
1743 struct fm_tcam_match_entry *match,
1744 struct fm_action *action)
1746 struct enic_fm_jump_flow *j;
1748 ENICPMD_FUNC_TRACE();
1749 j = calloc(1, sizeof(struct enic_fm_jump_flow));
1755 j->action = *action;
1756 TAILQ_INSERT_HEAD(&fm->jump_list, j, list);
1757 ENICPMD_LOG(DEBUG, "saved jump flow: flow=%p group=%u", flow, group);
1762 __enic_fm_flow_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
1764 if (fm_flow->entry_handle != FM_INVALID_HANDLE) {
1765 enic_fm_entry_free(fm, fm_flow->entry_handle);
1766 fm_flow->entry_handle = FM_INVALID_HANDLE;
1768 if (fm_flow->action_handle != FM_INVALID_HANDLE) {
1769 enic_fm_action_free(fm, fm_flow->action_handle);
1770 fm_flow->action_handle = FM_INVALID_HANDLE;
1772 enic_fm_counter_free(fm, fm_flow);
1774 enic_fet_put(fm, fm_flow->fet);
1775 fm_flow->fet = NULL;
1780 enic_fm_flow_free(struct enic_flowman *fm, struct rte_flow *flow)
1782 if (flow->fm->fet && flow->fm->fet->default_key)
1783 remove_jump_flow(fm, flow);
1784 __enic_fm_flow_free(fm, flow->fm);
1790 enic_fm_add_tcam_entry(struct enic_flowman *fm,
1791 struct fm_tcam_match_entry *match_in,
1792 uint64_t *entry_handle,
1794 struct rte_flow_error *error)
1796 struct fm_tcam_match_entry *ftm;
1800 ENICPMD_FUNC_TRACE();
1801 /* Copy entry to the command buffer */
1802 ftm = &fm->cmd.va->fm_tcam_match_entry;
1803 memcpy(ftm, match_in, sizeof(*ftm));
1804 /* Add TCAM entry */
1805 args[0] = FM_TCAM_ENTRY_INSTALL;
1806 args[1] = ingress ? fm->ig_tcam_hndl : fm->eg_tcam_hndl;
1807 args[2] = fm->cmd.pa;
1808 ret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 3);
1810 ENICPMD_LOG(ERR, "cannot add %s TCAM entry: rc=%d",
1811 ingress ? "ingress" : "egress", ret);
1812 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1813 NULL, "enic: devcmd(tcam-entry-install)");
1816 ENICPMD_LOG(DEBUG, "installed %s TCAM entry: handle=0x%" PRIx64,
1817 ingress ? "ingress" : "egress", (uint64_t)args[0]);
1818 *entry_handle = args[0];
1823 enic_fm_add_exact_entry(struct enic_flowman *fm,
1824 struct fm_tcam_match_entry *match_in,
1825 uint64_t *entry_handle,
1826 struct enic_fm_fet *fet,
1827 struct rte_flow_error *error)
1829 struct fm_exact_match_entry *fem;
1833 ENICPMD_FUNC_TRACE();
1834 /* The new entry must have the table's key */
1835 if (memcmp(fet->key.fk_hdrset, match_in->ftm_mask.fk_hdrset,
1836 sizeof(struct fm_header_set) * FM_HDRSET_MAX)) {
1837 return rte_flow_error_set(error, EINVAL,
1838 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1839 "enic: key does not match group's key");
1842 /* Copy entry to the command buffer */
1843 fem = &fm->cmd.va->fm_exact_match_entry;
1845 * Translate TCAM entry to exact entry. As is only need to drop
1846 * position and mask. The mask is part of the exact match table.
1847 * Position (aka priority) is not supported in the exact match table.
1849 fem->fem_data = match_in->ftm_data;
1850 fem->fem_flags = match_in->ftm_flags;
1851 fem->fem_action = match_in->ftm_action;
1852 fem->fem_counter = match_in->ftm_counter;
1854 /* Add exact entry */
1855 args[0] = FM_EXACT_ENTRY_INSTALL;
1856 args[1] = fet->handle;
1857 args[2] = fm->cmd.pa;
1858 ret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 3);
1860 ENICPMD_LOG(ERR, "cannot add %s exact entry: group=%u",
1861 fet->ingress ? "ingress" : "egress", fet->group);
1862 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1863 NULL, "enic: devcmd(exact-entry-install)");
1866 ENICPMD_LOG(DEBUG, "installed %s exact entry: group=%u"
1867 " handle=0x%" PRIx64,
1868 fet->ingress ? "ingress" : "egress", fet->group,
1870 *entry_handle = args[0];
1874 /* Push match-action to the NIC. */
1876 __enic_fm_flow_add_entry(struct enic_flowman *fm,
1877 struct enic_fm_flow *fm_flow,
1878 struct fm_tcam_match_entry *match_in,
1879 struct fm_action *action_in,
1882 struct rte_flow_error *error)
1884 struct enic_fm_counter *ctr;
1885 struct fm_action *fma;
1891 ENICPMD_FUNC_TRACE();
1892 /* Allocate action. */
1893 fma = &fm->cmd.va->fm_action;
1894 memcpy(fma, action_in, sizeof(*fma));
1895 args[0] = FM_ACTION_ALLOC;
1896 args[1] = fm->cmd.pa;
1897 ret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
1899 ENICPMD_LOG(ERR, "allocating TCAM table action rc=%d", ret);
1900 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1901 NULL, "enic: devcmd(action-alloc)");
1905 fm_flow->action_handle = action_h;
1906 match_in->ftm_action = action_h;
1907 ENICPMD_LOG(DEBUG, "action allocated: handle=0x%" PRIx64, action_h);
1909 /* Allocate counter if requested. */
1910 if (match_in->ftm_flags & FMEF_COUNTER) {
1911 ret = enic_fm_counter_alloc(fm, error, &ctr);
1912 if (ret) /* error has been filled in */
1914 fm_flow->counter_valid = true;
1915 fm_flow->counter = ctr;
1916 match_in->ftm_counter = ctr->handle;
1920 * Get the group's table (either TCAM or exact match table) and
1921 * add entry to it. If we use the exact match table, the handler
1922 * will translate the TCAM entry (match_in) to the appropriate
1923 * exact match entry and use that instead.
1925 entry_h = FM_INVALID_HANDLE;
1926 if (group == FM_TCAM_RTE_GROUP) {
1927 ret = enic_fm_add_tcam_entry(fm, match_in, &entry_h, ingress,
1931 /* Jump action might have a ref to fet */
1932 fm_flow->fet = fm->fet;
1935 struct enic_fm_fet *fet = NULL;
1937 ret = enic_fet_get(fm, group, ingress,
1938 &match_in->ftm_mask, &fet, error);
1942 ret = enic_fm_add_exact_entry(fm, match_in, &entry_h, fet,
1947 /* Clear counter after adding entry, as it requires in-use counter */
1948 if (fm_flow->counter_valid) {
1949 ret = enic_fm_counter_zero(fm, fm_flow->counter);
1953 fm_flow->entry_handle = entry_h;
1957 /* Push match-action to the NIC. */
1958 static struct rte_flow *
1959 enic_fm_flow_add_entry(struct enic_flowman *fm,
1960 struct fm_tcam_match_entry *match_in,
1961 struct fm_action *action_in,
1962 const struct rte_flow_attr *attrs,
1963 struct rte_flow_error *error)
1965 struct enic_fm_flow *fm_flow;
1966 struct rte_flow *flow;
1968 ENICPMD_FUNC_TRACE();
1969 enic_fm_dump_tcam_entry(match_in, action_in, attrs->ingress);
1970 flow = calloc(1, sizeof(*flow));
1971 fm_flow = calloc(1, sizeof(*fm_flow));
1972 if (flow == NULL || fm_flow == NULL) {
1973 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1974 NULL, "enic: cannot allocate rte_flow");
1980 fm_flow->action_handle = FM_INVALID_HANDLE;
1981 fm_flow->entry_handle = FM_INVALID_HANDLE;
1982 if (__enic_fm_flow_add_entry(fm, fm_flow, match_in, action_in,
1983 attrs->group, attrs->ingress, error)) {
1984 enic_fm_flow_free(fm, flow);
1991 convert_jump_flows(struct enic_flowman *fm, struct enic_fm_fet *fet,
1992 struct rte_flow_error *error)
1994 struct enic_fm_flow *fm_flow;
1995 struct enic_fm_jump_flow *j;
1996 struct fm_action *fma;
1999 ENICPMD_FUNC_TRACE();
2001 * Find the saved flows that should jump to the new table (fet).
2002 * Then delete the old TCAM entry that jumps to the default table,
2003 * and add a new one that jumps to the new table.
2006 j = find_jump_flow(fm, group);
2008 ENICPMD_LOG(DEBUG, "convert jump flow: flow=%p group=%u",
2010 /* Delete old entry */
2011 fm_flow = j->flow->fm;
2012 __enic_fm_flow_free(fm, fm_flow);
2016 fma->fma_action_ops[0].exact.handle = fet->handle;
2017 if (__enic_fm_flow_add_entry(fm, fm_flow, &j->match, fma,
2018 FM_TCAM_RTE_GROUP, fet->ingress, error)) {
2019 /* Cannot roll back changes at the moment */
2020 ENICPMD_LOG(ERR, "cannot convert jump flow: flow=%p",
2025 ENICPMD_LOG(DEBUG, "convert ok: group=%u ref=%u",
2026 fet->group, fet->ref);
2029 TAILQ_REMOVE(&fm->jump_list, j, list);
2031 j = find_jump_flow(fm, group);
2036 enic_fm_open_scratch(struct enic_flowman *fm)
2038 fm->action_op_count = 0;
2040 memset(&fm->tcam_entry, 0, sizeof(fm->tcam_entry));
2041 memset(&fm->action, 0, sizeof(fm->action));
2045 enic_fm_close_scratch(struct enic_flowman *fm)
2048 enic_fet_put(fm, fm->fet);
2051 fm->action_op_count = 0;
2055 enic_fm_flow_validate(struct rte_eth_dev *dev,
2056 const struct rte_flow_attr *attrs,
2057 const struct rte_flow_item pattern[],
2058 const struct rte_flow_action actions[],
2059 struct rte_flow_error *error)
2061 struct fm_tcam_match_entry *fm_tcam_entry;
2062 struct fm_action *fm_action;
2063 struct enic_flowman *fm;
2066 ENICPMD_FUNC_TRACE();
2067 fm = pmd_priv(dev)->fm;
2070 enic_fm_open_scratch(fm);
2071 ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2073 fm_tcam_entry = &fm->tcam_entry;
2074 fm_action = &fm->action;
2075 enic_fm_dump_tcam_entry(fm_tcam_entry, fm_action,
2078 enic_fm_close_scratch(fm);
2083 enic_fm_flow_query_count(struct rte_eth_dev *dev,
2084 struct rte_flow *flow, void *data,
2085 struct rte_flow_error *error)
2087 struct rte_flow_query_count *query;
2088 struct enic_fm_flow *fm_flow;
2093 ENICPMD_FUNC_TRACE();
2094 enic = pmd_priv(dev);
2097 if (!fm_flow->counter_valid)
2098 return rte_flow_error_set(error, ENOTSUP,
2099 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2100 "enic: flow does not have counter");
2102 args[0] = FM_COUNTER_QUERY;
2103 args[1] = fm_flow->counter->handle;
2104 args[2] = query->reset;
2105 rc = vnic_dev_flowman_cmd(enic->vdev, args, 3);
2107 ENICPMD_LOG(ERR, "cannot query counter: rc=%d handle=0x%x",
2108 rc, fm_flow->counter->handle);
2111 query->hits_set = 1;
2112 query->hits = args[0];
2113 query->bytes_set = 1;
2114 query->bytes = args[1];
2119 enic_fm_flow_query(struct rte_eth_dev *dev,
2120 struct rte_flow *flow,
2121 const struct rte_flow_action *actions,
2123 struct rte_flow_error *error)
2127 ENICPMD_FUNC_TRACE();
2128 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2129 switch (actions->type) {
2130 case RTE_FLOW_ACTION_TYPE_VOID:
2132 case RTE_FLOW_ACTION_TYPE_COUNT:
2133 ret = enic_fm_flow_query_count(dev, flow, data, error);
2136 return rte_flow_error_set(error, ENOTSUP,
2137 RTE_FLOW_ERROR_TYPE_ACTION,
2139 "action not supported");
2147 static struct rte_flow *
2148 enic_fm_flow_create(struct rte_eth_dev *dev,
2149 const struct rte_flow_attr *attrs,
2150 const struct rte_flow_item pattern[],
2151 const struct rte_flow_action actions[],
2152 struct rte_flow_error *error)
2154 struct fm_tcam_match_entry *fm_tcam_entry;
2155 struct fm_action *fm_action;
2156 struct enic_flowman *fm;
2157 struct enic_fm_fet *fet;
2158 struct rte_flow *flow;
2162 ENICPMD_FUNC_TRACE();
2163 enic = pmd_priv(dev);
2166 rte_flow_error_set(error, ENOTSUP,
2167 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2168 "flowman is not initialized");
2171 enic_fm_open_scratch(fm);
2173 ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2175 goto error_with_scratch;
2176 fm_tcam_entry = &fm->tcam_entry;
2177 fm_action = &fm->action;
2178 flow = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2181 LIST_INSERT_HEAD(&enic->flows, flow, next);
2182 fet = flow->fm->fet;
2183 if (fet && fet->default_key) {
2185 * Jump to non-existent group? Save the relevant info
2186 * so we can convert this flow when that group
2189 save_jump_flow(fm, flow, fet->group,
2190 fm_tcam_entry, fm_action);
2191 } else if (fet && fet->ref == 1) {
2193 * A new table is created. Convert the saved flows
2194 * that should jump to this group.
2196 convert_jump_flows(fm, fet, error);
2201 enic_fm_close_scratch(fm);
2206 enic_fm_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
2207 __rte_unused struct rte_flow_error *error)
2209 struct enic *enic = pmd_priv(dev);
2211 ENICPMD_FUNC_TRACE();
2212 if (enic->fm == NULL)
2214 LIST_REMOVE(flow, next);
2215 enic_fm_flow_free(enic->fm, flow);
2220 enic_fm_flow_flush(struct rte_eth_dev *dev,
2221 __rte_unused struct rte_flow_error *error)
2223 struct enic_fm_flow *fm_flow;
2224 struct enic_flowman *fm;
2225 struct rte_flow *flow;
2226 struct enic *enic = pmd_priv(dev);
2228 ENICPMD_FUNC_TRACE();
2229 if (enic->fm == NULL)
2232 while (!LIST_EMPTY(&enic->flows)) {
2233 flow = LIST_FIRST(&enic->flows);
2235 LIST_REMOVE(flow, next);
2237 * If tables are null, then vNIC is closing, and the firmware
2238 * has already cleaned up flowman state. So do not try to free
2239 * resources, as it only causes errors.
2241 if (fm->ig_tcam_hndl == FM_INVALID_HANDLE) {
2242 fm_flow->entry_handle = FM_INVALID_HANDLE;
2243 fm_flow->action_handle = FM_INVALID_HANDLE;
2244 fm_flow->fet = NULL;
2246 enic_fm_flow_free(fm, flow);
2252 enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle)
2257 args[0] = FM_MATCH_TABLE_FREE;
2259 rc = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
2261 ENICPMD_LOG(ERR, "cannot free table: rc=%d handle=0x%" PRIx64,
2267 enic_fm_tcam_tbl_alloc(struct enic_flowman *fm, uint32_t direction,
2268 uint32_t max_entries, uint64_t *handle)
2270 struct fm_tcam_match_table *tcam_tbl;
2275 ENICPMD_FUNC_TRACE();
2277 tcam_tbl = &fm->cmd.va->fm_tcam_match_table;
2278 tcam_tbl->ftt_direction = direction;
2279 tcam_tbl->ftt_stage = FM_STAGE_LAST;
2280 tcam_tbl->ftt_max_entries = max_entries;
2281 args[0] = FM_TCAM_TABLE_ALLOC;
2282 args[1] = fm->cmd.pa;
2283 rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
2285 ENICPMD_LOG(ERR, "cannot alloc %s TCAM table: rc=%d",
2286 (direction == FM_INGRESS) ? "IG" : "EG", rc);
2290 ENICPMD_LOG(DEBUG, "%s TCAM table allocated, handle=0x%" PRIx64,
2291 (direction == FM_INGRESS) ? "IG" : "EG", *handle);
2296 enic_fm_init_counters(struct enic_flowman *fm)
2298 ENICPMD_FUNC_TRACE();
2299 SLIST_INIT(&fm->counters);
2300 return enic_fm_more_counters(fm);
2304 enic_fm_free_all_counters(struct enic_flowman *fm)
2311 args[0] = FM_COUNTER_BRK;
2313 rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
2315 ENICPMD_LOG(ERR, "cannot free counters: rc=%d", rc);
2316 rte_free(fm->counter_stack);
2320 enic_fm_alloc_tcam_tables(struct enic_flowman *fm)
2324 ENICPMD_FUNC_TRACE();
2325 rc = enic_fm_tcam_tbl_alloc(fm, FM_INGRESS, FM_MAX_TCAM_TABLE_SIZE,
2329 rc = enic_fm_tcam_tbl_alloc(fm, FM_EGRESS, FM_MAX_TCAM_TABLE_SIZE,
2335 enic_fm_free_tcam_tables(struct enic_flowman *fm)
2337 ENICPMD_FUNC_TRACE();
2338 if (fm->ig_tcam_hndl) {
2339 ENICPMD_LOG(DEBUG, "free IG TCAM table handle=0x%" PRIx64,
2341 enic_fm_tbl_free(fm, fm->ig_tcam_hndl);
2342 fm->ig_tcam_hndl = FM_INVALID_HANDLE;
2344 if (fm->eg_tcam_hndl) {
2345 ENICPMD_LOG(DEBUG, "free EG TCAM table handle=0x%" PRIx64,
2347 enic_fm_tbl_free(fm, fm->eg_tcam_hndl);
2348 fm->eg_tcam_hndl = FM_INVALID_HANDLE;
2353 enic_fm_init(struct enic *enic)
2355 struct enic_flowman *fm;
2359 if (enic->flow_filter_mode != FILTER_FLOWMAN)
2361 ENICPMD_FUNC_TRACE();
2362 fm = calloc(1, sizeof(*fm));
2364 ENICPMD_LOG(ERR, "cannot alloc flowman struct");
2368 TAILQ_INIT(&fm->fet_list);
2369 TAILQ_INIT(&fm->jump_list);
2370 /* Allocate host memory for flowman commands */
2371 snprintf((char *)name, NAME_MAX, "fm-cmd-%s", enic->bdf_name);
2372 fm->cmd.va = enic_alloc_consistent(enic,
2373 sizeof(union enic_flowman_cmd_mem), &fm->cmd.pa, name);
2375 ENICPMD_LOG(ERR, "cannot allocate flowman command memory");
2379 /* Allocate TCAM tables upfront as they are the main tables */
2380 rc = enic_fm_alloc_tcam_tables(fm);
2382 ENICPMD_LOG(ERR, "cannot alloc TCAM tables");
2385 /* Then a number of counters */
2386 rc = enic_fm_init_counters(fm);
2388 ENICPMD_LOG(ERR, "cannot alloc counters");
2392 * One default exact match table for each direction. We hold onto
2395 rc = enic_fet_alloc(fm, 1, NULL, 128, &fm->default_ig_fet);
2397 ENICPMD_LOG(ERR, "cannot alloc default IG exact match table");
2398 goto error_counters;
2400 fm->default_ig_fet->ref = 1;
2401 rc = enic_fet_alloc(fm, 0, NULL, 128, &fm->default_eg_fet);
2403 ENICPMD_LOG(ERR, "cannot alloc default EG exact match table");
2406 fm->default_eg_fet->ref = 1;
2411 enic_fet_free(fm, fm->default_ig_fet);
2413 enic_fm_free_all_counters(fm);
2415 enic_fm_free_tcam_tables(fm);
2417 enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
2418 fm->cmd.va, fm->cmd.pa);
2425 enic_fm_destroy(struct enic *enic)
2427 struct enic_flowman *fm;
2428 struct enic_fm_fet *fet;
2430 if (enic->fm == NULL)
2432 ENICPMD_FUNC_TRACE();
2434 enic_fet_free(fm, fm->default_eg_fet);
2435 enic_fet_free(fm, fm->default_ig_fet);
2436 /* Free all exact match tables still open */
2437 while (!TAILQ_EMPTY(&fm->fet_list)) {
2438 fet = TAILQ_FIRST(&fm->fet_list);
2439 enic_fet_free(fm, fet);
2441 enic_fm_free_tcam_tables(fm);
2442 enic_fm_free_all_counters(fm);
2443 enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
2444 fm->cmd.va, fm->cmd.pa);
2450 const struct rte_flow_ops enic_fm_flow_ops = {
2451 .validate = enic_fm_flow_validate,
2452 .create = enic_fm_flow_create,
2453 .destroy = enic_fm_flow_destroy,
2454 .flush = enic_fm_flow_flush,
2455 .query = enic_fm_flow_query,