1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2008-2019 Cisco Systems, Inc. All rights reserved.
8 #include <rte_ethdev_driver.h>
9 #include <rte_flow_driver.h>
10 #include <rte_ether.h>
14 #include "enic_compat.h"
19 #define IP_DEFTTL 64 /* from RFC 1340. */
20 #define IP_VERSION 0x40
21 #define IP_HDRLEN 0x05 /* default IP header length == five 32-bits words. */
22 #define IP_VHL_DEF (IP_VERSION | IP_HDRLEN)
23 #define IP6_VTC_FLOW 0x60000000
25 /* Highest Item type supported by Flowman */
26 #define FM_MAX_ITEM_TYPE RTE_FLOW_ITEM_TYPE_VXLAN
28 /* Up to 1024 TCAM entries */
29 #define FM_MAX_TCAM_TABLE_SIZE 1024
31 /* Up to 4096 entries per exact match table */
32 #define FM_MAX_EXACT_TABLE_SIZE 4096
34 /* Number of counters to increase on for each increment */
35 #define FM_COUNTERS_EXPAND 100
37 #define FM_INVALID_HANDLE 0
40 * Flow exact match tables (FET) in the VIC and rte_flow groups.
41 * Use a simple scheme to map groups to tables.
42 * Group 0 uses the single TCAM tables, one for each direction.
43 * Group 1, 2, ... uses its own exact match table.
45 * The TCAM tables are allocated upfront during init.
47 * Exact match tables are allocated on demand. 3 paths that lead allocations.
49 * 1. Add a flow that jumps from group 0 to group N.
51 * If N does not exist, we allocate an exact match table for it, using
52 * a dummy key. A key is required for the table.
54 * 2. Add a flow that uses group N.
56 * If N does not exist, we allocate an exact match table for it, using
57 * the flow's key. Subsequent flows to the same group all should have
60 * Without a jump flow to N, N is not reachable in hardware. No packets
63 * 3. Add a flow to an empty group N.
65 * N has been created via (1) and the dummy key. We free that table, allocate
66 * a new table using the new flow's key. Also re-do the existing jump flow to
67 * point to the new table.
69 #define FM_TCAM_RTE_GROUP 0
72 TAILQ_ENTRY(enic_fm_fet) list;
73 uint32_t group; /* rte_flow group ID */
74 uint64_t handle; /* Exact match table handle from flowman */
77 int ref; /* Reference count via get/put */
78 struct fm_key_template key; /* Key associated with the table */
81 struct enic_fm_counter {
82 SLIST_ENTRY(enic_fm_counter) next;
89 uint64_t entry_handle;
90 uint64_t action_handle;
91 struct enic_fm_counter *counter;
92 struct enic_fm_fet *fet;
95 struct enic_fm_jump_flow {
96 TAILQ_ENTRY(enic_fm_jump_flow) list;
97 struct rte_flow *flow;
99 struct fm_tcam_match_entry match;
100 struct fm_action action;
104 * Flowman uses host memory for commands. This structure is allocated
105 * in DMA-able memory.
107 union enic_flowman_cmd_mem {
108 struct fm_tcam_match_table fm_tcam_match_table;
109 struct fm_exact_match_table fm_exact_match_table;
110 struct fm_tcam_match_entry fm_tcam_match_entry;
111 struct fm_exact_match_entry fm_exact_match_entry;
112 struct fm_action fm_action;
115 struct enic_flowman {
119 union enic_flowman_cmd_mem *va;
122 /* TCAM tables allocated upfront, used for group 0 */
123 uint64_t ig_tcam_hndl;
124 uint64_t eg_tcam_hndl;
126 SLIST_HEAD(enic_free_counters, enic_fm_counter) counters;
128 uint32_t counters_alloced;
129 /* Exact match tables for groups != 0, dynamically allocated */
130 TAILQ_HEAD(fet_list, enic_fm_fet) fet_list;
132 * Default exact match tables used for jump actions to
133 * non-existent groups.
135 struct enic_fm_fet *default_eg_fet;
136 struct enic_fm_fet *default_ig_fet;
137 /* Flows that jump to the default table above */
138 TAILQ_HEAD(jump_flow_list, enic_fm_jump_flow) jump_list;
140 * Scratch data used during each invocation of flow_create
143 struct enic_fm_fet *fet;
144 struct fm_tcam_match_entry tcam_entry;
145 struct fm_action action;
146 struct fm_action action_tmp; /* enic_fm_reorder_action_op */
150 static int enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle);
153 * Common arguments passed to copy_item functions. Use this structure
154 * so we can easily add new arguments.
155 * item: Item specification.
156 * fm_tcam_entry: Flowman TCAM match entry.
157 * header_level: 0 for outer header, 1 for inner header.
159 struct copy_item_args {
160 const struct rte_flow_item *item;
161 struct fm_tcam_match_entry *fm_tcam_entry;
162 uint8_t header_level;
165 /* functions for copying items into flowman match */
166 typedef int (enic_copy_item_fn)(struct copy_item_args *arg);
168 /* Info about how to copy items into flowman match */
169 struct enic_fm_items {
170 /* Function for copying and validating an item. */
171 enic_copy_item_fn * const copy_item;
172 /* List of valid previous items. */
173 const enum rte_flow_item_type * const prev_items;
175 * True if it's OK for this item to be the first item. For some NIC
176 * versions, it's invalid to start the stack above layer 3.
178 const uint8_t valid_start_item;
181 static enic_copy_item_fn enic_fm_copy_item_eth;
182 static enic_copy_item_fn enic_fm_copy_item_ipv4;
183 static enic_copy_item_fn enic_fm_copy_item_ipv6;
184 static enic_copy_item_fn enic_fm_copy_item_raw;
185 static enic_copy_item_fn enic_fm_copy_item_sctp;
186 static enic_copy_item_fn enic_fm_copy_item_tcp;
187 static enic_copy_item_fn enic_fm_copy_item_udp;
188 static enic_copy_item_fn enic_fm_copy_item_vlan;
189 static enic_copy_item_fn enic_fm_copy_item_vxlan;
191 /* Ingress actions */
192 static const enum rte_flow_action_type enic_fm_supported_ig_actions[] = {
193 RTE_FLOW_ACTION_TYPE_COUNT,
194 RTE_FLOW_ACTION_TYPE_DROP,
195 RTE_FLOW_ACTION_TYPE_FLAG,
196 RTE_FLOW_ACTION_TYPE_JUMP,
197 RTE_FLOW_ACTION_TYPE_MARK,
198 RTE_FLOW_ACTION_TYPE_PORT_ID,
199 RTE_FLOW_ACTION_TYPE_PASSTHRU,
200 RTE_FLOW_ACTION_TYPE_QUEUE,
201 RTE_FLOW_ACTION_TYPE_RSS,
202 RTE_FLOW_ACTION_TYPE_VOID,
203 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
204 RTE_FLOW_ACTION_TYPE_VXLAN_DECAP,
205 RTE_FLOW_ACTION_TYPE_END, /* END must be the last entry */
209 static const enum rte_flow_action_type enic_fm_supported_eg_actions[] = {
210 RTE_FLOW_ACTION_TYPE_COUNT,
211 RTE_FLOW_ACTION_TYPE_DROP,
212 RTE_FLOW_ACTION_TYPE_JUMP,
213 RTE_FLOW_ACTION_TYPE_PASSTHRU,
214 RTE_FLOW_ACTION_TYPE_VOID,
215 RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,
216 RTE_FLOW_ACTION_TYPE_END,
219 static const struct enic_fm_items enic_fm_items[] = {
220 [RTE_FLOW_ITEM_TYPE_RAW] = {
221 .copy_item = enic_fm_copy_item_raw,
222 .valid_start_item = 0,
223 .prev_items = (const enum rte_flow_item_type[]) {
224 RTE_FLOW_ITEM_TYPE_UDP,
225 RTE_FLOW_ITEM_TYPE_END,
228 [RTE_FLOW_ITEM_TYPE_ETH] = {
229 .copy_item = enic_fm_copy_item_eth,
230 .valid_start_item = 1,
231 .prev_items = (const enum rte_flow_item_type[]) {
232 RTE_FLOW_ITEM_TYPE_END,
235 [RTE_FLOW_ITEM_TYPE_VLAN] = {
236 .copy_item = enic_fm_copy_item_vlan,
237 .valid_start_item = 1,
238 .prev_items = (const enum rte_flow_item_type[]) {
239 RTE_FLOW_ITEM_TYPE_ETH,
240 RTE_FLOW_ITEM_TYPE_END,
243 [RTE_FLOW_ITEM_TYPE_IPV4] = {
244 .copy_item = enic_fm_copy_item_ipv4,
245 .valid_start_item = 1,
246 .prev_items = (const enum rte_flow_item_type[]) {
247 RTE_FLOW_ITEM_TYPE_ETH,
248 RTE_FLOW_ITEM_TYPE_VLAN,
249 RTE_FLOW_ITEM_TYPE_END,
252 [RTE_FLOW_ITEM_TYPE_IPV6] = {
253 .copy_item = enic_fm_copy_item_ipv6,
254 .valid_start_item = 1,
255 .prev_items = (const enum rte_flow_item_type[]) {
256 RTE_FLOW_ITEM_TYPE_ETH,
257 RTE_FLOW_ITEM_TYPE_VLAN,
258 RTE_FLOW_ITEM_TYPE_END,
261 [RTE_FLOW_ITEM_TYPE_UDP] = {
262 .copy_item = enic_fm_copy_item_udp,
263 .valid_start_item = 1,
264 .prev_items = (const enum rte_flow_item_type[]) {
265 RTE_FLOW_ITEM_TYPE_IPV4,
266 RTE_FLOW_ITEM_TYPE_IPV6,
267 RTE_FLOW_ITEM_TYPE_END,
270 [RTE_FLOW_ITEM_TYPE_TCP] = {
271 .copy_item = enic_fm_copy_item_tcp,
272 .valid_start_item = 1,
273 .prev_items = (const enum rte_flow_item_type[]) {
274 RTE_FLOW_ITEM_TYPE_IPV4,
275 RTE_FLOW_ITEM_TYPE_IPV6,
276 RTE_FLOW_ITEM_TYPE_END,
279 [RTE_FLOW_ITEM_TYPE_SCTP] = {
280 .copy_item = enic_fm_copy_item_sctp,
281 .valid_start_item = 0,
282 .prev_items = (const enum rte_flow_item_type[]) {
283 RTE_FLOW_ITEM_TYPE_IPV4,
284 RTE_FLOW_ITEM_TYPE_IPV6,
285 RTE_FLOW_ITEM_TYPE_END,
288 [RTE_FLOW_ITEM_TYPE_VXLAN] = {
289 .copy_item = enic_fm_copy_item_vxlan,
290 .valid_start_item = 1,
291 .prev_items = (const enum rte_flow_item_type[]) {
292 RTE_FLOW_ITEM_TYPE_UDP,
293 RTE_FLOW_ITEM_TYPE_END,
299 enic_fm_copy_item_eth(struct copy_item_args *arg)
301 const struct rte_flow_item *item = arg->item;
302 const struct rte_flow_item_eth *spec = item->spec;
303 const struct rte_flow_item_eth *mask = item->mask;
304 const uint8_t lvl = arg->header_level;
305 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
306 struct fm_header_set *fm_data, *fm_mask;
308 ENICPMD_FUNC_TRACE();
309 /* Match all if no spec */
313 mask = &rte_flow_item_eth_mask;
314 fm_data = &entry->ftm_data.fk_hdrset[lvl];
315 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
316 fm_data->fk_header_select |= FKH_ETHER;
317 fm_mask->fk_header_select |= FKH_ETHER;
318 memcpy(&fm_data->l2.eth, spec, sizeof(*spec));
319 memcpy(&fm_mask->l2.eth, mask, sizeof(*mask));
324 enic_fm_copy_item_vlan(struct copy_item_args *arg)
326 const struct rte_flow_item *item = arg->item;
327 const struct rte_flow_item_vlan *spec = item->spec;
328 const struct rte_flow_item_vlan *mask = item->mask;
329 const uint8_t lvl = arg->header_level;
330 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
331 struct fm_header_set *fm_data, *fm_mask;
332 struct rte_ether_hdr *eth_mask;
333 struct rte_ether_hdr *eth_val;
336 ENICPMD_FUNC_TRACE();
337 fm_data = &entry->ftm_data.fk_hdrset[lvl];
338 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
339 /* Outer and inner packet vlans need different flags */
340 meta = FKM_VLAN_PRES;
343 fm_data->fk_metadata |= meta;
344 fm_mask->fk_metadata |= meta;
346 /* Match all if no spec */
350 mask = &rte_flow_item_vlan_mask;
352 eth_mask = (void *)&fm_mask->l2.eth;
353 eth_val = (void *)&fm_data->l2.eth;
355 /* Outer TPID cannot be matched */
356 if (eth_mask->ether_type)
360 * When packet matching, the VIC always compares vlan-stripped
361 * L2, regardless of vlan stripping settings. So, the inner type
362 * from vlan becomes the ether type of the eth header.
364 eth_mask->ether_type = mask->inner_type;
365 eth_val->ether_type = spec->inner_type;
366 fm_data->fk_header_select |= FKH_ETHER | FKH_QTAG;
367 fm_mask->fk_header_select |= FKH_ETHER | FKH_QTAG;
368 fm_data->fk_vlan = rte_be_to_cpu_16(spec->tci);
369 fm_mask->fk_vlan = rte_be_to_cpu_16(mask->tci);
374 enic_fm_copy_item_ipv4(struct copy_item_args *arg)
376 const struct rte_flow_item *item = arg->item;
377 const struct rte_flow_item_ipv4 *spec = item->spec;
378 const struct rte_flow_item_ipv4 *mask = item->mask;
379 const uint8_t lvl = arg->header_level;
380 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
381 struct fm_header_set *fm_data, *fm_mask;
383 ENICPMD_FUNC_TRACE();
384 fm_data = &entry->ftm_data.fk_hdrset[lvl];
385 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
386 fm_data->fk_metadata |= FKM_IPV4;
387 fm_mask->fk_metadata |= FKM_IPV4;
392 mask = &rte_flow_item_ipv4_mask;
394 fm_data->fk_header_select |= FKH_IPV4;
395 fm_mask->fk_header_select |= FKH_IPV4;
396 memcpy(&fm_data->l3.ip4, spec, sizeof(*spec));
397 memcpy(&fm_mask->l3.ip4, mask, sizeof(*mask));
402 enic_fm_copy_item_ipv6(struct copy_item_args *arg)
404 const struct rte_flow_item *item = arg->item;
405 const struct rte_flow_item_ipv6 *spec = item->spec;
406 const struct rte_flow_item_ipv6 *mask = item->mask;
407 const uint8_t lvl = arg->header_level;
408 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
409 struct fm_header_set *fm_data, *fm_mask;
411 ENICPMD_FUNC_TRACE();
412 fm_data = &entry->ftm_data.fk_hdrset[lvl];
413 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
414 fm_data->fk_metadata |= FKM_IPV6;
415 fm_mask->fk_metadata |= FKM_IPV6;
420 mask = &rte_flow_item_ipv6_mask;
422 fm_data->fk_header_select |= FKH_IPV6;
423 fm_mask->fk_header_select |= FKH_IPV6;
424 memcpy(&fm_data->l3.ip6, spec, sizeof(*spec));
425 memcpy(&fm_mask->l3.ip6, mask, sizeof(*mask));
430 enic_fm_copy_item_udp(struct copy_item_args *arg)
432 const struct rte_flow_item *item = arg->item;
433 const struct rte_flow_item_udp *spec = item->spec;
434 const struct rte_flow_item_udp *mask = item->mask;
435 const uint8_t lvl = arg->header_level;
436 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
437 struct fm_header_set *fm_data, *fm_mask;
439 ENICPMD_FUNC_TRACE();
440 fm_data = &entry->ftm_data.fk_hdrset[lvl];
441 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
442 fm_data->fk_metadata |= FKM_UDP;
443 fm_mask->fk_metadata |= FKM_UDP;
448 mask = &rte_flow_item_udp_mask;
450 fm_data->fk_header_select |= FKH_UDP;
451 fm_mask->fk_header_select |= FKH_UDP;
452 memcpy(&fm_data->l4.udp, spec, sizeof(*spec));
453 memcpy(&fm_mask->l4.udp, mask, sizeof(*mask));
458 enic_fm_copy_item_tcp(struct copy_item_args *arg)
460 const struct rte_flow_item *item = arg->item;
461 const struct rte_flow_item_tcp *spec = item->spec;
462 const struct rte_flow_item_tcp *mask = item->mask;
463 const uint8_t lvl = arg->header_level;
464 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
465 struct fm_header_set *fm_data, *fm_mask;
467 ENICPMD_FUNC_TRACE();
468 fm_data = &entry->ftm_data.fk_hdrset[lvl];
469 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
470 fm_data->fk_metadata |= FKM_TCP;
471 fm_mask->fk_metadata |= FKM_TCP;
476 mask = &rte_flow_item_tcp_mask;
478 fm_data->fk_header_select |= FKH_TCP;
479 fm_mask->fk_header_select |= FKH_TCP;
480 memcpy(&fm_data->l4.tcp, spec, sizeof(*spec));
481 memcpy(&fm_mask->l4.tcp, mask, sizeof(*mask));
486 enic_fm_copy_item_sctp(struct copy_item_args *arg)
488 const struct rte_flow_item *item = arg->item;
489 const struct rte_flow_item_sctp *spec = item->spec;
490 const struct rte_flow_item_sctp *mask = item->mask;
491 const uint8_t lvl = arg->header_level;
492 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
493 struct fm_header_set *fm_data, *fm_mask;
494 uint8_t *ip_proto_mask = NULL;
495 uint8_t *ip_proto = NULL;
498 ENICPMD_FUNC_TRACE();
499 fm_data = &entry->ftm_data.fk_hdrset[lvl];
500 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
502 * The NIC filter API has no flags for "match sctp", so explicitly
503 * set the protocol number in the IP pattern.
505 if (fm_data->fk_metadata & FKM_IPV4) {
506 struct rte_ipv4_hdr *ip;
507 ip = (struct rte_ipv4_hdr *)&fm_mask->l3.ip4;
508 ip_proto_mask = &ip->next_proto_id;
509 ip = (struct rte_ipv4_hdr *)&fm_data->l3.ip4;
510 ip_proto = &ip->next_proto_id;
512 } else if (fm_data->fk_metadata & FKM_IPV6) {
513 struct rte_ipv6_hdr *ip;
514 ip = (struct rte_ipv6_hdr *)&fm_mask->l3.ip6;
515 ip_proto_mask = &ip->proto;
516 ip = (struct rte_ipv6_hdr *)&fm_data->l3.ip6;
517 ip_proto = &ip->proto;
520 /* Need IPv4/IPv6 pattern first */
523 *ip_proto = IPPROTO_SCTP;
524 *ip_proto_mask = 0xff;
525 fm_data->fk_header_select |= l3_fkh;
526 fm_mask->fk_header_select |= l3_fkh;
531 mask = &rte_flow_item_sctp_mask;
533 fm_data->fk_header_select |= FKH_L4RAW;
534 fm_mask->fk_header_select |= FKH_L4RAW;
535 memcpy(fm_data->l4.rawdata, spec, sizeof(*spec));
536 memcpy(fm_mask->l4.rawdata, mask, sizeof(*mask));
541 enic_fm_copy_item_vxlan(struct copy_item_args *arg)
543 const struct rte_flow_item *item = arg->item;
544 const struct rte_flow_item_vxlan *spec = item->spec;
545 const struct rte_flow_item_vxlan *mask = item->mask;
546 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
547 struct fm_header_set *fm_data, *fm_mask;
549 ENICPMD_FUNC_TRACE();
550 /* Only 2 header levels (outer and inner) allowed */
551 if (arg->header_level > 0)
554 fm_data = &entry->ftm_data.fk_hdrset[0];
555 fm_mask = &entry->ftm_mask.fk_hdrset[0];
556 fm_data->fk_metadata |= FKM_VXLAN;
557 fm_mask->fk_metadata |= FKM_VXLAN;
558 /* items from here on out are inner header items */
559 arg->header_level = 1;
561 /* Match all if no spec */
565 mask = &rte_flow_item_vxlan_mask;
567 fm_data->fk_header_select |= FKH_VXLAN;
568 fm_mask->fk_header_select |= FKH_VXLAN;
569 memcpy(&fm_data->vxlan, spec, sizeof(*spec));
570 memcpy(&fm_mask->vxlan, mask, sizeof(*mask));
575 * Currently, raw pattern match is very limited. It is intended for matching
576 * UDP tunnel header (e.g. vxlan or geneve).
579 enic_fm_copy_item_raw(struct copy_item_args *arg)
581 const struct rte_flow_item *item = arg->item;
582 const struct rte_flow_item_raw *spec = item->spec;
583 const struct rte_flow_item_raw *mask = item->mask;
584 const uint8_t lvl = arg->header_level;
585 struct fm_tcam_match_entry *entry = arg->fm_tcam_entry;
586 struct fm_header_set *fm_data, *fm_mask;
588 ENICPMD_FUNC_TRACE();
589 /* Cannot be used for inner packet */
592 /* Need both spec and mask */
595 /* Only supports relative with offset 0 */
596 if (!spec->relative || spec->offset != 0 || spec->search ||
599 /* Need non-null pattern that fits within the NIC's filter pattern */
600 if (spec->length == 0 ||
601 spec->length + sizeof(struct rte_udp_hdr) > FM_LAYER_SIZE ||
602 !spec->pattern || !mask->pattern)
605 * Mask fields, including length, are often set to zero. Assume that
606 * means "same as spec" to avoid breaking existing apps. If length
607 * is not zero, then it should be >= spec length.
609 * No more pattern follows this, so append to the L4 layer instead of
610 * L5 to work with both recent and older VICs.
612 if (mask->length != 0 && mask->length < spec->length)
615 fm_data = &entry->ftm_data.fk_hdrset[lvl];
616 fm_mask = &entry->ftm_mask.fk_hdrset[lvl];
617 fm_data->fk_header_select |= FKH_L4RAW;
618 fm_mask->fk_header_select |= FKH_L4RAW;
619 fm_data->fk_header_select &= ~FKH_UDP;
620 fm_mask->fk_header_select &= ~FKH_UDP;
621 memcpy(fm_data->l4.rawdata + sizeof(struct rte_udp_hdr),
622 spec->pattern, spec->length);
623 memcpy(fm_mask->l4.rawdata + sizeof(struct rte_udp_hdr),
624 mask->pattern, spec->length);
629 enic_fet_alloc(struct enic_flowman *fm, uint8_t ingress,
630 struct fm_key_template *key, int entries,
631 struct enic_fm_fet **fet_out)
633 struct fm_exact_match_table *cmd;
634 struct fm_header_set *hdr;
635 struct enic_fm_fet *fet;
639 ENICPMD_FUNC_TRACE();
640 fet = calloc(1, sizeof(struct enic_fm_fet));
643 cmd = &fm->cmd.va->fm_exact_match_table;
644 memset(cmd, 0, sizeof(*cmd));
645 cmd->fet_direction = ingress ? FM_INGRESS : FM_EGRESS;
646 cmd->fet_stage = FM_STAGE_LAST;
647 cmd->fet_max_entries = entries ? entries : FM_MAX_EXACT_TABLE_SIZE;
649 hdr = &cmd->fet_key.fk_hdrset[0];
650 memset(hdr, 0, sizeof(*hdr));
651 hdr->fk_header_select = FKH_IPV4 | FKH_UDP;
652 hdr->l3.ip4.fk_saddr = 0xFFFFFFFF;
653 hdr->l3.ip4.fk_daddr = 0xFFFFFFFF;
654 hdr->l4.udp.fk_source = 0xFFFF;
655 hdr->l4.udp.fk_dest = 0xFFFF;
656 fet->default_key = 1;
658 memcpy(&cmd->fet_key, key, sizeof(*key));
659 memcpy(&fet->key, key, sizeof(*key));
660 fet->default_key = 0;
662 cmd->fet_key.fk_packet_tag = 1;
664 args[0] = FM_EXACT_TABLE_ALLOC;
665 args[1] = fm->cmd.pa;
666 ret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
668 ENICPMD_LOG(ERR, "cannot alloc exact match table: rc=%d", ret);
672 fet->handle = args[0];
673 fet->ingress = ingress;
674 ENICPMD_LOG(DEBUG, "allocated exact match table: handle=0x%" PRIx64,
681 enic_fet_free(struct enic_flowman *fm, struct enic_fm_fet *fet)
683 ENICPMD_FUNC_TRACE();
684 enic_fm_tbl_free(fm, fet->handle);
685 if (!fet->default_key)
686 TAILQ_REMOVE(&fm->fet_list, fet, list);
691 * Get the exact match table for the given combination of
692 * <group, ingress, key>. Allocate one on the fly as necessary.
695 enic_fet_get(struct enic_flowman *fm,
698 struct fm_key_template *key,
699 struct enic_fm_fet **fet_out,
700 struct rte_flow_error *error)
702 struct enic_fm_fet *fet;
704 ENICPMD_FUNC_TRACE();
705 /* See if we already have this table open */
706 TAILQ_FOREACH(fet, &fm->fet_list, list) {
707 if (fet->group == group && fet->ingress == ingress)
711 /* Jumping to a non-existing group? Use the default table */
713 fet = ingress ? fm->default_ig_fet : fm->default_eg_fet;
714 } else if (enic_fet_alloc(fm, ingress, key, 0, &fet)) {
715 return rte_flow_error_set(error, EINVAL,
716 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
717 NULL, "enic: cannot get exact match table");
720 /* Default table is never on the open table list */
721 if (!fet->default_key)
722 TAILQ_INSERT_HEAD(&fm->fet_list, fet, list);
726 ENICPMD_LOG(DEBUG, "fet_get: %s %s group=%u ref=%u",
727 fet->default_key ? "default" : "",
728 fet->ingress ? "ingress" : "egress",
729 fet->group, fet->ref);
734 enic_fet_put(struct enic_flowman *fm, struct enic_fm_fet *fet)
736 ENICPMD_FUNC_TRACE();
737 RTE_ASSERT(fet->ref > 0);
739 ENICPMD_LOG(DEBUG, "fet_put: %s %s group=%u ref=%u",
740 fet->default_key ? "default" : "",
741 fet->ingress ? "ingress" : "egress",
742 fet->group, fet->ref);
744 enic_fet_free(fm, fet);
747 /* Return 1 if current item is valid on top of the previous one. */
749 fm_item_stacking_valid(enum rte_flow_item_type prev_item,
750 const struct enic_fm_items *item_info,
751 uint8_t is_first_item)
753 enum rte_flow_item_type const *allowed_items = item_info->prev_items;
755 ENICPMD_FUNC_TRACE();
756 for (; *allowed_items != RTE_FLOW_ITEM_TYPE_END; allowed_items++) {
757 if (prev_item == *allowed_items)
761 /* This is the first item in the stack. Check if that's cool */
762 if (is_first_item && item_info->valid_start_item)
768 * Build the flow manager match entry structure from the provided pattern.
769 * The pattern is validated as the items are copied.
772 enic_fm_copy_entry(struct enic_flowman *fm,
773 const struct rte_flow_item pattern[],
774 struct rte_flow_error *error)
776 const struct enic_fm_items *item_info;
777 enum rte_flow_item_type prev_item;
778 const struct rte_flow_item *item;
779 struct copy_item_args args;
780 uint8_t prev_header_level;
781 uint8_t is_first_item;
784 ENICPMD_FUNC_TRACE();
787 prev_item = RTE_FLOW_ITEM_TYPE_END;
789 args.fm_tcam_entry = &fm->tcam_entry;
790 args.header_level = 0;
791 prev_header_level = 0;
792 for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
794 * Get info about how to validate and copy the item. If NULL
795 * is returned the nic does not support the item.
797 if (item->type == RTE_FLOW_ITEM_TYPE_VOID)
800 item_info = &enic_fm_items[item->type];
802 if (item->type > FM_MAX_ITEM_TYPE ||
803 item_info->copy_item == NULL) {
804 return rte_flow_error_set(error, ENOTSUP,
805 RTE_FLOW_ERROR_TYPE_ITEM,
806 NULL, "enic: unsupported item");
809 /* check to see if item stacking is valid */
810 if (!fm_item_stacking_valid(prev_item, item_info,
815 ret = item_info->copy_item(&args);
817 goto item_not_supported;
818 /* Going from outer to inner? Treat it as a new packet start */
819 if (prev_header_level != args.header_level) {
820 prev_item = RTE_FLOW_ITEM_TYPE_END;
823 prev_item = item->type;
826 prev_header_level = args.header_level;
831 return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,
832 NULL, "enic: unsupported item type");
835 return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
836 item, "enic: unsupported item stack");
840 flow_item_skip_void(const struct rte_flow_item **item)
843 if ((*item)->type != RTE_FLOW_ITEM_TYPE_VOID)
848 append_template(void **template, uint8_t *off, const void *data, int len)
850 memcpy(*template, data, len);
851 *template = (char *)*template + len;
856 enic_fm_append_action_op(struct enic_flowman *fm,
857 struct fm_action_op *fm_op,
858 struct rte_flow_error *error)
862 count = fm->action_op_count;
863 ENICPMD_LOG(DEBUG, "append action op: idx=%d op=%u",
864 count, fm_op->fa_op);
865 if (count == FM_ACTION_OP_MAX) {
866 return rte_flow_error_set(error, EINVAL,
867 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
868 "too many action operations");
870 fm->action.fma_action_ops[count] = *fm_op;
871 fm->action_op_count = count + 1;
875 /* Steer operations need to appear before other ops */
877 enic_fm_reorder_action_op(struct enic_flowman *fm)
879 struct fm_action_op *dst, *dst_head, *src, *src_head;
881 ENICPMD_FUNC_TRACE();
882 /* Move steer ops to the front. */
883 src = fm->action.fma_action_ops;
885 dst = fm->action_tmp.fma_action_ops;
887 /* Copy steer ops to tmp */
888 while (src->fa_op != FMOP_END) {
889 if (src->fa_op == FMOP_RQ_STEER) {
890 ENICPMD_LOG(DEBUG, "move op: %ld -> dst %ld",
891 (long)(src - src_head),
892 (long)(dst - dst_head));
898 /* Then append non-steer ops */
900 while (src->fa_op != FMOP_END) {
901 if (src->fa_op != FMOP_RQ_STEER) {
902 ENICPMD_LOG(DEBUG, "move op: %ld -> dst %ld",
903 (long)(src - src_head),
904 (long)(dst - dst_head));
912 /* Finally replace the original action with the reordered one */
913 memcpy(fm->action.fma_action_ops, fm->action_tmp.fma_action_ops,
914 sizeof(fm->action.fma_action_ops));
917 /* VXLAN decap is done via flowman compound action */
919 enic_fm_copy_vxlan_decap(struct enic_flowman *fm,
920 struct fm_tcam_match_entry *fmt,
921 const struct rte_flow_action *action,
922 struct rte_flow_error *error)
924 struct fm_header_set *fm_data;
925 struct fm_action_op fm_op;
927 ENICPMD_FUNC_TRACE();
928 fm_data = &fmt->ftm_data.fk_hdrset[0];
929 if (!(fm_data->fk_metadata & FKM_VXLAN)) {
930 return rte_flow_error_set(error, EINVAL,
931 RTE_FLOW_ERROR_TYPE_ACTION, action,
932 "vxlan-decap: vxlan must be in pattern");
935 memset(&fm_op, 0, sizeof(fm_op));
936 fm_op.fa_op = FMOP_DECAP_NOSTRIP;
937 return enic_fm_append_action_op(fm, &fm_op, error);
940 /* VXLAN encap is done via flowman compound action */
942 enic_fm_copy_vxlan_encap(struct enic_flowman *fm,
943 const struct rte_flow_item *item,
944 struct rte_flow_error *error)
946 struct fm_action_op fm_op;
947 struct rte_ether_hdr *eth;
952 ENICPMD_FUNC_TRACE();
953 memset(&fm_op, 0, sizeof(fm_op));
954 fm_op.fa_op = FMOP_ENCAP;
955 template = fm->action.fma_data;
958 * Copy flow items to the flowman template starting L2.
959 * L2 must be ethernet.
961 flow_item_skip_void(&item);
962 if (item->type != RTE_FLOW_ITEM_TYPE_ETH)
963 return rte_flow_error_set(error, EINVAL,
964 RTE_FLOW_ERROR_TYPE_ITEM, item,
965 "vxlan-encap: first item should be ethernet");
966 eth = (struct rte_ether_hdr *)template;
967 ethertype = ð->ether_type;
968 append_template(&template, &off, item->spec,
969 sizeof(struct rte_flow_item_eth));
971 flow_item_skip_void(&item);
973 if (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {
974 const struct rte_flow_item_vlan *spec;
976 ENICPMD_LOG(DEBUG, "vxlan-encap: vlan");
978 fm_op.encap.outer_vlan = rte_be_to_cpu_16(spec->tci);
980 flow_item_skip_void(&item);
982 /* L3 must be IPv4, IPv6 */
983 switch (item->type) {
984 case RTE_FLOW_ITEM_TYPE_IPV4:
986 struct rte_ipv4_hdr *ip4;
988 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv4");
989 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
990 ip4 = (struct rte_ipv4_hdr *)template;
992 * Offset of IPv4 length field and its initial value
993 * (IP + UDP + VXLAN) are specified in the action. The NIC
994 * will add inner packet length.
996 fm_op.encap.len1_offset = off +
997 offsetof(struct rte_ipv4_hdr, total_length);
998 fm_op.encap.len1_delta = sizeof(struct rte_ipv4_hdr) +
999 sizeof(struct rte_udp_hdr) +
1000 sizeof(struct rte_vxlan_hdr);
1001 append_template(&template, &off, item->spec,
1002 sizeof(struct rte_ipv4_hdr));
1003 ip4->version_ihl = IP_VHL_DEF;
1004 if (ip4->time_to_live == 0)
1005 ip4->time_to_live = IP_DEFTTL;
1006 ip4->next_proto_id = IPPROTO_UDP;
1009 case RTE_FLOW_ITEM_TYPE_IPV6:
1011 struct rte_ipv6_hdr *ip6;
1013 ENICPMD_LOG(DEBUG, "vxlan-encap: ipv6");
1014 *ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
1015 ip6 = (struct rte_ipv6_hdr *)template;
1016 fm_op.encap.len1_offset = off +
1017 offsetof(struct rte_ipv6_hdr, payload_len);
1018 fm_op.encap.len1_delta = sizeof(struct rte_udp_hdr) +
1019 sizeof(struct rte_vxlan_hdr);
1020 append_template(&template, &off, item->spec,
1021 sizeof(struct rte_ipv6_hdr));
1022 ip6->vtc_flow |= rte_cpu_to_be_32(IP6_VTC_FLOW);
1023 if (ip6->hop_limits == 0)
1024 ip6->hop_limits = IP_DEFTTL;
1025 ip6->proto = IPPROTO_UDP;
1029 return rte_flow_error_set(error,
1030 EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1031 "vxlan-encap: L3 must be IPv4/IPv6");
1034 flow_item_skip_void(&item);
1037 if (item->type != RTE_FLOW_ITEM_TYPE_UDP)
1038 return rte_flow_error_set(error, EINVAL,
1039 RTE_FLOW_ERROR_TYPE_ITEM, item,
1040 "vxlan-encap: UDP must follow IPv4/IPv6");
1041 /* UDP length = UDP + VXLAN. NIC will add inner packet length. */
1042 fm_op.encap.len2_offset =
1043 off + offsetof(struct rte_udp_hdr, dgram_len);
1044 fm_op.encap.len2_delta =
1045 sizeof(struct rte_udp_hdr) + sizeof(struct rte_vxlan_hdr);
1046 append_template(&template, &off, item->spec,
1047 sizeof(struct rte_udp_hdr));
1049 flow_item_skip_void(&item);
1052 if (item->type != RTE_FLOW_ITEM_TYPE_VXLAN)
1053 return rte_flow_error_set(error,
1054 EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,
1055 "vxlan-encap: VXLAN must follow UDP");
1056 append_template(&template, &off, item->spec,
1057 sizeof(struct rte_flow_item_vxlan));
1060 * Fill in the rest of the action structure.
1061 * Indicate that we want to encap with vxlan at packet start.
1063 fm_op.encap.template_offset = 0;
1064 fm_op.encap.template_len = off;
1065 return enic_fm_append_action_op(fm, &fm_op, error);
1069 enic_fm_find_vnic(struct enic *enic, const struct rte_pci_addr *addr,
1076 ENICPMD_FUNC_TRACE();
1077 ENICPMD_LOG(DEBUG, "bdf=%x:%x:%x", addr->bus, addr->devid,
1079 bdf = addr->bus << 8 | addr->devid << 3 | addr->function;
1080 args[0] = FM_VNIC_FIND;
1082 rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
1084 ENICPMD_LOG(ERR, "allocating counters rc=%d", rc);
1088 ENICPMD_LOG(DEBUG, "found vnic: handle=0x%" PRIx64, *handle);
1092 /* Translate flow actions to flowman TCAM entry actions */
1094 enic_fm_copy_action(struct enic_flowman *fm,
1095 const struct rte_flow_action actions[],
1097 struct rte_flow_error *error)
1106 struct fm_tcam_match_entry *fmt;
1107 struct fm_action_op fm_op;
1114 ENICPMD_FUNC_TRACE();
1115 fmt = &fm->tcam_entry;
1119 vnic_h = 0; /* 0 = current vNIC */
1120 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
1121 switch (actions->type) {
1122 case RTE_FLOW_ACTION_TYPE_VOID:
1124 case RTE_FLOW_ACTION_TYPE_PASSTHRU: {
1125 if (overlap & PASSTHRU)
1127 overlap |= PASSTHRU;
1130 case RTE_FLOW_ACTION_TYPE_JUMP: {
1131 const struct rte_flow_action_jump *jump =
1133 struct enic_fm_fet *fet;
1137 ret = enic_fet_get(fm, jump->group, ingress, NULL,
1142 memset(&fm_op, 0, sizeof(fm_op));
1143 fm_op.fa_op = FMOP_EXACT_MATCH;
1144 fm_op.exact.handle = fet->handle;
1146 ret = enic_fm_append_action_op(fm, &fm_op, error);
1151 case RTE_FLOW_ACTION_TYPE_MARK: {
1152 const struct rte_flow_action_mark *mark =
1158 if (mark->id >= ENIC_MAGIC_FILTER_ID - 1)
1159 return rte_flow_error_set(error, EINVAL,
1160 RTE_FLOW_ERROR_TYPE_ACTION,
1161 NULL, "invalid mark id");
1162 memset(&fm_op, 0, sizeof(fm_op));
1163 fm_op.fa_op = FMOP_MARK;
1164 fm_op.mark.mark = mark->id + 1;
1165 ret = enic_fm_append_action_op(fm, &fm_op, error);
1170 case RTE_FLOW_ACTION_TYPE_FLAG: {
1174 /* ENIC_MAGIC_FILTER_ID is reserved for flagging */
1175 memset(&fm_op, 0, sizeof(fm_op));
1176 fm_op.fa_op = FMOP_MARK;
1177 fm_op.mark.mark = ENIC_MAGIC_FILTER_ID;
1178 ret = enic_fm_append_action_op(fm, &fm_op, error);
1183 case RTE_FLOW_ACTION_TYPE_QUEUE: {
1184 const struct rte_flow_action_queue *queue =
1188 * If other fate kind is set, fail. Multiple
1189 * queue actions are ok.
1191 if ((overlap & FATE) && first_rq)
1195 memset(&fm_op, 0, sizeof(fm_op));
1196 fm_op.fa_op = FMOP_RQ_STEER;
1197 fm_op.rq_steer.rq_index =
1198 enic_rte_rq_idx_to_sop_idx(queue->index);
1199 fm_op.rq_steer.vnic_handle = vnic_h;
1200 ret = enic_fm_append_action_op(fm, &fm_op, error);
1203 ENICPMD_LOG(DEBUG, "create QUEUE action rq: %u",
1204 fm_op.rq_steer.rq_index);
1207 case RTE_FLOW_ACTION_TYPE_DROP: {
1211 memset(&fm_op, 0, sizeof(fm_op));
1212 fm_op.fa_op = FMOP_DROP;
1213 ret = enic_fm_append_action_op(fm, &fm_op, error);
1216 ENICPMD_LOG(DEBUG, "create DROP action");
1219 case RTE_FLOW_ACTION_TYPE_COUNT: {
1220 if (overlap & COUNT)
1223 /* Count is associated with entry not action on VIC. */
1224 fmt->ftm_flags |= FMEF_COUNTER;
1227 case RTE_FLOW_ACTION_TYPE_RSS: {
1228 const struct rte_flow_action_rss *rss = actions->conf;
1233 * Hardware does not support general RSS actions, but
1234 * we can still support the dummy one that is used to
1235 * "receive normally".
1237 allow = rss->func == RTE_ETH_HASH_FUNCTION_DEFAULT &&
1240 rss->types == enic->rss_hf) &&
1241 rss->queue_num == enic->rq_count &&
1243 /* Identity queue map is ok */
1244 for (i = 0; i < rss->queue_num; i++)
1245 allow = allow && (i == rss->queue[i]);
1250 /* Need MARK or FLAG */
1251 if (!(overlap & MARK))
1256 case RTE_FLOW_ACTION_TYPE_PORT_ID: {
1257 const struct rte_flow_action_port_id *port;
1258 struct rte_pci_device *pdev;
1259 struct rte_eth_dev *dev;
1261 port = actions->conf;
1262 if (port->original) {
1263 vnic_h = 0; /* This port */
1266 ENICPMD_LOG(DEBUG, "port id %u", port->id);
1267 if (!rte_eth_dev_is_valid_port(port->id)) {
1268 return rte_flow_error_set(error, EINVAL,
1269 RTE_FLOW_ERROR_TYPE_ACTION,
1270 NULL, "invalid port_id");
1272 dev = &rte_eth_devices[port->id];
1273 if (!dev_is_enic(dev)) {
1274 return rte_flow_error_set(error, EINVAL,
1275 RTE_FLOW_ERROR_TYPE_ACTION,
1276 NULL, "port_id is not enic");
1278 pdev = RTE_ETH_DEV_TO_PCI(dev);
1279 if (enic_fm_find_vnic(enic, &pdev->addr, &vnic_h)) {
1280 return rte_flow_error_set(error, EINVAL,
1281 RTE_FLOW_ERROR_TYPE_ACTION,
1282 NULL, "port_id is not vnic");
1286 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: {
1287 ret = enic_fm_copy_vxlan_decap(fm, fmt, actions,
1293 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: {
1294 const struct rte_flow_action_vxlan_encap *encap;
1296 encap = actions->conf;
1297 if (overlap & ENCAP)
1300 ret = enic_fm_copy_vxlan_encap(fm, encap->definition,
1311 if (!(overlap & (FATE | PASSTHRU | COUNT)))
1313 memset(&fm_op, 0, sizeof(fm_op));
1314 fm_op.fa_op = FMOP_END;
1315 ret = enic_fm_append_action_op(fm, &fm_op, error);
1318 enic_fm_reorder_action_op(fm);
1322 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
1323 NULL, "enic: unsupported action");
1326 /** Check if the action is supported */
1328 enic_fm_match_action(const struct rte_flow_action *action,
1329 const enum rte_flow_action_type *supported_actions)
1331 for (; *supported_actions != RTE_FLOW_ACTION_TYPE_END;
1332 supported_actions++) {
1333 if (action->type == *supported_actions)
1339 /* Debug function to dump internal NIC action structure. */
1341 enic_fm_dump_tcam_actions(const struct fm_action *fm_action)
1343 /* Manually keep in sync with FMOP commands */
1344 const char *fmop_str[FMOP_OP_MAX] = {
1346 [FMOP_DROP] = "drop",
1347 [FMOP_RQ_STEER] = "steer",
1348 [FMOP_EXACT_MATCH] = "exmatch",
1349 [FMOP_MARK] = "mark",
1350 [FMOP_EXT_MARK] = "ext_mark",
1352 [FMOP_EG_HAIRPIN] = "eg_hairpin",
1353 [FMOP_IG_HAIRPIN] = "ig_hairpin",
1354 [FMOP_ENCAP_IVLAN] = "encap_ivlan",
1355 [FMOP_ENCAP_NOIVLAN] = "encap_noivlan",
1356 [FMOP_ENCAP] = "encap",
1357 [FMOP_SET_OVLAN] = "set_ovlan",
1358 [FMOP_DECAP_NOSTRIP] = "decap_nostrip",
1360 const struct fm_action_op *op = &fm_action->fma_action_ops[0];
1361 char buf[128], *bp = buf;
1366 buf_len = sizeof(buf);
1367 for (i = 0; i < FM_ACTION_OP_MAX; i++) {
1368 if (op->fa_op == FMOP_END)
1370 if (op->fa_op >= FMOP_OP_MAX)
1373 op_str = fmop_str[op->fa_op];
1374 n = snprintf(bp, buf_len, "%s,", op_str);
1375 if (n > 0 && n < buf_len) {
1381 /* Remove trailing comma */
1384 ENICPMD_LOG(DEBUG, " Acions: %s", buf);
1388 bits_to_str(uint32_t bits, const char *strings[], int max,
1389 char *buf, int buf_len)
1391 int i, n = 0, len = 0;
1393 for (i = 0; i < max; i++) {
1394 if (bits & (1 << i)) {
1395 n = snprintf(buf, buf_len, "%s,", strings[i]);
1396 if (n > 0 && n < buf_len) {
1403 /* Remove trailing comma */
1411 /* Debug function to dump internal NIC filter structure. */
1413 __enic_fm_dump_tcam_match(const struct fm_header_set *fk_hdrset, char *buf,
1416 /* Manually keep in sync with FKM_BITS */
1417 const char *fm_fkm_str[FKM_BIT_COUNT] = {
1418 [FKM_QTAG_BIT] = "qtag",
1419 [FKM_CMD_BIT] = "cmd",
1420 [FKM_IPV4_BIT] = "ip4",
1421 [FKM_IPV6_BIT] = "ip6",
1422 [FKM_ROCE_BIT] = "roce",
1423 [FKM_UDP_BIT] = "udp",
1424 [FKM_TCP_BIT] = "tcp",
1425 [FKM_TCPORUDP_BIT] = "tcpportudp",
1426 [FKM_IPFRAG_BIT] = "ipfrag",
1427 [FKM_NVGRE_BIT] = "nvgre",
1428 [FKM_VXLAN_BIT] = "vxlan",
1429 [FKM_GENEVE_BIT] = "geneve",
1430 [FKM_NSH_BIT] = "nsh",
1431 [FKM_ROCEV2_BIT] = "rocev2",
1432 [FKM_VLAN_PRES_BIT] = "vlan_pres",
1433 [FKM_IPOK_BIT] = "ipok",
1434 [FKM_L4OK_BIT] = "l4ok",
1435 [FKM_ROCEOK_BIT] = "roceok",
1436 [FKM_FCSOK_BIT] = "fcsok",
1437 [FKM_EG_SPAN_BIT] = "eg_span",
1438 [FKM_IG_SPAN_BIT] = "ig_span",
1439 [FKM_EG_HAIRPINNED_BIT] = "eg_hairpinned",
1441 /* Manually keep in sync with FKH_BITS */
1442 const char *fm_fkh_str[FKH_BIT_COUNT] = {
1443 [FKH_ETHER_BIT] = "eth",
1444 [FKH_QTAG_BIT] = "qtag",
1445 [FKH_L2RAW_BIT] = "l2raw",
1446 [FKH_IPV4_BIT] = "ip4",
1447 [FKH_IPV6_BIT] = "ip6",
1448 [FKH_L3RAW_BIT] = "l3raw",
1449 [FKH_UDP_BIT] = "udp",
1450 [FKH_TCP_BIT] = "tcp",
1451 [FKH_ICMP_BIT] = "icmp",
1452 [FKH_VXLAN_BIT] = "vxlan",
1453 [FKH_L4RAW_BIT] = "l4raw",
1455 uint32_t fkh_bits = fk_hdrset->fk_header_select;
1456 uint32_t fkm_bits = fk_hdrset->fk_metadata;
1459 if (!fkm_bits && !fkh_bits)
1461 n = snprintf(buf, buf_len, "metadata(");
1462 if (n > 0 && n < buf_len) {
1466 n = bits_to_str(fkm_bits, fm_fkm_str, FKM_BIT_COUNT, buf, buf_len);
1467 if (n > 0 && n < buf_len) {
1471 n = snprintf(buf, buf_len, ") valid hdr fields(");
1472 if (n > 0 && n < buf_len) {
1476 n = bits_to_str(fkh_bits, fm_fkh_str, FKH_BIT_COUNT, buf, buf_len);
1477 if (n > 0 && n < buf_len) {
1481 snprintf(buf, buf_len, ")");
1485 enic_fm_dump_tcam_match(const struct fm_tcam_match_entry *match,
1490 memset(buf, 0, sizeof(buf));
1491 __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[0],
1493 ENICPMD_LOG(DEBUG, " TCAM %s Outer: %s %scounter",
1494 (ingress) ? "IG" : "EG", buf,
1495 (match->ftm_flags & FMEF_COUNTER) ? "" : "no ");
1496 memset(buf, 0, sizeof(buf));
1497 __enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[1],
1500 ENICPMD_LOG(DEBUG, " Inner: %s", buf);
1503 /* Debug function to dump internal NIC flow structures. */
1505 enic_fm_dump_tcam_entry(const struct fm_tcam_match_entry *fm_match,
1506 const struct fm_action *fm_action,
1509 if (rte_log_get_level(enic_pmd_logtype) < (int)RTE_LOG_DEBUG)
1511 enic_fm_dump_tcam_match(fm_match, ingress);
1512 enic_fm_dump_tcam_actions(fm_action);
1516 enic_fm_flow_parse(struct enic_flowman *fm,
1517 const struct rte_flow_attr *attrs,
1518 const struct rte_flow_item pattern[],
1519 const struct rte_flow_action actions[],
1520 struct rte_flow_error *error)
1522 const struct rte_flow_action *action;
1524 static const enum rte_flow_action_type *sa;
1526 ENICPMD_FUNC_TRACE();
1529 rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM,
1530 NULL, "no pattern specified");
1535 rte_flow_error_set(error, EINVAL,
1536 RTE_FLOW_ERROR_TYPE_ACTION_NUM,
1537 NULL, "no action specified");
1542 if (attrs->priority) {
1543 rte_flow_error_set(error, ENOTSUP,
1544 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1546 "priorities are not supported");
1548 } else if (attrs->transfer) {
1549 rte_flow_error_set(error, ENOTSUP,
1550 RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1552 "transfer is not supported");
1554 } else if (attrs->ingress && attrs->egress) {
1555 rte_flow_error_set(error, ENOTSUP,
1556 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1558 "bidirectional rules not supported");
1563 rte_flow_error_set(error, EINVAL,
1564 RTE_FLOW_ERROR_TYPE_ATTR,
1565 NULL, "no attribute specified");
1569 /* Verify Actions. */
1570 sa = (attrs->ingress) ? enic_fm_supported_ig_actions :
1571 enic_fm_supported_eg_actions;
1572 for (action = &actions[0]; action->type != RTE_FLOW_ACTION_TYPE_END;
1574 if (action->type == RTE_FLOW_ACTION_TYPE_VOID)
1576 else if (!enic_fm_match_action(action, sa))
1579 if (action->type != RTE_FLOW_ACTION_TYPE_END) {
1580 rte_flow_error_set(error, EPERM, RTE_FLOW_ERROR_TYPE_ACTION,
1581 action, "invalid action");
1584 ret = enic_fm_copy_entry(fm, pattern, error);
1587 ret = enic_fm_copy_action(fm, actions, attrs->ingress, error);
1592 enic_fm_counter_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
1594 if (!fm_flow->counter_valid)
1596 SLIST_INSERT_HEAD(&fm->counters, fm_flow->counter, next);
1597 fm_flow->counter_valid = false;
1601 enic_fm_more_counters(struct enic_flowman *fm)
1603 struct enic_fm_counter *new_stack;
1604 struct enic_fm_counter *ctrs;
1609 ENICPMD_FUNC_TRACE();
1611 new_stack = rte_realloc(fm->counter_stack, (fm->counters_alloced +
1612 FM_COUNTERS_EXPAND) *
1613 sizeof(struct enic_fm_counter), 0);
1614 if (new_stack == NULL) {
1615 ENICPMD_LOG(ERR, "cannot alloc counter memory");
1618 fm->counter_stack = new_stack;
1620 args[0] = FM_COUNTER_BRK;
1621 args[1] = fm->counters_alloced + FM_COUNTERS_EXPAND;
1622 rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
1624 ENICPMD_LOG(ERR, "cannot alloc counters rc=%d", rc);
1627 ctrs = (struct enic_fm_counter *)fm->counter_stack +
1628 fm->counters_alloced;
1629 for (i = 0; i < FM_COUNTERS_EXPAND; i++, ctrs++) {
1630 ctrs->handle = fm->counters_alloced + i;
1631 SLIST_INSERT_HEAD(&fm->counters, ctrs, next);
1633 fm->counters_alloced += FM_COUNTERS_EXPAND;
1634 ENICPMD_LOG(DEBUG, "%u counters allocated, total: %u",
1635 FM_COUNTERS_EXPAND, fm->counters_alloced);
1640 enic_fm_counter_zero(struct enic_flowman *fm, struct enic_fm_counter *c)
1646 ENICPMD_FUNC_TRACE();
1648 args[0] = FM_COUNTER_QUERY;
1649 args[1] = c->handle;
1650 args[2] = 1; /* clear */
1651 ret = vnic_dev_flowman_cmd(enic->vdev, args, 3);
1653 ENICPMD_LOG(ERR, "counter init: rc=%d handle=0x%x",
1661 enic_fm_counter_alloc(struct enic_flowman *fm, struct rte_flow_error *error,
1662 struct enic_fm_counter **ctr)
1664 struct enic_fm_counter *c;
1667 ENICPMD_FUNC_TRACE();
1669 if (SLIST_EMPTY(&fm->counters)) {
1670 ret = enic_fm_more_counters(fm);
1672 return rte_flow_error_set(error, -ret,
1673 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1674 NULL, "enic: out of counters");
1676 c = SLIST_FIRST(&fm->counters);
1677 SLIST_REMOVE_HEAD(&fm->counters, next);
1683 enic_fm_action_free(struct enic_flowman *fm, uint64_t handle)
1688 ENICPMD_FUNC_TRACE();
1689 args[0] = FM_ACTION_FREE;
1691 rc = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
1693 ENICPMD_LOG(ERR, "cannot free action: rc=%d handle=0x%" PRIx64,
1699 enic_fm_entry_free(struct enic_flowman *fm, uint64_t handle)
1704 ENICPMD_FUNC_TRACE();
1705 args[0] = FM_MATCH_ENTRY_REMOVE;
1707 rc = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
1709 ENICPMD_LOG(ERR, "cannot free match entry: rc=%d"
1710 " handle=0x%" PRIx64, rc, handle);
1714 static struct enic_fm_jump_flow *
1715 find_jump_flow(struct enic_flowman *fm, uint32_t group)
1717 struct enic_fm_jump_flow *j;
1719 ENICPMD_FUNC_TRACE();
1720 TAILQ_FOREACH(j, &fm->jump_list, list) {
1721 if (j->group == group)
1728 remove_jump_flow(struct enic_flowman *fm, struct rte_flow *flow)
1730 struct enic_fm_jump_flow *j;
1732 ENICPMD_FUNC_TRACE();
1733 TAILQ_FOREACH(j, &fm->jump_list, list) {
1734 if (j->flow == flow) {
1735 TAILQ_REMOVE(&fm->jump_list, j, list);
1743 save_jump_flow(struct enic_flowman *fm,
1744 struct rte_flow *flow,
1746 struct fm_tcam_match_entry *match,
1747 struct fm_action *action)
1749 struct enic_fm_jump_flow *j;
1751 ENICPMD_FUNC_TRACE();
1752 j = calloc(1, sizeof(struct enic_fm_jump_flow));
1758 j->action = *action;
1759 TAILQ_INSERT_HEAD(&fm->jump_list, j, list);
1760 ENICPMD_LOG(DEBUG, "saved jump flow: flow=%p group=%u", flow, group);
1765 __enic_fm_flow_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)
1767 if (fm_flow->entry_handle != FM_INVALID_HANDLE) {
1768 enic_fm_entry_free(fm, fm_flow->entry_handle);
1769 fm_flow->entry_handle = FM_INVALID_HANDLE;
1771 if (fm_flow->action_handle != FM_INVALID_HANDLE) {
1772 enic_fm_action_free(fm, fm_flow->action_handle);
1773 fm_flow->action_handle = FM_INVALID_HANDLE;
1775 enic_fm_counter_free(fm, fm_flow);
1777 enic_fet_put(fm, fm_flow->fet);
1778 fm_flow->fet = NULL;
1783 enic_fm_flow_free(struct enic_flowman *fm, struct rte_flow *flow)
1785 if (flow->fm->fet && flow->fm->fet->default_key)
1786 remove_jump_flow(fm, flow);
1787 __enic_fm_flow_free(fm, flow->fm);
1793 enic_fm_add_tcam_entry(struct enic_flowman *fm,
1794 struct fm_tcam_match_entry *match_in,
1795 uint64_t *entry_handle,
1797 struct rte_flow_error *error)
1799 struct fm_tcam_match_entry *ftm;
1803 ENICPMD_FUNC_TRACE();
1804 /* Copy entry to the command buffer */
1805 ftm = &fm->cmd.va->fm_tcam_match_entry;
1806 memcpy(ftm, match_in, sizeof(*ftm));
1807 /* Add TCAM entry */
1808 args[0] = FM_TCAM_ENTRY_INSTALL;
1809 args[1] = ingress ? fm->ig_tcam_hndl : fm->eg_tcam_hndl;
1810 args[2] = fm->cmd.pa;
1811 ret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 3);
1813 ENICPMD_LOG(ERR, "cannot add %s TCAM entry: rc=%d",
1814 ingress ? "ingress" : "egress", ret);
1815 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1816 NULL, "enic: devcmd(tcam-entry-install)");
1819 ENICPMD_LOG(DEBUG, "installed %s TCAM entry: handle=0x%" PRIx64,
1820 ingress ? "ingress" : "egress", (uint64_t)args[0]);
1821 *entry_handle = args[0];
1826 enic_fm_add_exact_entry(struct enic_flowman *fm,
1827 struct fm_tcam_match_entry *match_in,
1828 uint64_t *entry_handle,
1829 struct enic_fm_fet *fet,
1830 struct rte_flow_error *error)
1832 struct fm_exact_match_entry *fem;
1836 ENICPMD_FUNC_TRACE();
1837 /* The new entry must have the table's key */
1838 if (memcmp(fet->key.fk_hdrset, match_in->ftm_mask.fk_hdrset,
1839 sizeof(struct fm_header_set) * FM_HDRSET_MAX)) {
1840 return rte_flow_error_set(error, EINVAL,
1841 RTE_FLOW_ERROR_TYPE_ITEM, NULL,
1842 "enic: key does not match group's key");
1845 /* Copy entry to the command buffer */
1846 fem = &fm->cmd.va->fm_exact_match_entry;
1848 * Translate TCAM entry to exact entry. As is only need to drop
1849 * position and mask. The mask is part of the exact match table.
1850 * Position (aka priority) is not supported in the exact match table.
1852 fem->fem_data = match_in->ftm_data;
1853 fem->fem_flags = match_in->ftm_flags;
1854 fem->fem_action = match_in->ftm_action;
1855 fem->fem_counter = match_in->ftm_counter;
1857 /* Add exact entry */
1858 args[0] = FM_EXACT_ENTRY_INSTALL;
1859 args[1] = fet->handle;
1860 args[2] = fm->cmd.pa;
1861 ret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 3);
1863 ENICPMD_LOG(ERR, "cannot add %s exact entry: group=%u",
1864 fet->ingress ? "ingress" : "egress", fet->group);
1865 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1866 NULL, "enic: devcmd(exact-entry-install)");
1869 ENICPMD_LOG(DEBUG, "installed %s exact entry: group=%u"
1870 " handle=0x%" PRIx64,
1871 fet->ingress ? "ingress" : "egress", fet->group,
1873 *entry_handle = args[0];
1877 /* Push match-action to the NIC. */
1879 __enic_fm_flow_add_entry(struct enic_flowman *fm,
1880 struct enic_fm_flow *fm_flow,
1881 struct fm_tcam_match_entry *match_in,
1882 struct fm_action *action_in,
1885 struct rte_flow_error *error)
1887 struct enic_fm_counter *ctr;
1888 struct fm_action *fma;
1894 ENICPMD_FUNC_TRACE();
1895 /* Allocate action. */
1896 fma = &fm->cmd.va->fm_action;
1897 memcpy(fma, action_in, sizeof(*fma));
1898 args[0] = FM_ACTION_ALLOC;
1899 args[1] = fm->cmd.pa;
1900 ret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
1902 ENICPMD_LOG(ERR, "allocating TCAM table action rc=%d", ret);
1903 rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1904 NULL, "enic: devcmd(action-alloc)");
1908 fm_flow->action_handle = action_h;
1909 match_in->ftm_action = action_h;
1910 ENICPMD_LOG(DEBUG, "action allocated: handle=0x%" PRIx64, action_h);
1912 /* Allocate counter if requested. */
1913 if (match_in->ftm_flags & FMEF_COUNTER) {
1914 ret = enic_fm_counter_alloc(fm, error, &ctr);
1915 if (ret) /* error has been filled in */
1917 fm_flow->counter_valid = true;
1918 fm_flow->counter = ctr;
1919 match_in->ftm_counter = ctr->handle;
1923 * Get the group's table (either TCAM or exact match table) and
1924 * add entry to it. If we use the exact match table, the handler
1925 * will translate the TCAM entry (match_in) to the appropriate
1926 * exact match entry and use that instead.
1928 entry_h = FM_INVALID_HANDLE;
1929 if (group == FM_TCAM_RTE_GROUP) {
1930 ret = enic_fm_add_tcam_entry(fm, match_in, &entry_h, ingress,
1934 /* Jump action might have a ref to fet */
1935 fm_flow->fet = fm->fet;
1938 struct enic_fm_fet *fet = NULL;
1940 ret = enic_fet_get(fm, group, ingress,
1941 &match_in->ftm_mask, &fet, error);
1945 ret = enic_fm_add_exact_entry(fm, match_in, &entry_h, fet,
1950 /* Clear counter after adding entry, as it requires in-use counter */
1951 if (fm_flow->counter_valid) {
1952 ret = enic_fm_counter_zero(fm, fm_flow->counter);
1956 fm_flow->entry_handle = entry_h;
1960 /* Push match-action to the NIC. */
1961 static struct rte_flow *
1962 enic_fm_flow_add_entry(struct enic_flowman *fm,
1963 struct fm_tcam_match_entry *match_in,
1964 struct fm_action *action_in,
1965 const struct rte_flow_attr *attrs,
1966 struct rte_flow_error *error)
1968 struct enic_fm_flow *fm_flow;
1969 struct rte_flow *flow;
1971 ENICPMD_FUNC_TRACE();
1972 enic_fm_dump_tcam_entry(match_in, action_in, attrs->ingress);
1973 flow = calloc(1, sizeof(*flow));
1974 fm_flow = calloc(1, sizeof(*fm_flow));
1975 if (flow == NULL || fm_flow == NULL) {
1976 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1977 NULL, "enic: cannot allocate rte_flow");
1983 fm_flow->action_handle = FM_INVALID_HANDLE;
1984 fm_flow->entry_handle = FM_INVALID_HANDLE;
1985 if (__enic_fm_flow_add_entry(fm, fm_flow, match_in, action_in,
1986 attrs->group, attrs->ingress, error)) {
1987 enic_fm_flow_free(fm, flow);
1994 convert_jump_flows(struct enic_flowman *fm, struct enic_fm_fet *fet,
1995 struct rte_flow_error *error)
1997 struct enic_fm_flow *fm_flow;
1998 struct enic_fm_jump_flow *j;
1999 struct fm_action *fma;
2002 ENICPMD_FUNC_TRACE();
2004 * Find the saved flows that should jump to the new table (fet).
2005 * Then delete the old TCAM entry that jumps to the default table,
2006 * and add a new one that jumps to the new table.
2009 j = find_jump_flow(fm, group);
2011 ENICPMD_LOG(DEBUG, "convert jump flow: flow=%p group=%u",
2013 /* Delete old entry */
2014 fm_flow = j->flow->fm;
2015 __enic_fm_flow_free(fm, fm_flow);
2019 fma->fma_action_ops[0].exact.handle = fet->handle;
2020 if (__enic_fm_flow_add_entry(fm, fm_flow, &j->match, fma,
2021 FM_TCAM_RTE_GROUP, fet->ingress, error)) {
2022 /* Cannot roll back changes at the moment */
2023 ENICPMD_LOG(ERR, "cannot convert jump flow: flow=%p",
2028 ENICPMD_LOG(DEBUG, "convert ok: group=%u ref=%u",
2029 fet->group, fet->ref);
2032 TAILQ_REMOVE(&fm->jump_list, j, list);
2034 j = find_jump_flow(fm, group);
2039 enic_fm_open_scratch(struct enic_flowman *fm)
2041 fm->action_op_count = 0;
2043 memset(&fm->tcam_entry, 0, sizeof(fm->tcam_entry));
2044 memset(&fm->action, 0, sizeof(fm->action));
2048 enic_fm_close_scratch(struct enic_flowman *fm)
2051 enic_fet_put(fm, fm->fet);
2054 fm->action_op_count = 0;
2058 enic_fm_flow_validate(struct rte_eth_dev *dev,
2059 const struct rte_flow_attr *attrs,
2060 const struct rte_flow_item pattern[],
2061 const struct rte_flow_action actions[],
2062 struct rte_flow_error *error)
2064 struct fm_tcam_match_entry *fm_tcam_entry;
2065 struct fm_action *fm_action;
2066 struct enic_flowman *fm;
2069 ENICPMD_FUNC_TRACE();
2070 fm = pmd_priv(dev)->fm;
2073 enic_fm_open_scratch(fm);
2074 ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2076 fm_tcam_entry = &fm->tcam_entry;
2077 fm_action = &fm->action;
2078 enic_fm_dump_tcam_entry(fm_tcam_entry, fm_action,
2081 enic_fm_close_scratch(fm);
2086 enic_fm_flow_query_count(struct rte_eth_dev *dev,
2087 struct rte_flow *flow, void *data,
2088 struct rte_flow_error *error)
2090 struct rte_flow_query_count *query;
2091 struct enic_fm_flow *fm_flow;
2096 ENICPMD_FUNC_TRACE();
2097 enic = pmd_priv(dev);
2100 if (!fm_flow->counter_valid)
2101 return rte_flow_error_set(error, ENOTSUP,
2102 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2103 "enic: flow does not have counter");
2105 args[0] = FM_COUNTER_QUERY;
2106 args[1] = fm_flow->counter->handle;
2107 args[2] = query->reset;
2108 rc = vnic_dev_flowman_cmd(enic->vdev, args, 3);
2110 ENICPMD_LOG(ERR, "cannot query counter: rc=%d handle=0x%x",
2111 rc, fm_flow->counter->handle);
2114 query->hits_set = 1;
2115 query->hits = args[0];
2116 query->bytes_set = 1;
2117 query->bytes = args[1];
2122 enic_fm_flow_query(struct rte_eth_dev *dev,
2123 struct rte_flow *flow,
2124 const struct rte_flow_action *actions,
2126 struct rte_flow_error *error)
2130 ENICPMD_FUNC_TRACE();
2131 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
2132 switch (actions->type) {
2133 case RTE_FLOW_ACTION_TYPE_VOID:
2135 case RTE_FLOW_ACTION_TYPE_COUNT:
2136 ret = enic_fm_flow_query_count(dev, flow, data, error);
2139 return rte_flow_error_set(error, ENOTSUP,
2140 RTE_FLOW_ERROR_TYPE_ACTION,
2142 "action not supported");
2150 static struct rte_flow *
2151 enic_fm_flow_create(struct rte_eth_dev *dev,
2152 const struct rte_flow_attr *attrs,
2153 const struct rte_flow_item pattern[],
2154 const struct rte_flow_action actions[],
2155 struct rte_flow_error *error)
2157 struct fm_tcam_match_entry *fm_tcam_entry;
2158 struct fm_action *fm_action;
2159 struct enic_flowman *fm;
2160 struct enic_fm_fet *fet;
2161 struct rte_flow *flow;
2165 ENICPMD_FUNC_TRACE();
2166 enic = pmd_priv(dev);
2169 rte_flow_error_set(error, ENOTSUP,
2170 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2171 "flowman is not initialized");
2174 enic_fm_open_scratch(fm);
2176 ret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);
2178 goto error_with_scratch;
2179 fm_tcam_entry = &fm->tcam_entry;
2180 fm_action = &fm->action;
2181 flow = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,
2184 LIST_INSERT_HEAD(&enic->flows, flow, next);
2185 fet = flow->fm->fet;
2186 if (fet && fet->default_key) {
2188 * Jump to non-existent group? Save the relevant info
2189 * so we can convert this flow when that group
2192 save_jump_flow(fm, flow, fet->group,
2193 fm_tcam_entry, fm_action);
2194 } else if (fet && fet->ref == 1) {
2196 * A new table is created. Convert the saved flows
2197 * that should jump to this group.
2199 convert_jump_flows(fm, fet, error);
2204 enic_fm_close_scratch(fm);
2209 enic_fm_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
2210 __rte_unused struct rte_flow_error *error)
2212 struct enic *enic = pmd_priv(dev);
2214 ENICPMD_FUNC_TRACE();
2215 if (enic->fm == NULL)
2217 LIST_REMOVE(flow, next);
2218 enic_fm_flow_free(enic->fm, flow);
2223 enic_fm_flow_flush(struct rte_eth_dev *dev,
2224 __rte_unused struct rte_flow_error *error)
2226 struct enic_fm_flow *fm_flow;
2227 struct enic_flowman *fm;
2228 struct rte_flow *flow;
2229 struct enic *enic = pmd_priv(dev);
2231 ENICPMD_FUNC_TRACE();
2232 if (enic->fm == NULL)
2235 while (!LIST_EMPTY(&enic->flows)) {
2236 flow = LIST_FIRST(&enic->flows);
2238 LIST_REMOVE(flow, next);
2240 * If tables are null, then vNIC is closing, and the firmware
2241 * has already cleaned up flowman state. So do not try to free
2242 * resources, as it only causes errors.
2244 if (fm->ig_tcam_hndl == FM_INVALID_HANDLE) {
2245 fm_flow->entry_handle = FM_INVALID_HANDLE;
2246 fm_flow->action_handle = FM_INVALID_HANDLE;
2247 fm_flow->fet = NULL;
2249 enic_fm_flow_free(fm, flow);
2255 enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle)
2260 args[0] = FM_MATCH_TABLE_FREE;
2262 rc = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);
2264 ENICPMD_LOG(ERR, "cannot free table: rc=%d handle=0x%" PRIx64,
2270 enic_fm_tcam_tbl_alloc(struct enic_flowman *fm, uint32_t direction,
2271 uint32_t max_entries, uint64_t *handle)
2273 struct fm_tcam_match_table *tcam_tbl;
2278 ENICPMD_FUNC_TRACE();
2280 tcam_tbl = &fm->cmd.va->fm_tcam_match_table;
2281 tcam_tbl->ftt_direction = direction;
2282 tcam_tbl->ftt_stage = FM_STAGE_LAST;
2283 tcam_tbl->ftt_max_entries = max_entries;
2284 args[0] = FM_TCAM_TABLE_ALLOC;
2285 args[1] = fm->cmd.pa;
2286 rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
2288 ENICPMD_LOG(ERR, "cannot alloc %s TCAM table: rc=%d",
2289 (direction == FM_INGRESS) ? "IG" : "EG", rc);
2293 ENICPMD_LOG(DEBUG, "%s TCAM table allocated, handle=0x%" PRIx64,
2294 (direction == FM_INGRESS) ? "IG" : "EG", *handle);
2299 enic_fm_init_counters(struct enic_flowman *fm)
2301 ENICPMD_FUNC_TRACE();
2302 SLIST_INIT(&fm->counters);
2303 return enic_fm_more_counters(fm);
2307 enic_fm_free_all_counters(struct enic_flowman *fm)
2314 args[0] = FM_COUNTER_BRK;
2316 rc = vnic_dev_flowman_cmd(enic->vdev, args, 2);
2318 ENICPMD_LOG(ERR, "cannot free counters: rc=%d", rc);
2319 rte_free(fm->counter_stack);
2323 enic_fm_alloc_tcam_tables(struct enic_flowman *fm)
2327 ENICPMD_FUNC_TRACE();
2328 rc = enic_fm_tcam_tbl_alloc(fm, FM_INGRESS, FM_MAX_TCAM_TABLE_SIZE,
2332 rc = enic_fm_tcam_tbl_alloc(fm, FM_EGRESS, FM_MAX_TCAM_TABLE_SIZE,
2338 enic_fm_free_tcam_tables(struct enic_flowman *fm)
2340 ENICPMD_FUNC_TRACE();
2341 if (fm->ig_tcam_hndl) {
2342 ENICPMD_LOG(DEBUG, "free IG TCAM table handle=0x%" PRIx64,
2344 enic_fm_tbl_free(fm, fm->ig_tcam_hndl);
2345 fm->ig_tcam_hndl = FM_INVALID_HANDLE;
2347 if (fm->eg_tcam_hndl) {
2348 ENICPMD_LOG(DEBUG, "free EG TCAM table handle=0x%" PRIx64,
2350 enic_fm_tbl_free(fm, fm->eg_tcam_hndl);
2351 fm->eg_tcam_hndl = FM_INVALID_HANDLE;
2356 enic_fm_init(struct enic *enic)
2358 struct enic_flowman *fm;
2362 if (enic->flow_filter_mode != FILTER_FLOWMAN)
2364 ENICPMD_FUNC_TRACE();
2365 fm = calloc(1, sizeof(*fm));
2367 ENICPMD_LOG(ERR, "cannot alloc flowman struct");
2371 TAILQ_INIT(&fm->fet_list);
2372 TAILQ_INIT(&fm->jump_list);
2373 /* Allocate host memory for flowman commands */
2374 snprintf((char *)name, NAME_MAX, "fm-cmd-%s", enic->bdf_name);
2375 fm->cmd.va = enic_alloc_consistent(enic,
2376 sizeof(union enic_flowman_cmd_mem), &fm->cmd.pa, name);
2378 ENICPMD_LOG(ERR, "cannot allocate flowman command memory");
2382 /* Allocate TCAM tables upfront as they are the main tables */
2383 rc = enic_fm_alloc_tcam_tables(fm);
2385 ENICPMD_LOG(ERR, "cannot alloc TCAM tables");
2388 /* Then a number of counters */
2389 rc = enic_fm_init_counters(fm);
2391 ENICPMD_LOG(ERR, "cannot alloc counters");
2395 * One default exact match table for each direction. We hold onto
2398 rc = enic_fet_alloc(fm, 1, NULL, 128, &fm->default_ig_fet);
2400 ENICPMD_LOG(ERR, "cannot alloc default IG exact match table");
2401 goto error_counters;
2403 fm->default_ig_fet->ref = 1;
2404 rc = enic_fet_alloc(fm, 0, NULL, 128, &fm->default_eg_fet);
2406 ENICPMD_LOG(ERR, "cannot alloc default EG exact match table");
2409 fm->default_eg_fet->ref = 1;
2414 enic_fet_free(fm, fm->default_ig_fet);
2416 enic_fm_free_all_counters(fm);
2418 enic_fm_free_tcam_tables(fm);
2420 enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
2421 fm->cmd.va, fm->cmd.pa);
2428 enic_fm_destroy(struct enic *enic)
2430 struct enic_flowman *fm;
2431 struct enic_fm_fet *fet;
2433 if (enic->fm == NULL)
2435 ENICPMD_FUNC_TRACE();
2437 enic_fet_free(fm, fm->default_eg_fet);
2438 enic_fet_free(fm, fm->default_ig_fet);
2439 /* Free all exact match tables still open */
2440 while (!TAILQ_EMPTY(&fm->fet_list)) {
2441 fet = TAILQ_FIRST(&fm->fet_list);
2442 enic_fet_free(fm, fet);
2444 enic_fm_free_tcam_tables(fm);
2445 enic_fm_free_all_counters(fm);
2446 enic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),
2447 fm->cmd.va, fm->cmd.pa);
2453 const struct rte_flow_ops enic_fm_flow_ops = {
2454 .validate = enic_fm_flow_validate,
2455 .create = enic_fm_flow_create,
2456 .destroy = enic_fm_flow_destroy,
2457 .flush = enic_fm_flow_flush,
2458 .query = enic_fm_flow_query,