X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fenic%2Fenic_flow.c;h=d188e582902be5798d0a24fccd7054cbf2d4f791;hb=e73e3547ce54d7ae48dff82d87efac0b7a30692a;hp=c20ff86b1bd408fa01df3ba34a9cdc3a541da310;hpb=aa3d2ff821986e5ecd266d280efaf7ae774998bc;p=dpdk.git diff --git a/drivers/net/enic/enic_flow.c b/drivers/net/enic/enic_flow.c index c20ff86b1b..d188e58290 100644 --- a/drivers/net/enic/enic_flow.c +++ b/drivers/net/enic/enic_flow.c @@ -1,37 +1,11 @@ -/* - * Copyright (c) 2017, Cisco Systems, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2008-2017 Cisco Systems, Inc. All rights reserved. */ #include +#include #include -#include +#include #include #include #include @@ -42,56 +16,75 @@ #include "vnic_dev.h" #include "vnic_nic.h" -#ifdef RTE_LIBRTE_ENIC_DEBUG_FLOW #define FLOW_TRACE() \ - RTE_LOG(DEBUG, PMD, "%s()\n", __func__) + rte_log(RTE_LOG_DEBUG, enicpmd_logtype_flow, \ + "%s()\n", __func__) #define FLOW_LOG(level, fmt, args...) \ - RTE_LOG(level, PMD, fmt, ## args) -#else -#define FLOW_TRACE() do { } while (0) -#define FLOW_LOG(level, fmt, args...) do { } while (0) -#endif + rte_log(RTE_LOG_ ## level, enicpmd_logtype_flow, \ + fmt "\n", ##args) + +/* + * Common arguments passed to copy_item functions. Use this structure + * so we can easily add new arguments. + * item: Item specification. + * filter: Partially filled in NIC filter structure. + * inner_ofst: If zero, this is an outer header. If non-zero, this is + * the offset into L5 where the header begins. + * l2_proto_off: offset to EtherType eth or vlan header. + * l3_proto_off: offset to next protocol field in IPv4 or 6 header. + */ +struct copy_item_args { + const struct rte_flow_item *item; + struct filter_v2 *filter; + uint8_t *inner_ofst; + uint8_t l2_proto_off; + uint8_t l3_proto_off; + struct enic *enic; +}; + +/* functions for copying items into enic filters */ +typedef int (enic_copy_item_fn)(struct copy_item_args *arg); /** Info about how to copy items into enic filters. */ struct enic_items { /** Function for copying and validating an item. */ - int (*copy_item)(const struct rte_flow_item *item, - struct filter_v2 *enic_filter, u8 *inner_ofst); + enic_copy_item_fn *copy_item; /** List of valid previous items. */ const enum rte_flow_item_type * const prev_items; /** True if it's OK for this item to be the first item. For some NIC * versions, it's invalid to start the stack above layer 3. */ const u8 valid_start_item; + /* Inner packet version of copy_item. */ + enic_copy_item_fn *inner_copy_item; }; /** Filtering capabilities for various NIC and firmware versions. */ struct enic_filter_cap { /** list of valid items and their handlers and attributes. */ const struct enic_items *item_info; + /* Max type in the above list, used to detect unsupported types */ + enum rte_flow_item_type max_item_type; }; /* functions for copying flow actions into enic actions */ -typedef int (copy_action_fn)(const struct rte_flow_action actions[], +typedef int (copy_action_fn)(struct enic *enic, + const struct rte_flow_action actions[], struct filter_action_v2 *enic_action); -/* functions for copying items into enic filters */ -typedef int(enic_copy_item_fn)(const struct rte_flow_item *item, - struct filter_v2 *enic_filter, u8 *inner_ofst); - /** Action capabilities for various NICs. */ struct enic_action_cap { /** list of valid actions */ const enum rte_flow_action_type *actions; /** copy function for a particular NIC */ - int (*copy_fn)(const struct rte_flow_action actions[], - struct filter_action_v2 *enic_action); + copy_action_fn *copy_fn; }; /* Forward declarations */ static enic_copy_item_fn enic_copy_item_ipv4_v1; static enic_copy_item_fn enic_copy_item_udp_v1; static enic_copy_item_fn enic_copy_item_tcp_v1; +static enic_copy_item_fn enic_copy_item_raw_v2; static enic_copy_item_fn enic_copy_item_eth_v2; static enic_copy_item_fn enic_copy_item_vlan_v2; static enic_copy_item_fn enic_copy_item_ipv4_v2; @@ -99,8 +92,13 @@ static enic_copy_item_fn enic_copy_item_ipv6_v2; static enic_copy_item_fn enic_copy_item_udp_v2; static enic_copy_item_fn enic_copy_item_tcp_v2; static enic_copy_item_fn enic_copy_item_sctp_v2; -static enic_copy_item_fn enic_copy_item_sctp_v2; static enic_copy_item_fn enic_copy_item_vxlan_v2; +static enic_copy_item_fn enic_copy_item_inner_eth_v2; +static enic_copy_item_fn enic_copy_item_inner_vlan_v2; +static enic_copy_item_fn enic_copy_item_inner_ipv4_v2; +static enic_copy_item_fn enic_copy_item_inner_ipv6_v2; +static enic_copy_item_fn enic_copy_item_inner_udp_v2; +static enic_copy_item_fn enic_copy_item_inner_tcp_v2; static copy_action_fn enic_copy_action_v1; static copy_action_fn enic_copy_action_v2; @@ -115,6 +113,7 @@ static const struct enic_items enic_items_v1[] = { .prev_items = (const enum rte_flow_item_type[]) { RTE_FLOW_ITEM_TYPE_END, }, + .inner_copy_item = NULL, }, [RTE_FLOW_ITEM_TYPE_UDP] = { .copy_item = enic_copy_item_udp_v1, @@ -123,6 +122,7 @@ static const struct enic_items enic_items_v1[] = { RTE_FLOW_ITEM_TYPE_IPV4, RTE_FLOW_ITEM_TYPE_END, }, + .inner_copy_item = NULL, }, [RTE_FLOW_ITEM_TYPE_TCP] = { .copy_item = enic_copy_item_tcp_v1, @@ -131,6 +131,7 @@ static const struct enic_items enic_items_v1[] = { RTE_FLOW_ITEM_TYPE_IPV4, RTE_FLOW_ITEM_TYPE_END, }, + .inner_copy_item = NULL, }, }; @@ -139,6 +140,15 @@ static const struct enic_items enic_items_v1[] = { * that layer 3 must be specified. */ static const struct enic_items enic_items_v2[] = { + [RTE_FLOW_ITEM_TYPE_RAW] = { + .copy_item = enic_copy_item_raw_v2, + .valid_start_item = 0, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_END, + }, + .inner_copy_item = NULL, + }, [RTE_FLOW_ITEM_TYPE_ETH] = { .copy_item = enic_copy_item_eth_v2, .valid_start_item = 1, @@ -146,6 +156,7 @@ static const struct enic_items enic_items_v2[] = { RTE_FLOW_ITEM_TYPE_VXLAN, RTE_FLOW_ITEM_TYPE_END, }, + .inner_copy_item = enic_copy_item_inner_eth_v2, }, [RTE_FLOW_ITEM_TYPE_VLAN] = { .copy_item = enic_copy_item_vlan_v2, @@ -154,6 +165,7 @@ static const struct enic_items enic_items_v2[] = { RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_END, }, + .inner_copy_item = enic_copy_item_inner_vlan_v2, }, [RTE_FLOW_ITEM_TYPE_IPV4] = { .copy_item = enic_copy_item_ipv4_v2, @@ -163,6 +175,7 @@ static const struct enic_items enic_items_v2[] = { RTE_FLOW_ITEM_TYPE_VLAN, RTE_FLOW_ITEM_TYPE_END, }, + .inner_copy_item = enic_copy_item_inner_ipv4_v2, }, [RTE_FLOW_ITEM_TYPE_IPV6] = { .copy_item = enic_copy_item_ipv6_v2, @@ -172,6 +185,7 @@ static const struct enic_items enic_items_v2[] = { RTE_FLOW_ITEM_TYPE_VLAN, RTE_FLOW_ITEM_TYPE_END, }, + .inner_copy_item = enic_copy_item_inner_ipv6_v2, }, [RTE_FLOW_ITEM_TYPE_UDP] = { .copy_item = enic_copy_item_udp_v2, @@ -181,6 +195,7 @@ static const struct enic_items enic_items_v2[] = { RTE_FLOW_ITEM_TYPE_IPV6, RTE_FLOW_ITEM_TYPE_END, }, + .inner_copy_item = enic_copy_item_inner_udp_v2, }, [RTE_FLOW_ITEM_TYPE_TCP] = { .copy_item = enic_copy_item_tcp_v2, @@ -190,6 +205,7 @@ static const struct enic_items enic_items_v2[] = { RTE_FLOW_ITEM_TYPE_IPV6, RTE_FLOW_ITEM_TYPE_END, }, + .inner_copy_item = enic_copy_item_inner_tcp_v2, }, [RTE_FLOW_ITEM_TYPE_SCTP] = { .copy_item = enic_copy_item_sctp_v2, @@ -199,6 +215,7 @@ static const struct enic_items enic_items_v2[] = { RTE_FLOW_ITEM_TYPE_IPV6, RTE_FLOW_ITEM_TYPE_END, }, + .inner_copy_item = NULL, }, [RTE_FLOW_ITEM_TYPE_VXLAN] = { .copy_item = enic_copy_item_vxlan_v2, @@ -207,11 +224,21 @@ static const struct enic_items enic_items_v2[] = { RTE_FLOW_ITEM_TYPE_UDP, RTE_FLOW_ITEM_TYPE_END, }, + .inner_copy_item = NULL, }, }; /** NICs with Advanced filters enabled */ static const struct enic_items enic_items_v3[] = { + [RTE_FLOW_ITEM_TYPE_RAW] = { + .copy_item = enic_copy_item_raw_v2, + .valid_start_item = 0, + .prev_items = (const enum rte_flow_item_type[]) { + RTE_FLOW_ITEM_TYPE_UDP, + RTE_FLOW_ITEM_TYPE_END, + }, + .inner_copy_item = NULL, + }, [RTE_FLOW_ITEM_TYPE_ETH] = { .copy_item = enic_copy_item_eth_v2, .valid_start_item = 1, @@ -219,6 +246,7 @@ static const struct enic_items enic_items_v3[] = { RTE_FLOW_ITEM_TYPE_VXLAN, RTE_FLOW_ITEM_TYPE_END, }, + .inner_copy_item = enic_copy_item_inner_eth_v2, }, [RTE_FLOW_ITEM_TYPE_VLAN] = { .copy_item = enic_copy_item_vlan_v2, @@ -227,6 +255,7 @@ static const struct enic_items enic_items_v3[] = { RTE_FLOW_ITEM_TYPE_ETH, RTE_FLOW_ITEM_TYPE_END, }, + .inner_copy_item = enic_copy_item_inner_vlan_v2, }, [RTE_FLOW_ITEM_TYPE_IPV4] = { .copy_item = enic_copy_item_ipv4_v2, @@ -236,6 +265,7 @@ static const struct enic_items enic_items_v3[] = { RTE_FLOW_ITEM_TYPE_VLAN, RTE_FLOW_ITEM_TYPE_END, }, + .inner_copy_item = enic_copy_item_inner_ipv4_v2, }, [RTE_FLOW_ITEM_TYPE_IPV6] = { .copy_item = enic_copy_item_ipv6_v2, @@ -245,6 +275,7 @@ static const struct enic_items enic_items_v3[] = { RTE_FLOW_ITEM_TYPE_VLAN, RTE_FLOW_ITEM_TYPE_END, }, + .inner_copy_item = enic_copy_item_inner_ipv6_v2, }, [RTE_FLOW_ITEM_TYPE_UDP] = { .copy_item = enic_copy_item_udp_v2, @@ -254,6 +285,7 @@ static const struct enic_items enic_items_v3[] = { RTE_FLOW_ITEM_TYPE_IPV6, RTE_FLOW_ITEM_TYPE_END, }, + .inner_copy_item = enic_copy_item_inner_udp_v2, }, [RTE_FLOW_ITEM_TYPE_TCP] = { .copy_item = enic_copy_item_tcp_v2, @@ -263,15 +295,17 @@ static const struct enic_items enic_items_v3[] = { RTE_FLOW_ITEM_TYPE_IPV6, RTE_FLOW_ITEM_TYPE_END, }, + .inner_copy_item = enic_copy_item_inner_tcp_v2, }, [RTE_FLOW_ITEM_TYPE_SCTP] = { .copy_item = enic_copy_item_sctp_v2, - .valid_start_item = 1, + .valid_start_item = 0, .prev_items = (const enum rte_flow_item_type[]) { RTE_FLOW_ITEM_TYPE_IPV4, RTE_FLOW_ITEM_TYPE_IPV6, RTE_FLOW_ITEM_TYPE_END, }, + .inner_copy_item = NULL, }, [RTE_FLOW_ITEM_TYPE_VXLAN] = { .copy_item = enic_copy_item_vxlan_v2, @@ -280,6 +314,7 @@ static const struct enic_items enic_items_v3[] = { RTE_FLOW_ITEM_TYPE_UDP, RTE_FLOW_ITEM_TYPE_END, }, + .inner_copy_item = NULL, }, }; @@ -287,12 +322,15 @@ static const struct enic_items enic_items_v3[] = { static const struct enic_filter_cap enic_filter_cap[] = { [FILTER_IPV4_5TUPLE] = { .item_info = enic_items_v1, + .max_item_type = RTE_FLOW_ITEM_TYPE_TCP, }, [FILTER_USNIC_IP] = { .item_info = enic_items_v2, + .max_item_type = RTE_FLOW_ITEM_TYPE_VXLAN, }, [FILTER_DPDK_1] = { .item_info = enic_items_v3, + .max_item_type = RTE_FLOW_ITEM_TYPE_VXLAN, }, }; @@ -303,10 +341,33 @@ static const enum rte_flow_action_type enic_supported_actions_v1[] = { }; /** Supported actions for newer NICs */ -static const enum rte_flow_action_type enic_supported_actions_v2[] = { +static const enum rte_flow_action_type enic_supported_actions_v2_id[] = { + RTE_FLOW_ACTION_TYPE_QUEUE, + RTE_FLOW_ACTION_TYPE_MARK, + RTE_FLOW_ACTION_TYPE_FLAG, + RTE_FLOW_ACTION_TYPE_RSS, + RTE_FLOW_ACTION_TYPE_PASSTHRU, + RTE_FLOW_ACTION_TYPE_END, +}; + +static const enum rte_flow_action_type enic_supported_actions_v2_drop[] = { + RTE_FLOW_ACTION_TYPE_QUEUE, + RTE_FLOW_ACTION_TYPE_MARK, + RTE_FLOW_ACTION_TYPE_FLAG, + RTE_FLOW_ACTION_TYPE_DROP, + RTE_FLOW_ACTION_TYPE_RSS, + RTE_FLOW_ACTION_TYPE_PASSTHRU, + RTE_FLOW_ACTION_TYPE_END, +}; + +static const enum rte_flow_action_type enic_supported_actions_v2_count[] = { RTE_FLOW_ACTION_TYPE_QUEUE, RTE_FLOW_ACTION_TYPE_MARK, RTE_FLOW_ACTION_TYPE_FLAG, + RTE_FLOW_ACTION_TYPE_DROP, + RTE_FLOW_ACTION_TYPE_COUNT, + RTE_FLOW_ACTION_TYPE_RSS, + RTE_FLOW_ACTION_TYPE_PASSTHRU, RTE_FLOW_ACTION_TYPE_END, }; @@ -316,8 +377,16 @@ static const struct enic_action_cap enic_action_cap[] = { .actions = enic_supported_actions_v1, .copy_fn = enic_copy_action_v1, }, - [FILTER_ACTION_V2_ALL] = { - .actions = enic_supported_actions_v2, + [FILTER_ACTION_FILTER_ID_FLAG] = { + .actions = enic_supported_actions_v2_id, + .copy_fn = enic_copy_action_v2, + }, + [FILTER_ACTION_DROP_FLAG] = { + .actions = enic_supported_actions_v2_drop, + .copy_fn = enic_copy_action_v2, + }, + [FILTER_ACTION_COUNTER_FLAG] = { + .actions = enic_supported_actions_v2_count, .copy_fn = enic_copy_action_v2, }, }; @@ -334,33 +403,21 @@ mask_exact_match(const u8 *supported, const u8 *supplied, return 1; } -/** - * Copy IPv4 item into version 1 NIC filter. - * - * @param item[in] - * Item specification. - * @param enic_filter[out] - * Partially filled in NIC filter structure. - * @param inner_ofst[in] - * Should always be 0 for version 1. - */ static int -enic_copy_item_ipv4_v1(const struct rte_flow_item *item, - struct filter_v2 *enic_filter, u8 *inner_ofst) +enic_copy_item_ipv4_v1(struct copy_item_args *arg) { + const struct rte_flow_item *item = arg->item; + struct filter_v2 *enic_filter = arg->filter; const struct rte_flow_item_ipv4 *spec = item->spec; const struct rte_flow_item_ipv4 *mask = item->mask; struct filter_ipv4_5tuple *enic_5tup = &enic_filter->u.ipv4; - struct ipv4_hdr supported_mask = { + struct rte_ipv4_hdr supported_mask = { .src_addr = 0xffffffff, .dst_addr = 0xffffffff, }; FLOW_TRACE(); - if (*inner_ofst) - return ENOTSUP; - if (!mask) mask = &rte_flow_item_ipv4_mask; @@ -384,33 +441,21 @@ enic_copy_item_ipv4_v1(const struct rte_flow_item *item, return 0; } -/** - * Copy UDP item into version 1 NIC filter. - * - * @param item[in] - * Item specification. - * @param enic_filter[out] - * Partially filled in NIC filter structure. - * @param inner_ofst[in] - * Should always be 0 for version 1. - */ static int -enic_copy_item_udp_v1(const struct rte_flow_item *item, - struct filter_v2 *enic_filter, u8 *inner_ofst) +enic_copy_item_udp_v1(struct copy_item_args *arg) { + const struct rte_flow_item *item = arg->item; + struct filter_v2 *enic_filter = arg->filter; const struct rte_flow_item_udp *spec = item->spec; const struct rte_flow_item_udp *mask = item->mask; struct filter_ipv4_5tuple *enic_5tup = &enic_filter->u.ipv4; - struct udp_hdr supported_mask = { + struct rte_udp_hdr supported_mask = { .src_port = 0xffff, .dst_port = 0xffff, }; FLOW_TRACE(); - if (*inner_ofst) - return ENOTSUP; - if (!mask) mask = &rte_flow_item_udp_mask; @@ -435,33 +480,21 @@ enic_copy_item_udp_v1(const struct rte_flow_item *item, return 0; } -/** - * Copy TCP item into version 1 NIC filter. - * - * @param item[in] - * Item specification. - * @param enic_filter[out] - * Partially filled in NIC filter structure. - * @param inner_ofst[in] - * Should always be 0 for version 1. - */ static int -enic_copy_item_tcp_v1(const struct rte_flow_item *item, - struct filter_v2 *enic_filter, u8 *inner_ofst) +enic_copy_item_tcp_v1(struct copy_item_args *arg) { + const struct rte_flow_item *item = arg->item; + struct filter_v2 *enic_filter = arg->filter; const struct rte_flow_item_tcp *spec = item->spec; const struct rte_flow_item_tcp *mask = item->mask; struct filter_ipv4_5tuple *enic_5tup = &enic_filter->u.ipv4; - struct tcp_hdr supported_mask = { + struct rte_tcp_hdr supported_mask = { .src_port = 0xffff, .dst_port = 0xffff, }; FLOW_TRACE(); - if (*inner_ofst) - return ENOTSUP; - if (!mask) mask = &rte_flow_item_tcp_mask; @@ -486,23 +519,152 @@ enic_copy_item_tcp_v1(const struct rte_flow_item *item, return 0; } -/** - * Copy ETH item into version 2 NIC filter. - * - * @param item[in] - * Item specification. - * @param enic_filter[out] - * Partially filled in NIC filter structure. - * @param inner_ofst[in] - * If zero, this is an outer header. If non-zero, this is the offset into L5 - * where the header begins. +/* + * The common 'copy' function for all inner packet patterns. Patterns are + * first appended to the L5 pattern buffer. Then, since the NIC filter + * API has no special support for inner packet matching at the moment, + * we set EtherType and IP proto as necessary. */ static int -enic_copy_item_eth_v2(const struct rte_flow_item *item, - struct filter_v2 *enic_filter, u8 *inner_ofst) +copy_inner_common(struct filter_generic_1 *gp, uint8_t *inner_ofst, + const void *val, const void *mask, uint8_t val_size, + uint8_t proto_off, uint16_t proto_val, uint8_t proto_size) +{ + uint8_t *l5_mask, *l5_val; + uint8_t start_off; + + /* No space left in the L5 pattern buffer. */ + start_off = *inner_ofst; + if ((start_off + val_size) > FILTER_GENERIC_1_KEY_LEN) + return ENOTSUP; + l5_mask = gp->layer[FILTER_GENERIC_1_L5].mask; + l5_val = gp->layer[FILTER_GENERIC_1_L5].val; + /* Copy the pattern into the L5 buffer. */ + if (val) { + memcpy(l5_mask + start_off, mask, val_size); + memcpy(l5_val + start_off, val, val_size); + } + /* Set the protocol field in the previous header. */ + if (proto_off) { + void *m, *v; + + m = l5_mask + proto_off; + v = l5_val + proto_off; + if (proto_size == 1) { + *(uint8_t *)m = 0xff; + *(uint8_t *)v = (uint8_t)proto_val; + } else if (proto_size == 2) { + *(uint16_t *)m = 0xffff; + *(uint16_t *)v = proto_val; + } + } + /* All inner headers land in L5 buffer even if their spec is null. */ + *inner_ofst += val_size; + return 0; +} + +static int +enic_copy_item_inner_eth_v2(struct copy_item_args *arg) +{ + const void *mask = arg->item->mask; + uint8_t *off = arg->inner_ofst; + + FLOW_TRACE(); + if (!mask) + mask = &rte_flow_item_eth_mask; + arg->l2_proto_off = *off + offsetof(struct rte_ether_hdr, ether_type); + return copy_inner_common(&arg->filter->u.generic_1, off, + arg->item->spec, mask, sizeof(struct rte_ether_hdr), + 0 /* no previous protocol */, 0, 0); +} + +static int +enic_copy_item_inner_vlan_v2(struct copy_item_args *arg) +{ + const void *mask = arg->item->mask; + uint8_t *off = arg->inner_ofst; + uint8_t eth_type_off; + + FLOW_TRACE(); + if (!mask) + mask = &rte_flow_item_vlan_mask; + /* Append vlan header to L5 and set ether type = TPID */ + eth_type_off = arg->l2_proto_off; + arg->l2_proto_off = *off + offsetof(struct rte_vlan_hdr, eth_proto); + return copy_inner_common(&arg->filter->u.generic_1, off, + arg->item->spec, mask, sizeof(struct rte_vlan_hdr), + eth_type_off, rte_cpu_to_be_16(RTE_ETHER_TYPE_VLAN), 2); +} + +static int +enic_copy_item_inner_ipv4_v2(struct copy_item_args *arg) +{ + const void *mask = arg->item->mask; + uint8_t *off = arg->inner_ofst; + + FLOW_TRACE(); + if (!mask) + mask = &rte_flow_item_ipv4_mask; + /* Append ipv4 header to L5 and set ether type = ipv4 */ + arg->l3_proto_off = *off + offsetof(struct rte_ipv4_hdr, next_proto_id); + return copy_inner_common(&arg->filter->u.generic_1, off, + arg->item->spec, mask, sizeof(struct rte_ipv4_hdr), + arg->l2_proto_off, rte_cpu_to_be_16(RTE_ETHER_TYPE_IPv4), 2); +} + +static int +enic_copy_item_inner_ipv6_v2(struct copy_item_args *arg) +{ + const void *mask = arg->item->mask; + uint8_t *off = arg->inner_ofst; + + FLOW_TRACE(); + if (!mask) + mask = &rte_flow_item_ipv6_mask; + /* Append ipv6 header to L5 and set ether type = ipv6 */ + arg->l3_proto_off = *off + offsetof(struct rte_ipv6_hdr, proto); + return copy_inner_common(&arg->filter->u.generic_1, off, + arg->item->spec, mask, sizeof(struct rte_ipv6_hdr), + arg->l2_proto_off, rte_cpu_to_be_16(RTE_ETHER_TYPE_IPv6), 2); +} + +static int +enic_copy_item_inner_udp_v2(struct copy_item_args *arg) +{ + const void *mask = arg->item->mask; + uint8_t *off = arg->inner_ofst; + + FLOW_TRACE(); + if (!mask) + mask = &rte_flow_item_udp_mask; + /* Append udp header to L5 and set ip proto = udp */ + return copy_inner_common(&arg->filter->u.generic_1, off, + arg->item->spec, mask, sizeof(struct rte_udp_hdr), + arg->l3_proto_off, IPPROTO_UDP, 1); +} + +static int +enic_copy_item_inner_tcp_v2(struct copy_item_args *arg) +{ + const void *mask = arg->item->mask; + uint8_t *off = arg->inner_ofst; + + FLOW_TRACE(); + if (!mask) + mask = &rte_flow_item_tcp_mask; + /* Append tcp header to L5 and set ip proto = tcp */ + return copy_inner_common(&arg->filter->u.generic_1, off, + arg->item->spec, mask, sizeof(struct rte_tcp_hdr), + arg->l3_proto_off, IPPROTO_TCP, 1); +} + +static int +enic_copy_item_eth_v2(struct copy_item_args *arg) { - struct ether_hdr enic_spec; - struct ether_hdr enic_mask; + const struct rte_flow_item *item = arg->item; + struct filter_v2 *enic_filter = arg->filter; + struct rte_ether_hdr enic_spec; + struct rte_ether_hdr enic_mask; const struct rte_flow_item_eth *spec = item->spec; const struct rte_flow_item_eth *mask = item->mask; struct filter_generic_1 *gp = &enic_filter->u.generic_1; @@ -517,56 +679,35 @@ enic_copy_item_eth_v2(const struct rte_flow_item *item, mask = &rte_flow_item_eth_mask; memcpy(enic_spec.d_addr.addr_bytes, spec->dst.addr_bytes, - ETHER_ADDR_LEN); + RTE_ETHER_ADDR_LEN); memcpy(enic_spec.s_addr.addr_bytes, spec->src.addr_bytes, - ETHER_ADDR_LEN); + RTE_ETHER_ADDR_LEN); memcpy(enic_mask.d_addr.addr_bytes, mask->dst.addr_bytes, - ETHER_ADDR_LEN); + RTE_ETHER_ADDR_LEN); memcpy(enic_mask.s_addr.addr_bytes, mask->src.addr_bytes, - ETHER_ADDR_LEN); + RTE_ETHER_ADDR_LEN); enic_spec.ether_type = spec->type; enic_mask.ether_type = mask->type; - if (*inner_ofst == 0) { - /* outer header */ - memcpy(gp->layer[FILTER_GENERIC_1_L2].mask, &enic_mask, - sizeof(struct ether_hdr)); - memcpy(gp->layer[FILTER_GENERIC_1_L2].val, &enic_spec, - sizeof(struct ether_hdr)); - } else { - /* inner header */ - if ((*inner_ofst + sizeof(struct ether_hdr)) > - FILTER_GENERIC_1_KEY_LEN) - return ENOTSUP; - /* Offset into L5 where inner Ethernet header goes */ - memcpy(&gp->layer[FILTER_GENERIC_1_L5].mask[*inner_ofst], - &enic_mask, sizeof(struct ether_hdr)); - memcpy(&gp->layer[FILTER_GENERIC_1_L5].val[*inner_ofst], - &enic_spec, sizeof(struct ether_hdr)); - *inner_ofst += sizeof(struct ether_hdr); - } + /* outer header */ + memcpy(gp->layer[FILTER_GENERIC_1_L2].mask, &enic_mask, + sizeof(struct rte_ether_hdr)); + memcpy(gp->layer[FILTER_GENERIC_1_L2].val, &enic_spec, + sizeof(struct rte_ether_hdr)); return 0; } -/** - * Copy VLAN item into version 2 NIC filter. - * - * @param item[in] - * Item specification. - * @param enic_filter[out] - * Partially filled in NIC filter structure. - * @param inner_ofst[in] - * If zero, this is an outer header. If non-zero, this is the offset into L5 - * where the header begins. - */ static int -enic_copy_item_vlan_v2(const struct rte_flow_item *item, - struct filter_v2 *enic_filter, u8 *inner_ofst) +enic_copy_item_vlan_v2(struct copy_item_args *arg) { + const struct rte_flow_item *item = arg->item; + struct filter_v2 *enic_filter = arg->filter; const struct rte_flow_item_vlan *spec = item->spec; const struct rte_flow_item_vlan *mask = item->mask; struct filter_generic_1 *gp = &enic_filter->u.generic_1; + struct rte_ether_hdr *eth_mask; + struct rte_ether_hdr *eth_val; FLOW_TRACE(); @@ -574,97 +715,75 @@ enic_copy_item_vlan_v2(const struct rte_flow_item *item, if (!spec) return 0; - /* Don't support filtering in tpid */ - if (mask) { - if (mask->tpid != 0) - return ENOTSUP; - } else { + if (!mask) mask = &rte_flow_item_vlan_mask; - RTE_ASSERT(mask->tpid == 0); - } - if (*inner_ofst == 0) { - /* Outer header. Use the vlan mask/val fields */ - gp->mask_vlan = mask->tci; - gp->val_vlan = spec->tci; + eth_mask = (void *)gp->layer[FILTER_GENERIC_1_L2].mask; + eth_val = (void *)gp->layer[FILTER_GENERIC_1_L2].val; + /* Outer TPID cannot be matched */ + if (eth_mask->ether_type) + return ENOTSUP; + /* + * For recent models: + * When packet matching, the VIC always compares vlan-stripped + * L2, regardless of vlan stripping settings. So, the inner type + * from vlan becomes the ether type of the eth header. + * + * Older models w/o hardware vxlan parser have a different + * behavior when vlan stripping is disabled. In this case, + * vlan tag remains in the L2 buffer. + */ + if (!arg->enic->vxlan && !arg->enic->ig_vlan_strip_en) { + struct rte_vlan_hdr *vlan; + + vlan = (struct rte_vlan_hdr *)(eth_mask + 1); + vlan->eth_proto = mask->inner_type; + vlan = (struct rte_vlan_hdr *)(eth_val + 1); + vlan->eth_proto = spec->inner_type; } else { - /* Inner header. Mask/Val start at *inner_ofst into L5 */ - if ((*inner_ofst + sizeof(struct vlan_hdr)) > - FILTER_GENERIC_1_KEY_LEN) - return ENOTSUP; - memcpy(&gp->layer[FILTER_GENERIC_1_L5].mask[*inner_ofst], - mask, sizeof(struct vlan_hdr)); - memcpy(&gp->layer[FILTER_GENERIC_1_L5].val[*inner_ofst], - spec, sizeof(struct vlan_hdr)); - *inner_ofst += sizeof(struct vlan_hdr); + eth_mask->ether_type = mask->inner_type; + eth_val->ether_type = spec->inner_type; } + /* For TCI, use the vlan mask/val fields (little endian). */ + gp->mask_vlan = rte_be_to_cpu_16(mask->tci); + gp->val_vlan = rte_be_to_cpu_16(spec->tci); return 0; } -/** - * Copy IPv4 item into version 2 NIC filter. - * - * @param item[in] - * Item specification. - * @param enic_filter[out] - * Partially filled in NIC filter structure. - * @param inner_ofst[in] - * Must be 0. Don't support inner IPv4 filtering. - */ static int -enic_copy_item_ipv4_v2(const struct rte_flow_item *item, - struct filter_v2 *enic_filter, u8 *inner_ofst) +enic_copy_item_ipv4_v2(struct copy_item_args *arg) { + const struct rte_flow_item *item = arg->item; + struct filter_v2 *enic_filter = arg->filter; const struct rte_flow_item_ipv4 *spec = item->spec; const struct rte_flow_item_ipv4 *mask = item->mask; struct filter_generic_1 *gp = &enic_filter->u.generic_1; FLOW_TRACE(); - if (*inner_ofst == 0) { - /* Match IPv4 */ - gp->mask_flags |= FILTER_GENERIC_1_IPV4; - gp->val_flags |= FILTER_GENERIC_1_IPV4; + /* Match IPv4 */ + gp->mask_flags |= FILTER_GENERIC_1_IPV4; + gp->val_flags |= FILTER_GENERIC_1_IPV4; - /* Match all if no spec */ - if (!spec) - return 0; + /* Match all if no spec */ + if (!spec) + return 0; - if (!mask) - mask = &rte_flow_item_ipv4_mask; + if (!mask) + mask = &rte_flow_item_ipv4_mask; - memcpy(gp->layer[FILTER_GENERIC_1_L3].mask, &mask->hdr, - sizeof(struct ipv4_hdr)); - memcpy(gp->layer[FILTER_GENERIC_1_L3].val, &spec->hdr, - sizeof(struct ipv4_hdr)); - } else { - /* Inner IPv4 header. Mask/Val start at *inner_ofst into L5 */ - if ((*inner_ofst + sizeof(struct ipv4_hdr)) > - FILTER_GENERIC_1_KEY_LEN) - return ENOTSUP; - memcpy(&gp->layer[FILTER_GENERIC_1_L5].mask[*inner_ofst], - mask, sizeof(struct ipv4_hdr)); - memcpy(&gp->layer[FILTER_GENERIC_1_L5].val[*inner_ofst], - spec, sizeof(struct ipv4_hdr)); - *inner_ofst += sizeof(struct ipv4_hdr); - } + memcpy(gp->layer[FILTER_GENERIC_1_L3].mask, &mask->hdr, + sizeof(struct rte_ipv4_hdr)); + memcpy(gp->layer[FILTER_GENERIC_1_L3].val, &spec->hdr, + sizeof(struct rte_ipv4_hdr)); return 0; } -/** - * Copy IPv6 item into version 2 NIC filter. - * - * @param item[in] - * Item specification. - * @param enic_filter[out] - * Partially filled in NIC filter structure. - * @param inner_ofst[in] - * Must be 0. Don't support inner IPv6 filtering. - */ static int -enic_copy_item_ipv6_v2(const struct rte_flow_item *item, - struct filter_v2 *enic_filter, u8 *inner_ofst) +enic_copy_item_ipv6_v2(struct copy_item_args *arg) { + const struct rte_flow_item *item = arg->item; + struct filter_v2 *enic_filter = arg->filter; const struct rte_flow_item_ipv6 *spec = item->spec; const struct rte_flow_item_ipv6 *mask = item->mask; struct filter_generic_1 *gp = &enic_filter->u.generic_1; @@ -682,39 +801,18 @@ enic_copy_item_ipv6_v2(const struct rte_flow_item *item, if (!mask) mask = &rte_flow_item_ipv6_mask; - if (*inner_ofst == 0) { - memcpy(gp->layer[FILTER_GENERIC_1_L3].mask, &mask->hdr, - sizeof(struct ipv6_hdr)); - memcpy(gp->layer[FILTER_GENERIC_1_L3].val, &spec->hdr, - sizeof(struct ipv6_hdr)); - } else { - /* Inner IPv6 header. Mask/Val start at *inner_ofst into L5 */ - if ((*inner_ofst + sizeof(struct ipv6_hdr)) > - FILTER_GENERIC_1_KEY_LEN) - return ENOTSUP; - memcpy(&gp->layer[FILTER_GENERIC_1_L5].mask[*inner_ofst], - mask, sizeof(struct ipv6_hdr)); - memcpy(&gp->layer[FILTER_GENERIC_1_L5].val[*inner_ofst], - spec, sizeof(struct ipv6_hdr)); - *inner_ofst += sizeof(struct ipv6_hdr); - } + memcpy(gp->layer[FILTER_GENERIC_1_L3].mask, &mask->hdr, + sizeof(struct rte_ipv6_hdr)); + memcpy(gp->layer[FILTER_GENERIC_1_L3].val, &spec->hdr, + sizeof(struct rte_ipv6_hdr)); return 0; } -/** - * Copy UDP item into version 2 NIC filter. - * - * @param item[in] - * Item specification. - * @param enic_filter[out] - * Partially filled in NIC filter structure. - * @param inner_ofst[in] - * Must be 0. Don't support inner UDP filtering. - */ static int -enic_copy_item_udp_v2(const struct rte_flow_item *item, - struct filter_v2 *enic_filter, u8 *inner_ofst) +enic_copy_item_udp_v2(struct copy_item_args *arg) { + const struct rte_flow_item *item = arg->item; + struct filter_v2 *enic_filter = arg->filter; const struct rte_flow_item_udp *spec = item->spec; const struct rte_flow_item_udp *mask = item->mask; struct filter_generic_1 *gp = &enic_filter->u.generic_1; @@ -732,39 +830,18 @@ enic_copy_item_udp_v2(const struct rte_flow_item *item, if (!mask) mask = &rte_flow_item_udp_mask; - if (*inner_ofst == 0) { - memcpy(gp->layer[FILTER_GENERIC_1_L4].mask, &mask->hdr, - sizeof(struct udp_hdr)); - memcpy(gp->layer[FILTER_GENERIC_1_L4].val, &spec->hdr, - sizeof(struct udp_hdr)); - } else { - /* Inner IPv6 header. Mask/Val start at *inner_ofst into L5 */ - if ((*inner_ofst + sizeof(struct udp_hdr)) > - FILTER_GENERIC_1_KEY_LEN) - return ENOTSUP; - memcpy(&gp->layer[FILTER_GENERIC_1_L5].mask[*inner_ofst], - mask, sizeof(struct udp_hdr)); - memcpy(&gp->layer[FILTER_GENERIC_1_L5].val[*inner_ofst], - spec, sizeof(struct udp_hdr)); - *inner_ofst += sizeof(struct udp_hdr); - } + memcpy(gp->layer[FILTER_GENERIC_1_L4].mask, &mask->hdr, + sizeof(struct rte_udp_hdr)); + memcpy(gp->layer[FILTER_GENERIC_1_L4].val, &spec->hdr, + sizeof(struct rte_udp_hdr)); return 0; } -/** - * Copy TCP item into version 2 NIC filter. - * - * @param item[in] - * Item specification. - * @param enic_filter[out] - * Partially filled in NIC filter structure. - * @param inner_ofst[in] - * Must be 0. Don't support inner TCP filtering. - */ static int -enic_copy_item_tcp_v2(const struct rte_flow_item *item, - struct filter_v2 *enic_filter, u8 *inner_ofst) +enic_copy_item_tcp_v2(struct copy_item_args *arg) { + const struct rte_flow_item *item = arg->item; + struct filter_v2 *enic_filter = arg->filter; const struct rte_flow_item_tcp *spec = item->spec; const struct rte_flow_item_tcp *mask = item->mask; struct filter_generic_1 *gp = &enic_filter->u.generic_1; @@ -782,47 +859,48 @@ enic_copy_item_tcp_v2(const struct rte_flow_item *item, if (!mask) return ENOTSUP; - if (*inner_ofst == 0) { - memcpy(gp->layer[FILTER_GENERIC_1_L4].mask, &mask->hdr, - sizeof(struct tcp_hdr)); - memcpy(gp->layer[FILTER_GENERIC_1_L4].val, &spec->hdr, - sizeof(struct tcp_hdr)); - } else { - /* Inner IPv6 header. Mask/Val start at *inner_ofst into L5 */ - if ((*inner_ofst + sizeof(struct tcp_hdr)) > - FILTER_GENERIC_1_KEY_LEN) - return ENOTSUP; - memcpy(&gp->layer[FILTER_GENERIC_1_L5].mask[*inner_ofst], - mask, sizeof(struct tcp_hdr)); - memcpy(&gp->layer[FILTER_GENERIC_1_L5].val[*inner_ofst], - spec, sizeof(struct tcp_hdr)); - *inner_ofst += sizeof(struct tcp_hdr); - } + memcpy(gp->layer[FILTER_GENERIC_1_L4].mask, &mask->hdr, + sizeof(struct rte_tcp_hdr)); + memcpy(gp->layer[FILTER_GENERIC_1_L4].val, &spec->hdr, + sizeof(struct rte_tcp_hdr)); return 0; } -/** - * Copy SCTP item into version 2 NIC filter. - * - * @param item[in] - * Item specification. - * @param enic_filter[out] - * Partially filled in NIC filter structure. - * @param inner_ofst[in] - * Must be 0. Don't support inner SCTP filtering. - */ static int -enic_copy_item_sctp_v2(const struct rte_flow_item *item, - struct filter_v2 *enic_filter, u8 *inner_ofst) +enic_copy_item_sctp_v2(struct copy_item_args *arg) { + const struct rte_flow_item *item = arg->item; + struct filter_v2 *enic_filter = arg->filter; const struct rte_flow_item_sctp *spec = item->spec; const struct rte_flow_item_sctp *mask = item->mask; struct filter_generic_1 *gp = &enic_filter->u.generic_1; + uint8_t *ip_proto_mask = NULL; + uint8_t *ip_proto = NULL; FLOW_TRACE(); - if (*inner_ofst) - return ENOTSUP; + /* + * The NIC filter API has no flags for "match sctp", so explicitly set + * the protocol number in the IP pattern. + */ + if (gp->val_flags & FILTER_GENERIC_1_IPV4) { + struct rte_ipv4_hdr *ip; + ip = (struct rte_ipv4_hdr *)gp->layer[FILTER_GENERIC_1_L3].mask; + ip_proto_mask = &ip->next_proto_id; + ip = (struct rte_ipv4_hdr *)gp->layer[FILTER_GENERIC_1_L3].val; + ip_proto = &ip->next_proto_id; + } else if (gp->val_flags & FILTER_GENERIC_1_IPV6) { + struct rte_ipv6_hdr *ip; + ip = (struct rte_ipv6_hdr *)gp->layer[FILTER_GENERIC_1_L3].mask; + ip_proto_mask = &ip->proto; + ip = (struct rte_ipv6_hdr *)gp->layer[FILTER_GENERIC_1_L3].val; + ip_proto = &ip->proto; + } else { + /* Need IPv4/IPv6 pattern first */ + return EINVAL; + } + *ip_proto = IPPROTO_SCTP; + *ip_proto_mask = 0xff; /* Match all if no spec */ if (!spec) @@ -832,35 +910,35 @@ enic_copy_item_sctp_v2(const struct rte_flow_item *item, mask = &rte_flow_item_sctp_mask; memcpy(gp->layer[FILTER_GENERIC_1_L4].mask, &mask->hdr, - sizeof(struct sctp_hdr)); + sizeof(struct rte_sctp_hdr)); memcpy(gp->layer[FILTER_GENERIC_1_L4].val, &spec->hdr, - sizeof(struct sctp_hdr)); + sizeof(struct rte_sctp_hdr)); return 0; } -/** - * Copy UDP item into version 2 NIC filter. - * - * @param item[in] - * Item specification. - * @param enic_filter[out] - * Partially filled in NIC filter structure. - * @param inner_ofst[in] - * Must be 0. VxLAN headers always start at the beginning of L5. - */ static int -enic_copy_item_vxlan_v2(const struct rte_flow_item *item, - struct filter_v2 *enic_filter, u8 *inner_ofst) +enic_copy_item_vxlan_v2(struct copy_item_args *arg) { + const struct rte_flow_item *item = arg->item; + struct filter_v2 *enic_filter = arg->filter; + uint8_t *inner_ofst = arg->inner_ofst; const struct rte_flow_item_vxlan *spec = item->spec; const struct rte_flow_item_vxlan *mask = item->mask; struct filter_generic_1 *gp = &enic_filter->u.generic_1; + struct rte_udp_hdr *udp; FLOW_TRACE(); - if (*inner_ofst) - return EINVAL; - + /* + * The NIC filter API has no flags for "match vxlan". Set UDP port to + * avoid false positives. + */ + gp->mask_flags |= FILTER_GENERIC_1_UDP; + gp->val_flags |= FILTER_GENERIC_1_UDP; + udp = (struct rte_udp_hdr *)gp->layer[FILTER_GENERIC_1_L4].mask; + udp->dst_port = 0xffff; + udp = (struct rte_udp_hdr *)gp->layer[FILTER_GENERIC_1_L4].val; + udp->dst_port = RTE_BE16(4789); /* Match all if no spec */ if (!spec) return 0; @@ -869,11 +947,60 @@ enic_copy_item_vxlan_v2(const struct rte_flow_item *item, mask = &rte_flow_item_vxlan_mask; memcpy(gp->layer[FILTER_GENERIC_1_L5].mask, mask, - sizeof(struct vxlan_hdr)); + sizeof(struct rte_vxlan_hdr)); memcpy(gp->layer[FILTER_GENERIC_1_L5].val, spec, - sizeof(struct vxlan_hdr)); + sizeof(struct rte_vxlan_hdr)); + + *inner_ofst = sizeof(struct rte_vxlan_hdr); + return 0; +} + +/* + * Copy raw item into version 2 NIC filter. Currently, raw pattern match is + * very limited. It is intended for matching UDP tunnel header (e.g. vxlan + * or geneve). + */ +static int +enic_copy_item_raw_v2(struct copy_item_args *arg) +{ + const struct rte_flow_item *item = arg->item; + struct filter_v2 *enic_filter = arg->filter; + uint8_t *inner_ofst = arg->inner_ofst; + const struct rte_flow_item_raw *spec = item->spec; + const struct rte_flow_item_raw *mask = item->mask; + struct filter_generic_1 *gp = &enic_filter->u.generic_1; + + FLOW_TRACE(); + + /* Cannot be used for inner packet */ + if (*inner_ofst) + return EINVAL; + /* Need both spec and mask */ + if (!spec || !mask) + return EINVAL; + /* Only supports relative with offset 0 */ + if (!spec->relative || spec->offset != 0 || spec->search || spec->limit) + return EINVAL; + /* Need non-null pattern that fits within the NIC's filter pattern */ + if (spec->length == 0 || + spec->length + sizeof(struct rte_udp_hdr) > FILTER_GENERIC_1_KEY_LEN || + !spec->pattern || !mask->pattern) + return EINVAL; + /* + * Mask fields, including length, are often set to zero. Assume that + * means "same as spec" to avoid breaking existing apps. If length + * is not zero, then it should be >= spec length. + * + * No more pattern follows this, so append to the L4 layer instead of + * L5 to work with both recent and older VICs. + */ + if (mask->length != 0 && mask->length < spec->length) + return EINVAL; + memcpy(gp->layer[FILTER_GENERIC_1_L4].mask + sizeof(struct rte_udp_hdr), + mask->pattern, spec->length); + memcpy(gp->layer[FILTER_GENERIC_1_L4].val + sizeof(struct rte_udp_hdr), + spec->pattern, spec->length); - *inner_ofst = sizeof(struct vxlan_hdr); return 0; } @@ -908,6 +1035,36 @@ item_stacking_valid(enum rte_flow_item_type prev_item, return 0; } +/* + * Fix up the L5 layer.. HW vxlan parsing removes vxlan header from L5. + * Instead it is in L4 following the UDP header. Append the vxlan + * pattern to L4 (udp) and shift any inner packet pattern in L5. + */ +static void +fixup_l5_layer(struct enic *enic, struct filter_generic_1 *gp, + uint8_t inner_ofst) +{ + uint8_t layer[FILTER_GENERIC_1_KEY_LEN]; + uint8_t inner; + uint8_t vxlan; + + if (!(inner_ofst > 0 && enic->vxlan)) + return; + FLOW_TRACE(); + vxlan = sizeof(struct rte_vxlan_hdr); + memcpy(gp->layer[FILTER_GENERIC_1_L4].mask + sizeof(struct rte_udp_hdr), + gp->layer[FILTER_GENERIC_1_L5].mask, vxlan); + memcpy(gp->layer[FILTER_GENERIC_1_L4].val + sizeof(struct rte_udp_hdr), + gp->layer[FILTER_GENERIC_1_L5].val, vxlan); + inner = inner_ofst - vxlan; + memset(layer, 0, sizeof(layer)); + memcpy(layer, gp->layer[FILTER_GENERIC_1_L5].mask + vxlan, inner); + memcpy(gp->layer[FILTER_GENERIC_1_L5].mask, layer, sizeof(layer)); + memset(layer, 0, sizeof(layer)); + memcpy(layer, gp->layer[FILTER_GENERIC_1_L5].val + vxlan, inner); + memcpy(gp->layer[FILTER_GENERIC_1_L5].val, layer, sizeof(layer)); +} + /** * Build the intenal enic filter structure from the provided pattern. The * pattern is validated as the items are copied. @@ -921,7 +1078,8 @@ item_stacking_valid(enum rte_flow_item_type prev_item, */ static int enic_copy_filter(const struct rte_flow_item pattern[], - const struct enic_items *items_info, + const struct enic_filter_cap *cap, + struct enic *enic, struct filter_v2 *enic_filter, struct rte_flow_error *error) { @@ -930,13 +1088,17 @@ enic_copy_filter(const struct rte_flow_item pattern[], u8 inner_ofst = 0; /* If encapsulated, ofst into L5 */ enum rte_flow_item_type prev_item; const struct enic_items *item_info; - + struct copy_item_args args; + enic_copy_item_fn *copy_fn; u8 is_first_item = 1; FLOW_TRACE(); prev_item = 0; + args.filter = enic_filter; + args.inner_ofst = &inner_ofst; + args.enic = enic; for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) { /* Get info about how to validate and copy the item. If NULL * is returned the nic does not support the item. @@ -944,18 +1106,31 @@ enic_copy_filter(const struct rte_flow_item pattern[], if (item->type == RTE_FLOW_ITEM_TYPE_VOID) continue; - item_info = &items_info[item->type]; + item_info = &cap->item_info[item->type]; + if (item->type > cap->max_item_type || + item_info->copy_item == NULL || + (inner_ofst > 0 && item_info->inner_copy_item == NULL)) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "Unsupported item."); + return -rte_errno; + } /* check to see if item stacking is valid */ if (!item_stacking_valid(prev_item, item_info, is_first_item)) goto stacking_error; - ret = item_info->copy_item(item, enic_filter, &inner_ofst); + args.item = item; + copy_fn = inner_ofst > 0 ? item_info->inner_copy_item : + item_info->copy_item; + ret = copy_fn(&args); if (ret) goto item_not_supported; prev_item = item->type; is_first_item = 0; } + fixup_l5_layer(enic, &enic_filter->u.generic_1, inner_ofst); + return 0; item_not_supported: @@ -979,9 +1154,13 @@ stacking_error: * @param error[out] */ static int -enic_copy_action_v1(const struct rte_flow_action actions[], +enic_copy_action_v1(__rte_unused struct enic *enic, + const struct rte_flow_action actions[], struct filter_action_v2 *enic_action) { + enum { FATE = 1, }; + uint32_t overlap = 0; + FLOW_TRACE(); for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { @@ -993,6 +1172,10 @@ enic_copy_action_v1(const struct rte_flow_action actions[], const struct rte_flow_action_queue *queue = (const struct rte_flow_action_queue *) actions->conf; + + if (overlap & FATE) + return ENOTSUP; + overlap |= FATE; enic_action->rq_idx = enic_rte_rq_idx_to_sop_idx(queue->index); break; @@ -1002,6 +1185,8 @@ enic_copy_action_v1(const struct rte_flow_action actions[], break; } } + if (!(overlap & FATE)) + return ENOTSUP; enic_action->type = FILTER_ACTION_RQ_STEERING; return 0; } @@ -1016,9 +1201,14 @@ enic_copy_action_v1(const struct rte_flow_action actions[], * @param error[out] */ static int -enic_copy_action_v2(const struct rte_flow_action actions[], +enic_copy_action_v2(struct enic *enic, + const struct rte_flow_action actions[], struct filter_action_v2 *enic_action) { + enum { FATE = 1, MARK = 2, }; + uint32_t overlap = 0; + bool passthru = false; + FLOW_TRACE(); for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { @@ -1027,6 +1217,10 @@ enic_copy_action_v2(const struct rte_flow_action actions[], const struct rte_flow_action_queue *queue = (const struct rte_flow_action_queue *) actions->conf; + + if (overlap & FATE) + return ENOTSUP; + overlap |= FATE; enic_action->rq_idx = enic_rte_rq_idx_to_sop_idx(queue->index); enic_action->flags |= FILTER_ACTION_RQ_STEERING_FLAG; @@ -1037,20 +1231,88 @@ enic_copy_action_v2(const struct rte_flow_action actions[], (const struct rte_flow_action_mark *) actions->conf; - /* ENIC_MAGIC_FILTER_ID is reserved and is the highest - * in the range of allows mark ids. + if (overlap & MARK) + return ENOTSUP; + overlap |= MARK; + /* + * Map mark ID (32-bit) to filter ID (16-bit): + * - Reject values > 16 bits + * - Filter ID 0 is reserved for filters that steer + * but not mark. So add 1 to the mark ID to avoid + * using 0. + * - Filter ID (ENIC_MAGIC_FILTER_ID = 0xffff) is + * reserved for the "flag" action below. */ - if (mark->id >= ENIC_MAGIC_FILTER_ID) + if (mark->id >= ENIC_MAGIC_FILTER_ID - 1) return EINVAL; - enic_action->filter_id = mark->id; + enic_action->filter_id = mark->id + 1; enic_action->flags |= FILTER_ACTION_FILTER_ID_FLAG; break; } case RTE_FLOW_ACTION_TYPE_FLAG: { + if (overlap & MARK) + return ENOTSUP; + overlap |= MARK; + /* ENIC_MAGIC_FILTER_ID is reserved for flagging */ enic_action->filter_id = ENIC_MAGIC_FILTER_ID; enic_action->flags |= FILTER_ACTION_FILTER_ID_FLAG; break; } + case RTE_FLOW_ACTION_TYPE_DROP: { + if (overlap & FATE) + return ENOTSUP; + overlap |= FATE; + enic_action->flags |= FILTER_ACTION_DROP_FLAG; + break; + } + case RTE_FLOW_ACTION_TYPE_COUNT: { + enic_action->flags |= FILTER_ACTION_COUNTER_FLAG; + break; + } + case RTE_FLOW_ACTION_TYPE_RSS: { + const struct rte_flow_action_rss *rss = + (const struct rte_flow_action_rss *) + actions->conf; + bool allow; + uint16_t i; + + /* + * Hardware does not support general RSS actions, but + * we can still support the dummy one that is used to + * "receive normally". + */ + allow = rss->func == RTE_ETH_HASH_FUNCTION_DEFAULT && + rss->level == 0 && + (rss->types == 0 || + rss->types == enic->rss_hf) && + rss->queue_num == enic->rq_count && + rss->key_len == 0; + /* Identity queue map is ok */ + for (i = 0; i < rss->queue_num; i++) + allow = allow && (i == rss->queue[i]); + if (!allow) + return ENOTSUP; + if (overlap & FATE) + return ENOTSUP; + /* Need MARK or FLAG */ + if (!(overlap & MARK)) + return ENOTSUP; + overlap |= FATE; + break; + } + case RTE_FLOW_ACTION_TYPE_PASSTHRU: { + /* + * Like RSS above, PASSTHRU + MARK may be used to + * "mark and then receive normally". MARK usually comes + * after PASSTHRU, so remember we have seen passthru + * and check for mark later. + */ + if (overlap & FATE) + return ENOTSUP; + overlap |= FATE; + passthru = true; + break; + } case RTE_FLOW_ACTION_TYPE_VOID: continue; default: @@ -1058,6 +1320,11 @@ enic_copy_action_v2(const struct rte_flow_action actions[], break; } } + /* Only PASSTHRU + MARK is allowed */ + if (passthru && !(overlap & MARK)) + return ENOTSUP; + if (!(overlap & FATE)) + return ENOTSUP; enic_action->type = FILTER_ACTION_V2; return 0; } @@ -1089,14 +1356,156 @@ enic_get_filter_cap(struct enic *enic) static const struct enic_action_cap * enic_get_action_cap(struct enic *enic) { - static const struct enic_action_cap *ea; - - if (enic->filter_tags) - ea = &enic_action_cap[FILTER_ACTION_V2_ALL]; + const struct enic_action_cap *ea; + uint8_t actions; + + actions = enic->filter_actions; + if (actions & FILTER_ACTION_COUNTER_FLAG) + ea = &enic_action_cap[FILTER_ACTION_COUNTER_FLAG]; + else if (actions & FILTER_ACTION_DROP_FLAG) + ea = &enic_action_cap[FILTER_ACTION_DROP_FLAG]; + else if (actions & FILTER_ACTION_FILTER_ID_FLAG) + ea = &enic_action_cap[FILTER_ACTION_FILTER_ID_FLAG]; else ea = &enic_action_cap[FILTER_ACTION_RQ_STEERING_FLAG]; return ea; } + +/* Debug function to dump internal NIC action structure. */ +static void +enic_dump_actions(const struct filter_action_v2 *ea) +{ + if (ea->type == FILTER_ACTION_RQ_STEERING) { + FLOW_LOG(INFO, "Action(V1), queue: %u\n", ea->rq_idx); + } else if (ea->type == FILTER_ACTION_V2) { + FLOW_LOG(INFO, "Actions(V2)\n"); + if (ea->flags & FILTER_ACTION_RQ_STEERING_FLAG) + FLOW_LOG(INFO, "\tqueue: %u\n", + enic_sop_rq_idx_to_rte_idx(ea->rq_idx)); + if (ea->flags & FILTER_ACTION_FILTER_ID_FLAG) + FLOW_LOG(INFO, "\tfilter_id: %u\n", ea->filter_id); + } +} + +/* Debug function to dump internal NIC filter structure. */ +static void +enic_dump_filter(const struct filter_v2 *filt) +{ + const struct filter_generic_1 *gp; + int i, j, mbyte; + char buf[128], *bp; + char ip4[16], ip6[16], udp[16], tcp[16], tcpudp[16], ip4csum[16]; + char l4csum[16], ipfrag[16]; + + switch (filt->type) { + case FILTER_IPV4_5TUPLE: + FLOW_LOG(INFO, "FILTER_IPV4_5TUPLE\n"); + break; + case FILTER_USNIC_IP: + case FILTER_DPDK_1: + /* FIXME: this should be a loop */ + gp = &filt->u.generic_1; + FLOW_LOG(INFO, "Filter: vlan: 0x%04x, mask: 0x%04x\n", + gp->val_vlan, gp->mask_vlan); + + if (gp->mask_flags & FILTER_GENERIC_1_IPV4) + sprintf(ip4, "%s ", + (gp->val_flags & FILTER_GENERIC_1_IPV4) + ? "ip4(y)" : "ip4(n)"); + else + sprintf(ip4, "%s ", "ip4(x)"); + + if (gp->mask_flags & FILTER_GENERIC_1_IPV6) + sprintf(ip6, "%s ", + (gp->val_flags & FILTER_GENERIC_1_IPV4) + ? "ip6(y)" : "ip6(n)"); + else + sprintf(ip6, "%s ", "ip6(x)"); + + if (gp->mask_flags & FILTER_GENERIC_1_UDP) + sprintf(udp, "%s ", + (gp->val_flags & FILTER_GENERIC_1_UDP) + ? "udp(y)" : "udp(n)"); + else + sprintf(udp, "%s ", "udp(x)"); + + if (gp->mask_flags & FILTER_GENERIC_1_TCP) + sprintf(tcp, "%s ", + (gp->val_flags & FILTER_GENERIC_1_TCP) + ? "tcp(y)" : "tcp(n)"); + else + sprintf(tcp, "%s ", "tcp(x)"); + + if (gp->mask_flags & FILTER_GENERIC_1_TCP_OR_UDP) + sprintf(tcpudp, "%s ", + (gp->val_flags & FILTER_GENERIC_1_TCP_OR_UDP) + ? "tcpudp(y)" : "tcpudp(n)"); + else + sprintf(tcpudp, "%s ", "tcpudp(x)"); + + if (gp->mask_flags & FILTER_GENERIC_1_IP4SUM_OK) + sprintf(ip4csum, "%s ", + (gp->val_flags & FILTER_GENERIC_1_IP4SUM_OK) + ? "ip4csum(y)" : "ip4csum(n)"); + else + sprintf(ip4csum, "%s ", "ip4csum(x)"); + + if (gp->mask_flags & FILTER_GENERIC_1_L4SUM_OK) + sprintf(l4csum, "%s ", + (gp->val_flags & FILTER_GENERIC_1_L4SUM_OK) + ? "l4csum(y)" : "l4csum(n)"); + else + sprintf(l4csum, "%s ", "l4csum(x)"); + + if (gp->mask_flags & FILTER_GENERIC_1_IPFRAG) + sprintf(ipfrag, "%s ", + (gp->val_flags & FILTER_GENERIC_1_IPFRAG) + ? "ipfrag(y)" : "ipfrag(n)"); + else + sprintf(ipfrag, "%s ", "ipfrag(x)"); + FLOW_LOG(INFO, "\tFlags: %s%s%s%s%s%s%s%s\n", ip4, ip6, udp, + tcp, tcpudp, ip4csum, l4csum, ipfrag); + + for (i = 0; i < FILTER_GENERIC_1_NUM_LAYERS; i++) { + mbyte = FILTER_GENERIC_1_KEY_LEN - 1; + while (mbyte && !gp->layer[i].mask[mbyte]) + mbyte--; + if (mbyte == 0) + continue; + + bp = buf; + for (j = 0; j <= mbyte; j++) { + sprintf(bp, "%02x", + gp->layer[i].mask[j]); + bp += 2; + } + *bp = '\0'; + FLOW_LOG(INFO, "\tL%u mask: %s\n", i + 2, buf); + bp = buf; + for (j = 0; j <= mbyte; j++) { + sprintf(bp, "%02x", + gp->layer[i].val[j]); + bp += 2; + } + *bp = '\0'; + FLOW_LOG(INFO, "\tL%u val: %s\n", i + 2, buf); + } + break; + default: + FLOW_LOG(INFO, "FILTER UNKNOWN\n"); + break; + } +} + +/* Debug function to dump internal NIC flow structures. */ +static void +enic_dump_flow(const struct filter_action_v2 *ea, const struct filter_v2 *filt) +{ + enic_dump_filter(filt); + enic_dump_actions(ea); +} + + /** * Internal flow parse/validate function. * @@ -1162,6 +1571,12 @@ enic_flow_parse(struct rte_eth_dev *dev, NULL, "egress is not supported"); return -rte_errno; + } else if (attrs->transfer) { + rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, + NULL, + "transfer is not supported"); + return -rte_errno; } else if (!attrs->ingress) { rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_INGRESS, @@ -1191,7 +1606,7 @@ enic_flow_parse(struct rte_eth_dev *dev, action, "Invalid action."); return -rte_errno; } - ret = enic_action_cap->copy_fn(actions, enic_action); + ret = enic_action_cap->copy_fn(enic, actions, enic_action); if (ret) { rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, "Unsupported action."); @@ -1208,7 +1623,7 @@ enic_flow_parse(struct rte_eth_dev *dev, return -rte_errno; } enic_filter->type = enic->flow_filter_mode; - ret = enic_copy_filter(pattern, enic_filter_cap->item_info, + ret = enic_copy_filter(pattern, enic_filter_cap, enic, enic_filter, error); return ret; } @@ -1230,8 +1645,10 @@ enic_flow_add_filter(struct enic *enic, struct filter_v2 *enic_filter, struct rte_flow_error *error) { struct rte_flow *flow; - int ret; - u16 entry; + int err; + uint16_t entry; + int ctr_idx; + int last_max_flow_ctr; FLOW_TRACE(); @@ -1242,20 +1659,64 @@ enic_flow_add_filter(struct enic *enic, struct filter_v2 *enic_filter, return NULL; } + flow->counter_idx = -1; + last_max_flow_ctr = -1; + if (enic_action->flags & FILTER_ACTION_COUNTER_FLAG) { + if (!vnic_dev_counter_alloc(enic->vdev, (uint32_t *)&ctr_idx)) { + rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_ACTION_CONF, + NULL, "cannot allocate counter"); + goto unwind_flow_alloc; + } + flow->counter_idx = ctr_idx; + enic_action->counter_index = ctr_idx; + + /* If index is the largest, increase the counter DMA size */ + if (ctr_idx > enic->max_flow_counter) { + err = vnic_dev_counter_dma_cfg(enic->vdev, + VNIC_FLOW_COUNTER_UPDATE_MSECS, + ctr_idx + 1); + if (err) { + rte_flow_error_set(error, -err, + RTE_FLOW_ERROR_TYPE_ACTION_CONF, + NULL, "counter DMA config failed"); + goto unwind_ctr_alloc; + } + last_max_flow_ctr = enic->max_flow_counter; + enic->max_flow_counter = ctr_idx; + } + } + /* entry[in] is the queue id, entry[out] is the filter Id for delete */ entry = enic_action->rq_idx; - ret = vnic_dev_classifier(enic->vdev, CLSF_ADD, &entry, enic_filter, + err = vnic_dev_classifier(enic->vdev, CLSF_ADD, &entry, enic_filter, enic_action); - if (!ret) { - flow->enic_filter_id = entry; - flow->enic_filter = *enic_filter; - } else { - rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_HANDLE, + if (err) { + rte_flow_error_set(error, -err, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, "vnic_dev_classifier error"); - rte_free(flow); - return NULL; + goto unwind_ctr_dma_cfg; } + + flow->enic_filter_id = entry; + flow->enic_filter = *enic_filter; + return flow; + +/* unwind if there are errors */ +unwind_ctr_dma_cfg: + if (last_max_flow_ctr != -1) { + /* reduce counter DMA size */ + vnic_dev_counter_dma_cfg(enic->vdev, + VNIC_FLOW_COUNTER_UPDATE_MSECS, + last_max_flow_ctr + 1); + enic->max_flow_counter = last_max_flow_ctr; + } +unwind_ctr_alloc: + if (flow->counter_idx != -1) + vnic_dev_counter_free(enic->vdev, ctr_idx); +unwind_flow_alloc: + rte_free(flow); + return NULL; } /** @@ -1270,18 +1731,29 @@ enic_flow_add_filter(struct enic *enic, struct filter_v2 *enic_filter, * @param error[out] */ static int -enic_flow_del_filter(struct enic *enic, u16 filter_id, +enic_flow_del_filter(struct enic *enic, struct rte_flow *flow, struct rte_flow_error *error) { - int ret; + u16 filter_id; + int err; FLOW_TRACE(); - ret = vnic_dev_classifier(enic->vdev, CLSF_DEL, &filter_id, NULL, NULL); - if (!ret) - rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_HANDLE, + filter_id = flow->enic_filter_id; + err = vnic_dev_classifier(enic->vdev, CLSF_DEL, &filter_id, NULL, NULL); + if (err) { + rte_flow_error_set(error, -err, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, "vnic_dev_classifier failed"); - return ret; + return -err; + } + + if (flow->counter_idx != -1) { + if (!vnic_dev_counter_free(enic->vdev, flow->counter_idx)) + dev_err(enic, "counter free failed, idx: %d\n", + flow->counter_idx); + flow->counter_idx = -1; + } + return 0; } /* @@ -1308,6 +1780,8 @@ enic_flow_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attrs, ret = enic_flow_parse(dev, attrs, pattern, actions, error, &enic_filter, &enic_action); + if (!ret) + enic_dump_flow(&enic_action, &enic_filter); return ret; } @@ -1362,9 +1836,10 @@ enic_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow, FLOW_TRACE(); rte_spinlock_lock(&enic->flows_lock); - enic_flow_del_filter(enic, flow->enic_filter_id, error); + enic_flow_del_filter(enic, flow, error); LIST_REMOVE(flow, next); rte_spinlock_unlock(&enic->flows_lock); + rte_free(flow); return 0; } @@ -1386,13 +1861,77 @@ enic_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error) while (!LIST_EMPTY(&enic->flows)) { flow = LIST_FIRST(&enic->flows); - enic_flow_del_filter(enic, flow->enic_filter_id, error); + enic_flow_del_filter(enic, flow, error); LIST_REMOVE(flow, next); + rte_free(flow); } rte_spinlock_unlock(&enic->flows_lock); return 0; } +static int +enic_flow_query_count(struct rte_eth_dev *dev, + struct rte_flow *flow, void *data, + struct rte_flow_error *error) +{ + struct enic *enic = pmd_priv(dev); + struct rte_flow_query_count *query; + uint64_t packets, bytes; + + FLOW_TRACE(); + + if (flow->counter_idx == -1) { + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "flow does not have counter"); + } + query = (struct rte_flow_query_count *)data; + if (!vnic_dev_counter_query(enic->vdev, flow->counter_idx, + !!query->reset, &packets, &bytes)) { + return rte_flow_error_set + (error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "cannot read counter"); + } + query->hits_set = 1; + query->bytes_set = 1; + query->hits = packets; + query->bytes = bytes; + return 0; +} + +static int +enic_flow_query(struct rte_eth_dev *dev, + struct rte_flow *flow, + const struct rte_flow_action *actions, + void *data, + struct rte_flow_error *error) +{ + int ret = 0; + + FLOW_TRACE(); + + for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { + switch (actions->type) { + case RTE_FLOW_ACTION_TYPE_VOID: + break; + case RTE_FLOW_ACTION_TYPE_COUNT: + ret = enic_flow_query_count(dev, flow, data, error); + break; + default: + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ACTION, + actions, + "action not supported"); + } + if (ret < 0) + return ret; + } + return 0; +} + /** * Flow callback registration. * @@ -1403,4 +1942,5 @@ const struct rte_flow_ops enic_flow_ops = { .create = enic_flow_create, .destroy = enic_flow_destroy, .flush = enic_flow_flush, + .query = enic_flow_query, };