4 * Copyright 2017 6WIND S.A.
5 * Copyright 2017 Mellanox.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 #include <sys/queue.h>
38 #include <sys/resource.h>
40 #include <rte_byteorder.h>
41 #include <rte_jhash.h>
42 #include <rte_malloc.h>
43 #include <rte_eth_tap.h>
45 #include <tap_autoconf.h>
46 #include <tap_tcmsgs.h>
49 #ifndef HAVE_TC_FLOWER
51 * For kernels < 4.2, this enum is not defined. Runtime checks will be made to
52 * avoid sending TC messages the kernel cannot understand.
59 TCA_FLOWER_KEY_ETH_DST, /* ETH_ALEN */
60 TCA_FLOWER_KEY_ETH_DST_MASK, /* ETH_ALEN */
61 TCA_FLOWER_KEY_ETH_SRC, /* ETH_ALEN */
62 TCA_FLOWER_KEY_ETH_SRC_MASK, /* ETH_ALEN */
63 TCA_FLOWER_KEY_ETH_TYPE, /* be16 */
64 TCA_FLOWER_KEY_IP_PROTO, /* u8 */
65 TCA_FLOWER_KEY_IPV4_SRC, /* be32 */
66 TCA_FLOWER_KEY_IPV4_SRC_MASK, /* be32 */
67 TCA_FLOWER_KEY_IPV4_DST, /* be32 */
68 TCA_FLOWER_KEY_IPV4_DST_MASK, /* be32 */
69 TCA_FLOWER_KEY_IPV6_SRC, /* struct in6_addr */
70 TCA_FLOWER_KEY_IPV6_SRC_MASK, /* struct in6_addr */
71 TCA_FLOWER_KEY_IPV6_DST, /* struct in6_addr */
72 TCA_FLOWER_KEY_IPV6_DST_MASK, /* struct in6_addr */
73 TCA_FLOWER_KEY_TCP_SRC, /* be16 */
74 TCA_FLOWER_KEY_TCP_DST, /* be16 */
75 TCA_FLOWER_KEY_UDP_SRC, /* be16 */
76 TCA_FLOWER_KEY_UDP_DST, /* be16 */
79 #ifndef HAVE_TC_VLAN_ID
81 /* TCA_FLOWER_FLAGS, */
82 TCA_FLOWER_KEY_VLAN_ID = TCA_FLOWER_KEY_UDP_DST + 2, /* be16 */
83 TCA_FLOWER_KEY_VLAN_PRIO, /* u8 */
84 TCA_FLOWER_KEY_VLAN_ETH_TYPE, /* be16 */
88 * For kernels < 4.2 BPF related enums may not be defined.
89 * Runtime checks will be carried out to gracefully report on TC messages that
90 * are rejected by the kernel. Rejection reasons may be due to:
91 * 1. enum is not defined
92 * 2. enum is defined but kernel is not configured to support BPF system calls,
93 * BPF classifications or BPF actions.
105 #ifndef HAVE_TC_BPF_FD
107 TCA_BPF_FD = TCA_BPF_OPS + 1,
111 #ifndef HAVE_TC_ACT_BPF
132 #ifndef HAVE_TC_ACT_BPF_FD
134 TCA_ACT_BPF_FD = TCA_ACT_BPF_OPS + 1,
139 /* RSS key management */
153 #define ISOLATE_HANDLE 1
156 LIST_ENTRY(rte_flow) next; /* Pointer to the next rte_flow structure */
157 struct rte_flow *remote_flow; /* associated remote flow */
158 int bpf_fd[SEC_MAX]; /* list of bfs fds per ELF section */
159 uint32_t key_idx; /* RSS rule key index into BPF map */
163 struct convert_data {
167 struct rte_flow *flow;
171 struct rte_flow_attr attr;
172 struct rte_flow_item items[2];
173 struct rte_flow_action actions[2];
182 struct tc_mirred mirred;
184 struct tc_skbedit skbedit;
188 struct tc_act_bpf bpf;
190 const char *annotation;
195 static int tap_flow_create_eth(const struct rte_flow_item *item, void *data);
196 static int tap_flow_create_vlan(const struct rte_flow_item *item, void *data);
197 static int tap_flow_create_ipv4(const struct rte_flow_item *item, void *data);
198 static int tap_flow_create_ipv6(const struct rte_flow_item *item, void *data);
199 static int tap_flow_create_udp(const struct rte_flow_item *item, void *data);
200 static int tap_flow_create_tcp(const struct rte_flow_item *item, void *data);
202 tap_flow_validate(struct rte_eth_dev *dev,
203 const struct rte_flow_attr *attr,
204 const struct rte_flow_item items[],
205 const struct rte_flow_action actions[],
206 struct rte_flow_error *error);
208 static struct rte_flow *
209 tap_flow_create(struct rte_eth_dev *dev,
210 const struct rte_flow_attr *attr,
211 const struct rte_flow_item items[],
212 const struct rte_flow_action actions[],
213 struct rte_flow_error *error);
216 tap_flow_free(struct pmd_internals *pmd,
217 struct rte_flow *flow);
220 tap_flow_destroy(struct rte_eth_dev *dev,
221 struct rte_flow *flow,
222 struct rte_flow_error *error);
225 tap_flow_isolate(struct rte_eth_dev *dev,
227 struct rte_flow_error *error);
229 static int bpf_rss_key(enum bpf_rss_key_e cmd, __u32 *key_idx);
230 static int rss_enable(struct pmd_internals *pmd,
231 const struct rte_flow_attr *attr,
232 struct rte_flow_error *error);
233 static int rss_add_actions(struct rte_flow *flow, struct pmd_internals *pmd,
234 const struct rte_flow_action_rss *rss,
235 struct rte_flow_error *error);
237 static const struct rte_flow_ops tap_flow_ops = {
238 .validate = tap_flow_validate,
239 .create = tap_flow_create,
240 .destroy = tap_flow_destroy,
241 .flush = tap_flow_flush,
242 .isolate = tap_flow_isolate,
245 /* Static initializer for items. */
247 (const enum rte_flow_item_type []){ \
248 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
251 /* Structure to generate a simple graph of layers supported by the NIC. */
252 struct tap_flow_items {
253 /* Bit-mask corresponding to what is supported for this item. */
255 const unsigned int mask_sz; /* Bit-mask size in bytes. */
257 * Bit-mask corresponding to the default mask, if none is provided
258 * along with the item.
260 const void *default_mask;
262 * Conversion function from rte_flow to netlink attributes.
265 * rte_flow item to convert.
267 * Internal structure to store the conversion.
270 * 0 on success, negative value otherwise.
272 int (*convert)(const struct rte_flow_item *item, void *data);
273 /** List of possible following items. */
274 const enum rte_flow_item_type *const items;
277 /* Graph of supported items and associated actions. */
278 static const struct tap_flow_items tap_flow_items[] = {
279 [RTE_FLOW_ITEM_TYPE_END] = {
280 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
282 [RTE_FLOW_ITEM_TYPE_ETH] = {
284 RTE_FLOW_ITEM_TYPE_VLAN,
285 RTE_FLOW_ITEM_TYPE_IPV4,
286 RTE_FLOW_ITEM_TYPE_IPV6),
287 .mask = &(const struct rte_flow_item_eth){
288 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
289 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
292 .mask_sz = sizeof(struct rte_flow_item_eth),
293 .default_mask = &rte_flow_item_eth_mask,
294 .convert = tap_flow_create_eth,
296 [RTE_FLOW_ITEM_TYPE_VLAN] = {
297 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
298 RTE_FLOW_ITEM_TYPE_IPV6),
299 .mask = &(const struct rte_flow_item_vlan){
301 /* DEI matching is not supported */
302 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
308 .mask_sz = sizeof(struct rte_flow_item_vlan),
309 .default_mask = &rte_flow_item_vlan_mask,
310 .convert = tap_flow_create_vlan,
312 [RTE_FLOW_ITEM_TYPE_IPV4] = {
313 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
314 RTE_FLOW_ITEM_TYPE_TCP),
315 .mask = &(const struct rte_flow_item_ipv4){
322 .mask_sz = sizeof(struct rte_flow_item_ipv4),
323 .default_mask = &rte_flow_item_ipv4_mask,
324 .convert = tap_flow_create_ipv4,
326 [RTE_FLOW_ITEM_TYPE_IPV6] = {
327 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
328 RTE_FLOW_ITEM_TYPE_TCP),
329 .mask = &(const struct rte_flow_item_ipv6){
332 "\xff\xff\xff\xff\xff\xff\xff\xff"
333 "\xff\xff\xff\xff\xff\xff\xff\xff",
336 "\xff\xff\xff\xff\xff\xff\xff\xff"
337 "\xff\xff\xff\xff\xff\xff\xff\xff",
342 .mask_sz = sizeof(struct rte_flow_item_ipv6),
343 .default_mask = &rte_flow_item_ipv6_mask,
344 .convert = tap_flow_create_ipv6,
346 [RTE_FLOW_ITEM_TYPE_UDP] = {
347 .mask = &(const struct rte_flow_item_udp){
353 .mask_sz = sizeof(struct rte_flow_item_udp),
354 .default_mask = &rte_flow_item_udp_mask,
355 .convert = tap_flow_create_udp,
357 [RTE_FLOW_ITEM_TYPE_TCP] = {
358 .mask = &(const struct rte_flow_item_tcp){
364 .mask_sz = sizeof(struct rte_flow_item_tcp),
365 .default_mask = &rte_flow_item_tcp_mask,
366 .convert = tap_flow_create_tcp,
371 * TC rules, by growing priority
373 * Remote netdevice Tap netdevice
374 * +-------------+-------------+ +-------------+-------------+
375 * | Ingress | Egress | | Ingress | Egress |
376 * |-------------|-------------| |-------------|-------------|
377 * | | \ / | | | REMOTE TX | prio 1
378 * | | \ / | | | \ / | prio 2
379 * | EXPLICIT | \ / | | EXPLICIT | \ / | .
380 * | | \ / | | | \ / | .
381 * | RULES | X | | RULES | X | .
382 * | . | / \ | | . | / \ | .
383 * | . | / \ | | . | / \ | .
384 * | . | / \ | | . | / \ | .
385 * | . | / \ | | . | / \ | .
387 * .... .... .... ....
389 * | . | \ / | | . | \ / | .
390 * | . | \ / | | . | \ / | .
391 * | | \ / | | | \ / |
392 * | LOCAL_MAC | \ / | | \ / | \ / | last prio - 5
393 * | PROMISC | X | | \ / | X | last prio - 4
394 * | ALLMULTI | / \ | | X | / \ | last prio - 3
395 * | BROADCAST | / \ | | / \ | / \ | last prio - 2
396 * | BROADCASTV6 | / \ | | / \ | / \ | last prio - 1
397 * | xx | / \ | | ISOLATE | / \ | last prio
398 * +-------------+-------------+ +-------------+-------------+
400 * The implicit flow rules are stored in a list in with mandatorily the last two
401 * being the ISOLATE and REMOTE_TX rules. e.g.:
403 * LOCAL_MAC -> BROADCAST -> BROADCASTV6 -> REMOTE_TX -> ISOLATE -> NULL
405 * That enables tap_flow_isolate() to remove implicit rules by popping the list
406 * head and remove it as long as it applies on the remote netdevice. The
407 * implicit rule for TX redirection is not removed, as isolate concerns only
411 static struct remote_rule implicit_rte_flows[TAP_REMOTE_MAX_IDX] = {
412 [TAP_REMOTE_LOCAL_MAC] = {
415 .priority = PRIORITY_MASK - TAP_REMOTE_LOCAL_MAC,
419 .type = RTE_FLOW_ITEM_TYPE_ETH,
420 .mask = &(const struct rte_flow_item_eth){
421 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
425 .type = RTE_FLOW_ITEM_TYPE_END,
427 .mirred = TCA_EGRESS_REDIR,
429 [TAP_REMOTE_BROADCAST] = {
432 .priority = PRIORITY_MASK - TAP_REMOTE_BROADCAST,
436 .type = RTE_FLOW_ITEM_TYPE_ETH,
437 .mask = &(const struct rte_flow_item_eth){
438 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
440 .spec = &(const struct rte_flow_item_eth){
441 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
445 .type = RTE_FLOW_ITEM_TYPE_END,
447 .mirred = TCA_EGRESS_MIRROR,
449 [TAP_REMOTE_BROADCASTV6] = {
452 .priority = PRIORITY_MASK - TAP_REMOTE_BROADCASTV6,
456 .type = RTE_FLOW_ITEM_TYPE_ETH,
457 .mask = &(const struct rte_flow_item_eth){
458 .dst.addr_bytes = "\x33\x33\x00\x00\x00\x00",
460 .spec = &(const struct rte_flow_item_eth){
461 .dst.addr_bytes = "\x33\x33\x00\x00\x00\x00",
465 .type = RTE_FLOW_ITEM_TYPE_END,
467 .mirred = TCA_EGRESS_MIRROR,
469 [TAP_REMOTE_PROMISC] = {
472 .priority = PRIORITY_MASK - TAP_REMOTE_PROMISC,
476 .type = RTE_FLOW_ITEM_TYPE_VOID,
479 .type = RTE_FLOW_ITEM_TYPE_END,
481 .mirred = TCA_EGRESS_MIRROR,
483 [TAP_REMOTE_ALLMULTI] = {
486 .priority = PRIORITY_MASK - TAP_REMOTE_ALLMULTI,
490 .type = RTE_FLOW_ITEM_TYPE_ETH,
491 .mask = &(const struct rte_flow_item_eth){
492 .dst.addr_bytes = "\x01\x00\x00\x00\x00\x00",
494 .spec = &(const struct rte_flow_item_eth){
495 .dst.addr_bytes = "\x01\x00\x00\x00\x00\x00",
499 .type = RTE_FLOW_ITEM_TYPE_END,
501 .mirred = TCA_EGRESS_MIRROR,
506 .priority = TAP_REMOTE_TX,
510 .type = RTE_FLOW_ITEM_TYPE_VOID,
513 .type = RTE_FLOW_ITEM_TYPE_END,
515 .mirred = TCA_EGRESS_MIRROR,
520 .priority = PRIORITY_MASK - TAP_ISOLATE,
524 .type = RTE_FLOW_ITEM_TYPE_VOID,
527 .type = RTE_FLOW_ITEM_TYPE_END,
533 * Make as much checks as possible on an Ethernet item, and if a flow is
534 * provided, fill it appropriately with Ethernet info.
537 * Item specification.
538 * @param[in, out] data
539 * Additional data structure to tell next layers we've been here.
542 * 0 if checks are alright, -1 otherwise.
545 tap_flow_create_eth(const struct rte_flow_item *item, void *data)
547 struct convert_data *info = (struct convert_data *)data;
548 const struct rte_flow_item_eth *spec = item->spec;
549 const struct rte_flow_item_eth *mask = item->mask;
550 struct rte_flow *flow = info->flow;
553 /* use default mask if none provided */
555 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_ETH].default_mask;
556 /* TC does not support eth_type masking. Only accept if exact match. */
557 if (mask->type && mask->type != 0xffff)
561 /* store eth_type for consistency if ipv4/6 pattern item comes next */
562 if (spec->type & mask->type)
563 info->eth_type = spec->type;
567 if (!is_zero_ether_addr(&spec->dst)) {
568 tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_ETH_DST, ETHER_ADDR_LEN,
569 &spec->dst.addr_bytes);
570 tap_nlattr_add(&msg->nh,
571 TCA_FLOWER_KEY_ETH_DST_MASK, ETHER_ADDR_LEN,
572 &mask->dst.addr_bytes);
574 if (!is_zero_ether_addr(&mask->src)) {
575 tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_ETH_SRC, ETHER_ADDR_LEN,
576 &spec->src.addr_bytes);
577 tap_nlattr_add(&msg->nh,
578 TCA_FLOWER_KEY_ETH_SRC_MASK, ETHER_ADDR_LEN,
579 &mask->src.addr_bytes);
585 * Make as much checks as possible on a VLAN item, and if a flow is provided,
586 * fill it appropriately with VLAN info.
589 * Item specification.
590 * @param[in, out] data
591 * Additional data structure to tell next layers we've been here.
594 * 0 if checks are alright, -1 otherwise.
597 tap_flow_create_vlan(const struct rte_flow_item *item, void *data)
599 struct convert_data *info = (struct convert_data *)data;
600 const struct rte_flow_item_vlan *spec = item->spec;
601 const struct rte_flow_item_vlan *mask = item->mask;
602 struct rte_flow *flow = info->flow;
605 /* use default mask if none provided */
607 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_VLAN].default_mask;
608 /* TC does not support tpid masking. Only accept if exact match. */
609 if (mask->tpid && mask->tpid != 0xffff)
611 /* Double-tagging not supported. */
612 if (spec && mask->tpid && spec->tpid != htons(ETH_P_8021Q))
618 msg->t.tcm_info = TC_H_MAKE(msg->t.tcm_info, htons(ETH_P_8021Q));
619 #define VLAN_PRIO(tci) ((tci) >> 13)
620 #define VLAN_ID(tci) ((tci) & 0xfff)
624 uint16_t tci = ntohs(spec->tci) & mask->tci;
625 uint16_t prio = VLAN_PRIO(tci);
626 uint8_t vid = VLAN_ID(tci);
629 tap_nlattr_add8(&msg->nh,
630 TCA_FLOWER_KEY_VLAN_PRIO, prio);
632 tap_nlattr_add16(&msg->nh,
633 TCA_FLOWER_KEY_VLAN_ID, vid);
639 * Make as much checks as possible on an IPv4 item, and if a flow is provided,
640 * fill it appropriately with IPv4 info.
643 * Item specification.
644 * @param[in, out] data
645 * Additional data structure to tell next layers we've been here.
648 * 0 if checks are alright, -1 otherwise.
651 tap_flow_create_ipv4(const struct rte_flow_item *item, void *data)
653 struct convert_data *info = (struct convert_data *)data;
654 const struct rte_flow_item_ipv4 *spec = item->spec;
655 const struct rte_flow_item_ipv4 *mask = item->mask;
656 struct rte_flow *flow = info->flow;
659 /* use default mask if none provided */
661 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_IPV4].default_mask;
662 /* check that previous eth type is compatible with ipv4 */
663 if (info->eth_type && info->eth_type != htons(ETH_P_IP))
665 /* store ip_proto for consistency if udp/tcp pattern item comes next */
667 info->ip_proto = spec->hdr.next_proto_id;
672 info->eth_type = htons(ETH_P_IP);
675 if (spec->hdr.dst_addr) {
676 tap_nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_DST,
678 tap_nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_DST_MASK,
681 if (spec->hdr.src_addr) {
682 tap_nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_SRC,
684 tap_nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_SRC_MASK,
687 if (spec->hdr.next_proto_id)
688 tap_nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO,
689 spec->hdr.next_proto_id);
694 * Make as much checks as possible on an IPv6 item, and if a flow is provided,
695 * fill it appropriately with IPv6 info.
698 * Item specification.
699 * @param[in, out] data
700 * Additional data structure to tell next layers we've been here.
703 * 0 if checks are alright, -1 otherwise.
706 tap_flow_create_ipv6(const struct rte_flow_item *item, void *data)
708 struct convert_data *info = (struct convert_data *)data;
709 const struct rte_flow_item_ipv6 *spec = item->spec;
710 const struct rte_flow_item_ipv6 *mask = item->mask;
711 struct rte_flow *flow = info->flow;
712 uint8_t empty_addr[16] = { 0 };
715 /* use default mask if none provided */
717 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_IPV6].default_mask;
718 /* check that previous eth type is compatible with ipv6 */
719 if (info->eth_type && info->eth_type != htons(ETH_P_IPV6))
721 /* store ip_proto for consistency if udp/tcp pattern item comes next */
723 info->ip_proto = spec->hdr.proto;
728 info->eth_type = htons(ETH_P_IPV6);
731 if (memcmp(spec->hdr.dst_addr, empty_addr, 16)) {
732 tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_DST,
733 sizeof(spec->hdr.dst_addr), &spec->hdr.dst_addr);
734 tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_DST_MASK,
735 sizeof(mask->hdr.dst_addr), &mask->hdr.dst_addr);
737 if (memcmp(spec->hdr.src_addr, empty_addr, 16)) {
738 tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_SRC,
739 sizeof(spec->hdr.src_addr), &spec->hdr.src_addr);
740 tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_SRC_MASK,
741 sizeof(mask->hdr.src_addr), &mask->hdr.src_addr);
744 tap_nlattr_add8(&msg->nh,
745 TCA_FLOWER_KEY_IP_PROTO, spec->hdr.proto);
750 * Make as much checks as possible on a UDP item, and if a flow is provided,
751 * fill it appropriately with UDP info.
754 * Item specification.
755 * @param[in, out] data
756 * Additional data structure to tell next layers we've been here.
759 * 0 if checks are alright, -1 otherwise.
762 tap_flow_create_udp(const struct rte_flow_item *item, void *data)
764 struct convert_data *info = (struct convert_data *)data;
765 const struct rte_flow_item_udp *spec = item->spec;
766 const struct rte_flow_item_udp *mask = item->mask;
767 struct rte_flow *flow = info->flow;
770 /* use default mask if none provided */
772 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_UDP].default_mask;
773 /* check that previous ip_proto is compatible with udp */
774 if (info->ip_proto && info->ip_proto != IPPROTO_UDP)
776 /* TC does not support UDP port masking. Only accept if exact match. */
777 if ((mask->hdr.src_port && mask->hdr.src_port != 0xffff) ||
778 (mask->hdr.dst_port && mask->hdr.dst_port != 0xffff))
783 tap_nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, IPPROTO_UDP);
786 if (spec->hdr.dst_port & mask->hdr.dst_port)
787 tap_nlattr_add16(&msg->nh, TCA_FLOWER_KEY_UDP_DST,
789 if (spec->hdr.src_port & mask->hdr.src_port)
790 tap_nlattr_add16(&msg->nh, TCA_FLOWER_KEY_UDP_SRC,
796 * Make as much checks as possible on a TCP item, and if a flow is provided,
797 * fill it appropriately with TCP info.
800 * Item specification.
801 * @param[in, out] data
802 * Additional data structure to tell next layers we've been here.
805 * 0 if checks are alright, -1 otherwise.
808 tap_flow_create_tcp(const struct rte_flow_item *item, void *data)
810 struct convert_data *info = (struct convert_data *)data;
811 const struct rte_flow_item_tcp *spec = item->spec;
812 const struct rte_flow_item_tcp *mask = item->mask;
813 struct rte_flow *flow = info->flow;
816 /* use default mask if none provided */
818 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_TCP].default_mask;
819 /* check that previous ip_proto is compatible with tcp */
820 if (info->ip_proto && info->ip_proto != IPPROTO_TCP)
822 /* TC does not support TCP port masking. Only accept if exact match. */
823 if ((mask->hdr.src_port && mask->hdr.src_port != 0xffff) ||
824 (mask->hdr.dst_port && mask->hdr.dst_port != 0xffff))
829 tap_nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, IPPROTO_TCP);
832 if (spec->hdr.dst_port & mask->hdr.dst_port)
833 tap_nlattr_add16(&msg->nh, TCA_FLOWER_KEY_TCP_DST,
835 if (spec->hdr.src_port & mask->hdr.src_port)
836 tap_nlattr_add16(&msg->nh, TCA_FLOWER_KEY_TCP_SRC,
842 * Check support for a given item.
845 * Item specification.
847 * Bit-Mask size in bytes.
848 * @param[in] supported_mask
849 * Bit-mask covering supported fields to compare with spec, last and mask in
851 * @param[in] default_mask
852 * Bit-mask default mask if none is provided in \item.
858 tap_flow_item_validate(const struct rte_flow_item *item,
860 const uint8_t *supported_mask,
861 const uint8_t *default_mask)
865 /* An empty layer is allowed, as long as all fields are NULL */
866 if (!item->spec && (item->mask || item->last))
868 /* Is the item spec compatible with what the NIC supports? */
869 if (item->spec && !item->mask) {
871 const uint8_t *spec = item->spec;
873 for (i = 0; i < size; ++i)
874 if ((spec[i] | supported_mask[i]) != supported_mask[i])
876 /* Is the default mask compatible with what the NIC supports? */
877 for (i = 0; i < size; i++)
878 if ((default_mask[i] | supported_mask[i]) !=
882 /* Is the item last compatible with what the NIC supports? */
883 if (item->last && !item->mask) {
885 const uint8_t *spec = item->last;
887 for (i = 0; i < size; ++i)
888 if ((spec[i] | supported_mask[i]) != supported_mask[i])
891 /* Is the item mask compatible with what the NIC supports? */
894 const uint8_t *spec = item->mask;
896 for (i = 0; i < size; ++i)
897 if ((spec[i] | supported_mask[i]) != supported_mask[i])
901 * Once masked, Are item spec and item last equal?
902 * TC does not support range so anything else is invalid.
904 if (item->spec && item->last) {
907 const uint8_t *apply = default_mask;
912 for (i = 0; i < size; ++i) {
913 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
914 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
916 ret = memcmp(spec, last, size);
922 * Configure the kernel with a TC action and its configured parameters
923 * Handled actions: "gact", "mirred", "skbedit", "bpf"
926 * Pointer to rte flow containing the netlink message
928 * @param[in, out] act_index
929 * Pointer to action sequence number in the TC command
932 * Pointer to struct holding the action parameters
935 * -1 on failure, 0 on success
938 add_action(struct rte_flow *flow, size_t *act_index, struct action_data *adata)
940 struct nlmsg *msg = &flow->msg;
942 if (tap_nlattr_nested_start(msg, (*act_index)++) < 0)
945 tap_nlattr_add(&msg->nh, TCA_ACT_KIND,
946 strlen(adata->id) + 1, adata->id);
947 if (tap_nlattr_nested_start(msg, TCA_ACT_OPTIONS) < 0)
949 if (strcmp("gact", adata->id) == 0) {
950 tap_nlattr_add(&msg->nh, TCA_GACT_PARMS, sizeof(adata->gact),
952 } else if (strcmp("mirred", adata->id) == 0) {
953 if (adata->mirred.eaction == TCA_EGRESS_MIRROR)
954 adata->mirred.action = TC_ACT_PIPE;
956 adata->mirred.action = TC_ACT_STOLEN;
957 tap_nlattr_add(&msg->nh, TCA_MIRRED_PARMS,
958 sizeof(adata->mirred),
960 } else if (strcmp("skbedit", adata->id) == 0) {
961 tap_nlattr_add(&msg->nh, TCA_SKBEDIT_PARMS,
962 sizeof(adata->skbedit.skbedit),
963 &adata->skbedit.skbedit);
964 tap_nlattr_add16(&msg->nh, TCA_SKBEDIT_QUEUE_MAPPING,
965 adata->skbedit.queue);
966 } else if (strcmp("bpf", adata->id) == 0) {
967 tap_nlattr_add32(&msg->nh, TCA_ACT_BPF_FD, adata->bpf.bpf_fd);
968 tap_nlattr_add(&msg->nh, TCA_ACT_BPF_NAME,
969 strlen(adata->bpf.annotation) + 1,
970 adata->bpf.annotation);
971 tap_nlattr_add(&msg->nh, TCA_ACT_BPF_PARMS,
972 sizeof(adata->bpf.bpf),
977 tap_nlattr_nested_finish(msg); /* nested TCA_ACT_OPTIONS */
978 tap_nlattr_nested_finish(msg); /* nested act_index */
983 * Helper function to send a serie of TC actions to the kernel
986 * Pointer to rte flow containing the netlink message
988 * @param[in] nb_actions
989 * Number of actions in an array of action structs
992 * Pointer to an array of action structs
994 * @param[in] classifier_actions
995 * The classifier on behave of which the actions are configured
998 * -1 on failure, 0 on success
1001 add_actions(struct rte_flow *flow, int nb_actions, struct action_data *data,
1002 int classifier_action)
1004 struct nlmsg *msg = &flow->msg;
1005 size_t act_index = 1;
1008 if (tap_nlattr_nested_start(msg, classifier_action) < 0)
1010 for (i = 0; i < nb_actions; i++)
1011 if (add_action(flow, &act_index, data + i) < 0)
1013 tap_nlattr_nested_finish(msg); /* nested TCA_FLOWER_ACT */
1018 * Validate a flow supported by TC.
1019 * If flow param is not NULL, then also fill the netlink message inside.
1022 * Pointer to private structure.
1024 * Flow rule attributes.
1025 * @param[in] pattern
1026 * Pattern specification (list terminated by the END pattern item).
1027 * @param[in] actions
1028 * Associated actions (list terminated by the END action).
1030 * Perform verbose error reporting if not NULL.
1031 * @param[in, out] flow
1032 * Flow structure to update.
1034 * If set to TCA_EGRESS_REDIR, provided actions will be replaced with a
1035 * redirection to the tap netdevice, and the TC rule will be configured
1036 * on the remote netdevice in pmd.
1037 * If set to TCA_EGRESS_MIRROR, provided actions will be replaced with a
1038 * mirroring to the tap netdevice, and the TC rule will be configured
1039 * on the remote netdevice in pmd. Matching packets will thus be duplicated.
1040 * If set to 0, the standard behavior is to be used: set correct actions for
1041 * the TC rule, and apply it on the tap netdevice.
1044 * 0 on success, a negative errno value otherwise and rte_errno is set.
1047 priv_flow_process(struct pmd_internals *pmd,
1048 const struct rte_flow_attr *attr,
1049 const struct rte_flow_item items[],
1050 const struct rte_flow_action actions[],
1051 struct rte_flow_error *error,
1052 struct rte_flow *flow,
1055 const struct tap_flow_items *cur_item = tap_flow_items;
1056 struct convert_data data = {
1061 int action = 0; /* Only one action authorized for now */
1063 if (attr->group > MAX_GROUP) {
1065 error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
1066 NULL, "group value too big: cannot exceed 15");
1069 if (attr->priority > MAX_PRIORITY) {
1071 error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1072 NULL, "priority value too big");
1075 uint16_t group = attr->group << GROUP_SHIFT;
1076 uint16_t prio = group | (attr->priority +
1077 RSS_PRIORITY_OFFSET + PRIORITY_OFFSET);
1078 flow->msg.t.tcm_info = TC_H_MAKE(prio << 16,
1079 flow->msg.t.tcm_info);
1084 * If attr->ingress, the rule applies on remote ingress
1085 * to match incoming packets
1086 * If attr->egress, the rule applies on tap ingress (as
1087 * seen from the kernel) to deal with packets going out
1088 * from the DPDK app.
1090 flow->msg.t.tcm_parent = TC_H_MAKE(TC_H_INGRESS, 0);
1092 /* Standard rule on tap egress (kernel standpoint). */
1093 flow->msg.t.tcm_parent =
1094 TC_H_MAKE(MULTIQ_MAJOR_HANDLE, 0);
1096 /* use flower filter type */
1097 tap_nlattr_add(&flow->msg.nh, TCA_KIND, sizeof("flower"), "flower");
1098 if (tap_nlattr_nested_start(&flow->msg, TCA_OPTIONS) < 0)
1099 goto exit_item_not_supported;
1101 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
1102 const struct tap_flow_items *token = NULL;
1106 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
1110 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
1112 if (cur_item->items[i] == items->type) {
1113 token = &tap_flow_items[items->type];
1118 goto exit_item_not_supported;
1120 err = tap_flow_item_validate(
1121 items, cur_item->mask_sz,
1122 (const uint8_t *)cur_item->mask,
1123 (const uint8_t *)cur_item->default_mask);
1125 goto exit_item_not_supported;
1126 if (flow && cur_item->convert) {
1127 err = cur_item->convert(items, &data);
1129 goto exit_item_not_supported;
1134 tap_nlattr_add16(&flow->msg.nh, TCA_FLOWER_KEY_ETH_TYPE,
1135 htons(ETH_P_8021Q));
1136 tap_nlattr_add16(&flow->msg.nh,
1137 TCA_FLOWER_KEY_VLAN_ETH_TYPE,
1139 data.eth_type : htons(ETH_P_ALL));
1140 } else if (data.eth_type) {
1141 tap_nlattr_add16(&flow->msg.nh, TCA_FLOWER_KEY_ETH_TYPE,
1145 if (mirred && flow) {
1146 struct action_data adata = {
1154 * If attr->egress && mirred, then this is a special
1155 * case where the rule must be applied on the tap, to
1156 * redirect packets coming from the DPDK App, out
1157 * through the remote netdevice.
1159 adata.mirred.ifindex = attr->ingress ? pmd->if_index :
1160 pmd->remote_if_index;
1161 if (mirred == TCA_EGRESS_MIRROR)
1162 adata.mirred.action = TC_ACT_PIPE;
1164 adata.mirred.action = TC_ACT_STOLEN;
1165 if (add_actions(flow, 1, &adata, TCA_FLOWER_ACT) < 0)
1166 goto exit_action_not_supported;
1170 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
1173 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
1175 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
1177 goto exit_action_not_supported;
1180 struct action_data adata = {
1183 .action = TC_ACT_SHOT,
1187 err = add_actions(flow, 1, &adata,
1190 } else if (actions->type == RTE_FLOW_ACTION_TYPE_PASSTHRU) {
1192 goto exit_action_not_supported;
1195 struct action_data adata = {
1199 .action = TC_ACT_UNSPEC,
1203 err = add_actions(flow, 1, &adata,
1206 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
1207 const struct rte_flow_action_queue *queue =
1208 (const struct rte_flow_action_queue *)
1212 goto exit_action_not_supported;
1215 (queue->index > pmd->dev->data->nb_rx_queues - 1))
1216 goto exit_action_not_supported;
1218 struct action_data adata = {
1222 .action = TC_ACT_PIPE,
1224 .queue = queue->index,
1228 err = add_actions(flow, 1, &adata,
1231 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
1232 const struct rte_flow_action_rss *rss =
1233 (const struct rte_flow_action_rss *)
1237 goto exit_action_not_supported;
1239 if (!pmd->rss_enabled) {
1240 err = rss_enable(pmd, attr, error);
1242 goto exit_action_not_supported;
1245 err = rss_add_actions(flow, pmd, rss, error);
1247 goto exit_action_not_supported;
1250 goto exit_action_not_supported;
1254 tap_nlattr_nested_finish(&flow->msg); /* nested TCA_OPTIONS */
1256 exit_item_not_supported:
1257 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1258 items, "item not supported");
1260 exit_action_not_supported:
1261 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
1262 actions, "action not supported");
1271 * @see rte_flow_validate()
1275 tap_flow_validate(struct rte_eth_dev *dev,
1276 const struct rte_flow_attr *attr,
1277 const struct rte_flow_item items[],
1278 const struct rte_flow_action actions[],
1279 struct rte_flow_error *error)
1281 struct pmd_internals *pmd = dev->data->dev_private;
1283 return priv_flow_process(pmd, attr, items, actions, error, NULL, 0);
1287 * Set a unique handle in a flow.
1289 * The kernel supports TC rules with equal priority, as long as they use the
1290 * same matching fields (e.g.: dst mac and ipv4) with different values (and
1291 * full mask to ensure no collision is possible).
1292 * In those rules, the handle (uint32_t) is the part that would identify
1293 * specifically each rule.
1295 * On 32-bit architectures, the handle can simply be the flow's pointer address.
1296 * On 64-bit architectures, we rely on jhash(flow) to find a (sufficiently)
1299 * @param[in, out] flow
1300 * The flow that needs its handle set.
1303 tap_flow_set_handle(struct rte_flow *flow)
1305 uint32_t handle = 0;
1307 if (sizeof(flow) > 4)
1308 handle = rte_jhash(&flow, sizeof(flow), 1);
1310 handle = (uintptr_t)flow;
1311 /* must be at least 1 to avoid letting the kernel choose one for us */
1314 flow->msg.t.tcm_handle = handle;
1318 * Free the flow opened file descriptors and allocated memory
1321 * Pointer to the flow to free
1325 tap_flow_free(struct pmd_internals *pmd, struct rte_flow *flow)
1332 if (pmd->rss_enabled) {
1333 /* Close flow BPF file descriptors */
1334 for (i = 0; i < SEC_MAX; i++)
1335 if (flow->bpf_fd[i] != 0) {
1336 close(flow->bpf_fd[i]);
1337 flow->bpf_fd[i] = 0;
1340 /* Release the map key for this RSS rule */
1341 bpf_rss_key(KEY_CMD_RELEASE, &flow->key_idx);
1345 /* Free flow allocated memory */
1352 * @see rte_flow_create()
1355 static struct rte_flow *
1356 tap_flow_create(struct rte_eth_dev *dev,
1357 const struct rte_flow_attr *attr,
1358 const struct rte_flow_item items[],
1359 const struct rte_flow_action actions[],
1360 struct rte_flow_error *error)
1362 struct pmd_internals *pmd = dev->data->dev_private;
1363 struct rte_flow *remote_flow = NULL;
1364 struct rte_flow *flow = NULL;
1365 struct nlmsg *msg = NULL;
1368 if (!pmd->if_index) {
1369 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
1371 "can't create rule, ifindex not found");
1375 * No rules configured through standard rte_flow should be set on the
1376 * priorities used by implicit rules.
1378 if ((attr->group == MAX_GROUP) &&
1379 attr->priority > (MAX_PRIORITY - TAP_REMOTE_MAX_IDX)) {
1381 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1382 NULL, "priority value too big");
1385 flow = rte_malloc(__func__, sizeof(struct rte_flow), 0);
1387 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1388 NULL, "cannot allocate memory for rte_flow");
1392 tc_init_msg(msg, pmd->if_index, RTM_NEWTFILTER,
1393 NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE);
1394 msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL));
1395 tap_flow_set_handle(flow);
1396 if (priv_flow_process(pmd, attr, items, actions, error, flow, 0))
1398 err = tap_nl_send(pmd->nlsk_fd, &msg->nh);
1400 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
1401 NULL, "couldn't send request to kernel");
1404 err = tap_nl_recv_ack(pmd->nlsk_fd);
1407 "Kernel refused TC filter rule creation (%d): %s\n",
1408 errno, strerror(errno));
1409 rte_flow_error_set(error, EEXIST, RTE_FLOW_ERROR_TYPE_HANDLE,
1411 "overlapping rules or Kernel too old for flower support");
1414 LIST_INSERT_HEAD(&pmd->flows, flow, next);
1416 * If a remote device is configured, a TC rule with identical items for
1417 * matching must be set on that device, with a single action: redirect
1418 * to the local pmd->if_index.
1420 if (pmd->remote_if_index) {
1421 remote_flow = rte_malloc(__func__, sizeof(struct rte_flow), 0);
1424 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1425 "cannot allocate memory for rte_flow");
1428 msg = &remote_flow->msg;
1429 /* set the rule if_index for the remote netdevice */
1431 msg, pmd->remote_if_index, RTM_NEWTFILTER,
1432 NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE);
1433 msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL));
1434 tap_flow_set_handle(remote_flow);
1435 if (priv_flow_process(pmd, attr, items, NULL,
1436 error, remote_flow, TCA_EGRESS_REDIR)) {
1438 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1439 NULL, "rte flow rule validation failed");
1442 err = tap_nl_send(pmd->nlsk_fd, &msg->nh);
1445 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1446 NULL, "Failure sending nl request");
1449 err = tap_nl_recv_ack(pmd->nlsk_fd);
1452 "Kernel refused TC filter rule creation (%d): %s\n",
1453 errno, strerror(errno));
1455 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1457 "overlapping rules or Kernel too old for flower support");
1460 flow->remote_flow = remote_flow;
1465 rte_free(remote_flow);
1467 tap_flow_free(pmd, flow);
1472 * Destroy a flow using pointer to pmd_internal.
1474 * @param[in, out] pmd
1475 * Pointer to private structure.
1477 * Pointer to the flow to destroy.
1478 * @param[in, out] error
1479 * Pointer to the flow error handler
1481 * @return 0 if the flow could be destroyed, -1 otherwise.
1484 tap_flow_destroy_pmd(struct pmd_internals *pmd,
1485 struct rte_flow *flow,
1486 struct rte_flow_error *error)
1488 struct rte_flow *remote_flow = flow->remote_flow;
1491 LIST_REMOVE(flow, next);
1492 flow->msg.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK;
1493 flow->msg.nh.nlmsg_type = RTM_DELTFILTER;
1495 ret = tap_nl_send(pmd->nlsk_fd, &flow->msg.nh);
1497 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
1498 NULL, "couldn't send request to kernel");
1501 ret = tap_nl_recv_ack(pmd->nlsk_fd);
1502 /* If errno is ENOENT, the rule is already no longer in the kernel. */
1503 if (ret < 0 && errno == ENOENT)
1507 "Kernel refused TC filter rule deletion (%d): %s\n",
1508 errno, strerror(errno));
1510 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1511 "couldn't receive kernel ack to our request");
1516 remote_flow->msg.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK;
1517 remote_flow->msg.nh.nlmsg_type = RTM_DELTFILTER;
1519 ret = tap_nl_send(pmd->nlsk_fd, &remote_flow->msg.nh);
1522 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1523 NULL, "Failure sending nl request");
1526 ret = tap_nl_recv_ack(pmd->nlsk_fd);
1527 if (ret < 0 && errno == ENOENT)
1531 "Kernel refused TC filter rule deletion (%d): %s\n",
1532 errno, strerror(errno));
1534 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1535 NULL, "Failure trying to receive nl ack");
1541 rte_free(remote_flow);
1542 tap_flow_free(pmd, flow);
1549 * @see rte_flow_destroy()
1553 tap_flow_destroy(struct rte_eth_dev *dev,
1554 struct rte_flow *flow,
1555 struct rte_flow_error *error)
1557 struct pmd_internals *pmd = dev->data->dev_private;
1559 return tap_flow_destroy_pmd(pmd, flow, error);
1563 * Enable/disable flow isolation.
1565 * @see rte_flow_isolate()
1569 tap_flow_isolate(struct rte_eth_dev *dev,
1571 struct rte_flow_error *error __rte_unused)
1573 struct pmd_internals *pmd = dev->data->dev_private;
1576 pmd->flow_isolate = 1;
1578 pmd->flow_isolate = 0;
1580 * If netdevice is there, setup appropriate flow rules immediately.
1581 * Otherwise it will be set when bringing up the netdevice (tun_alloc).
1583 if (!pmd->rxq[0].fd)
1586 struct rte_flow *flow;
1589 flow = LIST_FIRST(&pmd->implicit_flows);
1593 * Remove all implicit rules on the remote.
1594 * Keep the local rule to redirect packets on TX.
1595 * Keep also the last implicit local rule: ISOLATE.
1597 if (flow->msg.t.tcm_ifindex == pmd->if_index)
1599 if (tap_flow_destroy_pmd(pmd, flow, NULL) < 0)
1602 /* Switch the TC rule according to pmd->flow_isolate */
1603 if (tap_flow_implicit_create(pmd, TAP_ISOLATE) == -1)
1606 /* Switch the TC rule according to pmd->flow_isolate */
1607 if (tap_flow_implicit_create(pmd, TAP_ISOLATE) == -1)
1609 if (!pmd->remote_if_index)
1611 if (tap_flow_implicit_create(pmd, TAP_REMOTE_TX) < 0)
1613 if (tap_flow_implicit_create(pmd, TAP_REMOTE_LOCAL_MAC) < 0)
1615 if (tap_flow_implicit_create(pmd, TAP_REMOTE_BROADCAST) < 0)
1617 if (tap_flow_implicit_create(pmd, TAP_REMOTE_BROADCASTV6) < 0)
1619 if (dev->data->promiscuous &&
1620 tap_flow_implicit_create(pmd, TAP_REMOTE_PROMISC) < 0)
1622 if (dev->data->all_multicast &&
1623 tap_flow_implicit_create(pmd, TAP_REMOTE_ALLMULTI) < 0)
1628 pmd->flow_isolate = 0;
1629 return rte_flow_error_set(
1630 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1631 "TC rule creation failed");
1635 * Destroy all flows.
1637 * @see rte_flow_flush()
1641 tap_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error)
1643 struct pmd_internals *pmd = dev->data->dev_private;
1644 struct rte_flow *flow;
1646 while (!LIST_EMPTY(&pmd->flows)) {
1647 flow = LIST_FIRST(&pmd->flows);
1648 if (tap_flow_destroy(dev, flow, error) < 0)
1655 * Add an implicit flow rule on the remote device to make sure traffic gets to
1656 * the tap netdevice from there.
1659 * Pointer to private structure.
1661 * The idx in the implicit_rte_flows array specifying which rule to apply.
1663 * @return -1 if the rule couldn't be applied, 0 otherwise.
1665 int tap_flow_implicit_create(struct pmd_internals *pmd,
1666 enum implicit_rule_index idx)
1668 uint16_t flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE;
1669 struct rte_flow_action *actions = implicit_rte_flows[idx].actions;
1670 struct rte_flow_action isolate_actions[2] = {
1672 .type = RTE_FLOW_ACTION_TYPE_END,
1675 struct rte_flow_item *items = implicit_rte_flows[idx].items;
1676 struct rte_flow_attr *attr = &implicit_rte_flows[idx].attr;
1677 struct rte_flow_item_eth eth_local = { .type = 0 };
1678 uint16_t if_index = pmd->remote_if_index;
1679 struct rte_flow *remote_flow = NULL;
1680 struct nlmsg *msg = NULL;
1682 struct rte_flow_item items_local[2] = {
1684 .type = items[0].type,
1686 .mask = items[0].mask,
1689 .type = items[1].type,
1693 remote_flow = rte_malloc(__func__, sizeof(struct rte_flow), 0);
1695 RTE_LOG(ERR, PMD, "Cannot allocate memory for rte_flow\n");
1698 msg = &remote_flow->msg;
1699 if (idx == TAP_REMOTE_TX) {
1700 if_index = pmd->if_index;
1701 } else if (idx == TAP_ISOLATE) {
1702 if_index = pmd->if_index;
1703 /* Don't be exclusive for this rule, it can be changed later. */
1704 flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_CREATE;
1705 isolate_actions[0].type = pmd->flow_isolate ?
1706 RTE_FLOW_ACTION_TYPE_DROP :
1707 RTE_FLOW_ACTION_TYPE_PASSTHRU;
1708 actions = isolate_actions;
1709 } else if (idx == TAP_REMOTE_LOCAL_MAC) {
1711 * eth addr couldn't be set in implicit_rte_flows[] as it is not
1712 * known at compile time.
1714 memcpy(ð_local.dst, &pmd->eth_addr, sizeof(pmd->eth_addr));
1715 items = items_local;
1717 tc_init_msg(msg, if_index, RTM_NEWTFILTER, flags);
1718 msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL));
1720 * The ISOLATE rule is always present and must have a static handle, as
1721 * the action is changed whether the feature is enabled (DROP) or
1722 * disabled (PASSTHRU).
1724 if (idx == TAP_ISOLATE)
1725 remote_flow->msg.t.tcm_handle = ISOLATE_HANDLE;
1727 tap_flow_set_handle(remote_flow);
1728 if (priv_flow_process(pmd, attr, items, actions, NULL,
1729 remote_flow, implicit_rte_flows[idx].mirred)) {
1730 RTE_LOG(ERR, PMD, "rte flow rule validation failed\n");
1733 err = tap_nl_send(pmd->nlsk_fd, &msg->nh);
1735 RTE_LOG(ERR, PMD, "Failure sending nl request\n");
1738 err = tap_nl_recv_ack(pmd->nlsk_fd);
1741 "Kernel refused TC filter rule creation (%d): %s\n",
1742 errno, strerror(errno));
1745 LIST_INSERT_HEAD(&pmd->implicit_flows, remote_flow, next);
1749 rte_free(remote_flow);
1754 * Remove specific implicit flow rule on the remote device.
1756 * @param[in, out] pmd
1757 * Pointer to private structure.
1759 * The idx in the implicit_rte_flows array specifying which rule to remove.
1761 * @return -1 if one of the implicit rules couldn't be created, 0 otherwise.
1763 int tap_flow_implicit_destroy(struct pmd_internals *pmd,
1764 enum implicit_rule_index idx)
1766 struct rte_flow *remote_flow;
1768 int idx_prio = implicit_rte_flows[idx].attr.priority + PRIORITY_OFFSET;
1770 for (remote_flow = LIST_FIRST(&pmd->implicit_flows);
1772 remote_flow = LIST_NEXT(remote_flow, next)) {
1773 cur_prio = (remote_flow->msg.t.tcm_info >> 16) & PRIORITY_MASK;
1774 if (cur_prio != idx_prio)
1776 return tap_flow_destroy_pmd(pmd, remote_flow, NULL);
1782 * Destroy all implicit flows.
1784 * @see rte_flow_flush()
1787 tap_flow_implicit_flush(struct pmd_internals *pmd, struct rte_flow_error *error)
1789 struct rte_flow *remote_flow;
1791 while (!LIST_EMPTY(&pmd->implicit_flows)) {
1792 remote_flow = LIST_FIRST(&pmd->implicit_flows);
1793 if (tap_flow_destroy_pmd(pmd, remote_flow, error) < 0)
1799 #define MAX_RSS_KEYS 256
1800 #define KEY_IDX_OFFSET (3 * MAX_RSS_KEYS)
1801 #define SEC_NAME_CLS_Q "cls_q"
1803 const char *sec_name[SEC_MAX] = {
1804 [SEC_L3_L4] = "l3_l4",
1808 * Enable RSS on tap: create TC rules for queuing.
1810 * @param[in, out] pmd
1811 * Pointer to private structure.
1814 * Pointer to rte_flow to get flow group
1817 * Pointer to error reporting if not NULL.
1819 * @return 0 on success, negative value on failure.
1821 static int rss_enable(struct pmd_internals *pmd,
1822 const struct rte_flow_attr *attr,
1823 struct rte_flow_error *error)
1825 struct rte_flow *rss_flow = NULL;
1826 struct nlmsg *msg = NULL;
1827 /* 4096 is the maximum number of instructions for a BPF program */
1828 char annotation[64];
1832 /* unlimit locked memory */
1833 struct rlimit memlock_limit = {
1834 .rlim_cur = RLIM_INFINITY,
1835 .rlim_max = RLIM_INFINITY,
1837 setrlimit(RLIMIT_MEMLOCK, &memlock_limit);
1839 /* Get a new map key for a new RSS rule */
1840 err = bpf_rss_key(KEY_CMD_INIT, NULL);
1843 error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1844 "Failed to initialize BPF RSS keys");
1850 * Create BPF RSS MAP
1852 pmd->map_fd = tap_flow_bpf_rss_map_create(sizeof(__u32), /* key size */
1853 sizeof(struct rss_key),
1855 if (pmd->map_fd < 0) {
1857 "Failed to create BPF map (%d): %s\n",
1858 errno, strerror(errno));
1860 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1861 "Kernel too old or not configured "
1862 "to support BPF maps");
1868 * Add a rule per queue to match reclassified packets and direct them to
1869 * the correct queue.
1871 for (i = 0; i < pmd->dev->data->nb_rx_queues; i++) {
1872 pmd->bpf_fd[i] = tap_flow_bpf_cls_q(i);
1873 if (pmd->bpf_fd[i] < 0) {
1875 "Failed to load BPF section %s for queue %d",
1878 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
1880 "Kernel too old or not configured "
1881 "to support BPF programs loading");
1886 rss_flow = rte_malloc(__func__, sizeof(struct rte_flow), 0);
1889 "Cannot allocate memory for rte_flow");
1892 msg = &rss_flow->msg;
1893 tc_init_msg(msg, pmd->if_index, RTM_NEWTFILTER, NLM_F_REQUEST |
1894 NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE);
1895 msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL));
1896 tap_flow_set_handle(rss_flow);
1897 uint16_t group = attr->group << GROUP_SHIFT;
1898 uint16_t prio = group | (i + PRIORITY_OFFSET);
1899 msg->t.tcm_info = TC_H_MAKE(prio << 16, msg->t.tcm_info);
1900 msg->t.tcm_parent = TC_H_MAKE(MULTIQ_MAJOR_HANDLE, 0);
1902 tap_nlattr_add(&msg->nh, TCA_KIND, sizeof("bpf"), "bpf");
1903 if (tap_nlattr_nested_start(msg, TCA_OPTIONS) < 0)
1905 tap_nlattr_add32(&msg->nh, TCA_BPF_FD, pmd->bpf_fd[i]);
1906 snprintf(annotation, sizeof(annotation), "[%s%d]",
1908 tap_nlattr_add(&msg->nh, TCA_BPF_NAME, strlen(annotation) + 1,
1912 struct action_data adata = {
1916 .action = TC_ACT_PIPE,
1921 if (add_actions(rss_flow, 1, &adata, TCA_BPF_ACT) < 0)
1924 tap_nlattr_nested_finish(msg); /* nested TCA_OPTIONS */
1926 /* Netlink message is now ready to be sent */
1927 if (tap_nl_send(pmd->nlsk_fd, &msg->nh) < 0)
1929 err = tap_nl_recv_ack(pmd->nlsk_fd);
1932 "Kernel refused TC filter rule creation (%d): %s\n",
1933 errno, strerror(errno));
1936 LIST_INSERT_HEAD(&pmd->rss_flows, rss_flow, next);
1939 pmd->rss_enabled = 1;
1944 * Manage bpf RSS keys repository with operations: init, get, release
1947 * Command on RSS keys: init, get, release
1949 * @param[in, out] key_idx
1950 * Pointer to RSS Key index (out for get command, in for release command)
1952 * @return -1 if couldn't get, release or init the RSS keys, 0 otherwise.
1954 static int bpf_rss_key(enum bpf_rss_key_e cmd, __u32 *key_idx)
1958 static __u32 num_used_keys;
1959 static __u32 rss_keys[MAX_RSS_KEYS] = {KEY_STAT_UNSPEC};
1960 static __u32 rss_keys_initialized;
1964 if (!rss_keys_initialized) {
1969 if (num_used_keys == RTE_DIM(rss_keys)) {
1974 *key_idx = num_used_keys % RTE_DIM(rss_keys);
1975 while (rss_keys[*key_idx] == KEY_STAT_USED)
1976 *key_idx = (*key_idx + 1) % RTE_DIM(rss_keys);
1978 rss_keys[*key_idx] = KEY_STAT_USED;
1981 * Add an offset to key_idx in order to handle a case of
1982 * RSS and non RSS flows mixture.
1983 * If a non RSS flow is destroyed it has an eBPF map
1984 * index 0 (initialized on flow creation) and might
1985 * unintentionally remove RSS entry 0 from eBPF map.
1986 * To avoid this issue, add an offset to the real index
1987 * during a KEY_CMD_GET operation and subtract this offset
1988 * during a KEY_CMD_RELEASE operation in order to restore
1991 *key_idx += KEY_IDX_OFFSET;
1995 case KEY_CMD_RELEASE:
1996 if (!rss_keys_initialized)
2000 * Subtract offest to restore real key index
2001 * If a non RSS flow is falsely trying to release map
2002 * entry 0 - the offset subtraction will calculate the real
2003 * map index as an out-of-range value and the release operation
2004 * will be silently ignored.
2006 __u32 key = *key_idx - KEY_IDX_OFFSET;
2007 if (key >= RTE_DIM(rss_keys))
2010 if (rss_keys[key] == KEY_STAT_USED) {
2011 rss_keys[key] = KEY_STAT_AVAILABLE;
2017 for (i = 0; i < RTE_DIM(rss_keys); i++)
2018 rss_keys[i] = KEY_STAT_AVAILABLE;
2020 rss_keys_initialized = 1;
2024 case KEY_CMD_DEINIT:
2025 for (i = 0; i < RTE_DIM(rss_keys); i++)
2026 rss_keys[i] = KEY_STAT_UNSPEC;
2028 rss_keys_initialized = 0;
2040 * Add RSS hash calculations and queue selection
2042 * @param[in, out] pmd
2043 * Pointer to internal structure. Used to set/get RSS map fd
2046 * Pointer to RSS flow actions
2049 * Pointer to error reporting if not NULL.
2051 * @return 0 on success, negative value on failure
2053 static int rss_add_actions(struct rte_flow *flow, struct pmd_internals *pmd,
2054 const struct rte_flow_action_rss *rss,
2055 struct rte_flow_error *error)
2057 /* 4096 is the maximum number of instructions for a BPF program */
2060 struct rss_key rss_entry = { .hash_fields = 0,
2063 /* Get a new map key for a new RSS rule */
2064 err = bpf_rss_key(KEY_CMD_GET, &flow->key_idx);
2067 error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
2068 "Failed to get BPF RSS key");
2073 /* Update RSS map entry with queues */
2074 rss_entry.nb_queues = rss->num;
2075 for (i = 0; i < rss->num; i++)
2076 rss_entry.queues[i] = rss->queue[i];
2077 rss_entry.hash_fields =
2078 (1 << HASH_FIELD_IPV4_L3_L4) | (1 << HASH_FIELD_IPV6_L3_L4);
2080 /* Add this RSS entry to map */
2081 err = tap_flow_bpf_update_rss_elem(pmd->map_fd,
2082 &flow->key_idx, &rss_entry);
2086 "Failed to update BPF map entry #%u (%d): %s\n",
2087 flow->key_idx, errno, strerror(errno));
2089 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
2090 "Kernel too old or not configured "
2091 "to support BPF maps updates");
2098 * Load bpf rules to calculate hash for this key_idx
2101 flow->bpf_fd[SEC_L3_L4] =
2102 tap_flow_bpf_calc_l3_l4_hash(flow->key_idx, pmd->map_fd);
2103 if (flow->bpf_fd[SEC_L3_L4] < 0) {
2105 "Failed to load BPF section %s (%d): %s\n",
2106 sec_name[SEC_L3_L4], errno, strerror(errno));
2108 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
2109 "Kernel too old or not configured "
2110 "to support BPF program loading");
2117 struct action_data adata[] = {
2121 .bpf_fd = flow->bpf_fd[SEC_L3_L4],
2122 .annotation = sec_name[SEC_L3_L4],
2124 .action = TC_ACT_PIPE,
2130 if (add_actions(flow, RTE_DIM(adata), adata,
2131 TCA_FLOWER_ACT) < 0)
2139 * Manage filter operations.
2142 * Pointer to Ethernet device structure.
2143 * @param filter_type
2146 * Operation to perform.
2148 * Pointer to operation-specific structure.
2151 * 0 on success, negative errno value on failure.
2154 tap_dev_filter_ctrl(struct rte_eth_dev *dev,
2155 enum rte_filter_type filter_type,
2156 enum rte_filter_op filter_op,
2159 switch (filter_type) {
2160 case RTE_ETH_FILTER_GENERIC:
2161 if (filter_op != RTE_ETH_FILTER_GET)
2163 *(const void **)arg = &tap_flow_ops;
2166 RTE_LOG(ERR, PMD, "%p: filter type (%d) not supported\n",
2167 (void *)dev, filter_type);