1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2017 6WIND S.A.
3 * Copyright 2017 Mellanox Technologies, Ltd
10 #include <sys/resource.h>
12 #include <rte_byteorder.h>
13 #include <rte_jhash.h>
14 #include <rte_malloc.h>
15 #include <rte_eth_tap.h>
17 #include <tap_autoconf.h>
18 #include <tap_tcmsgs.h>
21 #ifndef HAVE_TC_FLOWER
23 * For kernels < 4.2, this enum is not defined. Runtime checks will be made to
24 * avoid sending TC messages the kernel cannot understand.
31 TCA_FLOWER_KEY_ETH_DST, /* ETH_ALEN */
32 TCA_FLOWER_KEY_ETH_DST_MASK, /* ETH_ALEN */
33 TCA_FLOWER_KEY_ETH_SRC, /* ETH_ALEN */
34 TCA_FLOWER_KEY_ETH_SRC_MASK, /* ETH_ALEN */
35 TCA_FLOWER_KEY_ETH_TYPE, /* be16 */
36 TCA_FLOWER_KEY_IP_PROTO, /* u8 */
37 TCA_FLOWER_KEY_IPV4_SRC, /* be32 */
38 TCA_FLOWER_KEY_IPV4_SRC_MASK, /* be32 */
39 TCA_FLOWER_KEY_IPV4_DST, /* be32 */
40 TCA_FLOWER_KEY_IPV4_DST_MASK, /* be32 */
41 TCA_FLOWER_KEY_IPV6_SRC, /* struct in6_addr */
42 TCA_FLOWER_KEY_IPV6_SRC_MASK, /* struct in6_addr */
43 TCA_FLOWER_KEY_IPV6_DST, /* struct in6_addr */
44 TCA_FLOWER_KEY_IPV6_DST_MASK, /* struct in6_addr */
45 TCA_FLOWER_KEY_TCP_SRC, /* be16 */
46 TCA_FLOWER_KEY_TCP_DST, /* be16 */
47 TCA_FLOWER_KEY_UDP_SRC, /* be16 */
48 TCA_FLOWER_KEY_UDP_DST, /* be16 */
51 #ifndef HAVE_TC_VLAN_ID
53 /* TCA_FLOWER_FLAGS, */
54 TCA_FLOWER_KEY_VLAN_ID = TCA_FLOWER_KEY_UDP_DST + 2, /* be16 */
55 TCA_FLOWER_KEY_VLAN_PRIO, /* u8 */
56 TCA_FLOWER_KEY_VLAN_ETH_TYPE, /* be16 */
60 * For kernels < 4.2 BPF related enums may not be defined.
61 * Runtime checks will be carried out to gracefully report on TC messages that
62 * are rejected by the kernel. Rejection reasons may be due to:
63 * 1. enum is not defined
64 * 2. enum is defined but kernel is not configured to support BPF system calls,
65 * BPF classifications or BPF actions.
77 #ifndef HAVE_TC_BPF_FD
79 TCA_BPF_FD = TCA_BPF_OPS + 1,
83 #ifndef HAVE_TC_ACT_BPF
104 #ifndef HAVE_TC_ACT_BPF_FD
106 TCA_ACT_BPF_FD = TCA_ACT_BPF_OPS + 1,
111 /* RSS key management */
125 #define ISOLATE_HANDLE 1
126 #define REMOTE_PROMISCUOUS_HANDLE 2
129 LIST_ENTRY(rte_flow) next; /* Pointer to the next rte_flow structure */
130 struct rte_flow *remote_flow; /* associated remote flow */
131 int bpf_fd[SEC_MAX]; /* list of bfs fds per ELF section */
132 uint32_t key_idx; /* RSS rule key index into BPF map */
136 struct convert_data {
140 struct rte_flow *flow;
144 struct rte_flow_attr attr;
145 struct rte_flow_item items[2];
146 struct rte_flow_action actions[2];
155 struct tc_mirred mirred;
157 struct tc_skbedit skbedit;
161 struct tc_act_bpf bpf;
163 const char *annotation;
168 static int tap_flow_create_eth(const struct rte_flow_item *item, void *data);
169 static int tap_flow_create_vlan(const struct rte_flow_item *item, void *data);
170 static int tap_flow_create_ipv4(const struct rte_flow_item *item, void *data);
171 static int tap_flow_create_ipv6(const struct rte_flow_item *item, void *data);
172 static int tap_flow_create_udp(const struct rte_flow_item *item, void *data);
173 static int tap_flow_create_tcp(const struct rte_flow_item *item, void *data);
175 tap_flow_validate(struct rte_eth_dev *dev,
176 const struct rte_flow_attr *attr,
177 const struct rte_flow_item items[],
178 const struct rte_flow_action actions[],
179 struct rte_flow_error *error);
181 static struct rte_flow *
182 tap_flow_create(struct rte_eth_dev *dev,
183 const struct rte_flow_attr *attr,
184 const struct rte_flow_item items[],
185 const struct rte_flow_action actions[],
186 struct rte_flow_error *error);
189 tap_flow_free(struct pmd_internals *pmd,
190 struct rte_flow *flow);
193 tap_flow_destroy(struct rte_eth_dev *dev,
194 struct rte_flow *flow,
195 struct rte_flow_error *error);
198 tap_flow_isolate(struct rte_eth_dev *dev,
200 struct rte_flow_error *error);
202 static int bpf_rss_key(enum bpf_rss_key_e cmd, __u32 *key_idx);
203 static int rss_enable(struct pmd_internals *pmd,
204 const struct rte_flow_attr *attr,
205 struct rte_flow_error *error);
206 static int rss_add_actions(struct rte_flow *flow, struct pmd_internals *pmd,
207 const struct rte_flow_action_rss *rss,
208 struct rte_flow_error *error);
210 static const struct rte_flow_ops tap_flow_ops = {
211 .validate = tap_flow_validate,
212 .create = tap_flow_create,
213 .destroy = tap_flow_destroy,
214 .flush = tap_flow_flush,
215 .isolate = tap_flow_isolate,
218 /* Static initializer for items. */
220 (const enum rte_flow_item_type []){ \
221 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
224 /* Structure to generate a simple graph of layers supported by the NIC. */
225 struct tap_flow_items {
226 /* Bit-mask corresponding to what is supported for this item. */
228 const unsigned int mask_sz; /* Bit-mask size in bytes. */
230 * Bit-mask corresponding to the default mask, if none is provided
231 * along with the item.
233 const void *default_mask;
235 * Conversion function from rte_flow to netlink attributes.
238 * rte_flow item to convert.
240 * Internal structure to store the conversion.
243 * 0 on success, negative value otherwise.
245 int (*convert)(const struct rte_flow_item *item, void *data);
246 /** List of possible following items. */
247 const enum rte_flow_item_type *const items;
250 /* Graph of supported items and associated actions. */
251 static const struct tap_flow_items tap_flow_items[] = {
252 [RTE_FLOW_ITEM_TYPE_END] = {
253 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
255 [RTE_FLOW_ITEM_TYPE_ETH] = {
257 RTE_FLOW_ITEM_TYPE_VLAN,
258 RTE_FLOW_ITEM_TYPE_IPV4,
259 RTE_FLOW_ITEM_TYPE_IPV6),
260 .mask = &(const struct rte_flow_item_eth){
261 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
262 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
265 .mask_sz = sizeof(struct rte_flow_item_eth),
266 .default_mask = &rte_flow_item_eth_mask,
267 .convert = tap_flow_create_eth,
269 [RTE_FLOW_ITEM_TYPE_VLAN] = {
270 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
271 RTE_FLOW_ITEM_TYPE_IPV6),
272 .mask = &(const struct rte_flow_item_vlan){
273 /* DEI matching is not supported */
274 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
281 .mask_sz = sizeof(struct rte_flow_item_vlan),
282 .default_mask = &rte_flow_item_vlan_mask,
283 .convert = tap_flow_create_vlan,
285 [RTE_FLOW_ITEM_TYPE_IPV4] = {
286 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
287 RTE_FLOW_ITEM_TYPE_TCP),
288 .mask = &(const struct rte_flow_item_ipv4){
295 .mask_sz = sizeof(struct rte_flow_item_ipv4),
296 .default_mask = &rte_flow_item_ipv4_mask,
297 .convert = tap_flow_create_ipv4,
299 [RTE_FLOW_ITEM_TYPE_IPV6] = {
300 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
301 RTE_FLOW_ITEM_TYPE_TCP),
302 .mask = &(const struct rte_flow_item_ipv6){
305 "\xff\xff\xff\xff\xff\xff\xff\xff"
306 "\xff\xff\xff\xff\xff\xff\xff\xff",
309 "\xff\xff\xff\xff\xff\xff\xff\xff"
310 "\xff\xff\xff\xff\xff\xff\xff\xff",
315 .mask_sz = sizeof(struct rte_flow_item_ipv6),
316 .default_mask = &rte_flow_item_ipv6_mask,
317 .convert = tap_flow_create_ipv6,
319 [RTE_FLOW_ITEM_TYPE_UDP] = {
320 .mask = &(const struct rte_flow_item_udp){
326 .mask_sz = sizeof(struct rte_flow_item_udp),
327 .default_mask = &rte_flow_item_udp_mask,
328 .convert = tap_flow_create_udp,
330 [RTE_FLOW_ITEM_TYPE_TCP] = {
331 .mask = &(const struct rte_flow_item_tcp){
337 .mask_sz = sizeof(struct rte_flow_item_tcp),
338 .default_mask = &rte_flow_item_tcp_mask,
339 .convert = tap_flow_create_tcp,
344 * TC rules, by growing priority
346 * Remote netdevice Tap netdevice
347 * +-------------+-------------+ +-------------+-------------+
348 * | Ingress | Egress | | Ingress | Egress |
349 * |-------------|-------------| |-------------|-------------|
350 * | | \ / | | | REMOTE TX | prio 1
351 * | | \ / | | | \ / | prio 2
352 * | EXPLICIT | \ / | | EXPLICIT | \ / | .
353 * | | \ / | | | \ / | .
354 * | RULES | X | | RULES | X | .
355 * | . | / \ | | . | / \ | .
356 * | . | / \ | | . | / \ | .
357 * | . | / \ | | . | / \ | .
358 * | . | / \ | | . | / \ | .
360 * .... .... .... ....
362 * | . | \ / | | . | \ / | .
363 * | . | \ / | | . | \ / | .
364 * | | \ / | | | \ / |
365 * | LOCAL_MAC | \ / | | \ / | \ / | last prio - 5
366 * | PROMISC | X | | \ / | X | last prio - 4
367 * | ALLMULTI | / \ | | X | / \ | last prio - 3
368 * | BROADCAST | / \ | | / \ | / \ | last prio - 2
369 * | BROADCASTV6 | / \ | | / \ | / \ | last prio - 1
370 * | xx | / \ | | ISOLATE | / \ | last prio
371 * +-------------+-------------+ +-------------+-------------+
373 * The implicit flow rules are stored in a list in with mandatorily the last two
374 * being the ISOLATE and REMOTE_TX rules. e.g.:
376 * LOCAL_MAC -> BROADCAST -> BROADCASTV6 -> REMOTE_TX -> ISOLATE -> NULL
378 * That enables tap_flow_isolate() to remove implicit rules by popping the list
379 * head and remove it as long as it applies on the remote netdevice. The
380 * implicit rule for TX redirection is not removed, as isolate concerns only
384 static struct remote_rule implicit_rte_flows[TAP_REMOTE_MAX_IDX] = {
385 [TAP_REMOTE_LOCAL_MAC] = {
388 .priority = PRIORITY_MASK - TAP_REMOTE_LOCAL_MAC,
392 .type = RTE_FLOW_ITEM_TYPE_ETH,
393 .mask = &(const struct rte_flow_item_eth){
394 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
398 .type = RTE_FLOW_ITEM_TYPE_END,
400 .mirred = TCA_EGRESS_REDIR,
402 [TAP_REMOTE_BROADCAST] = {
405 .priority = PRIORITY_MASK - TAP_REMOTE_BROADCAST,
409 .type = RTE_FLOW_ITEM_TYPE_ETH,
410 .mask = &(const struct rte_flow_item_eth){
411 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
413 .spec = &(const struct rte_flow_item_eth){
414 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
418 .type = RTE_FLOW_ITEM_TYPE_END,
420 .mirred = TCA_EGRESS_MIRROR,
422 [TAP_REMOTE_BROADCASTV6] = {
425 .priority = PRIORITY_MASK - TAP_REMOTE_BROADCASTV6,
429 .type = RTE_FLOW_ITEM_TYPE_ETH,
430 .mask = &(const struct rte_flow_item_eth){
431 .dst.addr_bytes = "\x33\x33\x00\x00\x00\x00",
433 .spec = &(const struct rte_flow_item_eth){
434 .dst.addr_bytes = "\x33\x33\x00\x00\x00\x00",
438 .type = RTE_FLOW_ITEM_TYPE_END,
440 .mirred = TCA_EGRESS_MIRROR,
442 [TAP_REMOTE_PROMISC] = {
445 .priority = PRIORITY_MASK - TAP_REMOTE_PROMISC,
449 .type = RTE_FLOW_ITEM_TYPE_VOID,
452 .type = RTE_FLOW_ITEM_TYPE_END,
454 .mirred = TCA_EGRESS_MIRROR,
456 [TAP_REMOTE_ALLMULTI] = {
459 .priority = PRIORITY_MASK - TAP_REMOTE_ALLMULTI,
463 .type = RTE_FLOW_ITEM_TYPE_ETH,
464 .mask = &(const struct rte_flow_item_eth){
465 .dst.addr_bytes = "\x01\x00\x00\x00\x00\x00",
467 .spec = &(const struct rte_flow_item_eth){
468 .dst.addr_bytes = "\x01\x00\x00\x00\x00\x00",
472 .type = RTE_FLOW_ITEM_TYPE_END,
474 .mirred = TCA_EGRESS_MIRROR,
479 .priority = TAP_REMOTE_TX,
483 .type = RTE_FLOW_ITEM_TYPE_VOID,
486 .type = RTE_FLOW_ITEM_TYPE_END,
488 .mirred = TCA_EGRESS_MIRROR,
493 .priority = PRIORITY_MASK - TAP_ISOLATE,
497 .type = RTE_FLOW_ITEM_TYPE_VOID,
500 .type = RTE_FLOW_ITEM_TYPE_END,
506 * Make as much checks as possible on an Ethernet item, and if a flow is
507 * provided, fill it appropriately with Ethernet info.
510 * Item specification.
511 * @param[in, out] data
512 * Additional data structure to tell next layers we've been here.
515 * 0 if checks are alright, -1 otherwise.
518 tap_flow_create_eth(const struct rte_flow_item *item, void *data)
520 struct convert_data *info = (struct convert_data *)data;
521 const struct rte_flow_item_eth *spec = item->spec;
522 const struct rte_flow_item_eth *mask = item->mask;
523 struct rte_flow *flow = info->flow;
526 /* use default mask if none provided */
528 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_ETH].default_mask;
529 /* TC does not support eth_type masking. Only accept if exact match. */
530 if (mask->type && mask->type != 0xffff)
534 /* store eth_type for consistency if ipv4/6 pattern item comes next */
535 if (spec->type & mask->type)
536 info->eth_type = spec->type;
540 if (!is_zero_ether_addr(&spec->dst)) {
541 tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_ETH_DST, ETHER_ADDR_LEN,
542 &spec->dst.addr_bytes);
543 tap_nlattr_add(&msg->nh,
544 TCA_FLOWER_KEY_ETH_DST_MASK, ETHER_ADDR_LEN,
545 &mask->dst.addr_bytes);
547 if (!is_zero_ether_addr(&mask->src)) {
548 tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_ETH_SRC, ETHER_ADDR_LEN,
549 &spec->src.addr_bytes);
550 tap_nlattr_add(&msg->nh,
551 TCA_FLOWER_KEY_ETH_SRC_MASK, ETHER_ADDR_LEN,
552 &mask->src.addr_bytes);
558 * Make as much checks as possible on a VLAN item, and if a flow is provided,
559 * fill it appropriately with VLAN info.
562 * Item specification.
563 * @param[in, out] data
564 * Additional data structure to tell next layers we've been here.
567 * 0 if checks are alright, -1 otherwise.
570 tap_flow_create_vlan(const struct rte_flow_item *item, void *data)
572 struct convert_data *info = (struct convert_data *)data;
573 const struct rte_flow_item_vlan *spec = item->spec;
574 const struct rte_flow_item_vlan *mask = item->mask;
575 struct rte_flow *flow = info->flow;
578 /* use default mask if none provided */
580 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_VLAN].default_mask;
581 /* Outer TPID cannot be matched. */
584 /* Double-tagging not supported. */
588 if (mask->inner_type) {
589 /* TC does not support partial eth_type masking */
590 if (mask->inner_type != RTE_BE16(0xffff))
592 info->eth_type = spec->inner_type;
597 msg->t.tcm_info = TC_H_MAKE(msg->t.tcm_info, htons(ETH_P_8021Q));
598 #define VLAN_PRIO(tci) ((tci) >> 13)
599 #define VLAN_ID(tci) ((tci) & 0xfff)
603 uint16_t tci = ntohs(spec->tci) & mask->tci;
604 uint16_t prio = VLAN_PRIO(tci);
605 uint8_t vid = VLAN_ID(tci);
608 tap_nlattr_add8(&msg->nh,
609 TCA_FLOWER_KEY_VLAN_PRIO, prio);
611 tap_nlattr_add16(&msg->nh,
612 TCA_FLOWER_KEY_VLAN_ID, vid);
618 * Make as much checks as possible on an IPv4 item, and if a flow is provided,
619 * fill it appropriately with IPv4 info.
622 * Item specification.
623 * @param[in, out] data
624 * Additional data structure to tell next layers we've been here.
627 * 0 if checks are alright, -1 otherwise.
630 tap_flow_create_ipv4(const struct rte_flow_item *item, void *data)
632 struct convert_data *info = (struct convert_data *)data;
633 const struct rte_flow_item_ipv4 *spec = item->spec;
634 const struct rte_flow_item_ipv4 *mask = item->mask;
635 struct rte_flow *flow = info->flow;
638 /* use default mask if none provided */
640 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_IPV4].default_mask;
641 /* check that previous eth type is compatible with ipv4 */
642 if (info->eth_type && info->eth_type != htons(ETH_P_IP))
644 /* store ip_proto for consistency if udp/tcp pattern item comes next */
646 info->ip_proto = spec->hdr.next_proto_id;
651 info->eth_type = htons(ETH_P_IP);
654 if (spec->hdr.dst_addr) {
655 tap_nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_DST,
657 tap_nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_DST_MASK,
660 if (spec->hdr.src_addr) {
661 tap_nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_SRC,
663 tap_nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_SRC_MASK,
666 if (spec->hdr.next_proto_id)
667 tap_nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO,
668 spec->hdr.next_proto_id);
673 * Make as much checks as possible on an IPv6 item, and if a flow is provided,
674 * fill it appropriately with IPv6 info.
677 * Item specification.
678 * @param[in, out] data
679 * Additional data structure to tell next layers we've been here.
682 * 0 if checks are alright, -1 otherwise.
685 tap_flow_create_ipv6(const struct rte_flow_item *item, void *data)
687 struct convert_data *info = (struct convert_data *)data;
688 const struct rte_flow_item_ipv6 *spec = item->spec;
689 const struct rte_flow_item_ipv6 *mask = item->mask;
690 struct rte_flow *flow = info->flow;
691 uint8_t empty_addr[16] = { 0 };
694 /* use default mask if none provided */
696 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_IPV6].default_mask;
697 /* check that previous eth type is compatible with ipv6 */
698 if (info->eth_type && info->eth_type != htons(ETH_P_IPV6))
700 /* store ip_proto for consistency if udp/tcp pattern item comes next */
702 info->ip_proto = spec->hdr.proto;
707 info->eth_type = htons(ETH_P_IPV6);
710 if (memcmp(spec->hdr.dst_addr, empty_addr, 16)) {
711 tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_DST,
712 sizeof(spec->hdr.dst_addr), &spec->hdr.dst_addr);
713 tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_DST_MASK,
714 sizeof(mask->hdr.dst_addr), &mask->hdr.dst_addr);
716 if (memcmp(spec->hdr.src_addr, empty_addr, 16)) {
717 tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_SRC,
718 sizeof(spec->hdr.src_addr), &spec->hdr.src_addr);
719 tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_SRC_MASK,
720 sizeof(mask->hdr.src_addr), &mask->hdr.src_addr);
723 tap_nlattr_add8(&msg->nh,
724 TCA_FLOWER_KEY_IP_PROTO, spec->hdr.proto);
729 * Make as much checks as possible on a UDP item, and if a flow is provided,
730 * fill it appropriately with UDP info.
733 * Item specification.
734 * @param[in, out] data
735 * Additional data structure to tell next layers we've been here.
738 * 0 if checks are alright, -1 otherwise.
741 tap_flow_create_udp(const struct rte_flow_item *item, void *data)
743 struct convert_data *info = (struct convert_data *)data;
744 const struct rte_flow_item_udp *spec = item->spec;
745 const struct rte_flow_item_udp *mask = item->mask;
746 struct rte_flow *flow = info->flow;
749 /* use default mask if none provided */
751 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_UDP].default_mask;
752 /* check that previous ip_proto is compatible with udp */
753 if (info->ip_proto && info->ip_proto != IPPROTO_UDP)
755 /* TC does not support UDP port masking. Only accept if exact match. */
756 if ((mask->hdr.src_port && mask->hdr.src_port != 0xffff) ||
757 (mask->hdr.dst_port && mask->hdr.dst_port != 0xffff))
762 tap_nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, IPPROTO_UDP);
765 if (spec->hdr.dst_port & mask->hdr.dst_port)
766 tap_nlattr_add16(&msg->nh, TCA_FLOWER_KEY_UDP_DST,
768 if (spec->hdr.src_port & mask->hdr.src_port)
769 tap_nlattr_add16(&msg->nh, TCA_FLOWER_KEY_UDP_SRC,
775 * Make as much checks as possible on a TCP item, and if a flow is provided,
776 * fill it appropriately with TCP info.
779 * Item specification.
780 * @param[in, out] data
781 * Additional data structure to tell next layers we've been here.
784 * 0 if checks are alright, -1 otherwise.
787 tap_flow_create_tcp(const struct rte_flow_item *item, void *data)
789 struct convert_data *info = (struct convert_data *)data;
790 const struct rte_flow_item_tcp *spec = item->spec;
791 const struct rte_flow_item_tcp *mask = item->mask;
792 struct rte_flow *flow = info->flow;
795 /* use default mask if none provided */
797 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_TCP].default_mask;
798 /* check that previous ip_proto is compatible with tcp */
799 if (info->ip_proto && info->ip_proto != IPPROTO_TCP)
801 /* TC does not support TCP port masking. Only accept if exact match. */
802 if ((mask->hdr.src_port && mask->hdr.src_port != 0xffff) ||
803 (mask->hdr.dst_port && mask->hdr.dst_port != 0xffff))
808 tap_nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, IPPROTO_TCP);
811 if (spec->hdr.dst_port & mask->hdr.dst_port)
812 tap_nlattr_add16(&msg->nh, TCA_FLOWER_KEY_TCP_DST,
814 if (spec->hdr.src_port & mask->hdr.src_port)
815 tap_nlattr_add16(&msg->nh, TCA_FLOWER_KEY_TCP_SRC,
821 * Check support for a given item.
824 * Item specification.
826 * Bit-Mask size in bytes.
827 * @param[in] supported_mask
828 * Bit-mask covering supported fields to compare with spec, last and mask in
830 * @param[in] default_mask
831 * Bit-mask default mask if none is provided in \item.
837 tap_flow_item_validate(const struct rte_flow_item *item,
839 const uint8_t *supported_mask,
840 const uint8_t *default_mask)
844 /* An empty layer is allowed, as long as all fields are NULL */
845 if (!item->spec && (item->mask || item->last))
847 /* Is the item spec compatible with what the NIC supports? */
848 if (item->spec && !item->mask) {
850 const uint8_t *spec = item->spec;
852 for (i = 0; i < size; ++i)
853 if ((spec[i] | supported_mask[i]) != supported_mask[i])
855 /* Is the default mask compatible with what the NIC supports? */
856 for (i = 0; i < size; i++)
857 if ((default_mask[i] | supported_mask[i]) !=
861 /* Is the item last compatible with what the NIC supports? */
862 if (item->last && !item->mask) {
864 const uint8_t *spec = item->last;
866 for (i = 0; i < size; ++i)
867 if ((spec[i] | supported_mask[i]) != supported_mask[i])
870 /* Is the item mask compatible with what the NIC supports? */
873 const uint8_t *spec = item->mask;
875 for (i = 0; i < size; ++i)
876 if ((spec[i] | supported_mask[i]) != supported_mask[i])
880 * Once masked, Are item spec and item last equal?
881 * TC does not support range so anything else is invalid.
883 if (item->spec && item->last) {
886 const uint8_t *apply = default_mask;
891 for (i = 0; i < size; ++i) {
892 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
893 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
895 ret = memcmp(spec, last, size);
901 * Configure the kernel with a TC action and its configured parameters
902 * Handled actions: "gact", "mirred", "skbedit", "bpf"
905 * Pointer to rte flow containing the netlink message
907 * @param[in, out] act_index
908 * Pointer to action sequence number in the TC command
911 * Pointer to struct holding the action parameters
914 * -1 on failure, 0 on success
917 add_action(struct rte_flow *flow, size_t *act_index, struct action_data *adata)
919 struct nlmsg *msg = &flow->msg;
921 if (tap_nlattr_nested_start(msg, (*act_index)++) < 0)
924 tap_nlattr_add(&msg->nh, TCA_ACT_KIND,
925 strlen(adata->id) + 1, adata->id);
926 if (tap_nlattr_nested_start(msg, TCA_ACT_OPTIONS) < 0)
928 if (strcmp("gact", adata->id) == 0) {
929 tap_nlattr_add(&msg->nh, TCA_GACT_PARMS, sizeof(adata->gact),
931 } else if (strcmp("mirred", adata->id) == 0) {
932 if (adata->mirred.eaction == TCA_EGRESS_MIRROR)
933 adata->mirred.action = TC_ACT_PIPE;
935 adata->mirred.action = TC_ACT_STOLEN;
936 tap_nlattr_add(&msg->nh, TCA_MIRRED_PARMS,
937 sizeof(adata->mirred),
939 } else if (strcmp("skbedit", adata->id) == 0) {
940 tap_nlattr_add(&msg->nh, TCA_SKBEDIT_PARMS,
941 sizeof(adata->skbedit.skbedit),
942 &adata->skbedit.skbedit);
943 tap_nlattr_add16(&msg->nh, TCA_SKBEDIT_QUEUE_MAPPING,
944 adata->skbedit.queue);
945 } else if (strcmp("bpf", adata->id) == 0) {
946 tap_nlattr_add32(&msg->nh, TCA_ACT_BPF_FD, adata->bpf.bpf_fd);
947 tap_nlattr_add(&msg->nh, TCA_ACT_BPF_NAME,
948 strlen(adata->bpf.annotation) + 1,
949 adata->bpf.annotation);
950 tap_nlattr_add(&msg->nh, TCA_ACT_BPF_PARMS,
951 sizeof(adata->bpf.bpf),
956 tap_nlattr_nested_finish(msg); /* nested TCA_ACT_OPTIONS */
957 tap_nlattr_nested_finish(msg); /* nested act_index */
962 * Helper function to send a serie of TC actions to the kernel
965 * Pointer to rte flow containing the netlink message
967 * @param[in] nb_actions
968 * Number of actions in an array of action structs
971 * Pointer to an array of action structs
973 * @param[in] classifier_actions
974 * The classifier on behave of which the actions are configured
977 * -1 on failure, 0 on success
980 add_actions(struct rte_flow *flow, int nb_actions, struct action_data *data,
981 int classifier_action)
983 struct nlmsg *msg = &flow->msg;
984 size_t act_index = 1;
987 if (tap_nlattr_nested_start(msg, classifier_action) < 0)
989 for (i = 0; i < nb_actions; i++)
990 if (add_action(flow, &act_index, data + i) < 0)
992 tap_nlattr_nested_finish(msg); /* nested TCA_FLOWER_ACT */
997 * Validate a flow supported by TC.
998 * If flow param is not NULL, then also fill the netlink message inside.
1001 * Pointer to private structure.
1003 * Flow rule attributes.
1004 * @param[in] pattern
1005 * Pattern specification (list terminated by the END pattern item).
1006 * @param[in] actions
1007 * Associated actions (list terminated by the END action).
1009 * Perform verbose error reporting if not NULL.
1010 * @param[in, out] flow
1011 * Flow structure to update.
1013 * If set to TCA_EGRESS_REDIR, provided actions will be replaced with a
1014 * redirection to the tap netdevice, and the TC rule will be configured
1015 * on the remote netdevice in pmd.
1016 * If set to TCA_EGRESS_MIRROR, provided actions will be replaced with a
1017 * mirroring to the tap netdevice, and the TC rule will be configured
1018 * on the remote netdevice in pmd. Matching packets will thus be duplicated.
1019 * If set to 0, the standard behavior is to be used: set correct actions for
1020 * the TC rule, and apply it on the tap netdevice.
1023 * 0 on success, a negative errno value otherwise and rte_errno is set.
1026 priv_flow_process(struct pmd_internals *pmd,
1027 const struct rte_flow_attr *attr,
1028 const struct rte_flow_item items[],
1029 const struct rte_flow_action actions[],
1030 struct rte_flow_error *error,
1031 struct rte_flow *flow,
1034 const struct tap_flow_items *cur_item = tap_flow_items;
1035 struct convert_data data = {
1040 int action = 0; /* Only one action authorized for now */
1042 if (attr->transfer) {
1044 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1045 NULL, "transfer is not supported");
1048 if (attr->group > MAX_GROUP) {
1050 error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
1051 NULL, "group value too big: cannot exceed 15");
1054 if (attr->priority > MAX_PRIORITY) {
1056 error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1057 NULL, "priority value too big");
1060 uint16_t group = attr->group << GROUP_SHIFT;
1061 uint16_t prio = group | (attr->priority +
1062 RSS_PRIORITY_OFFSET + PRIORITY_OFFSET);
1063 flow->msg.t.tcm_info = TC_H_MAKE(prio << 16,
1064 flow->msg.t.tcm_info);
1069 * If attr->ingress, the rule applies on remote ingress
1070 * to match incoming packets
1071 * If attr->egress, the rule applies on tap ingress (as
1072 * seen from the kernel) to deal with packets going out
1073 * from the DPDK app.
1075 flow->msg.t.tcm_parent = TC_H_MAKE(TC_H_INGRESS, 0);
1077 /* Standard rule on tap egress (kernel standpoint). */
1078 flow->msg.t.tcm_parent =
1079 TC_H_MAKE(MULTIQ_MAJOR_HANDLE, 0);
1081 /* use flower filter type */
1082 tap_nlattr_add(&flow->msg.nh, TCA_KIND, sizeof("flower"), "flower");
1083 if (tap_nlattr_nested_start(&flow->msg, TCA_OPTIONS) < 0)
1084 goto exit_item_not_supported;
1086 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
1087 const struct tap_flow_items *token = NULL;
1091 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
1095 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
1097 if (cur_item->items[i] == items->type) {
1098 token = &tap_flow_items[items->type];
1103 goto exit_item_not_supported;
1105 err = tap_flow_item_validate(
1106 items, cur_item->mask_sz,
1107 (const uint8_t *)cur_item->mask,
1108 (const uint8_t *)cur_item->default_mask);
1110 goto exit_item_not_supported;
1111 if (flow && cur_item->convert) {
1112 err = cur_item->convert(items, &data);
1114 goto exit_item_not_supported;
1119 tap_nlattr_add16(&flow->msg.nh, TCA_FLOWER_KEY_ETH_TYPE,
1120 htons(ETH_P_8021Q));
1121 tap_nlattr_add16(&flow->msg.nh,
1122 TCA_FLOWER_KEY_VLAN_ETH_TYPE,
1124 data.eth_type : htons(ETH_P_ALL));
1125 } else if (data.eth_type) {
1126 tap_nlattr_add16(&flow->msg.nh, TCA_FLOWER_KEY_ETH_TYPE,
1130 if (mirred && flow) {
1131 struct action_data adata = {
1139 * If attr->egress && mirred, then this is a special
1140 * case where the rule must be applied on the tap, to
1141 * redirect packets coming from the DPDK App, out
1142 * through the remote netdevice.
1144 adata.mirred.ifindex = attr->ingress ? pmd->if_index :
1145 pmd->remote_if_index;
1146 if (mirred == TCA_EGRESS_MIRROR)
1147 adata.mirred.action = TC_ACT_PIPE;
1149 adata.mirred.action = TC_ACT_STOLEN;
1150 if (add_actions(flow, 1, &adata, TCA_FLOWER_ACT) < 0)
1151 goto exit_action_not_supported;
1156 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
1159 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
1161 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
1163 goto exit_action_not_supported;
1166 struct action_data adata = {
1169 .action = TC_ACT_SHOT,
1173 err = add_actions(flow, 1, &adata,
1176 } else if (actions->type == RTE_FLOW_ACTION_TYPE_PASSTHRU) {
1178 goto exit_action_not_supported;
1181 struct action_data adata = {
1185 .action = TC_ACT_UNSPEC,
1189 err = add_actions(flow, 1, &adata,
1192 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
1193 const struct rte_flow_action_queue *queue =
1194 (const struct rte_flow_action_queue *)
1198 goto exit_action_not_supported;
1201 (queue->index > pmd->dev->data->nb_rx_queues - 1))
1202 goto exit_action_not_supported;
1204 struct action_data adata = {
1208 .action = TC_ACT_PIPE,
1210 .queue = queue->index,
1214 err = add_actions(flow, 1, &adata,
1217 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
1218 const struct rte_flow_action_rss *rss =
1219 (const struct rte_flow_action_rss *)
1223 goto exit_action_not_supported;
1225 if (!pmd->rss_enabled) {
1226 err = rss_enable(pmd, attr, error);
1228 goto exit_action_not_supported;
1231 err = rss_add_actions(flow, pmd, rss, error);
1233 goto exit_action_not_supported;
1236 goto exit_action_not_supported;
1238 /* When fate is unknown, drop traffic. */
1240 static const struct rte_flow_action drop[] = {
1241 { .type = RTE_FLOW_ACTION_TYPE_DROP, },
1242 { .type = RTE_FLOW_ACTION_TYPE_END, },
1250 tap_nlattr_nested_finish(&flow->msg); /* nested TCA_OPTIONS */
1252 exit_item_not_supported:
1253 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1254 items, "item not supported");
1256 exit_action_not_supported:
1257 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
1258 actions, "action not supported");
1267 * @see rte_flow_validate()
1271 tap_flow_validate(struct rte_eth_dev *dev,
1272 const struct rte_flow_attr *attr,
1273 const struct rte_flow_item items[],
1274 const struct rte_flow_action actions[],
1275 struct rte_flow_error *error)
1277 struct pmd_internals *pmd = dev->data->dev_private;
1279 return priv_flow_process(pmd, attr, items, actions, error, NULL, 0);
1283 * Set a unique handle in a flow.
1285 * The kernel supports TC rules with equal priority, as long as they use the
1286 * same matching fields (e.g.: dst mac and ipv4) with different values (and
1287 * full mask to ensure no collision is possible).
1288 * In those rules, the handle (uint32_t) is the part that would identify
1289 * specifically each rule.
1291 * On 32-bit architectures, the handle can simply be the flow's pointer address.
1292 * On 64-bit architectures, we rely on jhash(flow) to find a (sufficiently)
1295 * @param[in, out] flow
1296 * The flow that needs its handle set.
1299 tap_flow_set_handle(struct rte_flow *flow)
1301 uint32_t handle = 0;
1303 if (sizeof(flow) > 4)
1304 handle = rte_jhash(&flow, sizeof(flow), 1);
1306 handle = (uintptr_t)flow;
1307 /* must be at least 1 to avoid letting the kernel choose one for us */
1310 flow->msg.t.tcm_handle = handle;
1314 * Free the flow opened file descriptors and allocated memory
1317 * Pointer to the flow to free
1321 tap_flow_free(struct pmd_internals *pmd, struct rte_flow *flow)
1328 if (pmd->rss_enabled) {
1329 /* Close flow BPF file descriptors */
1330 for (i = 0; i < SEC_MAX; i++)
1331 if (flow->bpf_fd[i] != 0) {
1332 close(flow->bpf_fd[i]);
1333 flow->bpf_fd[i] = 0;
1336 /* Release the map key for this RSS rule */
1337 bpf_rss_key(KEY_CMD_RELEASE, &flow->key_idx);
1341 /* Free flow allocated memory */
1348 * @see rte_flow_create()
1351 static struct rte_flow *
1352 tap_flow_create(struct rte_eth_dev *dev,
1353 const struct rte_flow_attr *attr,
1354 const struct rte_flow_item items[],
1355 const struct rte_flow_action actions[],
1356 struct rte_flow_error *error)
1358 struct pmd_internals *pmd = dev->data->dev_private;
1359 struct rte_flow *remote_flow = NULL;
1360 struct rte_flow *flow = NULL;
1361 struct nlmsg *msg = NULL;
1364 if (!pmd->if_index) {
1365 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
1367 "can't create rule, ifindex not found");
1371 * No rules configured through standard rte_flow should be set on the
1372 * priorities used by implicit rules.
1374 if ((attr->group == MAX_GROUP) &&
1375 attr->priority > (MAX_PRIORITY - TAP_REMOTE_MAX_IDX)) {
1377 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1378 NULL, "priority value too big");
1381 flow = rte_malloc(__func__, sizeof(struct rte_flow), 0);
1383 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1384 NULL, "cannot allocate memory for rte_flow");
1388 tc_init_msg(msg, pmd->if_index, RTM_NEWTFILTER,
1389 NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE);
1390 msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL));
1391 tap_flow_set_handle(flow);
1392 if (priv_flow_process(pmd, attr, items, actions, error, flow, 0))
1394 err = tap_nl_send(pmd->nlsk_fd, &msg->nh);
1396 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
1397 NULL, "couldn't send request to kernel");
1400 err = tap_nl_recv_ack(pmd->nlsk_fd);
1403 "Kernel refused TC filter rule creation (%d): %s",
1404 errno, strerror(errno));
1405 rte_flow_error_set(error, EEXIST, RTE_FLOW_ERROR_TYPE_HANDLE,
1407 "overlapping rules or Kernel too old for flower support");
1410 LIST_INSERT_HEAD(&pmd->flows, flow, next);
1412 * If a remote device is configured, a TC rule with identical items for
1413 * matching must be set on that device, with a single action: redirect
1414 * to the local pmd->if_index.
1416 if (pmd->remote_if_index) {
1417 remote_flow = rte_malloc(__func__, sizeof(struct rte_flow), 0);
1420 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1421 "cannot allocate memory for rte_flow");
1424 msg = &remote_flow->msg;
1425 /* set the rule if_index for the remote netdevice */
1427 msg, pmd->remote_if_index, RTM_NEWTFILTER,
1428 NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE);
1429 msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL));
1430 tap_flow_set_handle(remote_flow);
1431 if (priv_flow_process(pmd, attr, items, NULL,
1432 error, remote_flow, TCA_EGRESS_REDIR)) {
1434 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1435 NULL, "rte flow rule validation failed");
1438 err = tap_nl_send(pmd->nlsk_fd, &msg->nh);
1441 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1442 NULL, "Failure sending nl request");
1445 err = tap_nl_recv_ack(pmd->nlsk_fd);
1448 "Kernel refused TC filter rule creation (%d): %s",
1449 errno, strerror(errno));
1451 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1453 "overlapping rules or Kernel too old for flower support");
1456 flow->remote_flow = remote_flow;
1461 rte_free(remote_flow);
1463 tap_flow_free(pmd, flow);
1468 * Destroy a flow using pointer to pmd_internal.
1470 * @param[in, out] pmd
1471 * Pointer to private structure.
1473 * Pointer to the flow to destroy.
1474 * @param[in, out] error
1475 * Pointer to the flow error handler
1477 * @return 0 if the flow could be destroyed, -1 otherwise.
1480 tap_flow_destroy_pmd(struct pmd_internals *pmd,
1481 struct rte_flow *flow,
1482 struct rte_flow_error *error)
1484 struct rte_flow *remote_flow = flow->remote_flow;
1487 LIST_REMOVE(flow, next);
1488 flow->msg.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK;
1489 flow->msg.nh.nlmsg_type = RTM_DELTFILTER;
1491 ret = tap_nl_send(pmd->nlsk_fd, &flow->msg.nh);
1493 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
1494 NULL, "couldn't send request to kernel");
1497 ret = tap_nl_recv_ack(pmd->nlsk_fd);
1498 /* If errno is ENOENT, the rule is already no longer in the kernel. */
1499 if (ret < 0 && errno == ENOENT)
1503 "Kernel refused TC filter rule deletion (%d): %s",
1504 errno, strerror(errno));
1506 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1507 "couldn't receive kernel ack to our request");
1512 remote_flow->msg.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK;
1513 remote_flow->msg.nh.nlmsg_type = RTM_DELTFILTER;
1515 ret = tap_nl_send(pmd->nlsk_fd, &remote_flow->msg.nh);
1518 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1519 NULL, "Failure sending nl request");
1522 ret = tap_nl_recv_ack(pmd->nlsk_fd);
1523 if (ret < 0 && errno == ENOENT)
1527 "Kernel refused TC filter rule deletion (%d): %s",
1528 errno, strerror(errno));
1530 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1531 NULL, "Failure trying to receive nl ack");
1537 rte_free(remote_flow);
1538 tap_flow_free(pmd, flow);
1545 * @see rte_flow_destroy()
1549 tap_flow_destroy(struct rte_eth_dev *dev,
1550 struct rte_flow *flow,
1551 struct rte_flow_error *error)
1553 struct pmd_internals *pmd = dev->data->dev_private;
1555 return tap_flow_destroy_pmd(pmd, flow, error);
1559 * Enable/disable flow isolation.
1561 * @see rte_flow_isolate()
1565 tap_flow_isolate(struct rte_eth_dev *dev,
1567 struct rte_flow_error *error __rte_unused)
1569 struct pmd_internals *pmd = dev->data->dev_private;
1571 /* normalize 'set' variable to contain 0 or 1 values */
1574 /* if already in the right isolation mode - nothing to do */
1575 if ((set ^ pmd->flow_isolate) == 0)
1577 /* mark the isolation mode for tap_flow_implicit_create() */
1578 pmd->flow_isolate = set;
1580 * If netdevice is there, setup appropriate flow rules immediately.
1581 * Otherwise it will be set when bringing up the netdevice (tun_alloc).
1583 if (!pmd->rxq[0].fd)
1586 struct rte_flow *remote_flow;
1589 remote_flow = LIST_FIRST(&pmd->implicit_flows);
1593 * Remove all implicit rules on the remote.
1594 * Keep the local rule to redirect packets on TX.
1595 * Keep also the last implicit local rule: ISOLATE.
1597 if (remote_flow->msg.t.tcm_ifindex == pmd->if_index)
1599 if (tap_flow_destroy_pmd(pmd, remote_flow, NULL) < 0)
1602 /* Switch the TC rule according to pmd->flow_isolate */
1603 if (tap_flow_implicit_create(pmd, TAP_ISOLATE) == -1)
1606 /* Switch the TC rule according to pmd->flow_isolate */
1607 if (tap_flow_implicit_create(pmd, TAP_ISOLATE) == -1)
1609 if (!pmd->remote_if_index)
1611 if (tap_flow_implicit_create(pmd, TAP_REMOTE_TX) < 0)
1613 if (tap_flow_implicit_create(pmd, TAP_REMOTE_LOCAL_MAC) < 0)
1615 if (tap_flow_implicit_create(pmd, TAP_REMOTE_BROADCAST) < 0)
1617 if (tap_flow_implicit_create(pmd, TAP_REMOTE_BROADCASTV6) < 0)
1619 if (dev->data->promiscuous &&
1620 tap_flow_implicit_create(pmd, TAP_REMOTE_PROMISC) < 0)
1622 if (dev->data->all_multicast &&
1623 tap_flow_implicit_create(pmd, TAP_REMOTE_ALLMULTI) < 0)
1628 pmd->flow_isolate = 0;
1629 return rte_flow_error_set(
1630 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1631 "TC rule creation failed");
1635 * Destroy all flows.
1637 * @see rte_flow_flush()
1641 tap_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error)
1643 struct pmd_internals *pmd = dev->data->dev_private;
1644 struct rte_flow *flow;
1646 while (!LIST_EMPTY(&pmd->flows)) {
1647 flow = LIST_FIRST(&pmd->flows);
1648 if (tap_flow_destroy(dev, flow, error) < 0)
1655 * Add an implicit flow rule on the remote device to make sure traffic gets to
1656 * the tap netdevice from there.
1659 * Pointer to private structure.
1661 * The idx in the implicit_rte_flows array specifying which rule to apply.
1663 * @return -1 if the rule couldn't be applied, 0 otherwise.
1665 int tap_flow_implicit_create(struct pmd_internals *pmd,
1666 enum implicit_rule_index idx)
1668 uint16_t flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE;
1669 struct rte_flow_action *actions = implicit_rte_flows[idx].actions;
1670 struct rte_flow_action isolate_actions[2] = {
1672 .type = RTE_FLOW_ACTION_TYPE_END,
1675 struct rte_flow_item *items = implicit_rte_flows[idx].items;
1676 struct rte_flow_attr *attr = &implicit_rte_flows[idx].attr;
1677 struct rte_flow_item_eth eth_local = { .type = 0 };
1678 uint16_t if_index = pmd->remote_if_index;
1679 struct rte_flow *remote_flow = NULL;
1680 struct nlmsg *msg = NULL;
1682 struct rte_flow_item items_local[2] = {
1684 .type = items[0].type,
1686 .mask = items[0].mask,
1689 .type = items[1].type,
1693 remote_flow = rte_malloc(__func__, sizeof(struct rte_flow), 0);
1695 TAP_LOG(ERR, "Cannot allocate memory for rte_flow");
1698 msg = &remote_flow->msg;
1699 if (idx == TAP_REMOTE_TX) {
1700 if_index = pmd->if_index;
1701 } else if (idx == TAP_ISOLATE) {
1702 if_index = pmd->if_index;
1703 /* Don't be exclusive for this rule, it can be changed later. */
1704 flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_CREATE;
1705 isolate_actions[0].type = pmd->flow_isolate ?
1706 RTE_FLOW_ACTION_TYPE_DROP :
1707 RTE_FLOW_ACTION_TYPE_PASSTHRU;
1708 actions = isolate_actions;
1709 } else if (idx == TAP_REMOTE_LOCAL_MAC) {
1711 * eth addr couldn't be set in implicit_rte_flows[] as it is not
1712 * known at compile time.
1714 memcpy(ð_local.dst, &pmd->eth_addr, sizeof(pmd->eth_addr));
1715 items = items_local;
1717 tc_init_msg(msg, if_index, RTM_NEWTFILTER, flags);
1718 msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL));
1720 * The ISOLATE rule is always present and must have a static handle, as
1721 * the action is changed whether the feature is enabled (DROP) or
1722 * disabled (PASSTHRU).
1723 * There is just one REMOTE_PROMISCUOUS rule in all cases. It should
1724 * have a static handle such that adding it twice will fail with EEXIST
1725 * with any kernel version. Remark: old kernels may falsely accept the
1726 * same REMOTE_PROMISCUOUS rules if they had different handles.
1728 if (idx == TAP_ISOLATE)
1729 remote_flow->msg.t.tcm_handle = ISOLATE_HANDLE;
1730 else if (idx == TAP_REMOTE_PROMISC)
1731 remote_flow->msg.t.tcm_handle = REMOTE_PROMISCUOUS_HANDLE;
1733 tap_flow_set_handle(remote_flow);
1734 if (priv_flow_process(pmd, attr, items, actions, NULL,
1735 remote_flow, implicit_rte_flows[idx].mirred)) {
1736 TAP_LOG(ERR, "rte flow rule validation failed");
1739 err = tap_nl_send(pmd->nlsk_fd, &msg->nh);
1741 TAP_LOG(ERR, "Failure sending nl request");
1744 err = tap_nl_recv_ack(pmd->nlsk_fd);
1746 /* Silently ignore re-entering existing rule */
1747 if (errno == EEXIST)
1750 "Kernel refused TC filter rule creation (%d): %s",
1751 errno, strerror(errno));
1754 LIST_INSERT_HEAD(&pmd->implicit_flows, remote_flow, next);
1759 rte_free(remote_flow);
1764 * Remove specific implicit flow rule on the remote device.
1766 * @param[in, out] pmd
1767 * Pointer to private structure.
1769 * The idx in the implicit_rte_flows array specifying which rule to remove.
1771 * @return -1 if one of the implicit rules couldn't be created, 0 otherwise.
1773 int tap_flow_implicit_destroy(struct pmd_internals *pmd,
1774 enum implicit_rule_index idx)
1776 struct rte_flow *remote_flow;
1778 int idx_prio = implicit_rte_flows[idx].attr.priority + PRIORITY_OFFSET;
1780 for (remote_flow = LIST_FIRST(&pmd->implicit_flows);
1782 remote_flow = LIST_NEXT(remote_flow, next)) {
1783 cur_prio = (remote_flow->msg.t.tcm_info >> 16) & PRIORITY_MASK;
1784 if (cur_prio != idx_prio)
1786 return tap_flow_destroy_pmd(pmd, remote_flow, NULL);
1792 * Destroy all implicit flows.
1794 * @see rte_flow_flush()
1797 tap_flow_implicit_flush(struct pmd_internals *pmd, struct rte_flow_error *error)
1799 struct rte_flow *remote_flow;
1801 while (!LIST_EMPTY(&pmd->implicit_flows)) {
1802 remote_flow = LIST_FIRST(&pmd->implicit_flows);
1803 if (tap_flow_destroy_pmd(pmd, remote_flow, error) < 0)
1809 #define MAX_RSS_KEYS 256
1810 #define KEY_IDX_OFFSET (3 * MAX_RSS_KEYS)
1811 #define SEC_NAME_CLS_Q "cls_q"
1813 const char *sec_name[SEC_MAX] = {
1814 [SEC_L3_L4] = "l3_l4",
1818 * Enable RSS on tap: create TC rules for queuing.
1820 * @param[in, out] pmd
1821 * Pointer to private structure.
1824 * Pointer to rte_flow to get flow group
1827 * Pointer to error reporting if not NULL.
1829 * @return 0 on success, negative value on failure.
1831 static int rss_enable(struct pmd_internals *pmd,
1832 const struct rte_flow_attr *attr,
1833 struct rte_flow_error *error)
1835 struct rte_flow *rss_flow = NULL;
1836 struct nlmsg *msg = NULL;
1837 /* 4096 is the maximum number of instructions for a BPF program */
1838 char annotation[64];
1842 /* unlimit locked memory */
1843 struct rlimit memlock_limit = {
1844 .rlim_cur = RLIM_INFINITY,
1845 .rlim_max = RLIM_INFINITY,
1847 setrlimit(RLIMIT_MEMLOCK, &memlock_limit);
1849 /* Get a new map key for a new RSS rule */
1850 err = bpf_rss_key(KEY_CMD_INIT, NULL);
1853 error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1854 "Failed to initialize BPF RSS keys");
1860 * Create BPF RSS MAP
1862 pmd->map_fd = tap_flow_bpf_rss_map_create(sizeof(__u32), /* key size */
1863 sizeof(struct rss_key),
1865 if (pmd->map_fd < 0) {
1867 "Failed to create BPF map (%d): %s",
1868 errno, strerror(errno));
1870 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1871 "Kernel too old or not configured "
1872 "to support BPF maps");
1878 * Add a rule per queue to match reclassified packets and direct them to
1879 * the correct queue.
1881 for (i = 0; i < pmd->dev->data->nb_rx_queues; i++) {
1882 pmd->bpf_fd[i] = tap_flow_bpf_cls_q(i);
1883 if (pmd->bpf_fd[i] < 0) {
1885 "Failed to load BPF section %s for queue %d",
1888 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
1890 "Kernel too old or not configured "
1891 "to support BPF programs loading");
1896 rss_flow = rte_malloc(__func__, sizeof(struct rte_flow), 0);
1899 "Cannot allocate memory for rte_flow");
1902 msg = &rss_flow->msg;
1903 tc_init_msg(msg, pmd->if_index, RTM_NEWTFILTER, NLM_F_REQUEST |
1904 NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE);
1905 msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL));
1906 tap_flow_set_handle(rss_flow);
1907 uint16_t group = attr->group << GROUP_SHIFT;
1908 uint16_t prio = group | (i + PRIORITY_OFFSET);
1909 msg->t.tcm_info = TC_H_MAKE(prio << 16, msg->t.tcm_info);
1910 msg->t.tcm_parent = TC_H_MAKE(MULTIQ_MAJOR_HANDLE, 0);
1912 tap_nlattr_add(&msg->nh, TCA_KIND, sizeof("bpf"), "bpf");
1913 if (tap_nlattr_nested_start(msg, TCA_OPTIONS) < 0)
1915 tap_nlattr_add32(&msg->nh, TCA_BPF_FD, pmd->bpf_fd[i]);
1916 snprintf(annotation, sizeof(annotation), "[%s%d]",
1918 tap_nlattr_add(&msg->nh, TCA_BPF_NAME, strlen(annotation) + 1,
1922 struct action_data adata = {
1926 .action = TC_ACT_PIPE,
1931 if (add_actions(rss_flow, 1, &adata, TCA_BPF_ACT) < 0)
1934 tap_nlattr_nested_finish(msg); /* nested TCA_OPTIONS */
1936 /* Netlink message is now ready to be sent */
1937 if (tap_nl_send(pmd->nlsk_fd, &msg->nh) < 0)
1939 err = tap_nl_recv_ack(pmd->nlsk_fd);
1942 "Kernel refused TC filter rule creation (%d): %s",
1943 errno, strerror(errno));
1946 LIST_INSERT_HEAD(&pmd->rss_flows, rss_flow, next);
1949 pmd->rss_enabled = 1;
1954 * Manage bpf RSS keys repository with operations: init, get, release
1957 * Command on RSS keys: init, get, release
1959 * @param[in, out] key_idx
1960 * Pointer to RSS Key index (out for get command, in for release command)
1962 * @return -1 if couldn't get, release or init the RSS keys, 0 otherwise.
1964 static int bpf_rss_key(enum bpf_rss_key_e cmd, __u32 *key_idx)
1968 static __u32 num_used_keys;
1969 static __u32 rss_keys[MAX_RSS_KEYS] = {KEY_STAT_UNSPEC};
1970 static __u32 rss_keys_initialized;
1975 if (!rss_keys_initialized) {
1980 if (num_used_keys == RTE_DIM(rss_keys)) {
1985 *key_idx = num_used_keys % RTE_DIM(rss_keys);
1986 while (rss_keys[*key_idx] == KEY_STAT_USED)
1987 *key_idx = (*key_idx + 1) % RTE_DIM(rss_keys);
1989 rss_keys[*key_idx] = KEY_STAT_USED;
1992 * Add an offset to key_idx in order to handle a case of
1993 * RSS and non RSS flows mixture.
1994 * If a non RSS flow is destroyed it has an eBPF map
1995 * index 0 (initialized on flow creation) and might
1996 * unintentionally remove RSS entry 0 from eBPF map.
1997 * To avoid this issue, add an offset to the real index
1998 * during a KEY_CMD_GET operation and subtract this offset
1999 * during a KEY_CMD_RELEASE operation in order to restore
2002 *key_idx += KEY_IDX_OFFSET;
2006 case KEY_CMD_RELEASE:
2007 if (!rss_keys_initialized)
2011 * Subtract offest to restore real key index
2012 * If a non RSS flow is falsely trying to release map
2013 * entry 0 - the offset subtraction will calculate the real
2014 * map index as an out-of-range value and the release operation
2015 * will be silently ignored.
2017 key = *key_idx - KEY_IDX_OFFSET;
2018 if (key >= RTE_DIM(rss_keys))
2021 if (rss_keys[key] == KEY_STAT_USED) {
2022 rss_keys[key] = KEY_STAT_AVAILABLE;
2028 for (i = 0; i < RTE_DIM(rss_keys); i++)
2029 rss_keys[i] = KEY_STAT_AVAILABLE;
2031 rss_keys_initialized = 1;
2035 case KEY_CMD_DEINIT:
2036 for (i = 0; i < RTE_DIM(rss_keys); i++)
2037 rss_keys[i] = KEY_STAT_UNSPEC;
2039 rss_keys_initialized = 0;
2051 * Add RSS hash calculations and queue selection
2053 * @param[in, out] pmd
2054 * Pointer to internal structure. Used to set/get RSS map fd
2057 * Pointer to RSS flow actions
2060 * Pointer to error reporting if not NULL.
2062 * @return 0 on success, negative value on failure
2064 static int rss_add_actions(struct rte_flow *flow, struct pmd_internals *pmd,
2065 const struct rte_flow_action_rss *rss,
2066 struct rte_flow_error *error)
2068 /* 4096 is the maximum number of instructions for a BPF program */
2071 struct rss_key rss_entry = { .hash_fields = 0,
2074 /* Check supported RSS features */
2075 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT)
2076 return rte_flow_error_set
2077 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2078 "non-default RSS hash functions are not supported");
2080 return rte_flow_error_set
2081 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2082 "a nonzero RSS encapsulation level is not supported");
2084 /* Get a new map key for a new RSS rule */
2085 err = bpf_rss_key(KEY_CMD_GET, &flow->key_idx);
2088 error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
2089 "Failed to get BPF RSS key");
2094 /* Update RSS map entry with queues */
2095 rss_entry.nb_queues = rss->queue_num;
2096 for (i = 0; i < rss->queue_num; i++)
2097 rss_entry.queues[i] = rss->queue[i];
2098 rss_entry.hash_fields =
2099 (1 << HASH_FIELD_IPV4_L3_L4) | (1 << HASH_FIELD_IPV6_L3_L4);
2101 /* Add this RSS entry to map */
2102 err = tap_flow_bpf_update_rss_elem(pmd->map_fd,
2103 &flow->key_idx, &rss_entry);
2107 "Failed to update BPF map entry #%u (%d): %s",
2108 flow->key_idx, errno, strerror(errno));
2110 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
2111 "Kernel too old or not configured "
2112 "to support BPF maps updates");
2119 * Load bpf rules to calculate hash for this key_idx
2122 flow->bpf_fd[SEC_L3_L4] =
2123 tap_flow_bpf_calc_l3_l4_hash(flow->key_idx, pmd->map_fd);
2124 if (flow->bpf_fd[SEC_L3_L4] < 0) {
2126 "Failed to load BPF section %s (%d): %s",
2127 sec_name[SEC_L3_L4], errno, strerror(errno));
2129 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
2130 "Kernel too old or not configured "
2131 "to support BPF program loading");
2138 struct action_data adata[] = {
2142 .bpf_fd = flow->bpf_fd[SEC_L3_L4],
2143 .annotation = sec_name[SEC_L3_L4],
2145 .action = TC_ACT_PIPE,
2151 if (add_actions(flow, RTE_DIM(adata), adata,
2152 TCA_FLOWER_ACT) < 0)
2160 * Manage filter operations.
2163 * Pointer to Ethernet device structure.
2164 * @param filter_type
2167 * Operation to perform.
2169 * Pointer to operation-specific structure.
2172 * 0 on success, negative errno value on failure.
2175 tap_dev_filter_ctrl(struct rte_eth_dev *dev,
2176 enum rte_filter_type filter_type,
2177 enum rte_filter_op filter_op,
2180 switch (filter_type) {
2181 case RTE_ETH_FILTER_GENERIC:
2182 if (filter_op != RTE_ETH_FILTER_GET)
2184 *(const void **)arg = &tap_flow_ops;
2187 TAP_LOG(ERR, "%p: filter type (%d) not supported",