4 * Copyright 2017 6WIND S.A.
5 * Copyright 2017 Mellanox.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 #include <sys/queue.h>
38 #include <rte_byteorder.h>
39 #include <rte_jhash.h>
40 #include <rte_malloc.h>
41 #include <rte_eth_tap.h>
43 #include <tap_autoconf.h>
44 #include <tap_tcmsgs.h>
46 #ifndef HAVE_TC_FLOWER
48 * For kernels < 4.2, this enum is not defined. Runtime checks will be made to
49 * avoid sending TC messages the kernel cannot understand.
56 TCA_FLOWER_KEY_ETH_DST, /* ETH_ALEN */
57 TCA_FLOWER_KEY_ETH_DST_MASK, /* ETH_ALEN */
58 TCA_FLOWER_KEY_ETH_SRC, /* ETH_ALEN */
59 TCA_FLOWER_KEY_ETH_SRC_MASK, /* ETH_ALEN */
60 TCA_FLOWER_KEY_ETH_TYPE, /* be16 */
61 TCA_FLOWER_KEY_IP_PROTO, /* u8 */
62 TCA_FLOWER_KEY_IPV4_SRC, /* be32 */
63 TCA_FLOWER_KEY_IPV4_SRC_MASK, /* be32 */
64 TCA_FLOWER_KEY_IPV4_DST, /* be32 */
65 TCA_FLOWER_KEY_IPV4_DST_MASK, /* be32 */
66 TCA_FLOWER_KEY_IPV6_SRC, /* struct in6_addr */
67 TCA_FLOWER_KEY_IPV6_SRC_MASK, /* struct in6_addr */
68 TCA_FLOWER_KEY_IPV6_DST, /* struct in6_addr */
69 TCA_FLOWER_KEY_IPV6_DST_MASK, /* struct in6_addr */
70 TCA_FLOWER_KEY_TCP_SRC, /* be16 */
71 TCA_FLOWER_KEY_TCP_DST, /* be16 */
72 TCA_FLOWER_KEY_UDP_SRC, /* be16 */
73 TCA_FLOWER_KEY_UDP_DST, /* be16 */
76 #ifndef HAVE_TC_VLAN_ID
78 /* TCA_FLOWER_FLAGS, */
79 TCA_FLOWER_KEY_VLAN_ID = TCA_FLOWER_KEY_UDP_DST + 2, /* be16 */
80 TCA_FLOWER_KEY_VLAN_PRIO, /* u8 */
81 TCA_FLOWER_KEY_VLAN_ETH_TYPE, /* be16 */
86 LIST_ENTRY(rte_flow) next; /* Pointer to the next rte_flow structure */
87 struct rte_flow *remote_flow; /* associated remote flow */
95 struct rte_flow *flow;
99 struct rte_flow_attr attr;
100 struct rte_flow_item items[2];
104 static int tap_flow_create_eth(const struct rte_flow_item *item, void *data);
105 static int tap_flow_create_vlan(const struct rte_flow_item *item, void *data);
106 static int tap_flow_create_ipv4(const struct rte_flow_item *item, void *data);
107 static int tap_flow_create_ipv6(const struct rte_flow_item *item, void *data);
108 static int tap_flow_create_udp(const struct rte_flow_item *item, void *data);
109 static int tap_flow_create_tcp(const struct rte_flow_item *item, void *data);
111 tap_flow_validate(struct rte_eth_dev *dev,
112 const struct rte_flow_attr *attr,
113 const struct rte_flow_item items[],
114 const struct rte_flow_action actions[],
115 struct rte_flow_error *error);
117 static struct rte_flow *
118 tap_flow_create(struct rte_eth_dev *dev,
119 const struct rte_flow_attr *attr,
120 const struct rte_flow_item items[],
121 const struct rte_flow_action actions[],
122 struct rte_flow_error *error);
125 tap_flow_destroy(struct rte_eth_dev *dev,
126 struct rte_flow *flow,
127 struct rte_flow_error *error);
129 static const struct rte_flow_ops tap_flow_ops = {
130 .validate = tap_flow_validate,
131 .create = tap_flow_create,
132 .destroy = tap_flow_destroy,
133 .flush = tap_flow_flush,
136 /* Static initializer for items. */
138 (const enum rte_flow_item_type []){ \
139 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
142 /* Structure to generate a simple graph of layers supported by the NIC. */
143 struct tap_flow_items {
144 /* Bit-mask corresponding to what is supported for this item. */
146 const unsigned int mask_sz; /* Bit-mask size in bytes. */
148 * Bit-mask corresponding to the default mask, if none is provided
149 * along with the item.
151 const void *default_mask;
153 * Conversion function from rte_flow to netlink attributes.
156 * rte_flow item to convert.
158 * Internal structure to store the conversion.
161 * 0 on success, negative value otherwise.
163 int (*convert)(const struct rte_flow_item *item, void *data);
164 /** List of possible following items. */
165 const enum rte_flow_item_type *const items;
168 /* Graph of supported items and associated actions. */
169 static const struct tap_flow_items tap_flow_items[] = {
170 [RTE_FLOW_ITEM_TYPE_END] = {
171 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
173 [RTE_FLOW_ITEM_TYPE_ETH] = {
175 RTE_FLOW_ITEM_TYPE_VLAN,
176 RTE_FLOW_ITEM_TYPE_IPV4,
177 RTE_FLOW_ITEM_TYPE_IPV6),
178 .mask = &(const struct rte_flow_item_eth){
179 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
180 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
183 .mask_sz = sizeof(struct rte_flow_item_eth),
184 .default_mask = &rte_flow_item_eth_mask,
185 .convert = tap_flow_create_eth,
187 [RTE_FLOW_ITEM_TYPE_VLAN] = {
188 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
189 RTE_FLOW_ITEM_TYPE_IPV6),
190 .mask = &(const struct rte_flow_item_vlan){
192 /* DEI matching is not supported */
193 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
199 .mask_sz = sizeof(struct rte_flow_item_vlan),
200 .default_mask = &rte_flow_item_vlan_mask,
201 .convert = tap_flow_create_vlan,
203 [RTE_FLOW_ITEM_TYPE_IPV4] = {
204 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
205 RTE_FLOW_ITEM_TYPE_TCP),
206 .mask = &(const struct rte_flow_item_ipv4){
213 .mask_sz = sizeof(struct rte_flow_item_ipv4),
214 .default_mask = &rte_flow_item_ipv4_mask,
215 .convert = tap_flow_create_ipv4,
217 [RTE_FLOW_ITEM_TYPE_IPV6] = {
218 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
219 RTE_FLOW_ITEM_TYPE_TCP),
220 .mask = &(const struct rte_flow_item_ipv6){
223 "\xff\xff\xff\xff\xff\xff\xff\xff"
224 "\xff\xff\xff\xff\xff\xff\xff\xff",
227 "\xff\xff\xff\xff\xff\xff\xff\xff"
228 "\xff\xff\xff\xff\xff\xff\xff\xff",
233 .mask_sz = sizeof(struct rte_flow_item_ipv6),
234 .default_mask = &rte_flow_item_ipv6_mask,
235 .convert = tap_flow_create_ipv6,
237 [RTE_FLOW_ITEM_TYPE_UDP] = {
238 .mask = &(const struct rte_flow_item_udp){
244 .mask_sz = sizeof(struct rte_flow_item_udp),
245 .default_mask = &rte_flow_item_udp_mask,
246 .convert = tap_flow_create_udp,
248 [RTE_FLOW_ITEM_TYPE_TCP] = {
249 .mask = &(const struct rte_flow_item_tcp){
255 .mask_sz = sizeof(struct rte_flow_item_tcp),
256 .default_mask = &rte_flow_item_tcp_mask,
257 .convert = tap_flow_create_tcp,
261 static struct remote_rule implicit_rte_flows[TAP_REMOTE_MAX_IDX] = {
262 [TAP_REMOTE_LOCAL_MAC] = {
265 .priority = PRIORITY_MASK - TAP_REMOTE_LOCAL_MAC,
269 .type = RTE_FLOW_ITEM_TYPE_ETH,
270 .mask = &(const struct rte_flow_item_eth){
271 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
275 .type = RTE_FLOW_ITEM_TYPE_END,
277 .mirred = TCA_EGRESS_REDIR,
279 [TAP_REMOTE_BROADCAST] = {
282 .priority = PRIORITY_MASK - TAP_REMOTE_BROADCAST,
286 .type = RTE_FLOW_ITEM_TYPE_ETH,
287 .mask = &(const struct rte_flow_item_eth){
288 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
290 .spec = &(const struct rte_flow_item_eth){
291 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
295 .type = RTE_FLOW_ITEM_TYPE_END,
297 .mirred = TCA_EGRESS_MIRROR,
299 [TAP_REMOTE_BROADCASTV6] = {
302 .priority = PRIORITY_MASK - TAP_REMOTE_BROADCASTV6,
306 .type = RTE_FLOW_ITEM_TYPE_ETH,
307 .mask = &(const struct rte_flow_item_eth){
308 .dst.addr_bytes = "\x33\x33\x00\x00\x00\x00",
310 .spec = &(const struct rte_flow_item_eth){
311 .dst.addr_bytes = "\x33\x33\x00\x00\x00\x00",
315 .type = RTE_FLOW_ITEM_TYPE_END,
317 .mirred = TCA_EGRESS_MIRROR,
319 [TAP_REMOTE_PROMISC] = {
322 .priority = PRIORITY_MASK - TAP_REMOTE_PROMISC,
326 .type = RTE_FLOW_ITEM_TYPE_VOID,
329 .type = RTE_FLOW_ITEM_TYPE_END,
331 .mirred = TCA_EGRESS_MIRROR,
333 [TAP_REMOTE_ALLMULTI] = {
336 .priority = PRIORITY_MASK - TAP_REMOTE_ALLMULTI,
340 .type = RTE_FLOW_ITEM_TYPE_ETH,
341 .mask = &(const struct rte_flow_item_eth){
342 .dst.addr_bytes = "\x01\x00\x00\x00\x00\x00",
344 .spec = &(const struct rte_flow_item_eth){
345 .dst.addr_bytes = "\x01\x00\x00\x00\x00\x00",
349 .type = RTE_FLOW_ITEM_TYPE_END,
351 .mirred = TCA_EGRESS_MIRROR,
356 .priority = TAP_REMOTE_TX,
360 .type = RTE_FLOW_ITEM_TYPE_VOID,
363 .type = RTE_FLOW_ITEM_TYPE_END,
365 .mirred = TCA_EGRESS_MIRROR,
370 * Make as much checks as possible on an Ethernet item, and if a flow is
371 * provided, fill it appropriately with Ethernet info.
374 * Item specification.
375 * @param[in, out] data
376 * Additional data structure to tell next layers we've been here.
379 * 0 if checks are alright, -1 otherwise.
382 tap_flow_create_eth(const struct rte_flow_item *item, void *data)
384 struct convert_data *info = (struct convert_data *)data;
385 const struct rte_flow_item_eth *spec = item->spec;
386 const struct rte_flow_item_eth *mask = item->mask;
387 struct rte_flow *flow = info->flow;
390 /* use default mask if none provided */
392 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_ETH].default_mask;
393 /* TC does not support eth_type masking. Only accept if exact match. */
394 if (mask->type && mask->type != 0xffff)
398 /* store eth_type for consistency if ipv4/6 pattern item comes next */
399 if (spec->type & mask->type)
400 info->eth_type = spec->type;
404 if (spec->type & mask->type)
405 msg->t.tcm_info = TC_H_MAKE(msg->t.tcm_info,
406 (spec->type & mask->type));
407 if (!is_zero_ether_addr(&spec->dst)) {
408 nlattr_add(&msg->nh, TCA_FLOWER_KEY_ETH_DST, ETHER_ADDR_LEN,
409 &spec->dst.addr_bytes);
411 TCA_FLOWER_KEY_ETH_DST_MASK, ETHER_ADDR_LEN,
412 &mask->dst.addr_bytes);
414 if (!is_zero_ether_addr(&mask->src)) {
415 nlattr_add(&msg->nh, TCA_FLOWER_KEY_ETH_SRC, ETHER_ADDR_LEN,
416 &spec->src.addr_bytes);
418 TCA_FLOWER_KEY_ETH_SRC_MASK, ETHER_ADDR_LEN,
419 &mask->src.addr_bytes);
425 * Make as much checks as possible on a VLAN item, and if a flow is provided,
426 * fill it appropriately with VLAN info.
429 * Item specification.
430 * @param[in, out] data
431 * Additional data structure to tell next layers we've been here.
434 * 0 if checks are alright, -1 otherwise.
437 tap_flow_create_vlan(const struct rte_flow_item *item, void *data)
439 struct convert_data *info = (struct convert_data *)data;
440 const struct rte_flow_item_vlan *spec = item->spec;
441 const struct rte_flow_item_vlan *mask = item->mask;
442 struct rte_flow *flow = info->flow;
445 /* use default mask if none provided */
447 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_VLAN].default_mask;
448 /* TC does not support tpid masking. Only accept if exact match. */
449 if (mask->tpid && mask->tpid != 0xffff)
451 /* Double-tagging not supported. */
452 if (spec && mask->tpid && spec->tpid != htons(ETH_P_8021Q))
458 msg->t.tcm_info = TC_H_MAKE(msg->t.tcm_info, htons(ETH_P_8021Q));
459 #define VLAN_PRIO(tci) ((tci) >> 13)
460 #define VLAN_ID(tci) ((tci) & 0xfff)
464 uint16_t tci = ntohs(spec->tci) & mask->tci;
465 uint16_t prio = VLAN_PRIO(tci);
466 uint8_t vid = VLAN_ID(tci);
469 nlattr_add8(&msg->nh, TCA_FLOWER_KEY_VLAN_PRIO, prio);
471 nlattr_add16(&msg->nh, TCA_FLOWER_KEY_VLAN_ID, vid);
477 * Make as much checks as possible on an IPv4 item, and if a flow is provided,
478 * fill it appropriately with IPv4 info.
481 * Item specification.
482 * @param[in, out] data
483 * Additional data structure to tell next layers we've been here.
486 * 0 if checks are alright, -1 otherwise.
489 tap_flow_create_ipv4(const struct rte_flow_item *item, void *data)
491 struct convert_data *info = (struct convert_data *)data;
492 const struct rte_flow_item_ipv4 *spec = item->spec;
493 const struct rte_flow_item_ipv4 *mask = item->mask;
494 struct rte_flow *flow = info->flow;
497 /* use default mask if none provided */
499 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_IPV4].default_mask;
500 /* check that previous eth type is compatible with ipv4 */
501 if (info->eth_type && info->eth_type != htons(ETH_P_IP))
503 /* store ip_proto for consistency if udp/tcp pattern item comes next */
505 info->ip_proto = spec->hdr.next_proto_id;
510 info->eth_type = htons(ETH_P_IP);
512 msg->t.tcm_info = TC_H_MAKE(msg->t.tcm_info, htons(ETH_P_IP));
515 if (spec->hdr.dst_addr) {
516 nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_DST,
518 nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_DST_MASK,
521 if (spec->hdr.src_addr) {
522 nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_SRC,
524 nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_SRC_MASK,
527 if (spec->hdr.next_proto_id)
528 nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO,
529 spec->hdr.next_proto_id);
534 * Make as much checks as possible on an IPv6 item, and if a flow is provided,
535 * fill it appropriately with IPv6 info.
538 * Item specification.
539 * @param[in, out] data
540 * Additional data structure to tell next layers we've been here.
543 * 0 if checks are alright, -1 otherwise.
546 tap_flow_create_ipv6(const struct rte_flow_item *item, void *data)
548 struct convert_data *info = (struct convert_data *)data;
549 const struct rte_flow_item_ipv6 *spec = item->spec;
550 const struct rte_flow_item_ipv6 *mask = item->mask;
551 struct rte_flow *flow = info->flow;
552 uint8_t empty_addr[16] = { 0 };
555 /* use default mask if none provided */
557 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_IPV6].default_mask;
558 /* check that previous eth type is compatible with ipv6 */
559 if (info->eth_type && info->eth_type != htons(ETH_P_IPV6))
561 /* store ip_proto for consistency if udp/tcp pattern item comes next */
563 info->ip_proto = spec->hdr.proto;
568 info->eth_type = htons(ETH_P_IPV6);
570 msg->t.tcm_info = TC_H_MAKE(msg->t.tcm_info, htons(ETH_P_IPV6));
573 if (memcmp(spec->hdr.dst_addr, empty_addr, 16)) {
574 nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_DST,
575 sizeof(spec->hdr.dst_addr), &spec->hdr.dst_addr);
576 nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_DST_MASK,
577 sizeof(mask->hdr.dst_addr), &mask->hdr.dst_addr);
579 if (memcmp(spec->hdr.src_addr, empty_addr, 16)) {
580 nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_SRC,
581 sizeof(spec->hdr.src_addr), &spec->hdr.src_addr);
582 nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_SRC_MASK,
583 sizeof(mask->hdr.src_addr), &mask->hdr.src_addr);
586 nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, spec->hdr.proto);
591 * Make as much checks as possible on a UDP item, and if a flow is provided,
592 * fill it appropriately with UDP info.
595 * Item specification.
596 * @param[in, out] data
597 * Additional data structure to tell next layers we've been here.
600 * 0 if checks are alright, -1 otherwise.
603 tap_flow_create_udp(const struct rte_flow_item *item, void *data)
605 struct convert_data *info = (struct convert_data *)data;
606 const struct rte_flow_item_udp *spec = item->spec;
607 const struct rte_flow_item_udp *mask = item->mask;
608 struct rte_flow *flow = info->flow;
611 /* use default mask if none provided */
613 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_UDP].default_mask;
614 /* check that previous ip_proto is compatible with udp */
615 if (info->ip_proto && info->ip_proto != IPPROTO_UDP)
617 /* TC does not support UDP port masking. Only accept if exact match. */
618 if ((mask->hdr.src_port && mask->hdr.src_port != 0xffff) ||
619 (mask->hdr.dst_port && mask->hdr.dst_port != 0xffff))
624 nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, IPPROTO_UDP);
627 if (spec->hdr.dst_port & mask->hdr.dst_port)
628 nlattr_add16(&msg->nh, TCA_FLOWER_KEY_UDP_DST,
630 if (spec->hdr.src_port & mask->hdr.src_port)
631 nlattr_add16(&msg->nh, TCA_FLOWER_KEY_UDP_SRC,
637 * Make as much checks as possible on a TCP item, and if a flow is provided,
638 * fill it appropriately with TCP info.
641 * Item specification.
642 * @param[in, out] data
643 * Additional data structure to tell next layers we've been here.
646 * 0 if checks are alright, -1 otherwise.
649 tap_flow_create_tcp(const struct rte_flow_item *item, void *data)
651 struct convert_data *info = (struct convert_data *)data;
652 const struct rte_flow_item_tcp *spec = item->spec;
653 const struct rte_flow_item_tcp *mask = item->mask;
654 struct rte_flow *flow = info->flow;
657 /* use default mask if none provided */
659 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_TCP].default_mask;
660 /* check that previous ip_proto is compatible with tcp */
661 if (info->ip_proto && info->ip_proto != IPPROTO_TCP)
663 /* TC does not support TCP port masking. Only accept if exact match. */
664 if ((mask->hdr.src_port && mask->hdr.src_port != 0xffff) ||
665 (mask->hdr.dst_port && mask->hdr.dst_port != 0xffff))
670 nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, IPPROTO_TCP);
673 if (spec->hdr.dst_port & mask->hdr.dst_port)
674 nlattr_add16(&msg->nh, TCA_FLOWER_KEY_TCP_DST,
676 if (spec->hdr.src_port & mask->hdr.src_port)
677 nlattr_add16(&msg->nh, TCA_FLOWER_KEY_TCP_SRC,
683 * Check support for a given item.
686 * Item specification.
688 * Bit-Mask size in bytes.
689 * @param[in] supported_mask
690 * Bit-mask covering supported fields to compare with spec, last and mask in
692 * @param[in] default_mask
693 * Bit-mask default mask if none is provided in \item.
699 tap_flow_item_validate(const struct rte_flow_item *item,
701 const uint8_t *supported_mask,
702 const uint8_t *default_mask)
706 /* An empty layer is allowed, as long as all fields are NULL */
707 if (!item->spec && (item->mask || item->last))
709 /* Is the item spec compatible with what the NIC supports? */
710 if (item->spec && !item->mask) {
712 const uint8_t *spec = item->spec;
714 for (i = 0; i < size; ++i)
715 if ((spec[i] | supported_mask[i]) != supported_mask[i])
717 /* Is the default mask compatible with what the NIC supports? */
718 for (i = 0; i < size; i++)
719 if ((default_mask[i] | supported_mask[i]) !=
723 /* Is the item last compatible with what the NIC supports? */
724 if (item->last && !item->mask) {
726 const uint8_t *spec = item->last;
728 for (i = 0; i < size; ++i)
729 if ((spec[i] | supported_mask[i]) != supported_mask[i])
732 /* Is the item mask compatible with what the NIC supports? */
735 const uint8_t *spec = item->mask;
737 for (i = 0; i < size; ++i)
738 if ((spec[i] | supported_mask[i]) != supported_mask[i])
742 * Once masked, Are item spec and item last equal?
743 * TC does not support range so anything else is invalid.
745 if (item->spec && item->last) {
748 const uint8_t *apply = default_mask;
753 for (i = 0; i < size; ++i) {
754 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
755 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
757 ret = memcmp(spec, last, size);
763 * Transform a DROP/PASSTHRU action item in the provided flow for TC.
765 * @param[in, out] flow
768 * Appropriate action to be set in the TCA_GACT_PARMS structure.
771 * 0 if checks are alright, -1 otherwise.
774 add_action_gact(struct rte_flow *flow, int action)
776 struct nlmsg *msg = &flow->msg;
777 size_t act_index = 1;
782 if (nlattr_nested_start(msg, TCA_FLOWER_ACT) < 0)
784 if (nlattr_nested_start(msg, act_index++) < 0)
786 nlattr_add(&msg->nh, TCA_ACT_KIND, sizeof("gact"), "gact");
787 if (nlattr_nested_start(msg, TCA_ACT_OPTIONS) < 0)
789 nlattr_add(&msg->nh, TCA_GACT_PARMS, sizeof(p), &p);
790 nlattr_nested_finish(msg); /* nested TCA_ACT_OPTIONS */
791 nlattr_nested_finish(msg); /* nested act_index */
792 nlattr_nested_finish(msg); /* nested TCA_FLOWER_ACT */
797 * Transform a MIRRED action item in the provided flow for TC.
799 * @param[in, out] flow
802 * Netdevice ifindex, where to mirror/redirect packet to.
803 * @param[in] action_type
804 * Either TCA_EGRESS_REDIR for redirection or TCA_EGRESS_MIRROR for mirroring.
807 * 0 if checks are alright, -1 otherwise.
810 add_action_mirred(struct rte_flow *flow, uint16_t ifindex, uint16_t action_type)
812 struct nlmsg *msg = &flow->msg;
813 size_t act_index = 1;
814 struct tc_mirred p = {
815 .eaction = action_type,
819 if (nlattr_nested_start(msg, TCA_FLOWER_ACT) < 0)
821 if (nlattr_nested_start(msg, act_index++) < 0)
823 nlattr_add(&msg->nh, TCA_ACT_KIND, sizeof("mirred"), "mirred");
824 if (nlattr_nested_start(msg, TCA_ACT_OPTIONS) < 0)
826 if (action_type == TCA_EGRESS_MIRROR)
827 p.action = TC_ACT_PIPE;
829 p.action = TC_ACT_STOLEN;
830 nlattr_add(&msg->nh, TCA_MIRRED_PARMS, sizeof(p), &p);
831 nlattr_nested_finish(msg); /* nested TCA_ACT_OPTIONS */
832 nlattr_nested_finish(msg); /* nested act_index */
833 nlattr_nested_finish(msg); /* nested TCA_FLOWER_ACT */
838 * Transform a QUEUE action item in the provided flow for TC.
840 * @param[in, out] flow
846 * 0 if checks are alright, -1 otherwise.
849 add_action_skbedit(struct rte_flow *flow, uint16_t queue)
851 struct nlmsg *msg = &flow->msg;
852 size_t act_index = 1;
853 struct tc_skbedit p = {
854 .action = TC_ACT_PIPE
857 if (nlattr_nested_start(msg, TCA_FLOWER_ACT) < 0)
859 if (nlattr_nested_start(msg, act_index++) < 0)
861 nlattr_add(&msg->nh, TCA_ACT_KIND, sizeof("skbedit"), "skbedit");
862 if (nlattr_nested_start(msg, TCA_ACT_OPTIONS) < 0)
864 nlattr_add(&msg->nh, TCA_SKBEDIT_PARMS, sizeof(p), &p);
865 nlattr_add16(&msg->nh, TCA_SKBEDIT_QUEUE_MAPPING, queue);
866 nlattr_nested_finish(msg); /* nested TCA_ACT_OPTIONS */
867 nlattr_nested_finish(msg); /* nested act_index */
868 nlattr_nested_finish(msg); /* nested TCA_FLOWER_ACT */
873 * Validate a flow supported by TC.
874 * If flow param is not NULL, then also fill the netlink message inside.
877 * Pointer to private structure.
879 * Flow rule attributes.
881 * Pattern specification (list terminated by the END pattern item).
883 * Associated actions (list terminated by the END action).
885 * Perform verbose error reporting if not NULL.
886 * @param[in, out] flow
887 * Flow structure to update.
889 * If set to TCA_EGRESS_REDIR, provided actions will be replaced with a
890 * redirection to the tap netdevice, and the TC rule will be configured
891 * on the remote netdevice in pmd.
892 * If set to TCA_EGRESS_MIRROR, provided actions will be replaced with a
893 * mirroring to the tap netdevice, and the TC rule will be configured
894 * on the remote netdevice in pmd. Matching packets will thus be duplicated.
895 * If set to 0, the standard behavior is to be used: set correct actions for
896 * the TC rule, and apply it on the tap netdevice.
899 * 0 on success, a negative errno value otherwise and rte_errno is set.
902 priv_flow_process(struct pmd_internals *pmd,
903 const struct rte_flow_attr *attr,
904 const struct rte_flow_item items[],
905 const struct rte_flow_action actions[],
906 struct rte_flow_error *error,
907 struct rte_flow *flow,
910 const struct tap_flow_items *cur_item = tap_flow_items;
911 struct convert_data data = {
916 int action = 0; /* Only one action authorized for now */
918 if (attr->group > MAX_GROUP) {
920 error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
921 NULL, "group value too big: cannot exceed 15");
924 if (attr->priority > MAX_PRIORITY) {
926 error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
927 NULL, "priority value too big");
930 uint16_t group = attr->group << GROUP_SHIFT;
931 uint16_t prio = group | (attr->priority + PRIORITY_OFFSET);
932 flow->msg.t.tcm_info = TC_H_MAKE(prio << 16,
933 flow->msg.t.tcm_info);
938 * If attr->ingress, the rule applies on remote ingress
939 * to match incoming packets
940 * If attr->egress, the rule applies on tap ingress (as
941 * seen from the kernel) to deal with packets going out
944 flow->msg.t.tcm_parent = TC_H_MAKE(TC_H_INGRESS, 0);
946 /* Standard rule on tap egress (kernel standpoint). */
947 flow->msg.t.tcm_parent =
948 TC_H_MAKE(MULTIQ_MAJOR_HANDLE, 0);
950 /* use flower filter type */
951 nlattr_add(&flow->msg.nh, TCA_KIND, sizeof("flower"), "flower");
952 if (nlattr_nested_start(&flow->msg, TCA_OPTIONS) < 0)
953 goto exit_item_not_supported;
955 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
956 const struct tap_flow_items *token = NULL;
960 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
964 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
966 if (cur_item->items[i] == items->type) {
967 token = &tap_flow_items[items->type];
972 goto exit_item_not_supported;
974 err = tap_flow_item_validate(
975 items, cur_item->mask_sz,
976 (const uint8_t *)cur_item->mask,
977 (const uint8_t *)cur_item->default_mask);
979 goto exit_item_not_supported;
980 if (flow && cur_item->convert) {
981 if (!pmd->flower_vlan_support &&
982 cur_item->convert == tap_flow_create_vlan)
983 goto exit_item_not_supported;
984 err = cur_item->convert(items, &data);
986 goto exit_item_not_supported;
990 if (pmd->flower_vlan_support && data.vlan) {
991 nlattr_add16(&flow->msg.nh, TCA_FLOWER_KEY_ETH_TYPE,
993 nlattr_add16(&flow->msg.nh,
994 TCA_FLOWER_KEY_VLAN_ETH_TYPE,
996 data.eth_type : htons(ETH_P_ALL));
997 } else if (data.eth_type) {
998 nlattr_add16(&flow->msg.nh, TCA_FLOWER_KEY_ETH_TYPE,
1002 if (mirred && flow) {
1003 uint16_t if_index = pmd->if_index;
1006 * If attr->egress && mirred, then this is a special
1007 * case where the rule must be applied on the tap, to
1008 * redirect packets coming from the DPDK App, out
1009 * through the remote netdevice.
1012 if_index = pmd->remote_if_index;
1013 if (add_action_mirred(flow, if_index, mirred) < 0)
1014 goto exit_action_not_supported;
1018 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
1021 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
1023 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
1025 goto exit_action_not_supported;
1028 err = add_action_gact(flow, TC_ACT_SHOT);
1029 } else if (actions->type == RTE_FLOW_ACTION_TYPE_PASSTHRU) {
1031 goto exit_action_not_supported;
1034 err = add_action_gact(flow, TC_ACT_UNSPEC);
1035 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
1036 const struct rte_flow_action_queue *queue =
1037 (const struct rte_flow_action_queue *)
1040 goto exit_action_not_supported;
1042 if (!queue || (queue->index >= pmd->nb_queues))
1043 goto exit_action_not_supported;
1045 err = add_action_skbedit(flow, queue->index);
1047 goto exit_action_not_supported;
1050 goto exit_action_not_supported;
1054 nlattr_nested_finish(&flow->msg); /* nested TCA_OPTIONS */
1056 exit_item_not_supported:
1057 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1058 items, "item not supported");
1060 exit_action_not_supported:
1061 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
1062 actions, "action not supported");
1071 * @see rte_flow_validate()
1075 tap_flow_validate(struct rte_eth_dev *dev,
1076 const struct rte_flow_attr *attr,
1077 const struct rte_flow_item items[],
1078 const struct rte_flow_action actions[],
1079 struct rte_flow_error *error)
1081 struct pmd_internals *pmd = dev->data->dev_private;
1083 return priv_flow_process(pmd, attr, items, actions, error, NULL, 0);
1087 * Set a unique handle in a flow.
1089 * The kernel supports TC rules with equal priority, as long as they use the
1090 * same matching fields (e.g.: dst mac and ipv4) with different values (and
1091 * full mask to ensure no collision is possible).
1092 * In those rules, the handle (uint32_t) is the part that would identify
1093 * specifically each rule.
1095 * On 32-bit architectures, the handle can simply be the flow's pointer address.
1096 * On 64-bit architectures, we rely on jhash(flow) to find a (sufficiently)
1099 * @param[in, out] flow
1100 * The flow that needs its handle set.
1103 tap_flow_set_handle(struct rte_flow *flow)
1105 uint32_t handle = 0;
1107 if (sizeof(flow) > 4)
1108 handle = rte_jhash(&flow, sizeof(flow), 1);
1110 handle = (uintptr_t)flow;
1111 /* must be at least 1 to avoid letting the kernel choose one for us */
1114 flow->msg.t.tcm_handle = handle;
1120 * @see rte_flow_create()
1123 static struct rte_flow *
1124 tap_flow_create(struct rte_eth_dev *dev,
1125 const struct rte_flow_attr *attr,
1126 const struct rte_flow_item items[],
1127 const struct rte_flow_action actions[],
1128 struct rte_flow_error *error)
1130 struct pmd_internals *pmd = dev->data->dev_private;
1131 struct rte_flow *remote_flow = NULL;
1132 struct rte_flow *flow = NULL;
1133 struct nlmsg *msg = NULL;
1136 if (!pmd->if_index) {
1137 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
1139 "can't create rule, ifindex not found");
1143 * No rules configured through standard rte_flow should be set on the
1144 * priorities used by implicit rules.
1146 if ((attr->group == MAX_GROUP) &&
1147 attr->priority > (MAX_PRIORITY - TAP_REMOTE_MAX_IDX)) {
1149 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1150 NULL, "priority value too big");
1153 flow = rte_malloc(__func__, sizeof(struct rte_flow), 0);
1155 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1156 NULL, "cannot allocate memory for rte_flow");
1160 tc_init_msg(msg, pmd->if_index, RTM_NEWTFILTER,
1161 NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE);
1162 msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL));
1163 tap_flow_set_handle(flow);
1164 if (priv_flow_process(pmd, attr, items, actions, error, flow, 0))
1166 err = nl_send(pmd->nlsk_fd, &msg->nh);
1168 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
1169 NULL, "couldn't send request to kernel");
1172 err = nl_recv_ack(pmd->nlsk_fd);
1175 "Kernel refused TC filter rule creation (%d): %s\n",
1176 errno, strerror(errno));
1177 rte_flow_error_set(error, EEXIST, RTE_FLOW_ERROR_TYPE_HANDLE,
1178 NULL, "overlapping rules");
1181 LIST_INSERT_HEAD(&pmd->flows, flow, next);
1183 * If a remote device is configured, a TC rule with identical items for
1184 * matching must be set on that device, with a single action: redirect
1185 * to the local pmd->if_index.
1187 if (pmd->remote_if_index) {
1188 remote_flow = rte_malloc(__func__, sizeof(struct rte_flow), 0);
1191 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1192 "cannot allocate memory for rte_flow");
1195 msg = &remote_flow->msg;
1196 /* set the rule if_index for the remote netdevice */
1198 msg, pmd->remote_if_index, RTM_NEWTFILTER,
1199 NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE);
1200 msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL));
1201 tap_flow_set_handle(remote_flow);
1202 if (priv_flow_process(pmd, attr, items, NULL,
1203 error, remote_flow, TCA_EGRESS_REDIR)) {
1205 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1206 NULL, "rte flow rule validation failed");
1209 err = nl_send(pmd->nlsk_fd, &msg->nh);
1212 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1213 NULL, "Failure sending nl request");
1216 err = nl_recv_ack(pmd->nlsk_fd);
1219 "Kernel refused TC filter rule creation (%d): %s\n",
1220 errno, strerror(errno));
1222 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1223 NULL, "overlapping rules");
1226 flow->remote_flow = remote_flow;
1231 rte_free(remote_flow);
1238 * Destroy a flow using pointer to pmd_internal.
1240 * @param[in, out] pmd
1241 * Pointer to private structure.
1243 * Pointer to the flow to destroy.
1244 * @param[in, out] error
1245 * Pointer to the flow error handler
1247 * @return 0 if the flow could be destroyed, -1 otherwise.
1250 tap_flow_destroy_pmd(struct pmd_internals *pmd,
1251 struct rte_flow *flow,
1252 struct rte_flow_error *error)
1254 struct rte_flow *remote_flow = flow->remote_flow;
1257 LIST_REMOVE(flow, next);
1258 flow->msg.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK;
1259 flow->msg.nh.nlmsg_type = RTM_DELTFILTER;
1261 ret = nl_send(pmd->nlsk_fd, &flow->msg.nh);
1263 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
1264 NULL, "couldn't send request to kernel");
1267 ret = nl_recv_ack(pmd->nlsk_fd);
1268 /* If errno is ENOENT, the rule is already no longer in the kernel. */
1269 if (ret < 0 && errno == ENOENT)
1273 "Kernel refused TC filter rule deletion (%d): %s\n",
1274 errno, strerror(errno));
1276 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1277 "couldn't receive kernel ack to our request");
1281 remote_flow->msg.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK;
1282 remote_flow->msg.nh.nlmsg_type = RTM_DELTFILTER;
1284 ret = nl_send(pmd->nlsk_fd, &remote_flow->msg.nh);
1287 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1288 NULL, "Failure sending nl request");
1291 ret = nl_recv_ack(pmd->nlsk_fd);
1292 if (ret < 0 && errno == ENOENT)
1296 "Kernel refused TC filter rule deletion (%d): %s\n",
1297 errno, strerror(errno));
1299 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1300 NULL, "Failure trying to receive nl ack");
1306 rte_free(remote_flow);
1314 * @see rte_flow_destroy()
1318 tap_flow_destroy(struct rte_eth_dev *dev,
1319 struct rte_flow *flow,
1320 struct rte_flow_error *error)
1322 struct pmd_internals *pmd = dev->data->dev_private;
1324 return tap_flow_destroy_pmd(pmd, flow, error);
1328 * Destroy all flows.
1330 * @see rte_flow_flush()
1334 tap_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error)
1336 struct pmd_internals *pmd = dev->data->dev_private;
1337 struct rte_flow *flow;
1339 while (!LIST_EMPTY(&pmd->flows)) {
1340 flow = LIST_FIRST(&pmd->flows);
1341 if (tap_flow_destroy(dev, flow, error) < 0)
1348 * Add an implicit flow rule on the remote device to make sure traffic gets to
1349 * the tap netdevice from there.
1352 * Pointer to private structure.
1354 * The idx in the implicit_rte_flows array specifying which rule to apply.
1356 * @return -1 if the rule couldn't be applied, 0 otherwise.
1358 int tap_flow_implicit_create(struct pmd_internals *pmd,
1359 enum implicit_rule_index idx)
1361 struct rte_flow_item *items = implicit_rte_flows[idx].items;
1362 struct rte_flow_attr *attr = &implicit_rte_flows[idx].attr;
1363 struct rte_flow_item_eth eth_local = { .type = 0 };
1364 uint16_t if_index = pmd->remote_if_index;
1365 struct rte_flow *remote_flow = NULL;
1366 struct nlmsg *msg = NULL;
1368 struct rte_flow_item items_local[2] = {
1370 .type = items[0].type,
1372 .mask = items[0].mask,
1375 .type = items[1].type,
1379 remote_flow = rte_malloc(__func__, sizeof(struct rte_flow), 0);
1381 RTE_LOG(ERR, PMD, "Cannot allocate memory for rte_flow");
1384 msg = &remote_flow->msg;
1385 if (idx == TAP_REMOTE_TX) {
1386 if_index = pmd->if_index;
1387 } else if (idx == TAP_REMOTE_LOCAL_MAC) {
1389 * eth addr couldn't be set in implicit_rte_flows[] as it is not
1390 * known at compile time.
1392 memcpy(ð_local.dst, &pmd->eth_addr, sizeof(pmd->eth_addr));
1393 items = items_local;
1395 tc_init_msg(msg, if_index, RTM_NEWTFILTER,
1396 NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE);
1397 msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL));
1398 tap_flow_set_handle(remote_flow);
1399 if (priv_flow_process(pmd, attr, items, NULL, NULL,
1400 remote_flow, implicit_rte_flows[idx].mirred)) {
1401 RTE_LOG(ERR, PMD, "rte flow rule validation failed\n");
1404 err = nl_send(pmd->nlsk_fd, &msg->nh);
1406 RTE_LOG(ERR, PMD, "Failure sending nl request");
1409 err = nl_recv_ack(pmd->nlsk_fd);
1412 "Kernel refused TC filter rule creation (%d): %s\n",
1413 errno, strerror(errno));
1416 LIST_INSERT_HEAD(&pmd->implicit_flows, remote_flow, next);
1420 rte_free(remote_flow);
1425 * Remove specific implicit flow rule on the remote device.
1427 * @param[in, out] pmd
1428 * Pointer to private structure.
1430 * The idx in the implicit_rte_flows array specifying which rule to remove.
1432 * @return -1 if one of the implicit rules couldn't be created, 0 otherwise.
1434 int tap_flow_implicit_destroy(struct pmd_internals *pmd,
1435 enum implicit_rule_index idx)
1437 struct rte_flow *remote_flow;
1439 int idx_prio = implicit_rte_flows[idx].attr.priority + PRIORITY_OFFSET;
1441 for (remote_flow = LIST_FIRST(&pmd->implicit_flows);
1443 remote_flow = LIST_NEXT(remote_flow, next)) {
1444 cur_prio = (remote_flow->msg.t.tcm_info >> 16) & PRIORITY_MASK;
1445 if (cur_prio != idx_prio)
1447 return tap_flow_destroy_pmd(pmd, remote_flow, NULL);
1453 * Destroy all implicit flows.
1455 * @see rte_flow_flush()
1458 tap_flow_implicit_flush(struct pmd_internals *pmd, struct rte_flow_error *error)
1460 struct rte_flow *remote_flow;
1462 while (!LIST_EMPTY(&pmd->implicit_flows)) {
1463 remote_flow = LIST_FIRST(&pmd->implicit_flows);
1464 if (tap_flow_destroy_pmd(pmd, remote_flow, error) < 0)
1471 * Manage filter operations.
1474 * Pointer to Ethernet device structure.
1475 * @param filter_type
1478 * Operation to perform.
1480 * Pointer to operation-specific structure.
1483 * 0 on success, negative errno value on failure.
1486 tap_dev_filter_ctrl(struct rte_eth_dev *dev,
1487 enum rte_filter_type filter_type,
1488 enum rte_filter_op filter_op,
1491 struct pmd_internals *pmd = dev->data->dev_private;
1493 if (!pmd->flower_support)
1495 switch (filter_type) {
1496 case RTE_ETH_FILTER_GENERIC:
1497 if (filter_op != RTE_ETH_FILTER_GET)
1499 *(const void **)arg = &tap_flow_ops;
1502 RTE_LOG(ERR, PMD, "%p: filter type (%d) not supported",
1503 (void *)dev, filter_type);