1 /* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
2 * Copyright 2017 Mellanox Technologies, Ltd.
8 #include <sys/socket.h>
12 #include <linux/if_ether.h>
14 #include <linux/ipv6.h>
15 #include <linux/if_tunnel.h>
16 #include <linux/filter.h>
17 #include <linux/bpf.h>
21 /** Create IPv4 address */
22 #define IPv4(a, b, c, d) ((__u32)(((a) & 0xff) << 24) | \
23 (((b) & 0xff) << 16) | \
24 (((c) & 0xff) << 8) | \
27 #define PORT(a, b) ((__u16)(((a) & 0xff) << 8) | \
31 * The queue number is offset by 1, to distinguish packets that have
32 * gone through this rule (skb->cb[1] != 0) from others.
34 #define QUEUE_OFFSET 1
35 #define PIN_GLOBAL_NS 2
38 #define BPF_MAP_ID_KEY 1
45 struct bpf_elf_map __attribute__((section("maps"), used))
47 .type = BPF_MAP_TYPE_HASH,
49 .size_key = sizeof(__u32),
50 .size_value = sizeof(struct rss_key),
52 .pinning = PIN_GLOBAL_NS,
55 __section("cls_q") int
56 match_q(struct __sk_buff *skb)
58 __u32 queue = skb->cb[1];
59 volatile __u32 q = 0xdeadbeef;
60 __u32 match_queue = QUEUE_OFFSET + q;
62 /* printt("match_q$i() queue = %d\n", queue); */
64 if (queue != match_queue)
70 struct ipv4_l3_l4_tuple {
75 } __attribute__((packed));
77 struct ipv6_l3_l4_tuple {
82 } __attribute__((packed));
84 static const __u8 def_rss_key[] = {
85 0xd1, 0x81, 0xc6, 0x2c,
86 0xf7, 0xf4, 0xdb, 0x5b,
87 0x19, 0x83, 0xa2, 0xfc,
88 0x94, 0x3e, 0x1a, 0xdb,
89 0xd9, 0x38, 0x9e, 0x6b,
90 0xd1, 0x03, 0x9c, 0x2c,
91 0xa7, 0x44, 0x99, 0xad,
92 0x59, 0x3d, 0x56, 0xd9,
93 0xf3, 0x25, 0x3c, 0x06,
94 0x2a, 0xdc, 0x1f, 0xfc,
97 static __u32 __attribute__((always_inline))
98 rte_softrss_be(const __u32 *input_tuple, const uint8_t *rss_key,
101 __u32 i, j, hash = 0;
103 for (j = 0; j < input_len; j++) {
105 for (i = 0; i < 32; i++) {
106 if (input_tuple[j] & (1 << (31 - i))) {
107 hash ^= ((const __u32 *)def_rss_key)[j] << i |
109 (((const __u32 *)def_rss_key)[j + 1])
117 static int __attribute__((always_inline))
118 rss_l3_l4(struct __sk_buff *skb)
120 void *data_end = (void *)(long)skb->data_end;
121 void *data = (void *)(long)skb->data;
122 __u16 proto = (__u16)skb->protocol;
123 __u32 key_idx = 0xdeadbeef;
125 struct rss_key *rsskey;
126 __u64 off = ETH_HLEN;
132 rsskey = map_lookup_elem(&map_keys, &key_idx);
134 printt("hash(): rss key is not configured\n");
137 key = (__u8 *)rsskey->key;
139 /* Get correct proto for 802.1ad */
140 if (skb->vlan_present && skb->vlan_proto == htons(ETH_P_8021AD)) {
141 if (data + ETH_ALEN * 2 + sizeof(struct vlan_hdr) +
142 sizeof(proto) > data_end)
144 proto = *(__u16 *)(data + ETH_ALEN * 2 +
145 sizeof(struct vlan_hdr));
146 off += sizeof(struct vlan_hdr);
149 if (proto == htons(ETH_P_IP)) {
150 if (data + off + sizeof(struct iphdr) + sizeof(__u32)
154 __u8 *src_dst_addr = data + off + offsetof(struct iphdr, saddr);
155 __u8 *src_dst_port = data + off + sizeof(struct iphdr);
156 struct ipv4_l3_l4_tuple v4_tuple = {
157 .src_addr = IPv4(*(src_dst_addr + 0),
160 *(src_dst_addr + 3)),
161 .dst_addr = IPv4(*(src_dst_addr + 4),
164 *(src_dst_addr + 7)),
165 .sport = PORT(*(src_dst_port + 0),
166 *(src_dst_port + 1)),
167 .dport = PORT(*(src_dst_port + 2),
168 *(src_dst_port + 3)),
170 __u8 input_len = sizeof(v4_tuple) / sizeof(__u32);
171 if (rsskey->hash_fields & (1 << HASH_FIELD_IPV4_L3))
173 hash = rte_softrss_be((__u32 *)&v4_tuple, key, 3);
174 } else if (proto == htons(ETH_P_IPV6)) {
175 if (data + off + sizeof(struct ipv6hdr) +
176 sizeof(__u32) > data_end)
178 __u8 *src_dst_addr = data + off +
179 offsetof(struct ipv6hdr, saddr);
180 __u8 *src_dst_port = data + off +
181 sizeof(struct ipv6hdr);
182 struct ipv6_l3_l4_tuple v6_tuple;
183 for (j = 0; j < 4; j++)
184 *((uint32_t *)&v6_tuple.src_addr + j) =
185 __builtin_bswap32(*((uint32_t *)
187 for (j = 0; j < 4; j++)
188 *((uint32_t *)&v6_tuple.dst_addr + j) =
189 __builtin_bswap32(*((uint32_t *)
190 src_dst_addr + 4 + j));
191 v6_tuple.sport = PORT(*(src_dst_port + 0),
192 *(src_dst_port + 1));
193 v6_tuple.dport = PORT(*(src_dst_port + 2),
194 *(src_dst_port + 3));
196 __u8 input_len = sizeof(v6_tuple) / sizeof(__u32);
197 if (rsskey->hash_fields & (1 << HASH_FIELD_IPV6_L3))
199 hash = rte_softrss_be((__u32 *)&v6_tuple, key, 9);
204 queue = rsskey->queues[(hash % rsskey->nb_queues) &
205 (TAP_MAX_QUEUES - 1)];
206 skb->cb[1] = QUEUE_OFFSET + queue;
207 /* printt(">>>>> rss_l3_l4 hash=0x%x queue=%u\n", hash, queue); */
209 return TC_ACT_RECLASSIFY;
214 L ## _hash(struct __sk_buff *skb) \
216 return rss_ ## L (skb); \
221 BPF_LICENSE("Dual BSD/GPL");