1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
13 #include <rte_debug.h>
14 #include <rte_ether.h>
15 #include <rte_ethdev_driver.h>
17 #include <rte_malloc.h>
18 #include <rte_eth_ctrl.h>
19 #include <rte_tailq.h>
20 #include <rte_flow_driver.h>
24 #include "iavf_generic_flow.h"
26 #define IAVF_PHINT_NONE 0
27 #define IAVF_PHINT_GTPU BIT_ULL(0)
28 #define IAVF_PHINT_GTPU_EH BIT_ULL(1)
29 #define IAVF_PHINT_GTPU_EH_DWN BIT_ULL(2)
30 #define IAVF_PHINT_GTPU_EH_UP BIT_ULL(3)
32 #define IAVF_PHINT_GTPU_MSK (IAVF_PHINT_GTPU | \
33 IAVF_PHINT_GTPU_EH | \
34 IAVF_PHINT_GTPU_EH_DWN | \
35 IAVF_PHINT_GTPU_EH_UP)
37 #define IAVF_GTPU_EH_DWNLINK 0
38 #define IAVF_GTPU_EH_UPLINK 1
40 struct iavf_hash_match_type {
42 struct virtchnl_proto_hdrs *proto_hdrs;
43 uint64_t pattern_hint;
46 struct iavf_rss_meta {
47 struct virtchnl_proto_hdrs proto_hdrs;
48 enum virtchnl_rss_algorithm rss_algorithm;
51 struct iavf_hash_flow_cfg {
52 struct virtchnl_rss_cfg *rss_cfg;
57 iavf_hash_init(struct iavf_adapter *ad);
59 iavf_hash_create(struct iavf_adapter *ad, struct rte_flow *flow, void *meta,
60 struct rte_flow_error *error);
62 iavf_hash_destroy(struct iavf_adapter *ad, struct rte_flow *flow,
63 struct rte_flow_error *error);
65 iavf_hash_uninit(struct iavf_adapter *ad);
67 iavf_hash_free(struct rte_flow *flow);
69 iavf_hash_parse_pattern_action(struct iavf_adapter *ad,
70 struct iavf_pattern_match_item *array,
72 const struct rte_flow_item pattern[],
73 const struct rte_flow_action actions[],
75 struct rte_flow_error *error);
77 #define FIELD_SELECTOR(proto_hdr_field) \
78 (1UL << ((proto_hdr_field) & PROTO_HDR_FIELD_MASK))
81 #define proto_hdr_eth { \
82 VIRTCHNL_PROTO_HDR_ETH, \
83 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_SRC) | \
84 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_DST), {BUFF_NOUSED} }
86 #define proto_hdr_svlan { \
87 VIRTCHNL_PROTO_HDR_S_VLAN, \
88 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_S_VLAN_ID), {BUFF_NOUSED} }
90 #define proto_hdr_cvlan { \
91 VIRTCHNL_PROTO_HDR_C_VLAN, \
92 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_C_VLAN_ID), {BUFF_NOUSED} }
94 #define proto_hdr_ipv4 { \
95 VIRTCHNL_PROTO_HDR_IPV4, \
96 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_SRC) | \
97 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_DST), {BUFF_NOUSED} }
99 #define proto_hdr_ipv6 { \
100 VIRTCHNL_PROTO_HDR_IPV6, \
101 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_SRC) | \
102 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_DST), {BUFF_NOUSED} }
104 #define proto_hdr_udp { \
105 VIRTCHNL_PROTO_HDR_UDP, \
106 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_UDP_SRC_PORT) | \
107 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_UDP_DST_PORT), {BUFF_NOUSED} }
109 #define proto_hdr_tcp { \
110 VIRTCHNL_PROTO_HDR_TCP, \
111 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_TCP_SRC_PORT) | \
112 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_TCP_DST_PORT), {BUFF_NOUSED} }
114 #define proto_hdr_sctp { \
115 VIRTCHNL_PROTO_HDR_SCTP, \
116 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_SCTP_SRC_PORT) | \
117 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_SCTP_DST_PORT), {BUFF_NOUSED} }
119 #define proto_hdr_esp { \
120 VIRTCHNL_PROTO_HDR_ESP, \
121 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ESP_SPI), {BUFF_NOUSED} }
123 #define proto_hdr_ah { \
124 VIRTCHNL_PROTO_HDR_AH, \
125 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_AH_SPI), {BUFF_NOUSED} }
127 #define proto_hdr_l2tpv3 { \
128 VIRTCHNL_PROTO_HDR_L2TPV3, \
129 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_L2TPV3_SESS_ID), {BUFF_NOUSED} }
131 #define proto_hdr_pfcp { \
132 VIRTCHNL_PROTO_HDR_PFCP, \
133 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_PFCP_SEID), {BUFF_NOUSED} }
135 #define TUNNEL_LEVEL_OUTER 0
136 #define TUNNEL_LEVEL_INNER 1
138 /* proto_hdrs template */
139 struct virtchnl_proto_hdrs outer_ipv4_tmplt = {
140 TUNNEL_LEVEL_OUTER, 4,
141 {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv4}
144 struct virtchnl_proto_hdrs outer_ipv4_udp_tmplt = {
145 TUNNEL_LEVEL_OUTER, 5,
146 {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv4,
150 struct virtchnl_proto_hdrs outer_ipv4_tcp_tmplt = {
151 TUNNEL_LEVEL_OUTER, 5,
152 {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv4,
156 struct virtchnl_proto_hdrs outer_ipv4_sctp_tmplt = {
157 TUNNEL_LEVEL_OUTER, 5,
158 {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv4,
162 struct virtchnl_proto_hdrs outer_ipv6_tmplt = {
163 TUNNEL_LEVEL_OUTER, 4,
164 {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv6}
167 struct virtchnl_proto_hdrs outer_ipv6_udp_tmplt = {
168 TUNNEL_LEVEL_OUTER, 5,
169 {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv6,
173 struct virtchnl_proto_hdrs outer_ipv6_tcp_tmplt = {
174 TUNNEL_LEVEL_OUTER, 5,
175 {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv6,
179 struct virtchnl_proto_hdrs outer_ipv6_sctp_tmplt = {
180 TUNNEL_LEVEL_OUTER, 5,
181 {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv6,
185 struct virtchnl_proto_hdrs inner_ipv4_tmplt = {
186 TUNNEL_LEVEL_INNER, 1, {proto_hdr_ipv4}
189 struct virtchnl_proto_hdrs inner_ipv4_udp_tmplt = {
190 TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv4, proto_hdr_udp}
193 struct virtchnl_proto_hdrs inner_ipv4_tcp_tmplt = {
194 TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv4, proto_hdr_tcp}
197 struct virtchnl_proto_hdrs inner_ipv4_sctp_tmplt = {
198 TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv4, proto_hdr_sctp}
201 struct virtchnl_proto_hdrs inner_ipv6_tmplt = {
202 TUNNEL_LEVEL_INNER, 1, {proto_hdr_ipv6}
205 struct virtchnl_proto_hdrs inner_ipv6_udp_tmplt = {
206 TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv6, proto_hdr_udp}
209 struct virtchnl_proto_hdrs inner_ipv6_tcp_tmplt = {
210 TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv6, proto_hdr_tcp}
213 struct virtchnl_proto_hdrs inner_ipv6_sctp_tmplt = {
214 TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv6, proto_hdr_sctp}
217 struct virtchnl_proto_hdrs ipv4_esp_tmplt = {
218 TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_esp}
221 struct virtchnl_proto_hdrs ipv4_udp_esp_tmplt = {
222 TUNNEL_LEVEL_OUTER, 3,
223 {proto_hdr_ipv4, proto_hdr_udp, proto_hdr_esp}
226 struct virtchnl_proto_hdrs ipv4_ah_tmplt = {
227 TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_ah}
230 struct virtchnl_proto_hdrs ipv6_esp_tmplt = {
231 TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_esp}
234 struct virtchnl_proto_hdrs ipv6_udp_esp_tmplt = {
235 TUNNEL_LEVEL_OUTER, 3,
236 {proto_hdr_ipv6, proto_hdr_udp, proto_hdr_esp}
239 struct virtchnl_proto_hdrs ipv6_ah_tmplt = {
240 TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_ah}
243 struct virtchnl_proto_hdrs ipv4_l2tpv3_tmplt = {
244 TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_l2tpv3}
247 struct virtchnl_proto_hdrs ipv6_l2tpv3_tmplt = {
248 TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_l2tpv3}
251 struct virtchnl_proto_hdrs ipv4_pfcp_tmplt = {
252 TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_pfcp}
255 struct virtchnl_proto_hdrs ipv6_pfcp_tmplt = {
256 TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_pfcp}
259 /* rss type super set */
262 #define IAVF_RSS_TYPE_OUTER_IPV4 (ETH_RSS_ETH | ETH_RSS_IPV4)
263 #define IAVF_RSS_TYPE_OUTER_IPV4_UDP (IAVF_RSS_TYPE_OUTER_IPV4 | \
264 ETH_RSS_NONFRAG_IPV4_UDP)
265 #define IAVF_RSS_TYPE_OUTER_IPV4_TCP (IAVF_RSS_TYPE_OUTER_IPV4 | \
266 ETH_RSS_NONFRAG_IPV4_TCP)
267 #define IAVF_RSS_TYPE_OUTER_IPV4_SCTP (IAVF_RSS_TYPE_OUTER_IPV4 | \
268 ETH_RSS_NONFRAG_IPV4_SCTP)
270 #define IAVF_RSS_TYPE_OUTER_IPV6 (ETH_RSS_ETH | ETH_RSS_IPV6)
271 #define IAVF_RSS_TYPE_OUTER_IPV6_UDP (IAVF_RSS_TYPE_OUTER_IPV6 | \
272 ETH_RSS_NONFRAG_IPV6_UDP)
273 #define IAVF_RSS_TYPE_OUTER_IPV6_TCP (IAVF_RSS_TYPE_OUTER_IPV6 | \
274 ETH_RSS_NONFRAG_IPV6_TCP)
275 #define IAVF_RSS_TYPE_OUTER_IPV6_SCTP (IAVF_RSS_TYPE_OUTER_IPV6 | \
276 ETH_RSS_NONFRAG_IPV6_SCTP)
278 #define IAVF_RSS_TYPE_VLAN_IPV4 (IAVF_RSS_TYPE_OUTER_IPV4 | \
279 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
280 #define IAVF_RSS_TYPE_VLAN_IPV4_UDP (IAVF_RSS_TYPE_OUTER_IPV4_UDP | \
281 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
282 #define IAVF_RSS_TYPE_VLAN_IPV4_TCP (IAVF_RSS_TYPE_OUTER_IPV4_TCP | \
283 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
284 #define IAVF_RSS_TYPE_VLAN_IPV4_SCTP (IAVF_RSS_TYPE_OUTER_IPV4_SCTP | \
285 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
287 #define IAVF_RSS_TYPE_VLAN_IPV6 (IAVF_RSS_TYPE_OUTER_IPV6 | \
288 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
289 #define IAVF_RSS_TYPE_VLAN_IPV6_UDP (IAVF_RSS_TYPE_OUTER_IPV6_UDP | \
290 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
291 #define IAVF_RSS_TYPE_VLAN_IPV6_TCP (IAVF_RSS_TYPE_OUTER_IPV6_TCP | \
292 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
293 #define IAVF_RSS_TYPE_VLAN_IPV6_SCTP (IAVF_RSS_TYPE_OUTER_IPV6_SCTP | \
294 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
296 #define IAVF_RSS_TYPE_INNER_IPV4 ETH_RSS_IPV4
297 #define IAVF_RSS_TYPE_INNER_IPV4_UDP (ETH_RSS_IPV4 | \
298 ETH_RSS_NONFRAG_IPV4_UDP)
299 #define IAVF_RSS_TYPE_INNER_IPV4_TCP (ETH_RSS_IPV4 | \
300 ETH_RSS_NONFRAG_IPV4_TCP)
301 #define IAVF_RSS_TYPE_INNER_IPV4_SCTP (ETH_RSS_IPV4 | \
302 ETH_RSS_NONFRAG_IPV4_SCTP)
304 #define IAVF_RSS_TYPE_INNER_IPV6 ETH_RSS_IPV6
305 #define IAVF_RSS_TYPE_INNER_IPV6_UDP (ETH_RSS_IPV6 | \
306 ETH_RSS_NONFRAG_IPV6_UDP)
307 #define IAVF_RSS_TYPE_INNER_IPV6_TCP (ETH_RSS_IPV6 | \
308 ETH_RSS_NONFRAG_IPV6_TCP)
309 #define IAVF_RSS_TYPE_INNER_IPV6_SCTP (ETH_RSS_IPV6 | \
310 ETH_RSS_NONFRAG_IPV6_SCTP)
312 #define IAVF_RSS_TYPE_GTPU_IPV4 (IAVF_RSS_TYPE_INNER_IPV4 | \
314 #define IAVF_RSS_TYPE_GTPU_IPV4_UDP (IAVF_RSS_TYPE_INNER_IPV4_UDP | \
316 #define IAVF_RSS_TYPE_GTPU_IPV4_TCP (IAVF_RSS_TYPE_INNER_IPV4_TCP | \
319 #define IAVF_RSS_TYPE_GTPU_IPV6 (IAVF_RSS_TYPE_INNER_IPV6 | \
321 #define IAVF_RSS_TYPE_GTPU_IPV6_UDP (IAVF_RSS_TYPE_INNER_IPV6_UDP | \
323 #define IAVF_RSS_TYPE_GTPU_IPV6_TCP (IAVF_RSS_TYPE_INNER_IPV6_TCP | \
325 /* ESP, AH, L2TPV3 and PFCP */
326 #define IAVF_RSS_TYPE_IPV4_ESP (ETH_RSS_ESP | ETH_RSS_IPV4)
327 #define IAVF_RSS_TYPE_IPV4_AH (ETH_RSS_AH | ETH_RSS_IPV4)
328 #define IAVF_RSS_TYPE_IPV6_ESP (ETH_RSS_ESP | ETH_RSS_IPV6)
329 #define IAVF_RSS_TYPE_IPV6_AH (ETH_RSS_AH | ETH_RSS_IPV6)
330 #define IAVF_RSS_TYPE_IPV4_L2TPV3 (ETH_RSS_L2TPV3 | ETH_RSS_IPV4)
331 #define IAVF_RSS_TYPE_IPV6_L2TPV3 (ETH_RSS_L2TPV3 | ETH_RSS_IPV6)
332 #define IAVF_RSS_TYPE_IPV4_PFCP (ETH_RSS_PFCP | ETH_RSS_IPV4)
333 #define IAVF_RSS_TYPE_IPV6_PFCP (ETH_RSS_PFCP | ETH_RSS_IPV6)
336 * Supported pattern for hash.
337 * The first member is pattern item type,
338 * the second member is input set mask,
339 * the third member is virtchnl_proto_hdrs template
341 static struct iavf_pattern_match_item iavf_hash_pattern_list[] = {
343 {iavf_pattern_eth_ipv4, IAVF_RSS_TYPE_OUTER_IPV4, &outer_ipv4_tmplt},
344 {iavf_pattern_eth_ipv4_udp, IAVF_RSS_TYPE_OUTER_IPV4_UDP, &outer_ipv4_udp_tmplt},
345 {iavf_pattern_eth_ipv4_tcp, IAVF_RSS_TYPE_OUTER_IPV4_TCP, &outer_ipv4_tcp_tmplt},
346 {iavf_pattern_eth_ipv4_sctp, IAVF_RSS_TYPE_OUTER_IPV4_SCTP, &outer_ipv4_sctp_tmplt},
347 {iavf_pattern_eth_vlan_ipv4, IAVF_RSS_TYPE_VLAN_IPV4, &outer_ipv4_tmplt},
348 {iavf_pattern_eth_vlan_ipv4_udp, IAVF_RSS_TYPE_VLAN_IPV4_UDP, &outer_ipv4_udp_tmplt},
349 {iavf_pattern_eth_vlan_ipv4_tcp, IAVF_RSS_TYPE_VLAN_IPV4_TCP, &outer_ipv4_tcp_tmplt},
350 {iavf_pattern_eth_vlan_ipv4_sctp, IAVF_RSS_TYPE_VLAN_IPV4_SCTP, &outer_ipv4_sctp_tmplt},
351 {iavf_pattern_eth_ipv4_gtpu_ipv4, IAVF_RSS_TYPE_GTPU_IPV4, &inner_ipv4_tmplt},
352 {iavf_pattern_eth_ipv4_gtpu_ipv4_udp, IAVF_RSS_TYPE_GTPU_IPV4_UDP, &inner_ipv4_udp_tmplt},
353 {iavf_pattern_eth_ipv4_gtpu_ipv4_tcp, IAVF_RSS_TYPE_GTPU_IPV4_TCP, &inner_ipv4_tcp_tmplt},
354 {iavf_pattern_eth_ipv6_gtpu_ipv4, IAVF_RSS_TYPE_GTPU_IPV4, &inner_ipv4_tmplt},
355 {iavf_pattern_eth_ipv6_gtpu_ipv4_udp, IAVF_RSS_TYPE_GTPU_IPV4_UDP, &inner_ipv4_udp_tmplt},
356 {iavf_pattern_eth_ipv6_gtpu_ipv4_tcp, IAVF_RSS_TYPE_GTPU_IPV4_TCP, &inner_ipv4_tcp_tmplt},
357 {iavf_pattern_eth_ipv4_gtpu_eh_ipv4, IAVF_RSS_TYPE_GTPU_IPV4, &inner_ipv4_tmplt},
358 {iavf_pattern_eth_ipv4_gtpu_eh_ipv4_udp, IAVF_RSS_TYPE_GTPU_IPV4_UDP, &inner_ipv4_udp_tmplt},
359 {iavf_pattern_eth_ipv4_gtpu_eh_ipv4_tcp, IAVF_RSS_TYPE_GTPU_IPV4_TCP, &inner_ipv4_tcp_tmplt},
360 {iavf_pattern_eth_ipv6_gtpu_eh_ipv4, IAVF_RSS_TYPE_GTPU_IPV4, &inner_ipv4_tmplt},
361 {iavf_pattern_eth_ipv6_gtpu_eh_ipv4_udp, IAVF_RSS_TYPE_GTPU_IPV4_UDP, &inner_ipv4_udp_tmplt},
362 {iavf_pattern_eth_ipv6_gtpu_eh_ipv4_tcp, IAVF_RSS_TYPE_GTPU_IPV4_TCP, &inner_ipv4_tcp_tmplt},
363 {iavf_pattern_eth_ipv4_esp, IAVF_RSS_TYPE_IPV4_ESP, &ipv4_esp_tmplt},
364 {iavf_pattern_eth_ipv4_udp_esp, IAVF_RSS_TYPE_IPV4_ESP, &ipv4_udp_esp_tmplt},
365 {iavf_pattern_eth_ipv4_ah, IAVF_RSS_TYPE_IPV4_AH, &ipv4_ah_tmplt},
366 {iavf_pattern_eth_ipv4_l2tpv3, IAVF_RSS_TYPE_IPV4_L2TPV3, &ipv4_l2tpv3_tmplt},
367 {iavf_pattern_eth_ipv4_pfcp, IAVF_RSS_TYPE_IPV4_PFCP, &ipv4_pfcp_tmplt},
369 {iavf_pattern_eth_ipv6, IAVF_RSS_TYPE_OUTER_IPV6, &outer_ipv6_tmplt},
370 {iavf_pattern_eth_ipv6_udp, IAVF_RSS_TYPE_OUTER_IPV6_UDP, &outer_ipv6_udp_tmplt},
371 {iavf_pattern_eth_ipv6_tcp, IAVF_RSS_TYPE_OUTER_IPV6_TCP, &outer_ipv6_tcp_tmplt},
372 {iavf_pattern_eth_ipv6_sctp, IAVF_RSS_TYPE_OUTER_IPV6_SCTP, &outer_ipv6_sctp_tmplt},
373 {iavf_pattern_eth_vlan_ipv6, IAVF_RSS_TYPE_VLAN_IPV6, &outer_ipv6_tmplt},
374 {iavf_pattern_eth_vlan_ipv6_udp, IAVF_RSS_TYPE_VLAN_IPV6_UDP, &outer_ipv6_udp_tmplt},
375 {iavf_pattern_eth_vlan_ipv6_tcp, IAVF_RSS_TYPE_VLAN_IPV6_TCP, &outer_ipv6_tcp_tmplt},
376 {iavf_pattern_eth_vlan_ipv6_sctp, IAVF_RSS_TYPE_VLAN_IPV6_SCTP, &outer_ipv6_sctp_tmplt},
377 {iavf_pattern_eth_ipv4_gtpu_ipv6, IAVF_RSS_TYPE_GTPU_IPV6, &inner_ipv6_tmplt},
378 {iavf_pattern_eth_ipv4_gtpu_ipv6_udp, IAVF_RSS_TYPE_GTPU_IPV6_UDP, &inner_ipv6_udp_tmplt},
379 {iavf_pattern_eth_ipv4_gtpu_ipv6_tcp, IAVF_RSS_TYPE_GTPU_IPV6_TCP, &inner_ipv6_tcp_tmplt},
380 {iavf_pattern_eth_ipv6_gtpu_ipv6, IAVF_RSS_TYPE_GTPU_IPV6, &inner_ipv6_tmplt},
381 {iavf_pattern_eth_ipv6_gtpu_ipv6_udp, IAVF_RSS_TYPE_GTPU_IPV6_UDP, &inner_ipv6_udp_tmplt},
382 {iavf_pattern_eth_ipv6_gtpu_ipv6_tcp, IAVF_RSS_TYPE_GTPU_IPV6_TCP, &inner_ipv6_tcp_tmplt},
383 {iavf_pattern_eth_ipv4_gtpu_eh_ipv6, IAVF_RSS_TYPE_GTPU_IPV6, &inner_ipv6_tmplt},
384 {iavf_pattern_eth_ipv4_gtpu_eh_ipv6_udp, IAVF_RSS_TYPE_GTPU_IPV6_UDP, &inner_ipv6_udp_tmplt},
385 {iavf_pattern_eth_ipv4_gtpu_eh_ipv6_tcp, IAVF_RSS_TYPE_GTPU_IPV6_TCP, &inner_ipv6_tcp_tmplt},
386 {iavf_pattern_eth_ipv6_gtpu_eh_ipv6, IAVF_RSS_TYPE_GTPU_IPV6, &inner_ipv6_tmplt},
387 {iavf_pattern_eth_ipv6_gtpu_eh_ipv6_udp, IAVF_RSS_TYPE_GTPU_IPV6_UDP, &inner_ipv6_udp_tmplt},
388 {iavf_pattern_eth_ipv6_gtpu_eh_ipv6_tcp, IAVF_RSS_TYPE_GTPU_IPV6_TCP, &inner_ipv6_tcp_tmplt},
389 {iavf_pattern_eth_ipv6_esp, IAVF_RSS_TYPE_IPV6_ESP, &ipv6_esp_tmplt},
390 {iavf_pattern_eth_ipv6_udp_esp, IAVF_RSS_TYPE_IPV6_ESP, &ipv6_udp_esp_tmplt},
391 {iavf_pattern_eth_ipv6_ah, IAVF_RSS_TYPE_IPV6_AH, &ipv6_ah_tmplt},
392 {iavf_pattern_eth_ipv6_l2tpv3, IAVF_RSS_TYPE_IPV6_L2TPV3, &ipv6_l2tpv3_tmplt},
393 {iavf_pattern_eth_ipv6_pfcp, IAVF_RSS_TYPE_IPV6_PFCP, &ipv6_pfcp_tmplt},
396 struct virtchnl_proto_hdrs *iavf_hash_default_hdrs[] = {
398 &inner_ipv4_udp_tmplt,
399 &inner_ipv4_tcp_tmplt,
400 &inner_ipv4_sctp_tmplt,
402 &inner_ipv6_udp_tmplt,
403 &inner_ipv6_tcp_tmplt,
404 &inner_ipv6_sctp_tmplt,
407 static struct iavf_flow_engine iavf_hash_engine = {
408 .init = iavf_hash_init,
409 .create = iavf_hash_create,
410 .destroy = iavf_hash_destroy,
411 .uninit = iavf_hash_uninit,
412 .free = iavf_hash_free,
413 .type = IAVF_FLOW_ENGINE_HASH,
416 /* Register parser for comms package. */
417 static struct iavf_flow_parser iavf_hash_parser = {
418 .engine = &iavf_hash_engine,
419 .array = iavf_hash_pattern_list,
420 .array_len = RTE_DIM(iavf_hash_pattern_list),
421 .parse_pattern_action = iavf_hash_parse_pattern_action,
422 .stage = IAVF_FLOW_STAGE_RSS,
426 iavf_hash_default_set(struct iavf_adapter *ad, bool add)
428 struct virtchnl_rss_cfg *rss_cfg;
431 rss_cfg = rte_zmalloc("iavf rss rule",
432 sizeof(struct virtchnl_rss_cfg), 0);
436 for (i = 0; i < RTE_DIM(iavf_hash_default_hdrs); i++) {
437 rss_cfg->proto_hdrs = *iavf_hash_default_hdrs[i];
438 rss_cfg->rss_algorithm = VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC;
440 iavf_add_del_rss_cfg(ad, rss_cfg, add);
446 RTE_INIT(iavf_hash_engine_init)
448 struct iavf_flow_engine *engine = &iavf_hash_engine;
450 iavf_register_flow_engine(engine);
454 iavf_hash_init(struct iavf_adapter *ad)
456 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
457 struct iavf_flow_parser *parser;
466 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF))
469 parser = &iavf_hash_parser;
471 ret = iavf_register_parser(parser, ad);
473 PMD_DRV_LOG(ERR, "fail to register hash parser");
477 ret = iavf_hash_default_set(ad, true);
479 PMD_DRV_LOG(ERR, "fail to set default RSS");
480 iavf_unregister_parser(parser, ad);
487 iavf_hash_parse_pattern(struct iavf_pattern_match_item *pattern_match_item,
488 const struct rte_flow_item pattern[], uint64_t *phint,
489 struct rte_flow_error *error)
491 const struct rte_flow_item *item = pattern;
492 const struct rte_flow_item_gtp_psc *psc;
494 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
496 rte_flow_error_set(error, EINVAL,
497 RTE_FLOW_ERROR_TYPE_ITEM, item,
498 "Not support range");
502 switch (item->type) {
503 case RTE_FLOW_ITEM_TYPE_GTPU:
504 *phint |= IAVF_PHINT_GTPU;
506 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
507 *phint |= IAVF_PHINT_GTPU_EH;
511 else if (psc->pdu_type == IAVF_GTPU_EH_UPLINK)
512 *phint |= IAVF_PHINT_GTPU_EH_UP;
513 else if (psc->pdu_type == IAVF_GTPU_EH_DWNLINK)
514 *phint |= IAVF_PHINT_GTPU_EH_DWN;
521 /* update and restore pattern hint */
522 *phint |= *(uint64_t *)(pattern_match_item->meta);
527 /* refine proto hdrs base on l2, l3, l4 rss type */
529 iavf_refine_proto_hdrs_l234(struct virtchnl_proto_hdrs *proto_hdrs,
532 struct virtchnl_proto_hdr *hdr;
535 for (i = 0; i < proto_hdrs->count; i++) {
536 hdr = &proto_hdrs->proto_hdr[i];
538 case VIRTCHNL_PROTO_HDR_ETH:
539 if (!(rss_type & ETH_RSS_ETH))
540 hdr->field_selector = 0;
541 else if (rss_type & ETH_RSS_L2_SRC_ONLY)
542 VIRTCHNL_DEL_PROTO_HDR_FIELD(hdr,
543 VIRTCHNL_PROTO_HDR_ETH_DST);
544 else if (rss_type & ETH_RSS_L2_DST_ONLY)
545 VIRTCHNL_DEL_PROTO_HDR_FIELD(hdr,
546 VIRTCHNL_PROTO_HDR_ETH_SRC);
548 case VIRTCHNL_PROTO_HDR_IPV4:
551 ETH_RSS_NONFRAG_IPV4_UDP |
552 ETH_RSS_NONFRAG_IPV4_TCP |
553 ETH_RSS_NONFRAG_IPV4_SCTP)) {
554 if (rss_type & ETH_RSS_L3_SRC_ONLY)
555 VIRTCHNL_DEL_PROTO_HDR_FIELD(hdr,
556 VIRTCHNL_PROTO_HDR_IPV4_DST);
557 else if (rss_type & ETH_RSS_L3_DST_ONLY)
558 VIRTCHNL_DEL_PROTO_HDR_FIELD(hdr,
559 VIRTCHNL_PROTO_HDR_IPV4_SRC);
561 (ETH_RSS_L4_SRC_ONLY |
562 ETH_RSS_L4_DST_ONLY))
563 hdr->field_selector = 0;
565 hdr->field_selector = 0;
568 case VIRTCHNL_PROTO_HDR_IPV6:
571 ETH_RSS_NONFRAG_IPV6_UDP |
572 ETH_RSS_NONFRAG_IPV6_TCP |
573 ETH_RSS_NONFRAG_IPV6_SCTP)) {
574 if (rss_type & ETH_RSS_L3_SRC_ONLY)
575 VIRTCHNL_DEL_PROTO_HDR_FIELD(hdr,
576 VIRTCHNL_PROTO_HDR_IPV6_DST);
577 else if (rss_type & ETH_RSS_L3_DST_ONLY)
578 VIRTCHNL_DEL_PROTO_HDR_FIELD(hdr,
579 VIRTCHNL_PROTO_HDR_IPV6_SRC);
581 (ETH_RSS_L4_SRC_ONLY |
582 ETH_RSS_L4_DST_ONLY))
583 hdr->field_selector = 0;
585 hdr->field_selector = 0;
588 case VIRTCHNL_PROTO_HDR_UDP:
590 (ETH_RSS_NONFRAG_IPV4_UDP |
591 ETH_RSS_NONFRAG_IPV6_UDP)) {
592 if (rss_type & ETH_RSS_L4_SRC_ONLY)
593 VIRTCHNL_DEL_PROTO_HDR_FIELD(hdr,
594 VIRTCHNL_PROTO_HDR_UDP_DST_PORT);
595 else if (rss_type & ETH_RSS_L4_DST_ONLY)
596 VIRTCHNL_DEL_PROTO_HDR_FIELD(hdr,
597 VIRTCHNL_PROTO_HDR_UDP_SRC_PORT);
599 (ETH_RSS_L3_SRC_ONLY |
600 ETH_RSS_L3_DST_ONLY))
601 hdr->field_selector = 0;
603 hdr->field_selector = 0;
606 case VIRTCHNL_PROTO_HDR_TCP:
608 (ETH_RSS_NONFRAG_IPV4_TCP |
609 ETH_RSS_NONFRAG_IPV6_TCP)) {
610 if (rss_type & ETH_RSS_L4_SRC_ONLY)
611 VIRTCHNL_DEL_PROTO_HDR_FIELD(hdr,
612 VIRTCHNL_PROTO_HDR_TCP_DST_PORT);
613 else if (rss_type & ETH_RSS_L4_DST_ONLY)
614 VIRTCHNL_DEL_PROTO_HDR_FIELD(hdr,
615 VIRTCHNL_PROTO_HDR_TCP_SRC_PORT);
617 (ETH_RSS_L3_SRC_ONLY |
618 ETH_RSS_L3_DST_ONLY))
619 hdr->field_selector = 0;
621 hdr->field_selector = 0;
624 case VIRTCHNL_PROTO_HDR_SCTP:
626 (ETH_RSS_NONFRAG_IPV4_SCTP |
627 ETH_RSS_NONFRAG_IPV6_SCTP)) {
628 if (rss_type & ETH_RSS_L4_SRC_ONLY)
629 VIRTCHNL_DEL_PROTO_HDR_FIELD(hdr,
630 VIRTCHNL_PROTO_HDR_SCTP_DST_PORT);
631 else if (rss_type & ETH_RSS_L4_DST_ONLY)
632 VIRTCHNL_DEL_PROTO_HDR_FIELD(hdr,
633 VIRTCHNL_PROTO_HDR_TCP_SRC_PORT);
635 (ETH_RSS_L3_SRC_ONLY |
636 ETH_RSS_L3_DST_ONLY))
637 hdr->field_selector = 0;
639 hdr->field_selector = 0;
642 case VIRTCHNL_PROTO_HDR_S_VLAN:
643 if (!(rss_type & ETH_RSS_S_VLAN))
644 hdr->field_selector = 0;
646 case VIRTCHNL_PROTO_HDR_C_VLAN:
647 if (!(rss_type & ETH_RSS_C_VLAN))
648 hdr->field_selector = 0;
650 case VIRTCHNL_PROTO_HDR_L2TPV3:
651 if (!(rss_type & ETH_RSS_L2TPV3))
652 hdr->field_selector = 0;
654 case VIRTCHNL_PROTO_HDR_ESP:
655 if (!(rss_type & ETH_RSS_ESP))
656 hdr->field_selector = 0;
658 case VIRTCHNL_PROTO_HDR_AH:
659 if (!(rss_type & ETH_RSS_AH))
660 hdr->field_selector = 0;
662 case VIRTCHNL_PROTO_HDR_PFCP:
663 if (!(rss_type & ETH_RSS_PFCP))
664 hdr->field_selector = 0;
672 /* refine proto hdrs base on gtpu rss type */
674 iavf_refine_proto_hdrs_gtpu(struct virtchnl_proto_hdrs *proto_hdrs,
677 struct virtchnl_proto_hdr *hdr;
680 if (!(rss_type & ETH_RSS_GTPU))
683 for (i = 0; i < proto_hdrs->count; i++) {
684 hdr = &proto_hdrs->proto_hdr[i];
686 case VIRTCHNL_PROTO_HDR_GTPU_IP:
687 VIRTCHNL_ADD_PROTO_HDR_FIELD(hdr,
688 VIRTCHNL_PROTO_HDR_GTPU_IP_TEID);
697 iavf_refine_proto_hdrs_by_pattern(struct virtchnl_proto_hdrs *proto_hdrs,
700 struct virtchnl_proto_hdr *hdr1;
701 struct virtchnl_proto_hdr *hdr2;
704 if (!(phint & IAVF_PHINT_GTPU_MSK) ||
705 proto_hdrs->tunnel_level == 0)
708 /* shift headers 1 layer */
709 for (i = proto_hdrs->count; i > 0; i--) {
710 hdr1 = &proto_hdrs->proto_hdr[i];
711 hdr2 = &proto_hdrs->proto_hdr[i - 1];
716 /* adding gtpu header at layer 0 */
718 hdr1 = &proto_hdrs->proto_hdr[0];
719 hdr1->field_selector = 0;
721 if (phint & IAVF_PHINT_GTPU_EH_DWN)
722 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_EH_PDU_DWN);
723 else if (phint & IAVF_PHINT_GTPU_EH_UP)
724 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_EH_PDU_UP);
725 else if (phint & IAVF_PHINT_GTPU_EH)
726 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_EH);
727 else if (phint & IAVF_PHINT_GTPU)
728 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_IP);
731 static void iavf_refine_proto_hdrs(struct virtchnl_proto_hdrs *proto_hdrs,
732 uint64_t rss_type, uint64_t phint)
734 iavf_refine_proto_hdrs_l234(proto_hdrs, rss_type);
735 iavf_refine_proto_hdrs_by_pattern(proto_hdrs, phint);
736 iavf_refine_proto_hdrs_gtpu(proto_hdrs, rss_type);
739 static uint64_t invalid_rss_comb[] = {
740 ETH_RSS_IPV4 | ETH_RSS_NONFRAG_IPV4_UDP,
741 ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_UDP,
742 RTE_ETH_RSS_L3_PRE32 | RTE_ETH_RSS_L3_PRE40 |
743 RTE_ETH_RSS_L3_PRE48 | RTE_ETH_RSS_L3_PRE56 |
748 iavf_any_invalid_rss_type(uint64_t rss_type, uint64_t allow_rss_type)
752 /* check invalid combination */
753 for (i = 0; i < RTE_DIM(invalid_rss_comb); i++) {
754 if (__builtin_popcountll(rss_type & invalid_rss_comb[i]) > 1)
758 /* check not allowed RSS type */
759 #define _RSS_ATTR_ (ETH_RSS_L3_SRC_ONLY | \
760 ETH_RSS_L3_DST_ONLY | \
761 ETH_RSS_L4_SRC_ONLY | \
762 ETH_RSS_L4_DST_ONLY | \
763 ETH_RSS_L2_SRC_ONLY | \
764 ETH_RSS_L2_DST_ONLY | \
765 RTE_ETH_RSS_L3_PRE32 | \
766 RTE_ETH_RSS_L3_PRE40 | \
767 RTE_ETH_RSS_L3_PRE48 | \
768 RTE_ETH_RSS_L3_PRE56 | \
769 RTE_ETH_RSS_L3_PRE64 | \
770 RTE_ETH_RSS_L3_PRE96)
772 rss_type &= ~_RSS_ATTR_;
774 return ((rss_type & allow_rss_type) != rss_type);
778 iavf_hash_parse_action(struct iavf_pattern_match_item *match_item,
779 const struct rte_flow_action actions[],
780 uint64_t pattern_hint, void **meta,
781 struct rte_flow_error *error)
783 struct iavf_rss_meta *rss_meta = (struct iavf_rss_meta *)*meta;
784 struct virtchnl_proto_hdrs *proto_hdrs;
785 enum rte_flow_action_type action_type;
786 const struct rte_flow_action_rss *rss;
787 const struct rte_flow_action *action;
790 /* Supported action is RSS. */
791 for (action = actions; action->type !=
792 RTE_FLOW_ACTION_TYPE_END; action++) {
793 action_type = action->type;
794 switch (action_type) {
795 case RTE_FLOW_ACTION_TYPE_RSS:
797 rss_type = rss->types;
800 RTE_ETH_HASH_FUNCTION_SIMPLE_XOR){
801 rss_meta->rss_algorithm =
802 VIRTCHNL_RSS_ALG_XOR_ASYMMETRIC;
803 return rte_flow_error_set(error, ENOTSUP,
804 RTE_FLOW_ERROR_TYPE_ACTION, action,
805 "function simple_xor is not supported");
806 } else if (rss->func ==
807 RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) {
808 rss_meta->rss_algorithm =
809 VIRTCHNL_RSS_ALG_TOEPLITZ_SYMMETRIC;
811 rss_meta->rss_algorithm =
812 VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC;
816 return rte_flow_error_set(error, ENOTSUP,
817 RTE_FLOW_ERROR_TYPE_ACTION, action,
818 "a nonzero RSS encapsulation level is not supported");
821 return rte_flow_error_set(error, ENOTSUP,
822 RTE_FLOW_ERROR_TYPE_ACTION, action,
823 "a nonzero RSS key_len is not supported");
826 return rte_flow_error_set(error, ENOTSUP,
827 RTE_FLOW_ERROR_TYPE_ACTION, action,
828 "a non-NULL RSS queue is not supported");
831 * Check simultaneous use of SRC_ONLY and DST_ONLY
834 rss_type = rte_eth_rss_hf_refine(rss_type);
836 if (iavf_any_invalid_rss_type(rss_type,
837 match_item->input_set_mask))
838 return rte_flow_error_set(error, ENOTSUP,
839 RTE_FLOW_ERROR_TYPE_ACTION,
840 action, "RSS type not supported");
841 proto_hdrs = match_item->meta;
842 rss_meta->proto_hdrs = *proto_hdrs;
843 iavf_refine_proto_hdrs(&rss_meta->proto_hdrs,
844 rss_type, pattern_hint);
847 case RTE_FLOW_ACTION_TYPE_END:
851 rte_flow_error_set(error, EINVAL,
852 RTE_FLOW_ERROR_TYPE_ACTION, action,
862 iavf_hash_parse_pattern_action(__rte_unused struct iavf_adapter *ad,
863 struct iavf_pattern_match_item *array,
865 const struct rte_flow_item pattern[],
866 const struct rte_flow_action actions[],
868 struct rte_flow_error *error)
870 struct iavf_pattern_match_item *pattern_match_item;
871 struct iavf_rss_meta *rss_meta_ptr;
872 uint64_t phint = IAVF_PHINT_NONE;
875 rss_meta_ptr = rte_zmalloc(NULL, sizeof(*rss_meta_ptr), 0);
877 rte_flow_error_set(error, EINVAL,
878 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
879 "No memory for rss_meta_ptr");
883 /* Check rss supported pattern and find matched pattern. */
885 iavf_search_pattern_match_item(pattern, array, array_len,
887 if (!pattern_match_item) {
892 ret = iavf_hash_parse_pattern(pattern_match_item, pattern, &phint,
897 ret = iavf_hash_parse_action(pattern_match_item, actions, phint,
898 (void **)&rss_meta_ptr, error);
902 *meta = rss_meta_ptr;
904 rte_free(rss_meta_ptr);
906 rte_free(pattern_match_item);
912 iavf_hash_create(__rte_unused struct iavf_adapter *ad,
913 __rte_unused struct rte_flow *flow, void *meta,
914 __rte_unused struct rte_flow_error *error)
916 struct iavf_rss_meta *rss_meta = (struct iavf_rss_meta *)meta;
917 struct virtchnl_rss_cfg *rss_cfg;
920 rss_cfg = rte_zmalloc("iavf rss rule",
921 sizeof(struct virtchnl_rss_cfg), 0);
923 rte_flow_error_set(error, EINVAL,
924 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
925 "No memory for rss rule");
929 rss_cfg->proto_hdrs = rss_meta->proto_hdrs;
930 rss_cfg->rss_algorithm = rss_meta->rss_algorithm;
932 ret = iavf_add_del_rss_cfg(ad, rss_cfg, true);
934 flow->rule = rss_cfg;
936 PMD_DRV_LOG(ERR, "fail to add RSS configure");
937 rte_flow_error_set(error, -ret,
938 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
939 "Failed to add rss rule.");
950 iavf_hash_destroy(__rte_unused struct iavf_adapter *ad,
951 struct rte_flow *flow,
952 __rte_unused struct rte_flow_error *error)
954 struct virtchnl_rss_cfg *rss_cfg;
957 rss_cfg = (struct virtchnl_rss_cfg *)flow->rule;
959 ret = iavf_add_del_rss_cfg(ad, rss_cfg, false);
961 PMD_DRV_LOG(ERR, "fail to del RSS configure");
962 rte_flow_error_set(error, -ret,
963 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
964 "Failed to delete rss rule.");
971 iavf_hash_uninit(struct iavf_adapter *ad)
973 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
978 if (iavf_hash_default_set(ad, false))
979 PMD_DRV_LOG(ERR, "fail to delete default RSS");
981 iavf_unregister_parser(&iavf_hash_parser, ad);
985 iavf_hash_free(struct rte_flow *flow)
987 rte_free(flow->rule);