1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
13 #include <rte_debug.h>
14 #include <rte_ether.h>
15 #include <ethdev_driver.h>
17 #include <rte_malloc.h>
18 #include <rte_eth_ctrl.h>
19 #include <rte_tailq.h>
20 #include <rte_flow_driver.h>
24 #include "iavf_generic_flow.h"
26 #define IAVF_PHINT_NONE 0
27 #define IAVF_PHINT_GTPU BIT_ULL(0)
28 #define IAVF_PHINT_GTPU_EH BIT_ULL(1)
29 #define IAVF_PHINT_GTPU_EH_DWN BIT_ULL(2)
30 #define IAVF_PHINT_GTPU_EH_UP BIT_ULL(3)
31 #define IAVF_PHINT_OUTER_IPV4 BIT_ULL(4)
32 #define IAVF_PHINT_OUTER_IPV6 BIT_ULL(5)
33 #define IAVF_PHINT_GRE BIT_ULL(6)
35 #define IAVF_PHINT_GTPU_MSK (IAVF_PHINT_GTPU | \
36 IAVF_PHINT_GTPU_EH | \
37 IAVF_PHINT_GTPU_EH_DWN | \
38 IAVF_PHINT_GTPU_EH_UP)
40 #define IAVF_PHINT_LAYERS_MSK (IAVF_PHINT_OUTER_IPV4 | \
41 IAVF_PHINT_OUTER_IPV6)
43 #define IAVF_GTPU_EH_DWNLINK 0
44 #define IAVF_GTPU_EH_UPLINK 1
46 struct iavf_hash_match_type {
48 struct virtchnl_proto_hdrs *proto_hdrs;
49 uint64_t pattern_hint;
52 struct iavf_rss_meta {
53 struct virtchnl_proto_hdrs proto_hdrs;
54 enum virtchnl_rss_algorithm rss_algorithm;
57 struct iavf_hash_flow_cfg {
58 struct virtchnl_rss_cfg *rss_cfg;
63 iavf_hash_init(struct iavf_adapter *ad);
65 iavf_hash_create(struct iavf_adapter *ad, struct rte_flow *flow, void *meta,
66 struct rte_flow_error *error);
68 iavf_hash_destroy(struct iavf_adapter *ad, struct rte_flow *flow,
69 struct rte_flow_error *error);
71 iavf_hash_uninit(struct iavf_adapter *ad);
73 iavf_hash_free(struct rte_flow *flow);
75 iavf_hash_parse_pattern_action(struct iavf_adapter *ad,
76 struct iavf_pattern_match_item *array,
78 const struct rte_flow_item pattern[],
79 const struct rte_flow_action actions[],
81 struct rte_flow_error *error);
83 #define FIELD_SELECTOR(proto_hdr_field) \
84 (1UL << ((proto_hdr_field) & PROTO_HDR_FIELD_MASK))
87 #define proto_hdr_eth { \
88 VIRTCHNL_PROTO_HDR_ETH, \
89 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_SRC) | \
90 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_DST), {BUFF_NOUSED} }
92 #define proto_hdr_svlan { \
93 VIRTCHNL_PROTO_HDR_S_VLAN, \
94 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_S_VLAN_ID), {BUFF_NOUSED} }
96 #define proto_hdr_cvlan { \
97 VIRTCHNL_PROTO_HDR_C_VLAN, \
98 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_C_VLAN_ID), {BUFF_NOUSED} }
100 #define proto_hdr_ipv4 { \
101 VIRTCHNL_PROTO_HDR_IPV4, \
102 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_SRC) | \
103 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_DST), {BUFF_NOUSED} }
105 #define proto_hdr_ipv4_with_prot { \
106 VIRTCHNL_PROTO_HDR_IPV4, \
107 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_SRC) | \
108 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_DST) | \
109 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_PROT), {BUFF_NOUSED} }
111 #define proto_hdr_ipv6 { \
112 VIRTCHNL_PROTO_HDR_IPV6, \
113 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_SRC) | \
114 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_DST), {BUFF_NOUSED} }
116 #define proto_hdr_ipv6_frag { \
117 VIRTCHNL_PROTO_HDR_IPV6_EH_FRAG, \
118 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_EH_FRAG_PKID), {BUFF_NOUSED} }
120 #define proto_hdr_ipv6_with_prot { \
121 VIRTCHNL_PROTO_HDR_IPV6, \
122 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_SRC) | \
123 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_DST) | \
124 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_PROT), {BUFF_NOUSED} }
126 #define proto_hdr_udp { \
127 VIRTCHNL_PROTO_HDR_UDP, \
128 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_UDP_SRC_PORT) | \
129 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_UDP_DST_PORT), {BUFF_NOUSED} }
131 #define proto_hdr_tcp { \
132 VIRTCHNL_PROTO_HDR_TCP, \
133 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_TCP_SRC_PORT) | \
134 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_TCP_DST_PORT), {BUFF_NOUSED} }
136 #define proto_hdr_sctp { \
137 VIRTCHNL_PROTO_HDR_SCTP, \
138 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_SCTP_SRC_PORT) | \
139 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_SCTP_DST_PORT), {BUFF_NOUSED} }
141 #define proto_hdr_esp { \
142 VIRTCHNL_PROTO_HDR_ESP, \
143 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ESP_SPI), {BUFF_NOUSED} }
145 #define proto_hdr_ah { \
146 VIRTCHNL_PROTO_HDR_AH, \
147 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_AH_SPI), {BUFF_NOUSED} }
149 #define proto_hdr_l2tpv3 { \
150 VIRTCHNL_PROTO_HDR_L2TPV3, \
151 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_L2TPV3_SESS_ID), {BUFF_NOUSED} }
153 #define proto_hdr_pfcp { \
154 VIRTCHNL_PROTO_HDR_PFCP, \
155 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_PFCP_SEID), {BUFF_NOUSED} }
157 #define proto_hdr_gtpc { \
158 VIRTCHNL_PROTO_HDR_GTPC, 0, {BUFF_NOUSED} }
160 #define proto_hdr_ecpri { \
161 VIRTCHNL_PROTO_HDR_ECPRI, \
162 FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ECPRI_PC_RTC_ID), {BUFF_NOUSED} }
164 #define TUNNEL_LEVEL_OUTER 0
165 #define TUNNEL_LEVEL_INNER 1
167 /* proto_hdrs template */
168 struct virtchnl_proto_hdrs outer_ipv4_tmplt = {
169 TUNNEL_LEVEL_OUTER, 4,
170 {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv4}
173 struct virtchnl_proto_hdrs outer_ipv4_udp_tmplt = {
174 TUNNEL_LEVEL_OUTER, 5,
175 {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan,
176 proto_hdr_ipv4_with_prot,
180 struct virtchnl_proto_hdrs outer_ipv4_tcp_tmplt = {
181 TUNNEL_LEVEL_OUTER, 5,
182 {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan,
183 proto_hdr_ipv4_with_prot,
187 struct virtchnl_proto_hdrs outer_ipv4_sctp_tmplt = {
188 TUNNEL_LEVEL_OUTER, 5,
189 {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv4,
193 struct virtchnl_proto_hdrs outer_ipv6_tmplt = {
194 TUNNEL_LEVEL_OUTER, 4,
195 {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv6}
198 struct virtchnl_proto_hdrs outer_ipv6_frag_tmplt = {
199 TUNNEL_LEVEL_OUTER, 5,
200 {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan,
201 proto_hdr_ipv6, proto_hdr_ipv6_frag}
204 struct virtchnl_proto_hdrs outer_ipv6_udp_tmplt = {
205 TUNNEL_LEVEL_OUTER, 5,
206 {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan,
207 proto_hdr_ipv6_with_prot,
211 struct virtchnl_proto_hdrs outer_ipv6_tcp_tmplt = {
212 TUNNEL_LEVEL_OUTER, 5,
213 {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan,
214 proto_hdr_ipv6_with_prot,
218 struct virtchnl_proto_hdrs outer_ipv6_sctp_tmplt = {
219 TUNNEL_LEVEL_OUTER, 5,
220 {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv6,
224 struct virtchnl_proto_hdrs inner_ipv4_tmplt = {
225 TUNNEL_LEVEL_INNER, 1, {proto_hdr_ipv4}
228 struct virtchnl_proto_hdrs inner_ipv4_udp_tmplt = {
229 TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv4_with_prot, proto_hdr_udp}
232 struct virtchnl_proto_hdrs inner_ipv4_tcp_tmplt = {
233 TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv4_with_prot, proto_hdr_tcp}
236 struct virtchnl_proto_hdrs inner_ipv4_sctp_tmplt = {
237 TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv4, proto_hdr_sctp}
240 struct virtchnl_proto_hdrs inner_ipv6_tmplt = {
241 TUNNEL_LEVEL_INNER, 1, {proto_hdr_ipv6}
244 struct virtchnl_proto_hdrs inner_ipv6_udp_tmplt = {
245 TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv6_with_prot, proto_hdr_udp}
248 struct virtchnl_proto_hdrs inner_ipv6_tcp_tmplt = {
249 TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv6_with_prot, proto_hdr_tcp}
252 struct virtchnl_proto_hdrs inner_ipv6_sctp_tmplt = {
253 TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv6, proto_hdr_sctp}
256 struct virtchnl_proto_hdrs ipv4_esp_tmplt = {
257 TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_esp}
260 struct virtchnl_proto_hdrs ipv4_udp_esp_tmplt = {
261 TUNNEL_LEVEL_OUTER, 3,
262 {proto_hdr_ipv4, proto_hdr_udp, proto_hdr_esp}
265 struct virtchnl_proto_hdrs ipv4_ah_tmplt = {
266 TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_ah}
269 struct virtchnl_proto_hdrs ipv6_esp_tmplt = {
270 TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_esp}
273 struct virtchnl_proto_hdrs ipv6_udp_esp_tmplt = {
274 TUNNEL_LEVEL_OUTER, 3,
275 {proto_hdr_ipv6, proto_hdr_udp, proto_hdr_esp}
278 struct virtchnl_proto_hdrs ipv6_ah_tmplt = {
279 TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_ah}
282 struct virtchnl_proto_hdrs ipv4_l2tpv3_tmplt = {
283 TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_l2tpv3}
286 struct virtchnl_proto_hdrs ipv6_l2tpv3_tmplt = {
287 TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_l2tpv3}
290 struct virtchnl_proto_hdrs ipv4_pfcp_tmplt = {
291 TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_pfcp}
294 struct virtchnl_proto_hdrs ipv6_pfcp_tmplt = {
295 TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_pfcp}
298 struct virtchnl_proto_hdrs ipv4_udp_gtpc_tmplt = {
299 TUNNEL_LEVEL_OUTER, 3, {proto_hdr_ipv4, proto_hdr_udp, proto_hdr_gtpc}
302 struct virtchnl_proto_hdrs ipv6_udp_gtpc_tmplt = {
303 TUNNEL_LEVEL_OUTER, 3, {proto_hdr_ipv6, proto_hdr_udp, proto_hdr_gtpc}
306 struct virtchnl_proto_hdrs eth_ecpri_tmplt = {
307 TUNNEL_LEVEL_OUTER, 2, {proto_hdr_eth, proto_hdr_ecpri}
310 struct virtchnl_proto_hdrs ipv4_ecpri_tmplt = {
311 TUNNEL_LEVEL_OUTER, 3, {proto_hdr_ipv4, proto_hdr_udp, proto_hdr_ecpri}
314 /* rss type super set */
317 #define IAVF_RSS_TYPE_OUTER_IPV4 (ETH_RSS_ETH | ETH_RSS_IPV4 | \
319 #define IAVF_RSS_TYPE_OUTER_IPV4_UDP (IAVF_RSS_TYPE_OUTER_IPV4 | \
320 ETH_RSS_NONFRAG_IPV4_UDP)
321 #define IAVF_RSS_TYPE_OUTER_IPV4_TCP (IAVF_RSS_TYPE_OUTER_IPV4 | \
322 ETH_RSS_NONFRAG_IPV4_TCP)
323 #define IAVF_RSS_TYPE_OUTER_IPV4_SCTP (IAVF_RSS_TYPE_OUTER_IPV4 | \
324 ETH_RSS_NONFRAG_IPV4_SCTP)
326 #define IAVF_RSS_TYPE_OUTER_IPV6 (ETH_RSS_ETH | ETH_RSS_IPV6)
327 #define IAVF_RSS_TYPE_OUTER_IPV6_FRAG (IAVF_RSS_TYPE_OUTER_IPV6 | \
329 #define IAVF_RSS_TYPE_OUTER_IPV6_UDP (IAVF_RSS_TYPE_OUTER_IPV6 | \
330 ETH_RSS_NONFRAG_IPV6_UDP)
331 #define IAVF_RSS_TYPE_OUTER_IPV6_TCP (IAVF_RSS_TYPE_OUTER_IPV6 | \
332 ETH_RSS_NONFRAG_IPV6_TCP)
333 #define IAVF_RSS_TYPE_OUTER_IPV6_SCTP (IAVF_RSS_TYPE_OUTER_IPV6 | \
334 ETH_RSS_NONFRAG_IPV6_SCTP)
336 #define IAVF_RSS_TYPE_VLAN_IPV4 (IAVF_RSS_TYPE_OUTER_IPV4 | \
337 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
338 #define IAVF_RSS_TYPE_VLAN_IPV4_UDP (IAVF_RSS_TYPE_OUTER_IPV4_UDP | \
339 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
340 #define IAVF_RSS_TYPE_VLAN_IPV4_TCP (IAVF_RSS_TYPE_OUTER_IPV4_TCP | \
341 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
342 #define IAVF_RSS_TYPE_VLAN_IPV4_SCTP (IAVF_RSS_TYPE_OUTER_IPV4_SCTP | \
343 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
345 #define IAVF_RSS_TYPE_VLAN_IPV6 (IAVF_RSS_TYPE_OUTER_IPV6 | \
346 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
347 #define IAVF_RSS_TYPE_VLAN_IPV6_FRAG (IAVF_RSS_TYPE_OUTER_IPV6_FRAG | \
348 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
349 #define IAVF_RSS_TYPE_VLAN_IPV6_UDP (IAVF_RSS_TYPE_OUTER_IPV6_UDP | \
350 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
351 #define IAVF_RSS_TYPE_VLAN_IPV6_TCP (IAVF_RSS_TYPE_OUTER_IPV6_TCP | \
352 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
353 #define IAVF_RSS_TYPE_VLAN_IPV6_SCTP (IAVF_RSS_TYPE_OUTER_IPV6_SCTP | \
354 ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
356 #define IAVF_RSS_TYPE_INNER_IPV4 ETH_RSS_IPV4
357 #define IAVF_RSS_TYPE_INNER_IPV4_UDP (ETH_RSS_IPV4 | \
358 ETH_RSS_NONFRAG_IPV4_UDP)
359 #define IAVF_RSS_TYPE_INNER_IPV4_TCP (ETH_RSS_IPV4 | \
360 ETH_RSS_NONFRAG_IPV4_TCP)
361 #define IAVF_RSS_TYPE_INNER_IPV4_SCTP (ETH_RSS_IPV4 | \
362 ETH_RSS_NONFRAG_IPV4_SCTP)
364 #define IAVF_RSS_TYPE_INNER_IPV6 ETH_RSS_IPV6
365 #define IAVF_RSS_TYPE_INNER_IPV6_UDP (ETH_RSS_IPV6 | \
366 ETH_RSS_NONFRAG_IPV6_UDP)
367 #define IAVF_RSS_TYPE_INNER_IPV6_TCP (ETH_RSS_IPV6 | \
368 ETH_RSS_NONFRAG_IPV6_TCP)
369 #define IAVF_RSS_TYPE_INNER_IPV6_SCTP (ETH_RSS_IPV6 | \
370 ETH_RSS_NONFRAG_IPV6_SCTP)
372 #define IAVF_RSS_TYPE_GTPU_IPV4 (IAVF_RSS_TYPE_INNER_IPV4 | \
374 #define IAVF_RSS_TYPE_GTPU_IPV4_UDP (IAVF_RSS_TYPE_INNER_IPV4_UDP | \
376 #define IAVF_RSS_TYPE_GTPU_IPV4_TCP (IAVF_RSS_TYPE_INNER_IPV4_TCP | \
379 #define IAVF_RSS_TYPE_GTPU_IPV6 (IAVF_RSS_TYPE_INNER_IPV6 | \
381 #define IAVF_RSS_TYPE_GTPU_IPV6_UDP (IAVF_RSS_TYPE_INNER_IPV6_UDP | \
383 #define IAVF_RSS_TYPE_GTPU_IPV6_TCP (IAVF_RSS_TYPE_INNER_IPV6_TCP | \
385 /* ESP, AH, L2TPV3 and PFCP */
386 #define IAVF_RSS_TYPE_IPV4_ESP (ETH_RSS_ESP | ETH_RSS_IPV4)
387 #define IAVF_RSS_TYPE_IPV4_AH (ETH_RSS_AH | ETH_RSS_IPV4)
388 #define IAVF_RSS_TYPE_IPV6_ESP (ETH_RSS_ESP | ETH_RSS_IPV6)
389 #define IAVF_RSS_TYPE_IPV6_AH (ETH_RSS_AH | ETH_RSS_IPV6)
390 #define IAVF_RSS_TYPE_IPV4_L2TPV3 (ETH_RSS_L2TPV3 | ETH_RSS_IPV4)
391 #define IAVF_RSS_TYPE_IPV6_L2TPV3 (ETH_RSS_L2TPV3 | ETH_RSS_IPV6)
392 #define IAVF_RSS_TYPE_IPV4_PFCP (ETH_RSS_PFCP | ETH_RSS_IPV4)
393 #define IAVF_RSS_TYPE_IPV6_PFCP (ETH_RSS_PFCP | ETH_RSS_IPV6)
396 * Supported pattern for hash.
397 * The first member is pattern item type,
398 * the second member is input set mask,
399 * the third member is virtchnl_proto_hdrs template
401 static struct iavf_pattern_match_item iavf_hash_pattern_list[] = {
403 {iavf_pattern_eth_ipv4, IAVF_RSS_TYPE_OUTER_IPV4, &outer_ipv4_tmplt},
404 {iavf_pattern_eth_ipv4_udp, IAVF_RSS_TYPE_OUTER_IPV4_UDP, &outer_ipv4_udp_tmplt},
405 {iavf_pattern_eth_ipv4_tcp, IAVF_RSS_TYPE_OUTER_IPV4_TCP, &outer_ipv4_tcp_tmplt},
406 {iavf_pattern_eth_ipv4_sctp, IAVF_RSS_TYPE_OUTER_IPV4_SCTP, &outer_ipv4_sctp_tmplt},
407 {iavf_pattern_eth_vlan_ipv4, IAVF_RSS_TYPE_VLAN_IPV4, &outer_ipv4_tmplt},
408 {iavf_pattern_eth_vlan_ipv4_udp, IAVF_RSS_TYPE_VLAN_IPV4_UDP, &outer_ipv4_udp_tmplt},
409 {iavf_pattern_eth_vlan_ipv4_tcp, IAVF_RSS_TYPE_VLAN_IPV4_TCP, &outer_ipv4_tcp_tmplt},
410 {iavf_pattern_eth_vlan_ipv4_sctp, IAVF_RSS_TYPE_VLAN_IPV4_SCTP, &outer_ipv4_sctp_tmplt},
411 {iavf_pattern_eth_ipv4_gtpu, ETH_RSS_IPV4, &outer_ipv4_udp_tmplt},
412 {iavf_pattern_eth_ipv4_gtpu_ipv4, IAVF_RSS_TYPE_GTPU_IPV4, &inner_ipv4_tmplt},
413 {iavf_pattern_eth_ipv4_gtpu_ipv4_udp, IAVF_RSS_TYPE_GTPU_IPV4_UDP, &inner_ipv4_udp_tmplt},
414 {iavf_pattern_eth_ipv4_gtpu_ipv4_tcp, IAVF_RSS_TYPE_GTPU_IPV4_TCP, &inner_ipv4_tcp_tmplt},
415 {iavf_pattern_eth_ipv6_gtpu_ipv4, IAVF_RSS_TYPE_GTPU_IPV4, &inner_ipv4_tmplt},
416 {iavf_pattern_eth_ipv6_gtpu_ipv4_udp, IAVF_RSS_TYPE_GTPU_IPV4_UDP, &inner_ipv4_udp_tmplt},
417 {iavf_pattern_eth_ipv6_gtpu_ipv4_tcp, IAVF_RSS_TYPE_GTPU_IPV4_TCP, &inner_ipv4_tcp_tmplt},
418 {iavf_pattern_eth_ipv4_gtpu_eh_ipv4, IAVF_RSS_TYPE_GTPU_IPV4, &inner_ipv4_tmplt},
419 {iavf_pattern_eth_ipv4_gtpu_eh_ipv4_udp, IAVF_RSS_TYPE_GTPU_IPV4_UDP, &inner_ipv4_udp_tmplt},
420 {iavf_pattern_eth_ipv4_gtpu_eh_ipv4_tcp, IAVF_RSS_TYPE_GTPU_IPV4_TCP, &inner_ipv4_tcp_tmplt},
421 {iavf_pattern_eth_ipv6_gtpu_eh_ipv4, IAVF_RSS_TYPE_GTPU_IPV4, &inner_ipv4_tmplt},
422 {iavf_pattern_eth_ipv6_gtpu_eh_ipv4_udp, IAVF_RSS_TYPE_GTPU_IPV4_UDP, &inner_ipv4_udp_tmplt},
423 {iavf_pattern_eth_ipv6_gtpu_eh_ipv4_tcp, IAVF_RSS_TYPE_GTPU_IPV4_TCP, &inner_ipv4_tcp_tmplt},
424 {iavf_pattern_eth_ipv4_esp, IAVF_RSS_TYPE_IPV4_ESP, &ipv4_esp_tmplt},
425 {iavf_pattern_eth_ipv4_udp_esp, IAVF_RSS_TYPE_IPV4_ESP, &ipv4_udp_esp_tmplt},
426 {iavf_pattern_eth_ipv4_ah, IAVF_RSS_TYPE_IPV4_AH, &ipv4_ah_tmplt},
427 {iavf_pattern_eth_ipv4_l2tpv3, IAVF_RSS_TYPE_IPV4_L2TPV3, &ipv4_l2tpv3_tmplt},
428 {iavf_pattern_eth_ipv4_pfcp, IAVF_RSS_TYPE_IPV4_PFCP, &ipv4_pfcp_tmplt},
429 {iavf_pattern_eth_ipv4_gtpc, ETH_RSS_IPV4, &ipv4_udp_gtpc_tmplt},
430 {iavf_pattern_eth_ecpri, ETH_RSS_ECPRI, ð_ecpri_tmplt},
431 {iavf_pattern_eth_ipv4_ecpri, ETH_RSS_ECPRI, &ipv4_ecpri_tmplt},
432 {iavf_pattern_eth_ipv4_gre_ipv4, IAVF_RSS_TYPE_INNER_IPV4, &inner_ipv4_tmplt},
433 {iavf_pattern_eth_ipv6_gre_ipv4, IAVF_RSS_TYPE_INNER_IPV4, &inner_ipv4_tmplt},
434 {iavf_pattern_eth_ipv4_gre_ipv4_tcp, IAVF_RSS_TYPE_INNER_IPV4_TCP, &inner_ipv4_tcp_tmplt},
435 {iavf_pattern_eth_ipv6_gre_ipv4_tcp, IAVF_RSS_TYPE_INNER_IPV4_TCP, &inner_ipv4_tcp_tmplt},
436 {iavf_pattern_eth_ipv4_gre_ipv4_udp, IAVF_RSS_TYPE_INNER_IPV4_UDP, &inner_ipv4_udp_tmplt},
437 {iavf_pattern_eth_ipv6_gre_ipv4_udp, IAVF_RSS_TYPE_INNER_IPV4_UDP, &inner_ipv4_udp_tmplt},
439 {iavf_pattern_eth_ipv6, IAVF_RSS_TYPE_OUTER_IPV6, &outer_ipv6_tmplt},
440 {iavf_pattern_eth_ipv6_frag_ext, IAVF_RSS_TYPE_OUTER_IPV6_FRAG, &outer_ipv6_frag_tmplt},
441 {iavf_pattern_eth_ipv6_udp, IAVF_RSS_TYPE_OUTER_IPV6_UDP, &outer_ipv6_udp_tmplt},
442 {iavf_pattern_eth_ipv6_tcp, IAVF_RSS_TYPE_OUTER_IPV6_TCP, &outer_ipv6_tcp_tmplt},
443 {iavf_pattern_eth_ipv6_sctp, IAVF_RSS_TYPE_OUTER_IPV6_SCTP, &outer_ipv6_sctp_tmplt},
444 {iavf_pattern_eth_vlan_ipv6, IAVF_RSS_TYPE_VLAN_IPV6, &outer_ipv6_tmplt},
445 {iavf_pattern_eth_vlan_ipv6_frag_ext, IAVF_RSS_TYPE_OUTER_IPV6_FRAG, &outer_ipv6_frag_tmplt},
446 {iavf_pattern_eth_vlan_ipv6_udp, IAVF_RSS_TYPE_VLAN_IPV6_UDP, &outer_ipv6_udp_tmplt},
447 {iavf_pattern_eth_vlan_ipv6_tcp, IAVF_RSS_TYPE_VLAN_IPV6_TCP, &outer_ipv6_tcp_tmplt},
448 {iavf_pattern_eth_vlan_ipv6_sctp, IAVF_RSS_TYPE_VLAN_IPV6_SCTP, &outer_ipv6_sctp_tmplt},
449 {iavf_pattern_eth_ipv6_gtpu, ETH_RSS_IPV6, &outer_ipv6_udp_tmplt},
450 {iavf_pattern_eth_ipv4_gtpu_ipv6, IAVF_RSS_TYPE_GTPU_IPV6, &inner_ipv6_tmplt},
451 {iavf_pattern_eth_ipv4_gtpu_ipv6_udp, IAVF_RSS_TYPE_GTPU_IPV6_UDP, &inner_ipv6_udp_tmplt},
452 {iavf_pattern_eth_ipv4_gtpu_ipv6_tcp, IAVF_RSS_TYPE_GTPU_IPV6_TCP, &inner_ipv6_tcp_tmplt},
453 {iavf_pattern_eth_ipv6_gtpu_ipv6, IAVF_RSS_TYPE_GTPU_IPV6, &inner_ipv6_tmplt},
454 {iavf_pattern_eth_ipv6_gtpu_ipv6_udp, IAVF_RSS_TYPE_GTPU_IPV6_UDP, &inner_ipv6_udp_tmplt},
455 {iavf_pattern_eth_ipv6_gtpu_ipv6_tcp, IAVF_RSS_TYPE_GTPU_IPV6_TCP, &inner_ipv6_tcp_tmplt},
456 {iavf_pattern_eth_ipv4_gtpu_eh_ipv6, IAVF_RSS_TYPE_GTPU_IPV6, &inner_ipv6_tmplt},
457 {iavf_pattern_eth_ipv4_gtpu_eh_ipv6_udp, IAVF_RSS_TYPE_GTPU_IPV6_UDP, &inner_ipv6_udp_tmplt},
458 {iavf_pattern_eth_ipv4_gtpu_eh_ipv6_tcp, IAVF_RSS_TYPE_GTPU_IPV6_TCP, &inner_ipv6_tcp_tmplt},
459 {iavf_pattern_eth_ipv6_gtpu_eh_ipv6, IAVF_RSS_TYPE_GTPU_IPV6, &inner_ipv6_tmplt},
460 {iavf_pattern_eth_ipv6_gtpu_eh_ipv6_udp, IAVF_RSS_TYPE_GTPU_IPV6_UDP, &inner_ipv6_udp_tmplt},
461 {iavf_pattern_eth_ipv6_gtpu_eh_ipv6_tcp, IAVF_RSS_TYPE_GTPU_IPV6_TCP, &inner_ipv6_tcp_tmplt},
462 {iavf_pattern_eth_ipv6_esp, IAVF_RSS_TYPE_IPV6_ESP, &ipv6_esp_tmplt},
463 {iavf_pattern_eth_ipv6_udp_esp, IAVF_RSS_TYPE_IPV6_ESP, &ipv6_udp_esp_tmplt},
464 {iavf_pattern_eth_ipv6_ah, IAVF_RSS_TYPE_IPV6_AH, &ipv6_ah_tmplt},
465 {iavf_pattern_eth_ipv6_l2tpv3, IAVF_RSS_TYPE_IPV6_L2TPV3, &ipv6_l2tpv3_tmplt},
466 {iavf_pattern_eth_ipv6_pfcp, IAVF_RSS_TYPE_IPV6_PFCP, &ipv6_pfcp_tmplt},
467 {iavf_pattern_eth_ipv6_gtpc, ETH_RSS_IPV6, &ipv6_udp_gtpc_tmplt},
468 {iavf_pattern_eth_ipv4_gre_ipv6, IAVF_RSS_TYPE_INNER_IPV6, &inner_ipv6_tmplt},
469 {iavf_pattern_eth_ipv6_gre_ipv6, IAVF_RSS_TYPE_INNER_IPV6, &inner_ipv6_tmplt},
470 {iavf_pattern_eth_ipv4_gre_ipv6_tcp, IAVF_RSS_TYPE_INNER_IPV6_TCP, &inner_ipv6_tcp_tmplt},
471 {iavf_pattern_eth_ipv6_gre_ipv6_tcp, IAVF_RSS_TYPE_INNER_IPV6_TCP, &inner_ipv6_tcp_tmplt},
472 {iavf_pattern_eth_ipv4_gre_ipv6_udp, IAVF_RSS_TYPE_INNER_IPV6_UDP, &inner_ipv6_udp_tmplt},
473 {iavf_pattern_eth_ipv6_gre_ipv6_udp, IAVF_RSS_TYPE_INNER_IPV6_UDP, &inner_ipv6_udp_tmplt},
476 static struct iavf_flow_engine iavf_hash_engine = {
477 .init = iavf_hash_init,
478 .create = iavf_hash_create,
479 .destroy = iavf_hash_destroy,
480 .uninit = iavf_hash_uninit,
481 .free = iavf_hash_free,
482 .type = IAVF_FLOW_ENGINE_HASH,
485 /* Register parser for comms package. */
486 static struct iavf_flow_parser iavf_hash_parser = {
487 .engine = &iavf_hash_engine,
488 .array = iavf_hash_pattern_list,
489 .array_len = RTE_DIM(iavf_hash_pattern_list),
490 .parse_pattern_action = iavf_hash_parse_pattern_action,
491 .stage = IAVF_FLOW_STAGE_RSS,
495 iavf_rss_hash_set(struct iavf_adapter *ad, uint64_t rss_hf, bool add)
497 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
498 struct virtchnl_rss_cfg rss_cfg;
500 #define IAVF_RSS_HF_ALL ( \
503 ETH_RSS_NONFRAG_IPV4_UDP | \
504 ETH_RSS_NONFRAG_IPV6_UDP | \
505 ETH_RSS_NONFRAG_IPV4_TCP | \
506 ETH_RSS_NONFRAG_IPV6_TCP | \
507 ETH_RSS_NONFRAG_IPV4_SCTP | \
508 ETH_RSS_NONFRAG_IPV6_SCTP)
510 rss_cfg.rss_algorithm = VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC;
511 if (rss_hf & ETH_RSS_IPV4) {
512 rss_cfg.proto_hdrs = inner_ipv4_tmplt;
513 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
516 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
517 rss_cfg.proto_hdrs = inner_ipv4_udp_tmplt;
518 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
521 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
522 rss_cfg.proto_hdrs = inner_ipv4_tcp_tmplt;
523 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
526 if (rss_hf & ETH_RSS_NONFRAG_IPV4_SCTP) {
527 rss_cfg.proto_hdrs = inner_ipv4_sctp_tmplt;
528 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
531 if (rss_hf & ETH_RSS_IPV6) {
532 rss_cfg.proto_hdrs = inner_ipv6_tmplt;
533 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
536 if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) {
537 rss_cfg.proto_hdrs = inner_ipv6_udp_tmplt;
538 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
541 if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) {
542 rss_cfg.proto_hdrs = inner_ipv6_tcp_tmplt;
543 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
546 if (rss_hf & ETH_RSS_NONFRAG_IPV6_SCTP) {
547 rss_cfg.proto_hdrs = inner_ipv6_sctp_tmplt;
548 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
551 vf->rss_hf = rss_hf & IAVF_RSS_HF_ALL;
555 RTE_INIT(iavf_hash_engine_init)
557 struct iavf_flow_engine *engine = &iavf_hash_engine;
559 iavf_register_flow_engine(engine);
563 iavf_hash_init(struct iavf_adapter *ad)
565 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
566 struct iavf_flow_parser *parser;
575 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF))
578 parser = &iavf_hash_parser;
580 ret = iavf_register_parser(parser, ad);
582 PMD_DRV_LOG(ERR, "fail to register hash parser");
590 iavf_hash_parse_pattern(const struct rte_flow_item pattern[], uint64_t *phint,
591 struct rte_flow_error *error)
593 const struct rte_flow_item *item = pattern;
594 const struct rte_flow_item_gtp_psc *psc;
595 const struct rte_flow_item_ecpri *ecpri;
596 struct rte_ecpri_common_hdr ecpri_common;
598 for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
600 rte_flow_error_set(error, EINVAL,
601 RTE_FLOW_ERROR_TYPE_ITEM, item,
602 "Not support range");
606 switch (item->type) {
607 case RTE_FLOW_ITEM_TYPE_IPV4:
608 if (!(*phint & IAVF_PHINT_GTPU_MSK) && !(*phint & IAVF_PHINT_GRE))
609 *phint |= IAVF_PHINT_OUTER_IPV4;
611 case RTE_FLOW_ITEM_TYPE_IPV6:
612 if (!(*phint & IAVF_PHINT_GTPU_MSK) && !(*phint & IAVF_PHINT_GRE))
613 *phint |= IAVF_PHINT_OUTER_IPV6;
615 case RTE_FLOW_ITEM_TYPE_GTPU:
616 *phint |= IAVF_PHINT_GTPU;
618 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
619 *phint |= IAVF_PHINT_GTPU_EH;
623 else if (psc->pdu_type == IAVF_GTPU_EH_UPLINK)
624 *phint |= IAVF_PHINT_GTPU_EH_UP;
625 else if (psc->pdu_type == IAVF_GTPU_EH_DWNLINK)
626 *phint |= IAVF_PHINT_GTPU_EH_DWN;
628 case RTE_FLOW_ITEM_TYPE_ECPRI:
633 ecpri_common.u32 = rte_be_to_cpu_32(ecpri->hdr.common.u32);
635 if (ecpri_common.type !=
636 RTE_ECPRI_MSG_TYPE_IQ_DATA) {
637 rte_flow_error_set(error, EINVAL,
638 RTE_FLOW_ERROR_TYPE_ITEM, item,
639 "Unsupported common type.");
643 case RTE_FLOW_ITEM_TYPE_GRE:
644 *phint |= IAVF_PHINT_GRE;
653 #define REFINE_PROTO_FLD(op, fld) \
654 VIRTCHNL_##op##_PROTO_HDR_FIELD(hdr, VIRTCHNL_PROTO_HDR_##fld)
655 #define REPALCE_PROTO_FLD(fld_1, fld_2) \
657 REFINE_PROTO_FLD(DEL, fld_1); \
658 REFINE_PROTO_FLD(ADD, fld_2); \
662 iavf_hash_add_fragment_hdr(struct virtchnl_proto_hdrs *hdrs, int layer)
664 struct virtchnl_proto_hdr *hdr1;
665 struct virtchnl_proto_hdr *hdr2;
668 if (layer < 0 || layer > hdrs->count)
671 /* shift headers layer */
672 for (i = hdrs->count; i >= layer; i--) {
673 hdr1 = &hdrs->proto_hdr[i];
674 hdr2 = &hdrs->proto_hdr[i - 1];
678 /* adding dummy fragment header */
679 hdr1 = &hdrs->proto_hdr[layer];
680 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, IPV4_FRAG);
681 hdrs->count = ++layer;
684 /* refine proto hdrs base on l2, l3, l4 rss type */
686 iavf_refine_proto_hdrs_l234(struct virtchnl_proto_hdrs *proto_hdrs,
689 struct virtchnl_proto_hdr *hdr;
692 for (i = 0; i < proto_hdrs->count; i++) {
693 hdr = &proto_hdrs->proto_hdr[i];
695 case VIRTCHNL_PROTO_HDR_ETH:
696 if (!(rss_type & ETH_RSS_ETH))
697 hdr->field_selector = 0;
698 else if (rss_type & ETH_RSS_L2_SRC_ONLY)
699 REFINE_PROTO_FLD(DEL, ETH_DST);
700 else if (rss_type & ETH_RSS_L2_DST_ONLY)
701 REFINE_PROTO_FLD(DEL, ETH_SRC);
703 case VIRTCHNL_PROTO_HDR_IPV4:
705 (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
706 ETH_RSS_NONFRAG_IPV4_UDP |
707 ETH_RSS_NONFRAG_IPV4_TCP |
708 ETH_RSS_NONFRAG_IPV4_SCTP)) {
709 if (rss_type & ETH_RSS_FRAG_IPV4) {
710 iavf_hash_add_fragment_hdr(proto_hdrs, i + 1);
711 } else if (rss_type & ETH_RSS_L3_SRC_ONLY) {
712 REFINE_PROTO_FLD(DEL, IPV4_DST);
713 } else if (rss_type & ETH_RSS_L3_DST_ONLY) {
714 REFINE_PROTO_FLD(DEL, IPV4_SRC);
715 } else if (rss_type &
716 (ETH_RSS_L4_SRC_ONLY |
717 ETH_RSS_L4_DST_ONLY)) {
718 REFINE_PROTO_FLD(DEL, IPV4_DST);
719 REFINE_PROTO_FLD(DEL, IPV4_SRC);
722 hdr->field_selector = 0;
725 case VIRTCHNL_PROTO_HDR_IPV4_FRAG:
727 (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
728 ETH_RSS_NONFRAG_IPV4_UDP |
729 ETH_RSS_NONFRAG_IPV4_TCP |
730 ETH_RSS_NONFRAG_IPV4_SCTP)) {
731 if (rss_type & ETH_RSS_FRAG_IPV4)
732 REFINE_PROTO_FLD(ADD, IPV4_FRAG_PKID);
734 hdr->field_selector = 0;
737 case VIRTCHNL_PROTO_HDR_IPV6:
739 (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
740 ETH_RSS_NONFRAG_IPV6_UDP |
741 ETH_RSS_NONFRAG_IPV6_TCP |
742 ETH_RSS_NONFRAG_IPV6_SCTP)) {
743 if (rss_type & ETH_RSS_L3_SRC_ONLY) {
744 REFINE_PROTO_FLD(DEL, IPV6_DST);
745 } else if (rss_type & ETH_RSS_L3_DST_ONLY) {
746 REFINE_PROTO_FLD(DEL, IPV6_SRC);
747 } else if (rss_type &
748 (ETH_RSS_L4_SRC_ONLY |
749 ETH_RSS_L4_DST_ONLY)) {
750 REFINE_PROTO_FLD(DEL, IPV6_DST);
751 REFINE_PROTO_FLD(DEL, IPV6_SRC);
754 hdr->field_selector = 0;
756 if (rss_type & RTE_ETH_RSS_L3_PRE64) {
757 if (REFINE_PROTO_FLD(TEST, IPV6_SRC))
758 REPALCE_PROTO_FLD(IPV6_SRC,
760 if (REFINE_PROTO_FLD(TEST, IPV6_DST))
761 REPALCE_PROTO_FLD(IPV6_DST,
765 case VIRTCHNL_PROTO_HDR_IPV6_EH_FRAG:
766 if (rss_type & ETH_RSS_FRAG_IPV6)
767 REFINE_PROTO_FLD(ADD, IPV6_EH_FRAG_PKID);
769 hdr->field_selector = 0;
772 case VIRTCHNL_PROTO_HDR_UDP:
774 (ETH_RSS_NONFRAG_IPV4_UDP |
775 ETH_RSS_NONFRAG_IPV6_UDP)) {
776 if (rss_type & ETH_RSS_L4_SRC_ONLY)
777 REFINE_PROTO_FLD(DEL, UDP_DST_PORT);
778 else if (rss_type & ETH_RSS_L4_DST_ONLY)
779 REFINE_PROTO_FLD(DEL, UDP_SRC_PORT);
781 (ETH_RSS_L3_SRC_ONLY |
782 ETH_RSS_L3_DST_ONLY))
783 hdr->field_selector = 0;
785 hdr->field_selector = 0;
788 case VIRTCHNL_PROTO_HDR_TCP:
790 (ETH_RSS_NONFRAG_IPV4_TCP |
791 ETH_RSS_NONFRAG_IPV6_TCP)) {
792 if (rss_type & ETH_RSS_L4_SRC_ONLY)
793 REFINE_PROTO_FLD(DEL, TCP_DST_PORT);
794 else if (rss_type & ETH_RSS_L4_DST_ONLY)
795 REFINE_PROTO_FLD(DEL, TCP_SRC_PORT);
797 (ETH_RSS_L3_SRC_ONLY |
798 ETH_RSS_L3_DST_ONLY))
799 hdr->field_selector = 0;
801 hdr->field_selector = 0;
804 case VIRTCHNL_PROTO_HDR_SCTP:
806 (ETH_RSS_NONFRAG_IPV4_SCTP |
807 ETH_RSS_NONFRAG_IPV6_SCTP)) {
808 if (rss_type & ETH_RSS_L4_SRC_ONLY)
809 REFINE_PROTO_FLD(DEL, SCTP_DST_PORT);
810 else if (rss_type & ETH_RSS_L4_DST_ONLY)
811 REFINE_PROTO_FLD(DEL, SCTP_SRC_PORT);
813 (ETH_RSS_L3_SRC_ONLY |
814 ETH_RSS_L3_DST_ONLY))
815 hdr->field_selector = 0;
817 hdr->field_selector = 0;
820 case VIRTCHNL_PROTO_HDR_S_VLAN:
821 if (!(rss_type & ETH_RSS_S_VLAN))
822 hdr->field_selector = 0;
824 case VIRTCHNL_PROTO_HDR_C_VLAN:
825 if (!(rss_type & ETH_RSS_C_VLAN))
826 hdr->field_selector = 0;
828 case VIRTCHNL_PROTO_HDR_L2TPV3:
829 if (!(rss_type & ETH_RSS_L2TPV3))
830 hdr->field_selector = 0;
832 case VIRTCHNL_PROTO_HDR_ESP:
833 if (!(rss_type & ETH_RSS_ESP))
834 hdr->field_selector = 0;
836 case VIRTCHNL_PROTO_HDR_AH:
837 if (!(rss_type & ETH_RSS_AH))
838 hdr->field_selector = 0;
840 case VIRTCHNL_PROTO_HDR_PFCP:
841 if (!(rss_type & ETH_RSS_PFCP))
842 hdr->field_selector = 0;
844 case VIRTCHNL_PROTO_HDR_ECPRI:
845 if (!(rss_type & ETH_RSS_ECPRI))
846 hdr->field_selector = 0;
854 /* refine proto hdrs base on gtpu rss type */
856 iavf_refine_proto_hdrs_gtpu(struct virtchnl_proto_hdrs *proto_hdrs,
859 struct virtchnl_proto_hdr *hdr;
862 if (!(rss_type & ETH_RSS_GTPU))
865 for (i = 0; i < proto_hdrs->count; i++) {
866 hdr = &proto_hdrs->proto_hdr[i];
868 case VIRTCHNL_PROTO_HDR_GTPU_IP:
869 REFINE_PROTO_FLD(ADD, GTPU_IP_TEID);
878 iavf_refine_proto_hdrs_by_pattern(struct virtchnl_proto_hdrs *proto_hdrs,
881 struct virtchnl_proto_hdr *hdr1;
882 struct virtchnl_proto_hdr *hdr2;
883 int i, shift_count = 1;
885 if (!(phint & IAVF_PHINT_GTPU_MSK) && !(phint & IAVF_PHINT_GRE))
888 if (phint & IAVF_PHINT_LAYERS_MSK)
891 if (proto_hdrs->tunnel_level == TUNNEL_LEVEL_INNER) {
892 /* shift headers layer */
893 for (i = proto_hdrs->count - 1 + shift_count;
894 i > shift_count - 1; i--) {
895 hdr1 = &proto_hdrs->proto_hdr[i];
896 hdr2 = &proto_hdrs->proto_hdr[i - shift_count];
900 if (shift_count == 1) {
901 /* adding tunnel header at layer 0 */
902 hdr1 = &proto_hdrs->proto_hdr[0];
904 /* adding tunnel header and outer ip header */
905 hdr1 = &proto_hdrs->proto_hdr[1];
906 hdr2 = &proto_hdrs->proto_hdr[0];
907 hdr2->field_selector = 0;
909 proto_hdrs->tunnel_level = TUNNEL_LEVEL_OUTER;
911 if (phint & IAVF_PHINT_OUTER_IPV4)
912 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr2, IPV4);
913 else if (phint & IAVF_PHINT_OUTER_IPV6)
914 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr2, IPV6);
917 hdr1 = &proto_hdrs->proto_hdr[proto_hdrs->count];
920 hdr1->field_selector = 0;
923 if (phint & IAVF_PHINT_GTPU_EH_DWN)
924 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_EH_PDU_DWN);
925 else if (phint & IAVF_PHINT_GTPU_EH_UP)
926 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_EH_PDU_UP);
927 else if (phint & IAVF_PHINT_GTPU_EH)
928 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_EH);
929 else if (phint & IAVF_PHINT_GTPU)
930 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_IP);
931 else if (phint & IAVF_PHINT_GRE)
932 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GRE);
935 static void iavf_refine_proto_hdrs(struct virtchnl_proto_hdrs *proto_hdrs,
936 uint64_t rss_type, uint64_t phint)
938 iavf_refine_proto_hdrs_l234(proto_hdrs, rss_type);
939 iavf_refine_proto_hdrs_by_pattern(proto_hdrs, phint);
940 iavf_refine_proto_hdrs_gtpu(proto_hdrs, rss_type);
943 static uint64_t invalid_rss_comb[] = {
944 ETH_RSS_IPV4 | ETH_RSS_NONFRAG_IPV4_UDP,
945 ETH_RSS_IPV4 | ETH_RSS_NONFRAG_IPV4_TCP,
946 ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_UDP,
947 ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_TCP,
948 RTE_ETH_RSS_L3_PRE32 | RTE_ETH_RSS_L3_PRE40 |
949 RTE_ETH_RSS_L3_PRE48 | RTE_ETH_RSS_L3_PRE56 |
953 struct rss_attr_type {
958 #define VALID_RSS_IPV4_L4 (ETH_RSS_NONFRAG_IPV4_UDP | \
959 ETH_RSS_NONFRAG_IPV4_TCP | \
960 ETH_RSS_NONFRAG_IPV4_SCTP)
962 #define VALID_RSS_IPV6_L4 (ETH_RSS_NONFRAG_IPV6_UDP | \
963 ETH_RSS_NONFRAG_IPV6_TCP | \
964 ETH_RSS_NONFRAG_IPV6_SCTP)
966 #define VALID_RSS_IPV4 (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | \
968 #define VALID_RSS_IPV6 (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | \
970 #define VALID_RSS_L3 (VALID_RSS_IPV4 | VALID_RSS_IPV6)
971 #define VALID_RSS_L4 (VALID_RSS_IPV4_L4 | VALID_RSS_IPV6_L4)
973 #define VALID_RSS_ATTR (ETH_RSS_L3_SRC_ONLY | \
974 ETH_RSS_L3_DST_ONLY | \
975 ETH_RSS_L4_SRC_ONLY | \
976 ETH_RSS_L4_DST_ONLY | \
977 ETH_RSS_L2_SRC_ONLY | \
978 ETH_RSS_L2_DST_ONLY | \
979 RTE_ETH_RSS_L3_PRE64)
981 #define INVALID_RSS_ATTR (RTE_ETH_RSS_L3_PRE32 | \
982 RTE_ETH_RSS_L3_PRE40 | \
983 RTE_ETH_RSS_L3_PRE48 | \
984 RTE_ETH_RSS_L3_PRE56 | \
985 RTE_ETH_RSS_L3_PRE96)
987 static struct rss_attr_type rss_attr_to_valid_type[] = {
988 {ETH_RSS_L2_SRC_ONLY | ETH_RSS_L2_DST_ONLY, ETH_RSS_ETH},
989 {ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY, VALID_RSS_L3},
990 {ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY, VALID_RSS_L4},
991 /* current ipv6 prefix only supports prefix 64 bits*/
992 {RTE_ETH_RSS_L3_PRE64, VALID_RSS_IPV6},
993 {INVALID_RSS_ATTR, 0}
997 iavf_any_invalid_rss_type(enum rte_eth_hash_function rss_func,
998 uint64_t rss_type, uint64_t allow_rss_type)
1003 * Check if l3/l4 SRC/DST_ONLY is set for SYMMETRIC_TOEPLITZ
1006 if (rss_func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) {
1007 if (rss_type & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY |
1008 ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY))
1012 (ETH_RSS_IPV4 | ETH_RSS_IPV6 |
1013 ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_NONFRAG_IPV6_UDP |
1014 ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_NONFRAG_IPV6_TCP |
1015 ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_NONFRAG_IPV6_SCTP)))
1019 /* check invalid combination */
1020 for (i = 0; i < RTE_DIM(invalid_rss_comb); i++) {
1021 if (__builtin_popcountll(rss_type & invalid_rss_comb[i]) > 1)
1025 /* check invalid RSS attribute */
1026 for (i = 0; i < RTE_DIM(rss_attr_to_valid_type); i++) {
1027 struct rss_attr_type *rat = &rss_attr_to_valid_type[i];
1029 if (rat->attr & rss_type && !(rat->type & rss_type))
1033 /* check not allowed RSS type */
1034 rss_type &= ~VALID_RSS_ATTR;
1036 return ((rss_type & allow_rss_type) != rss_type);
1040 iavf_hash_parse_action(struct iavf_pattern_match_item *match_item,
1041 const struct rte_flow_action actions[],
1042 uint64_t pattern_hint, void **meta,
1043 struct rte_flow_error *error)
1045 struct iavf_rss_meta *rss_meta = (struct iavf_rss_meta *)*meta;
1046 struct virtchnl_proto_hdrs *proto_hdrs;
1047 enum rte_flow_action_type action_type;
1048 const struct rte_flow_action_rss *rss;
1049 const struct rte_flow_action *action;
1052 /* Supported action is RSS. */
1053 for (action = actions; action->type !=
1054 RTE_FLOW_ACTION_TYPE_END; action++) {
1055 action_type = action->type;
1056 switch (action_type) {
1057 case RTE_FLOW_ACTION_TYPE_RSS:
1059 rss_type = rss->types;
1062 RTE_ETH_HASH_FUNCTION_SIMPLE_XOR){
1063 rss_meta->rss_algorithm =
1064 VIRTCHNL_RSS_ALG_XOR_ASYMMETRIC;
1065 return rte_flow_error_set(error, ENOTSUP,
1066 RTE_FLOW_ERROR_TYPE_ACTION, action,
1067 "function simple_xor is not supported");
1068 } else if (rss->func ==
1069 RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) {
1070 rss_meta->rss_algorithm =
1071 VIRTCHNL_RSS_ALG_TOEPLITZ_SYMMETRIC;
1073 rss_meta->rss_algorithm =
1074 VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC;
1078 return rte_flow_error_set(error, ENOTSUP,
1079 RTE_FLOW_ERROR_TYPE_ACTION, action,
1080 "a nonzero RSS encapsulation level is not supported");
1083 return rte_flow_error_set(error, ENOTSUP,
1084 RTE_FLOW_ERROR_TYPE_ACTION, action,
1085 "a nonzero RSS key_len is not supported");
1088 return rte_flow_error_set(error, ENOTSUP,
1089 RTE_FLOW_ERROR_TYPE_ACTION, action,
1090 "a non-NULL RSS queue is not supported");
1093 * Check simultaneous use of SRC_ONLY and DST_ONLY
1094 * of the same level.
1096 rss_type = rte_eth_rss_hf_refine(rss_type);
1098 if (iavf_any_invalid_rss_type(rss->func, rss_type,
1099 match_item->input_set_mask))
1100 return rte_flow_error_set(error, ENOTSUP,
1101 RTE_FLOW_ERROR_TYPE_ACTION,
1102 action, "RSS type not supported");
1103 proto_hdrs = match_item->meta;
1104 rss_meta->proto_hdrs = *proto_hdrs;
1105 iavf_refine_proto_hdrs(&rss_meta->proto_hdrs,
1106 rss_type, pattern_hint);
1109 case RTE_FLOW_ACTION_TYPE_END:
1113 rte_flow_error_set(error, EINVAL,
1114 RTE_FLOW_ERROR_TYPE_ACTION, action,
1124 iavf_hash_parse_pattern_action(__rte_unused struct iavf_adapter *ad,
1125 struct iavf_pattern_match_item *array,
1127 const struct rte_flow_item pattern[],
1128 const struct rte_flow_action actions[],
1130 struct rte_flow_error *error)
1132 struct iavf_pattern_match_item *pattern_match_item;
1133 struct iavf_rss_meta *rss_meta_ptr;
1134 uint64_t phint = IAVF_PHINT_NONE;
1137 rss_meta_ptr = rte_zmalloc(NULL, sizeof(*rss_meta_ptr), 0);
1138 if (!rss_meta_ptr) {
1139 rte_flow_error_set(error, EINVAL,
1140 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1141 "No memory for rss_meta_ptr");
1145 /* Check rss supported pattern and find matched pattern. */
1146 pattern_match_item =
1147 iavf_search_pattern_match_item(pattern, array, array_len,
1149 if (!pattern_match_item) {
1154 ret = iavf_hash_parse_pattern(pattern, &phint, error);
1158 ret = iavf_hash_parse_action(pattern_match_item, actions, phint,
1159 (void **)&rss_meta_ptr, error);
1163 *meta = rss_meta_ptr;
1165 rte_free(rss_meta_ptr);
1167 rte_free(pattern_match_item);
1173 iavf_hash_create(__rte_unused struct iavf_adapter *ad,
1174 __rte_unused struct rte_flow *flow, void *meta,
1175 __rte_unused struct rte_flow_error *error)
1177 struct iavf_rss_meta *rss_meta = (struct iavf_rss_meta *)meta;
1178 struct virtchnl_rss_cfg *rss_cfg;
1181 rss_cfg = rte_zmalloc("iavf rss rule",
1182 sizeof(struct virtchnl_rss_cfg), 0);
1184 rte_flow_error_set(error, EINVAL,
1185 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1186 "No memory for rss rule");
1190 rss_cfg->proto_hdrs = rss_meta->proto_hdrs;
1191 rss_cfg->rss_algorithm = rss_meta->rss_algorithm;
1193 ret = iavf_add_del_rss_cfg(ad, rss_cfg, true);
1195 flow->rule = rss_cfg;
1197 PMD_DRV_LOG(ERR, "fail to add RSS configure");
1198 rte_flow_error_set(error, -ret,
1199 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1200 "Failed to add rss rule.");
1211 iavf_hash_destroy(__rte_unused struct iavf_adapter *ad,
1212 struct rte_flow *flow,
1213 __rte_unused struct rte_flow_error *error)
1215 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
1216 struct virtchnl_rss_cfg *rss_cfg;
1222 rss_cfg = (struct virtchnl_rss_cfg *)flow->rule;
1224 ret = iavf_add_del_rss_cfg(ad, rss_cfg, false);
1226 PMD_DRV_LOG(ERR, "fail to del RSS configure");
1227 rte_flow_error_set(error, -ret,
1228 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1229 "Failed to delete rss rule.");
1236 iavf_hash_uninit(struct iavf_adapter *ad)
1238 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
1239 struct rte_eth_rss_conf *rss_conf;
1247 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF))
1250 rss_conf = &ad->eth_dev->data->dev_conf.rx_adv_conf.rss_conf;
1251 if (iavf_rss_hash_set(ad, rss_conf->rss_hf, false))
1252 PMD_DRV_LOG(ERR, "fail to delete default RSS");
1254 iavf_unregister_parser(&iavf_hash_parser, ad);
1258 iavf_hash_free(struct rte_flow *flow)
1260 rte_free(flow->rule);