vdpa/mlx5: support queue update
[dpdk.git] / drivers / net / ice / ice_hash.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2019 Intel Corporation
3  */
4
5 #include <sys/queue.h>
6 #include <stdio.h>
7 #include <errno.h>
8 #include <stdint.h>
9 #include <string.h>
10 #include <unistd.h>
11 #include <stdarg.h>
12
13 #include <rte_debug.h>
14 #include <rte_ether.h>
15 #include <rte_ethdev_driver.h>
16 #include <rte_log.h>
17 #include <rte_malloc.h>
18 #include <rte_eth_ctrl.h>
19 #include <rte_tailq.h>
20 #include <rte_flow_driver.h>
21
22 #include "ice_logs.h"
23 #include "base/ice_type.h"
24 #include "base/ice_flow.h"
25 #include "ice_ethdev.h"
26 #include "ice_generic_flow.h"
27
28 struct rss_type_match_hdr {
29         uint32_t hdr_mask;
30         uint64_t eth_rss_hint;
31 };
32
33 struct ice_hash_match_type {
34         uint64_t hash_type;
35         uint64_t hash_flds;
36 };
37
38 struct rss_meta {
39         uint32_t pkt_hdr;
40         uint64_t hash_flds;
41         uint8_t hash_function;
42 };
43
44 struct ice_hash_flow_cfg {
45         bool simple_xor;
46         struct ice_rss_cfg rss_cfg;
47 };
48
49 static int
50 ice_hash_init(struct ice_adapter *ad);
51
52 static int
53 ice_hash_create(struct ice_adapter *ad,
54                 struct rte_flow *flow,
55                 void *meta,
56                 struct rte_flow_error *error);
57
58 static int
59 ice_hash_destroy(struct ice_adapter *ad,
60                 struct rte_flow *flow,
61                 struct rte_flow_error *error);
62
63 static void
64 ice_hash_uninit(struct ice_adapter *ad);
65
66 static void
67 ice_hash_free(struct rte_flow *flow);
68
69 static int
70 ice_hash_parse_pattern_action(struct ice_adapter *ad,
71                         struct ice_pattern_match_item *array,
72                         uint32_t array_len,
73                         const struct rte_flow_item pattern[],
74                         const struct rte_flow_action actions[],
75                         void **meta,
76                         struct rte_flow_error *error);
77
78 /* The first member is protocol header, the second member is ETH_RSS_*. */
79 struct rss_type_match_hdr hint_0 = {
80         ICE_FLOW_SEG_HDR_NONE,  0};
81 struct rss_type_match_hdr hint_1 = {
82         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER, ETH_RSS_IPV4};
83 struct rss_type_match_hdr hint_2 = {
84         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_UDP |
85         ICE_FLOW_SEG_HDR_IPV_OTHER, ETH_RSS_NONFRAG_IPV4_UDP};
86 struct rss_type_match_hdr hint_3 = {
87         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_TCP |
88         ICE_FLOW_SEG_HDR_IPV_OTHER, ETH_RSS_NONFRAG_IPV4_TCP};
89 struct rss_type_match_hdr hint_4 = {
90         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_SCTP |
91         ICE_FLOW_SEG_HDR_IPV_OTHER, ETH_RSS_NONFRAG_IPV4_SCTP};
92 struct rss_type_match_hdr hint_5 = {
93         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER, ETH_RSS_IPV6};
94 struct rss_type_match_hdr hint_6 = {
95         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_UDP |
96         ICE_FLOW_SEG_HDR_IPV_OTHER, ETH_RSS_NONFRAG_IPV6_UDP};
97 struct rss_type_match_hdr hint_7 = {
98         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_TCP |
99         ICE_FLOW_SEG_HDR_IPV_OTHER, ETH_RSS_NONFRAG_IPV6_TCP};
100 struct rss_type_match_hdr hint_8 = {
101         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_SCTP |
102         ICE_FLOW_SEG_HDR_IPV_OTHER, ETH_RSS_NONFRAG_IPV6_SCTP};
103 struct rss_type_match_hdr hint_9 = {
104         ICE_FLOW_SEG_HDR_GTPU_EH, ETH_RSS_IPV4};
105 struct rss_type_match_hdr hint_10 = {
106         ICE_FLOW_SEG_HDR_PPPOE, ETH_RSS_IPV4};
107 struct rss_type_match_hdr hint_11 = {
108         ICE_FLOW_SEG_HDR_PPPOE, ETH_RSS_NONFRAG_IPV4_UDP};
109 struct rss_type_match_hdr hint_12 = {
110         ICE_FLOW_SEG_HDR_PPPOE, ETH_RSS_NONFRAG_IPV4_TCP};
111 struct rss_type_match_hdr hint_13 = {
112         ICE_FLOW_SEG_HDR_PPPOE, ETH_RSS_NONFRAG_IPV4_SCTP};
113 struct rss_type_match_hdr hint_14 = {
114         ICE_FLOW_SEG_HDR_GTPU_EH, ETH_RSS_NONFRAG_IPV4_UDP};
115 struct rss_type_match_hdr hint_15 = {
116         ICE_FLOW_SEG_HDR_GTPU_EH, ETH_RSS_NONFRAG_IPV4_TCP};
117
118 /* Supported pattern for os default package. */
119 static struct ice_pattern_match_item ice_hash_pattern_list_os[] = {
120         {pattern_eth_ipv4,      ICE_INSET_NONE, &hint_1},
121         {pattern_eth_ipv4_udp,  ICE_INSET_NONE, &hint_2},
122         {pattern_eth_ipv4_tcp,  ICE_INSET_NONE, &hint_3},
123         {pattern_eth_ipv4_sctp, ICE_INSET_NONE, &hint_4},
124         {pattern_eth_ipv6,      ICE_INSET_NONE, &hint_5},
125         {pattern_eth_ipv6_udp,  ICE_INSET_NONE, &hint_6},
126         {pattern_eth_ipv6_tcp,  ICE_INSET_NONE, &hint_7},
127         {pattern_eth_ipv6_sctp, ICE_INSET_NONE, &hint_8},
128         {pattern_empty,         ICE_INSET_NONE, &hint_0},
129 };
130
131 /* Supported pattern for comms package. */
132 static struct ice_pattern_match_item ice_hash_pattern_list_comms[] = {
133         {pattern_eth_ipv4,                  ICE_INSET_NONE,  &hint_1},
134         {pattern_eth_ipv4_udp,              ICE_INSET_NONE,  &hint_2},
135         {pattern_eth_ipv4_tcp,              ICE_INSET_NONE,  &hint_3},
136         {pattern_eth_ipv4_sctp,             ICE_INSET_NONE,  &hint_4},
137         {pattern_eth_ipv6,                  ICE_INSET_NONE,  &hint_5},
138         {pattern_eth_ipv6_udp,              ICE_INSET_NONE,  &hint_6},
139         {pattern_eth_ipv6_tcp,              ICE_INSET_NONE,  &hint_7},
140         {pattern_eth_ipv6_sctp,             ICE_INSET_NONE,  &hint_8},
141         {pattern_empty,                     ICE_INSET_NONE,  &hint_0},
142         {pattern_eth_ipv4_gtpu_eh_ipv4,     ICE_INSET_NONE,  &hint_9},
143         {pattern_eth_ipv4_gtpu_eh_ipv4_udp, ICE_INSET_NONE,  &hint_14},
144         {pattern_eth_ipv4_gtpu_eh_ipv4_tcp, ICE_INSET_NONE,  &hint_15},
145         {pattern_eth_pppoes_ipv4,           ICE_INSET_NONE,  &hint_10},
146         {pattern_eth_pppoes_ipv4_udp,       ICE_INSET_NONE,  &hint_11},
147         {pattern_eth_pppoes_ipv4_tcp,       ICE_INSET_NONE,  &hint_12},
148         {pattern_eth_pppoes_ipv4_sctp,      ICE_INSET_NONE,  &hint_13},
149 };
150
151 /**
152  * The first member is input set combination,
153  * the second member is hash fields.
154  */
155 struct ice_hash_match_type ice_hash_type_list[] = {
156         {ETH_RSS_IPV4 | ETH_RSS_L3_SRC_ONLY,                                    BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA)},
157         {ETH_RSS_IPV4 | ETH_RSS_L3_DST_ONLY,                                    BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA)},
158         {ETH_RSS_IPV4,                                                          ICE_FLOW_HASH_IPV4},
159         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)},
160         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)},
161         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L3_SRC_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA)},
162         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)},
163         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)},
164         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L3_DST_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA)},
165         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L4_SRC_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)},
166         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L4_DST_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)},
167         {ETH_RSS_NONFRAG_IPV4_UDP,                                              ICE_HASH_UDP_IPV4},
168         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)},
169         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)},
170         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L3_SRC_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA)},
171         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)},
172         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)},
173         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L3_DST_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA)},
174         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L4_SRC_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)},
175         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L4_DST_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)},
176         {ETH_RSS_NONFRAG_IPV4_TCP,                                              ICE_HASH_TCP_IPV4},
177         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY, BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)},
178         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY, BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)},
179         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L3_SRC_ONLY,                       BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA)},
180         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY, BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)},
181         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY, BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)},
182         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L3_DST_ONLY,                       BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA)},
183         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L4_SRC_ONLY,                       BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)},
184         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L4_DST_ONLY,                       BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)},
185         {ETH_RSS_NONFRAG_IPV4_SCTP,                                             ICE_HASH_SCTP_IPV4},
186         {ETH_RSS_IPV6 | ETH_RSS_L3_SRC_ONLY,                                    BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)},
187         {ETH_RSS_IPV6 | ETH_RSS_L3_DST_ONLY,                                    BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)},
188         {ETH_RSS_IPV6,                                                          ICE_FLOW_HASH_IPV6},
189         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)},
190         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)},
191         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L3_SRC_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)},
192         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)},
193         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)},
194         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L3_DST_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)},
195         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L4_SRC_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)},
196         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L4_DST_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)},
197         {ETH_RSS_NONFRAG_IPV6_UDP,                                              ICE_HASH_UDP_IPV6},
198         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)},
199         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)},
200         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L3_SRC_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)},
201         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)},
202         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)},
203         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L3_DST_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)},
204         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L4_SRC_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)},
205         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L4_DST_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)},
206         {ETH_RSS_NONFRAG_IPV6_TCP,                                              ICE_HASH_TCP_IPV6},
207         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY, BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)},
208         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY, BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)},
209         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L3_SRC_ONLY,                       BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)},
210         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY, BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)},
211         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY, BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)},
212         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L3_DST_ONLY,                       BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)},
213         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L4_SRC_ONLY,                       BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)},
214         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L4_DST_ONLY,                       BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)},
215         {ETH_RSS_NONFRAG_IPV6_SCTP,                                             ICE_HASH_SCTP_IPV6},
216 };
217
218 static struct ice_flow_engine ice_hash_engine = {
219         .init = ice_hash_init,
220         .create = ice_hash_create,
221         .destroy = ice_hash_destroy,
222         .uninit = ice_hash_uninit,
223         .free = ice_hash_free,
224         .type = ICE_FLOW_ENGINE_HASH,
225 };
226
227 /* Register parser for os package. */
228 static struct ice_flow_parser ice_hash_parser_os = {
229         .engine = &ice_hash_engine,
230         .array = ice_hash_pattern_list_os,
231         .array_len = RTE_DIM(ice_hash_pattern_list_os),
232         .parse_pattern_action = ice_hash_parse_pattern_action,
233         .stage = ICE_FLOW_STAGE_RSS,
234 };
235
236 /* Register parser for comms package. */
237 static struct ice_flow_parser ice_hash_parser_comms = {
238         .engine = &ice_hash_engine,
239         .array = ice_hash_pattern_list_comms,
240         .array_len = RTE_DIM(ice_hash_pattern_list_comms),
241         .parse_pattern_action = ice_hash_parse_pattern_action,
242         .stage = ICE_FLOW_STAGE_RSS,
243 };
244
245 RTE_INIT(ice_hash_engine_init)
246 {
247         struct ice_flow_engine *engine = &ice_hash_engine;
248         ice_register_flow_engine(engine);
249 }
250
251 static int
252 ice_hash_init(struct ice_adapter *ad)
253 {
254         struct ice_flow_parser *parser = NULL;
255
256         if (ad->hw.dcf_enabled)
257                 return 0;
258
259         if (ad->active_pkg_type == ICE_PKG_TYPE_OS_DEFAULT)
260                 parser = &ice_hash_parser_os;
261         else if (ad->active_pkg_type == ICE_PKG_TYPE_COMMS)
262                 parser = &ice_hash_parser_comms;
263         else
264                 return -EINVAL;
265
266         return ice_register_parser(parser, ad);
267 }
268
269 static int
270 ice_hash_check_inset(const struct rte_flow_item pattern[],
271                 struct rte_flow_error *error)
272 {
273         const struct rte_flow_item *item = pattern;
274
275         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
276                 if (item->last) {
277                         rte_flow_error_set(error, EINVAL,
278                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
279                                         "Not support range");
280                         return -rte_errno;
281                 }
282
283                 /* Ignore spec and mask. */
284                 if (item->spec || item->mask) {
285                         rte_flow_error_set(error, EINVAL,
286                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
287                                         "Invalid spec/mask.");
288                         return -rte_errno;
289                 }
290         }
291
292         return 0;
293 }
294
295 static int
296 ice_hash_parse_action(struct ice_pattern_match_item *pattern_match_item,
297                 const struct rte_flow_action actions[],
298                 void **meta,
299                 struct rte_flow_error *error)
300 {
301         const struct rte_flow_action *action;
302         enum rte_flow_action_type action_type;
303         const struct rte_flow_action_rss *rss;
304         struct rss_type_match_hdr *m = (struct rss_type_match_hdr *)
305                                 (pattern_match_item->meta);
306         uint32_t type_list_len = RTE_DIM(ice_hash_type_list);
307         struct ice_hash_match_type *type_match_item;
308         uint64_t rss_hf;
309         uint16_t i;
310
311         /* Supported action is RSS. */
312         for (action = actions; action->type !=
313                 RTE_FLOW_ACTION_TYPE_END; action++) {
314                 action_type = action->type;
315                 switch (action_type) {
316                 case RTE_FLOW_ACTION_TYPE_RSS:
317                         rss = action->conf;
318                         rss_hf = rss->types;
319
320                         /**
321                          * Check simultaneous use of SRC_ONLY and DST_ONLY
322                          * of the same level.
323                          */
324                         rss_hf = rte_eth_rss_hf_refine(rss_hf);
325
326                         /* Check if pattern is empty. */
327                         if (pattern_match_item->pattern_list !=
328                                 pattern_empty && rss->func ==
329                                 RTE_ETH_HASH_FUNCTION_SIMPLE_XOR)
330                                 return rte_flow_error_set(error, ENOTSUP,
331                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
332                                         "Not supported flow");
333
334                         /* Check if rss types match pattern. */
335                         if (rss->func != RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) {
336                                 if (((rss_hf & ETH_RSS_IPV4) != m->eth_rss_hint) &&
337                                 ((rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) != m->eth_rss_hint) &&
338                                 ((rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) != m->eth_rss_hint) &&
339                                 ((rss_hf & ETH_RSS_NONFRAG_IPV4_SCTP) != m->eth_rss_hint) &&
340                                 ((rss_hf & ETH_RSS_IPV6) != m->eth_rss_hint) &&
341                                 ((rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) != m->eth_rss_hint) &&
342                                 ((rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) != m->eth_rss_hint) &&
343                                 ((rss_hf & ETH_RSS_NONFRAG_IPV6_SCTP) != m->eth_rss_hint))
344                                         return rte_flow_error_set(error,
345                                         ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
346                                         action, "Not supported RSS types");
347                         }
348
349                         if (rss->level)
350                                 return rte_flow_error_set(error, ENOTSUP,
351                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
352                                         "a nonzero RSS encapsulation level is not supported");
353
354                         if (rss->key_len)
355                                 return rte_flow_error_set(error, ENOTSUP,
356                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
357                                         "a nonzero RSS key_len is not supported");
358
359                         if (rss->queue)
360                                 return rte_flow_error_set(error, ENOTSUP,
361                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
362                                         "a non-NULL RSS queue is not supported");
363
364                         /* Check hash function and save it to rss_meta. */
365                         if (rss->func ==
366                                 RTE_ETH_HASH_FUNCTION_SIMPLE_XOR)
367                                 ((struct rss_meta *)*meta)->hash_function =
368                                 RTE_ETH_HASH_FUNCTION_SIMPLE_XOR;
369
370                         if (rss->func ==
371                                 RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ)
372                                 ((struct rss_meta *)*meta)->hash_function =
373                                 RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ;
374
375                         type_match_item = rte_zmalloc("ice_type_match_item",
376                                         sizeof(struct ice_hash_match_type), 0);
377                         if (!type_match_item) {
378                                 rte_flow_error_set(error, EINVAL,
379                                         RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
380                                         "No memory for type_match_item");
381                                 return -ENOMEM;
382                         }
383
384                         /* Find matched hash fields according to hash type. */
385                         for (i = 0; i < type_list_len; i++) {
386                                 if (rss_hf ==
387                                         ice_hash_type_list[i].hash_type) {
388                                         type_match_item->hash_type =
389                                                 ice_hash_type_list[i].hash_type;
390                                         type_match_item->hash_flds =
391                                                 ice_hash_type_list[i].hash_flds;
392                                 }
393                         }
394
395                         /* Save hash fileds to rss_meta. */
396                         ((struct rss_meta *)*meta)->hash_flds =
397                                         type_match_item->hash_flds;
398
399                         rte_free(type_match_item);
400                         break;
401
402                 case RTE_FLOW_ACTION_TYPE_END:
403                         break;
404
405                 default:
406                         rte_flow_error_set(error, EINVAL,
407                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
408                                         "Invalid action.");
409                         return -rte_errno;
410                 }
411         }
412
413         return 0;
414 }
415
416 static int
417 ice_hash_parse_pattern_action(__rte_unused struct ice_adapter *ad,
418                         struct ice_pattern_match_item *array,
419                         uint32_t array_len,
420                         const struct rte_flow_item pattern[],
421                         const struct rte_flow_action actions[],
422                         void **meta,
423                         struct rte_flow_error *error)
424 {
425         int ret = 0;
426         struct ice_pattern_match_item *pattern_match_item;
427         struct rss_meta *rss_meta_ptr;
428
429         rss_meta_ptr = rte_zmalloc(NULL, sizeof(*rss_meta_ptr), 0);
430         if (!rss_meta_ptr) {
431                 rte_flow_error_set(error, EINVAL,
432                                 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
433                                 "No memory for rss_meta_ptr");
434                 return -ENOMEM;
435         }
436
437         /* Check rss supported pattern and find matched pattern. */
438         pattern_match_item = ice_search_pattern_match_item(pattern,
439                                         array, array_len, error);
440         if (!pattern_match_item) {
441                 ret = -rte_errno;
442                 goto error;
443         }
444
445         ret = ice_hash_check_inset(pattern, error);
446         if (ret)
447                 goto error;
448
449         /* Save protocol header to rss_meta. */
450         rss_meta_ptr->pkt_hdr = ((struct rss_type_match_hdr *)
451                 (pattern_match_item->meta))->hdr_mask;
452
453         /* Check rss action. */
454         ret = ice_hash_parse_action(pattern_match_item, actions,
455                                     (void **)&rss_meta_ptr, error);
456
457 error:
458         if (!ret && meta)
459                 *meta = rss_meta_ptr;
460         else
461                 rte_free(rss_meta_ptr);
462         rte_free(pattern_match_item);
463
464         return ret;
465 }
466
467 static int
468 ice_hash_create(struct ice_adapter *ad,
469                 struct rte_flow *flow,
470                 void *meta,
471                 struct rte_flow_error *error)
472 {
473         struct ice_pf *pf = &ad->pf;
474         struct ice_hw *hw = ICE_PF_TO_HW(pf);
475         struct ice_vsi *vsi = pf->main_vsi;
476         int ret;
477         uint32_t reg;
478         struct ice_hash_flow_cfg *filter_ptr;
479
480         uint32_t headermask = ((struct rss_meta *)meta)->pkt_hdr;
481         uint64_t hash_field = ((struct rss_meta *)meta)->hash_flds;
482         uint8_t hash_function = ((struct rss_meta *)meta)->hash_function;
483
484         filter_ptr = rte_zmalloc("ice_rss_filter",
485                                 sizeof(struct ice_hash_flow_cfg), 0);
486         if (!filter_ptr) {
487                 rte_flow_error_set(error, EINVAL,
488                                 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
489                                 "No memory for filter_ptr");
490                 return -ENOMEM;
491         }
492
493         if (hash_function == RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) {
494                 /* Enable registers for simple_xor hash function. */
495                 reg = ICE_READ_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id));
496                 reg = (reg & (~VSIQF_HASH_CTL_HASH_SCHEME_M)) |
497                         (2 << VSIQF_HASH_CTL_HASH_SCHEME_S);
498                 ICE_WRITE_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id), reg);
499
500                 filter_ptr->simple_xor = 1;
501
502                 goto out;
503         } else {
504                 filter_ptr->rss_cfg.packet_hdr = headermask;
505                 filter_ptr->rss_cfg.hashed_flds = hash_field;
506                 filter_ptr->rss_cfg.symm =
507                         (hash_function ==
508                                 RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ);
509
510                 ret = ice_add_rss_cfg(hw, vsi->idx,
511                                 filter_ptr->rss_cfg.hashed_flds,
512                                 filter_ptr->rss_cfg.packet_hdr,
513                                 filter_ptr->rss_cfg.symm);
514                 if (ret) {
515                         rte_flow_error_set(error, EINVAL,
516                                         RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
517                                         "rss flow create fail");
518                         goto error;
519                 }
520         }
521
522 out:
523         flow->rule = filter_ptr;
524         rte_free(meta);
525         return 0;
526
527 error:
528         rte_free(filter_ptr);
529         rte_free(meta);
530         return -rte_errno;
531 }
532
533 static int
534 ice_hash_destroy(struct ice_adapter *ad,
535                 struct rte_flow *flow,
536                 struct rte_flow_error *error)
537 {
538         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(ad);
539         struct ice_hw *hw = ICE_PF_TO_HW(pf);
540         struct ice_vsi *vsi = pf->main_vsi;
541         int ret;
542         uint32_t reg;
543         struct ice_hash_flow_cfg *filter_ptr;
544
545         filter_ptr = (struct ice_hash_flow_cfg *)flow->rule;
546
547         if (filter_ptr->simple_xor == 1) {
548                 /* Return to symmetric_toeplitz state. */
549                 reg = ICE_READ_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id));
550                 reg = (reg & (~VSIQF_HASH_CTL_HASH_SCHEME_M)) |
551                         (1 << VSIQF_HASH_CTL_HASH_SCHEME_S);
552                 ICE_WRITE_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id), reg);
553         } else {
554                 ret = ice_rem_rss_cfg(hw, vsi->idx,
555                                 filter_ptr->rss_cfg.hashed_flds,
556                                 filter_ptr->rss_cfg.packet_hdr);
557                 /* Fixme: Ignore the error if a rule does not exist.
558                  * Currently a rule for inputset change or symm turn on/off
559                  * will overwrite an exist rule, while application still
560                  * have 2 rte_flow handles.
561                  **/
562                 if (ret && ret != ICE_ERR_DOES_NOT_EXIST) {
563                         rte_flow_error_set(error, EINVAL,
564                                         RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
565                                         "rss flow destroy fail");
566                         goto error;
567                 }
568         }
569
570         rte_free(filter_ptr);
571         return 0;
572
573 error:
574         rte_free(filter_ptr);
575         return -rte_errno;
576 }
577
578 static void
579 ice_hash_uninit(struct ice_adapter *ad)
580 {
581         if (ad->hw.dcf_enabled)
582                 return;
583
584         if (ad->active_pkg_type == ICE_PKG_TYPE_OS_DEFAULT)
585                 ice_unregister_parser(&ice_hash_parser_os, ad);
586         else if (ad->active_pkg_type == ICE_PKG_TYPE_COMMS)
587                 ice_unregister_parser(&ice_hash_parser_comms, ad);
588 }
589
590 static void
591 ice_hash_free(struct rte_flow *flow)
592 {
593         rte_free(flow->rule);
594 }