drivers/net/iavf/iavf_hash.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10 #include <unistd.h>
  11 #include <stdarg.h>
  12
  13 #include <rte_debug.h>
  14 #include <rte_ether.h>
  15 #include <ethdev_driver.h>
  16 #include <rte_log.h>
  17 #include <rte_malloc.h>
  18 #include <rte_eth_ctrl.h>
  19 #include <rte_tailq.h>
  20 #include <rte_flow_driver.h>
  21
  22 #include "iavf_log.h"
  23 #include "iavf.h"
  24 #include "iavf_generic_flow.h"
  25
  26 #define IAVF_PHINT_NONE                         0
  27 #define IAVF_PHINT_GTPU                         BIT_ULL(0)
  28 #define IAVF_PHINT_GTPU_EH                      BIT_ULL(1)
  29 #define IAVF_PHINT_GTPU_EH_DWN                  BIT_ULL(2)
  30 #define IAVF_PHINT_GTPU_EH_UP                   BIT_ULL(3)
  31 #define IAVF_PHINT_OUTER_IPV4                   BIT_ULL(4)
  32 #define IAVF_PHINT_OUTER_IPV6                   BIT_ULL(5)
  33 #define IAVF_PHINT_GRE                          BIT_ULL(6)
  34 /* the second IP header of GTPoGRE */
  35 #define IAVF_PHINT_MID_IPV4                     BIT_ULL(7)
  36 #define IAVF_PHINT_MID_IPV6                     BIT_ULL(8)
  37
  38 #define IAVF_PHINT_GTPU_MSK     (IAVF_PHINT_GTPU        | \
  39                                  IAVF_PHINT_GTPU_EH     | \
  40                                  IAVF_PHINT_GTPU_EH_DWN | \
  41                                  IAVF_PHINT_GTPU_EH_UP)
  42
  43 #define IAVF_PHINT_LAYERS_MSK   (IAVF_PHINT_OUTER_IPV4  | \
  44                                  IAVF_PHINT_OUTER_IPV6)
  45
  46 #define IAVF_GTPU_EH_DWNLINK    0
  47 #define IAVF_GTPU_EH_UPLINK     1
  48
  49 struct iavf_hash_match_type {
  50         uint64_t hash_type;
  51         struct virtchnl_proto_hdrs *proto_hdrs;
  52         uint64_t pattern_hint;
  53 };
  54
  55 struct iavf_rss_meta {
  56         struct virtchnl_proto_hdrs proto_hdrs;
  57         enum virtchnl_rss_algorithm rss_algorithm;
  58 };
  59
  60 struct iavf_hash_flow_cfg {
  61         struct virtchnl_rss_cfg *rss_cfg;
  62         bool simple_xor;
  63 };
  64
  65 static int
  66 iavf_hash_init(struct iavf_adapter *ad);
  67 static int
  68 iavf_hash_create(struct iavf_adapter *ad, struct rte_flow *flow, void *meta,
  69                  struct rte_flow_error *error);
  70 static int
  71 iavf_hash_destroy(struct iavf_adapter *ad, struct rte_flow *flow,
  72                   struct rte_flow_error *error);
  73 static void
  74 iavf_hash_uninit(struct iavf_adapter *ad);
  75 static void
  76 iavf_hash_free(struct rte_flow *flow);
  77 static int
  78 iavf_hash_parse_pattern_action(struct iavf_adapter *ad,
  79                                struct iavf_pattern_match_item *array,
  80                                uint32_t array_len,
  81                                const struct rte_flow_item pattern[],
  82                                const struct rte_flow_action actions[],
  83                                void **meta,
  84                                struct rte_flow_error *error);
  85
  86 #define FIELD_SELECTOR(proto_hdr_field) \
  87                 (1UL << ((proto_hdr_field) & PROTO_HDR_FIELD_MASK))
  88 #define BUFF_NOUSED                     0
  89
  90 #define proto_hdr_eth { \
  91         VIRTCHNL_PROTO_HDR_ETH, \
  92         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_SRC) | \
  93         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_DST), {BUFF_NOUSED} }
  94
  95 #define proto_hdr_svlan { \
  96         VIRTCHNL_PROTO_HDR_S_VLAN, \
  97         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_S_VLAN_ID), {BUFF_NOUSED} }
  98
  99 #define proto_hdr_cvlan { \
 100         VIRTCHNL_PROTO_HDR_C_VLAN, \
 101         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_C_VLAN_ID), {BUFF_NOUSED} }
 102
 103 #define proto_hdr_ipv4 { \
 104         VIRTCHNL_PROTO_HDR_IPV4, \
 105         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_SRC) | \
 106         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_DST), {BUFF_NOUSED} }
 107
 108 #define proto_hdr_ipv4_with_prot { \
 109         VIRTCHNL_PROTO_HDR_IPV4, \
 110         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_SRC) | \
 111         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_DST) | \
 112         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_PROT), {BUFF_NOUSED} }
 113
 114 #define proto_hdr_ipv6 { \
 115         VIRTCHNL_PROTO_HDR_IPV6, \
 116         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_SRC) | \
 117         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_DST), {BUFF_NOUSED} }
 118
 119 #define proto_hdr_ipv6_frag { \
 120         VIRTCHNL_PROTO_HDR_IPV6_EH_FRAG, \
 121         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_EH_FRAG_PKID), {BUFF_NOUSED} }
 122
 123 #define proto_hdr_ipv6_with_prot { \
 124         VIRTCHNL_PROTO_HDR_IPV6, \
 125         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_SRC) | \
 126         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_DST) | \
 127         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_PROT), {BUFF_NOUSED} }
 128
 129 #define proto_hdr_udp { \
 130         VIRTCHNL_PROTO_HDR_UDP, \
 131         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_UDP_SRC_PORT) | \
 132         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_UDP_DST_PORT), {BUFF_NOUSED} }
 133
 134 #define proto_hdr_tcp { \
 135         VIRTCHNL_PROTO_HDR_TCP, \
 136         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_TCP_SRC_PORT) | \
 137         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_TCP_DST_PORT), {BUFF_NOUSED} }
 138
 139 #define proto_hdr_sctp { \
 140         VIRTCHNL_PROTO_HDR_SCTP, \
 141         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_SCTP_SRC_PORT) | \
 142         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_SCTP_DST_PORT), {BUFF_NOUSED} }
 143
 144 #define proto_hdr_esp { \
 145         VIRTCHNL_PROTO_HDR_ESP, \
 146         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ESP_SPI), {BUFF_NOUSED} }
 147
 148 #define proto_hdr_ah { \
 149         VIRTCHNL_PROTO_HDR_AH, \
 150         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_AH_SPI), {BUFF_NOUSED} }
 151
 152 #define proto_hdr_l2tpv3 { \
 153         VIRTCHNL_PROTO_HDR_L2TPV3, \
 154         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_L2TPV3_SESS_ID), {BUFF_NOUSED} }
 155
 156 #define proto_hdr_pfcp { \
 157         VIRTCHNL_PROTO_HDR_PFCP, \
 158         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_PFCP_SEID), {BUFF_NOUSED} }
 159
 160 #define proto_hdr_gtpc { \
 161         VIRTCHNL_PROTO_HDR_GTPC, 0, {BUFF_NOUSED} }
 162
 163 #define proto_hdr_ecpri { \
 164         VIRTCHNL_PROTO_HDR_ECPRI, \
 165         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ECPRI_PC_RTC_ID), {BUFF_NOUSED} }
 166
 167 #define TUNNEL_LEVEL_OUTER              0
 168 #define TUNNEL_LEVEL_INNER              1
 169
 170 /* proto_hdrs template */
 171 struct virtchnl_proto_hdrs outer_ipv4_tmplt = {
 172         TUNNEL_LEVEL_OUTER, 4,
 173         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv4}
 174 };
 175
 176 struct virtchnl_proto_hdrs outer_ipv4_udp_tmplt = {
 177         TUNNEL_LEVEL_OUTER, 5,
 178         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan,
 179          proto_hdr_ipv4_with_prot,
 180          proto_hdr_udp}
 181 };
 182
 183 struct virtchnl_proto_hdrs outer_ipv4_tcp_tmplt = {
 184         TUNNEL_LEVEL_OUTER, 5,
 185         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan,
 186          proto_hdr_ipv4_with_prot,
 187          proto_hdr_tcp}
 188 };
 189
 190 struct virtchnl_proto_hdrs outer_ipv4_sctp_tmplt = {
 191         TUNNEL_LEVEL_OUTER, 5,
 192         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv4,
 193          proto_hdr_sctp}
 194 };
 195
 196 struct virtchnl_proto_hdrs outer_ipv6_tmplt = {
 197         TUNNEL_LEVEL_OUTER, 4,
 198         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv6}
 199 };
 200
 201 struct virtchnl_proto_hdrs outer_ipv6_frag_tmplt = {
 202         TUNNEL_LEVEL_OUTER, 5,
 203         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan,
 204          proto_hdr_ipv6, proto_hdr_ipv6_frag}
 205 };
 206
 207 struct virtchnl_proto_hdrs outer_ipv6_udp_tmplt = {
 208         TUNNEL_LEVEL_OUTER, 5,
 209         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan,
 210          proto_hdr_ipv6_with_prot,
 211          proto_hdr_udp}
 212 };
 213
 214 struct virtchnl_proto_hdrs outer_ipv6_tcp_tmplt = {
 215         TUNNEL_LEVEL_OUTER, 5,
 216         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan,
 217          proto_hdr_ipv6_with_prot,
 218          proto_hdr_tcp}
 219 };
 220
 221 struct virtchnl_proto_hdrs outer_ipv6_sctp_tmplt = {
 222         TUNNEL_LEVEL_OUTER, 5,
 223         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv6,
 224          proto_hdr_sctp}
 225 };
 226
 227 struct virtchnl_proto_hdrs inner_ipv4_tmplt = {
 228         TUNNEL_LEVEL_INNER, 1, {proto_hdr_ipv4}
 229 };
 230
 231 struct virtchnl_proto_hdrs inner_ipv4_udp_tmplt = {
 232         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv4_with_prot, proto_hdr_udp}
 233 };
 234
 235 struct virtchnl_proto_hdrs inner_ipv4_tcp_tmplt = {
 236         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv4_with_prot, proto_hdr_tcp}
 237 };
 238
 239 struct virtchnl_proto_hdrs second_inner_ipv4_tmplt = {
 240         2, 1, {proto_hdr_ipv4}
 241 };
 242
 243 struct virtchnl_proto_hdrs second_inner_ipv4_udp_tmplt = {
 244         2, 2, {proto_hdr_ipv4_with_prot, proto_hdr_udp}
 245 };
 246
 247 struct virtchnl_proto_hdrs second_inner_ipv4_tcp_tmplt = {
 248         2, 2, {proto_hdr_ipv4_with_prot, proto_hdr_tcp}
 249 };
 250
 251 struct virtchnl_proto_hdrs second_inner_ipv6_tmplt = {
 252         2, 1, {proto_hdr_ipv6}
 253 };
 254
 255 struct virtchnl_proto_hdrs second_inner_ipv6_udp_tmplt = {
 256         2, 2, {proto_hdr_ipv6_with_prot, proto_hdr_udp}
 257 };
 258
 259 struct virtchnl_proto_hdrs second_inner_ipv6_tcp_tmplt = {
 260         2, 2, {proto_hdr_ipv6_with_prot, proto_hdr_tcp}
 261 };
 262
 263 struct virtchnl_proto_hdrs inner_ipv4_sctp_tmplt = {
 264         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv4, proto_hdr_sctp}
 265 };
 266
 267 struct virtchnl_proto_hdrs inner_ipv6_tmplt = {
 268         TUNNEL_LEVEL_INNER, 1, {proto_hdr_ipv6}
 269 };
 270
 271 struct virtchnl_proto_hdrs inner_ipv6_udp_tmplt = {
 272         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv6_with_prot, proto_hdr_udp}
 273 };
 274
 275 struct virtchnl_proto_hdrs inner_ipv6_tcp_tmplt = {
 276         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv6_with_prot, proto_hdr_tcp}
 277 };
 278
 279 struct virtchnl_proto_hdrs inner_ipv6_sctp_tmplt = {
 280         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv6, proto_hdr_sctp}
 281 };
 282
 283 struct virtchnl_proto_hdrs ipv4_esp_tmplt = {
 284         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_esp}
 285 };
 286
 287 struct virtchnl_proto_hdrs ipv4_udp_esp_tmplt = {
 288         TUNNEL_LEVEL_OUTER, 3,
 289         {proto_hdr_ipv4, proto_hdr_udp, proto_hdr_esp}
 290 };
 291
 292 struct virtchnl_proto_hdrs ipv4_ah_tmplt = {
 293         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_ah}
 294 };
 295
 296 struct virtchnl_proto_hdrs ipv6_esp_tmplt = {
 297         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_esp}
 298 };
 299
 300 struct virtchnl_proto_hdrs ipv6_udp_esp_tmplt = {
 301         TUNNEL_LEVEL_OUTER, 3,
 302         {proto_hdr_ipv6, proto_hdr_udp, proto_hdr_esp}
 303 };
 304
 305 struct virtchnl_proto_hdrs ipv6_ah_tmplt = {
 306         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_ah}
 307 };
 308
 309 struct virtchnl_proto_hdrs ipv4_l2tpv3_tmplt = {
 310         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_l2tpv3}
 311 };
 312
 313 struct virtchnl_proto_hdrs ipv6_l2tpv3_tmplt = {
 314         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_l2tpv3}
 315 };
 316
 317 struct virtchnl_proto_hdrs ipv4_pfcp_tmplt = {
 318         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_pfcp}
 319 };
 320
 321 struct virtchnl_proto_hdrs ipv6_pfcp_tmplt = {
 322         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_pfcp}
 323 };
 324
 325 struct virtchnl_proto_hdrs ipv4_udp_gtpc_tmplt = {
 326         TUNNEL_LEVEL_OUTER, 3, {proto_hdr_ipv4, proto_hdr_udp, proto_hdr_gtpc}
 327 };
 328
 329 struct virtchnl_proto_hdrs ipv6_udp_gtpc_tmplt = {
 330         TUNNEL_LEVEL_OUTER, 3, {proto_hdr_ipv6, proto_hdr_udp, proto_hdr_gtpc}
 331 };
 332
 333 struct virtchnl_proto_hdrs eth_ecpri_tmplt = {
 334         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_eth, proto_hdr_ecpri}
 335 };
 336
 337 struct virtchnl_proto_hdrs ipv4_ecpri_tmplt = {
 338         TUNNEL_LEVEL_OUTER, 3, {proto_hdr_ipv4, proto_hdr_udp, proto_hdr_ecpri}
 339 };
 340
 341 /* rss type super set */
 342
 343 /* IPv4 outer */
 344 #define IAVF_RSS_TYPE_OUTER_IPV4        (ETH_RSS_ETH | ETH_RSS_IPV4 | \
 345                                          ETH_RSS_FRAG_IPV4)
 346 #define IAVF_RSS_TYPE_OUTER_IPV4_UDP    (IAVF_RSS_TYPE_OUTER_IPV4 | \
 347                                          ETH_RSS_NONFRAG_IPV4_UDP)
 348 #define IAVF_RSS_TYPE_OUTER_IPV4_TCP    (IAVF_RSS_TYPE_OUTER_IPV4 | \
 349                                          ETH_RSS_NONFRAG_IPV4_TCP)
 350 #define IAVF_RSS_TYPE_OUTER_IPV4_SCTP   (IAVF_RSS_TYPE_OUTER_IPV4 | \
 351                                          ETH_RSS_NONFRAG_IPV4_SCTP)
 352 /* IPv6 outer */
 353 #define IAVF_RSS_TYPE_OUTER_IPV6        (ETH_RSS_ETH | ETH_RSS_IPV6)
 354 #define IAVF_RSS_TYPE_OUTER_IPV6_FRAG   (IAVF_RSS_TYPE_OUTER_IPV6 | \
 355                                          ETH_RSS_FRAG_IPV6)
 356 #define IAVF_RSS_TYPE_OUTER_IPV6_UDP    (IAVF_RSS_TYPE_OUTER_IPV6 | \
 357                                          ETH_RSS_NONFRAG_IPV6_UDP)
 358 #define IAVF_RSS_TYPE_OUTER_IPV6_TCP    (IAVF_RSS_TYPE_OUTER_IPV6 | \
 359                                          ETH_RSS_NONFRAG_IPV6_TCP)
 360 #define IAVF_RSS_TYPE_OUTER_IPV6_SCTP   (IAVF_RSS_TYPE_OUTER_IPV6 | \
 361                                          ETH_RSS_NONFRAG_IPV6_SCTP)
 362 /* VLAN IPV4 */
 363 #define IAVF_RSS_TYPE_VLAN_IPV4         (IAVF_RSS_TYPE_OUTER_IPV4 | \
 364                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 365 #define IAVF_RSS_TYPE_VLAN_IPV4_UDP     (IAVF_RSS_TYPE_OUTER_IPV4_UDP | \
 366                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 367 #define IAVF_RSS_TYPE_VLAN_IPV4_TCP     (IAVF_RSS_TYPE_OUTER_IPV4_TCP | \
 368                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 369 #define IAVF_RSS_TYPE_VLAN_IPV4_SCTP    (IAVF_RSS_TYPE_OUTER_IPV4_SCTP | \
 370                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 371 /* VLAN IPv6 */
 372 #define IAVF_RSS_TYPE_VLAN_IPV6         (IAVF_RSS_TYPE_OUTER_IPV6 | \
 373                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 374 #define IAVF_RSS_TYPE_VLAN_IPV6_FRAG    (IAVF_RSS_TYPE_OUTER_IPV6_FRAG | \
 375                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 376 #define IAVF_RSS_TYPE_VLAN_IPV6_UDP     (IAVF_RSS_TYPE_OUTER_IPV6_UDP | \
 377                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 378 #define IAVF_RSS_TYPE_VLAN_IPV6_TCP     (IAVF_RSS_TYPE_OUTER_IPV6_TCP | \
 379                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 380 #define IAVF_RSS_TYPE_VLAN_IPV6_SCTP    (IAVF_RSS_TYPE_OUTER_IPV6_SCTP | \
 381                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 382 /* IPv4 inner */
 383 #define IAVF_RSS_TYPE_INNER_IPV4        ETH_RSS_IPV4
 384 #define IAVF_RSS_TYPE_INNER_IPV4_UDP    (ETH_RSS_IPV4 | \
 385                                          ETH_RSS_NONFRAG_IPV4_UDP)
 386 #define IAVF_RSS_TYPE_INNER_IPV4_TCP    (ETH_RSS_IPV4 | \
 387                                          ETH_RSS_NONFRAG_IPV4_TCP)
 388 #define IAVF_RSS_TYPE_INNER_IPV4_SCTP   (ETH_RSS_IPV4 | \
 389                                          ETH_RSS_NONFRAG_IPV4_SCTP)
 390 /* IPv6 inner */
 391 #define IAVF_RSS_TYPE_INNER_IPV6        ETH_RSS_IPV6
 392 #define IAVF_RSS_TYPE_INNER_IPV6_UDP    (ETH_RSS_IPV6 | \
 393                                          ETH_RSS_NONFRAG_IPV6_UDP)
 394 #define IAVF_RSS_TYPE_INNER_IPV6_TCP    (ETH_RSS_IPV6 | \
 395                                          ETH_RSS_NONFRAG_IPV6_TCP)
 396 #define IAVF_RSS_TYPE_INNER_IPV6_SCTP   (ETH_RSS_IPV6 | \
 397                                          ETH_RSS_NONFRAG_IPV6_SCTP)
 398 /* GTPU IPv4 */
 399 #define IAVF_RSS_TYPE_GTPU_IPV4         (IAVF_RSS_TYPE_INNER_IPV4 | \
 400                                          ETH_RSS_GTPU)
 401 #define IAVF_RSS_TYPE_GTPU_IPV4_UDP     (IAVF_RSS_TYPE_INNER_IPV4_UDP | \
 402                                          ETH_RSS_GTPU)
 403 #define IAVF_RSS_TYPE_GTPU_IPV4_TCP     (IAVF_RSS_TYPE_INNER_IPV4_TCP | \
 404                                          ETH_RSS_GTPU)
 405 /* GTPU IPv6 */
 406 #define IAVF_RSS_TYPE_GTPU_IPV6         (IAVF_RSS_TYPE_INNER_IPV6 | \
 407                                          ETH_RSS_GTPU)
 408 #define IAVF_RSS_TYPE_GTPU_IPV6_UDP     (IAVF_RSS_TYPE_INNER_IPV6_UDP | \
 409                                          ETH_RSS_GTPU)
 410 #define IAVF_RSS_TYPE_GTPU_IPV6_TCP     (IAVF_RSS_TYPE_INNER_IPV6_TCP | \
 411                                          ETH_RSS_GTPU)
 412 /* ESP, AH, L2TPV3 and PFCP */
 413 #define IAVF_RSS_TYPE_IPV4_ESP          (ETH_RSS_ESP | ETH_RSS_IPV4)
 414 #define IAVF_RSS_TYPE_IPV4_AH           (ETH_RSS_AH | ETH_RSS_IPV4)
 415 #define IAVF_RSS_TYPE_IPV6_ESP          (ETH_RSS_ESP | ETH_RSS_IPV6)
 416 #define IAVF_RSS_TYPE_IPV6_AH           (ETH_RSS_AH | ETH_RSS_IPV6)
 417 #define IAVF_RSS_TYPE_IPV4_L2TPV3       (ETH_RSS_L2TPV3 | ETH_RSS_IPV4)
 418 #define IAVF_RSS_TYPE_IPV6_L2TPV3       (ETH_RSS_L2TPV3 | ETH_RSS_IPV6)
 419 #define IAVF_RSS_TYPE_IPV4_PFCP         (ETH_RSS_PFCP | ETH_RSS_IPV4)
 420 #define IAVF_RSS_TYPE_IPV6_PFCP         (ETH_RSS_PFCP | ETH_RSS_IPV6)
 421
 422 /**
 423  * Supported pattern for hash.
 424  * The first member is pattern item type,
 425  * the second member is input set mask,
 426  * the third member is virtchnl_proto_hdrs template
 427  */
 428 static struct iavf_pattern_match_item iavf_hash_pattern_list[] = {
 429         /* IPv4 */
 430         {iavf_pattern_eth_ipv4,                         IAVF_RSS_TYPE_OUTER_IPV4,       &outer_ipv4_tmplt},
 431         {iavf_pattern_eth_ipv4_udp,                     IAVF_RSS_TYPE_OUTER_IPV4_UDP,   &outer_ipv4_udp_tmplt},
 432         {iavf_pattern_eth_ipv4_tcp,                     IAVF_RSS_TYPE_OUTER_IPV4_TCP,   &outer_ipv4_tcp_tmplt},
 433         {iavf_pattern_eth_ipv4_sctp,                    IAVF_RSS_TYPE_OUTER_IPV4_SCTP,  &outer_ipv4_sctp_tmplt},
 434         {iavf_pattern_eth_vlan_ipv4,                    IAVF_RSS_TYPE_VLAN_IPV4,        &outer_ipv4_tmplt},
 435         {iavf_pattern_eth_vlan_ipv4_udp,                IAVF_RSS_TYPE_VLAN_IPV4_UDP,    &outer_ipv4_udp_tmplt},
 436         {iavf_pattern_eth_vlan_ipv4_tcp,                IAVF_RSS_TYPE_VLAN_IPV4_TCP,    &outer_ipv4_tcp_tmplt},
 437         {iavf_pattern_eth_vlan_ipv4_sctp,               IAVF_RSS_TYPE_VLAN_IPV4_SCTP,   &outer_ipv4_sctp_tmplt},
 438         {iavf_pattern_eth_ipv4_gtpu,                    ETH_RSS_IPV4,                   &outer_ipv4_udp_tmplt},
 439         {iavf_pattern_eth_ipv4_gtpu_ipv4,               IAVF_RSS_TYPE_GTPU_IPV4,        &inner_ipv4_tmplt},
 440         {iavf_pattern_eth_ipv4_gtpu_ipv4_udp,           IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &inner_ipv4_udp_tmplt},
 441         {iavf_pattern_eth_ipv4_gtpu_ipv4_tcp,           IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &inner_ipv4_tcp_tmplt},
 442         {iavf_pattern_eth_ipv6_gtpu_ipv4,               IAVF_RSS_TYPE_GTPU_IPV4,        &inner_ipv4_tmplt},
 443         {iavf_pattern_eth_ipv6_gtpu_ipv4_udp,           IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &inner_ipv4_udp_tmplt},
 444         {iavf_pattern_eth_ipv6_gtpu_ipv4_tcp,           IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &inner_ipv4_tcp_tmplt},
 445         {iavf_pattern_eth_ipv4_gtpu_eh_ipv4,            IAVF_RSS_TYPE_GTPU_IPV4,        &inner_ipv4_tmplt},
 446         {iavf_pattern_eth_ipv4_gtpu_eh_ipv4_udp,        IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &inner_ipv4_udp_tmplt},
 447         {iavf_pattern_eth_ipv4_gtpu_eh_ipv4_tcp,        IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &inner_ipv4_tcp_tmplt},
 448         {iavf_pattern_eth_ipv6_gtpu_eh_ipv4,            IAVF_RSS_TYPE_GTPU_IPV4,        &inner_ipv4_tmplt},
 449         {iavf_pattern_eth_ipv6_gtpu_eh_ipv4_udp,        IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &inner_ipv4_udp_tmplt},
 450         {iavf_pattern_eth_ipv6_gtpu_eh_ipv4_tcp,        IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &inner_ipv4_tcp_tmplt},
 451         {iavf_pattern_eth_ipv4_gre_ipv4_gtpu_ipv4,              IAVF_RSS_TYPE_GTPU_IPV4,        &second_inner_ipv4_tmplt},
 452         {iavf_pattern_eth_ipv4_gre_ipv4_gtpu_ipv4_udp,          IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &second_inner_ipv4_udp_tmplt},
 453         {iavf_pattern_eth_ipv4_gre_ipv4_gtpu_ipv4_tcp,          IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &second_inner_ipv4_tcp_tmplt},
 454         {iavf_pattern_eth_ipv4_gre_ipv6_gtpu_ipv4,              IAVF_RSS_TYPE_GTPU_IPV4,        &second_inner_ipv4_tmplt},
 455         {iavf_pattern_eth_ipv4_gre_ipv6_gtpu_ipv4_udp,          IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &second_inner_ipv4_udp_tmplt},
 456         {iavf_pattern_eth_ipv4_gre_ipv6_gtpu_ipv4_tcp,          IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &second_inner_ipv4_tcp_tmplt},
 457         {iavf_pattern_eth_ipv6_gre_ipv4_gtpu_ipv4,              IAVF_RSS_TYPE_GTPU_IPV4,        &second_inner_ipv4_tmplt},
 458         {iavf_pattern_eth_ipv6_gre_ipv4_gtpu_ipv4_udp,          IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &second_inner_ipv4_udp_tmplt},
 459         {iavf_pattern_eth_ipv6_gre_ipv4_gtpu_ipv4_tcp,          IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &second_inner_ipv4_tcp_tmplt},
 460         {iavf_pattern_eth_ipv6_gre_ipv6_gtpu_ipv4,              IAVF_RSS_TYPE_GTPU_IPV4,        &second_inner_ipv4_tmplt},
 461         {iavf_pattern_eth_ipv6_gre_ipv6_gtpu_ipv4_udp,          IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &second_inner_ipv4_udp_tmplt},
 462         {iavf_pattern_eth_ipv6_gre_ipv6_gtpu_ipv4_tcp,          IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &second_inner_ipv4_tcp_tmplt},
 463         {iavf_pattern_eth_ipv4_gre_ipv4_gtpu_eh_ipv4,           IAVF_RSS_TYPE_GTPU_IPV4,        &second_inner_ipv4_tmplt},
 464         {iavf_pattern_eth_ipv4_gre_ipv4_gtpu_eh_ipv4_udp,       IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &second_inner_ipv4_udp_tmplt},
 465         {iavf_pattern_eth_ipv4_gre_ipv4_gtpu_eh_ipv4_tcp,       IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &second_inner_ipv4_tcp_tmplt},
 466         {iavf_pattern_eth_ipv4_gre_ipv6_gtpu_eh_ipv4,           IAVF_RSS_TYPE_GTPU_IPV4,        &second_inner_ipv4_tmplt},
 467         {iavf_pattern_eth_ipv4_gre_ipv6_gtpu_eh_ipv4_udp,       IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &second_inner_ipv4_udp_tmplt},
 468         {iavf_pattern_eth_ipv4_gre_ipv6_gtpu_eh_ipv4_tcp,       IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &second_inner_ipv4_tcp_tmplt},
 469         {iavf_pattern_eth_ipv6_gre_ipv4_gtpu_eh_ipv4,           IAVF_RSS_TYPE_GTPU_IPV4,        &second_inner_ipv4_tmplt},
 470         {iavf_pattern_eth_ipv6_gre_ipv4_gtpu_eh_ipv4_udp,       IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &second_inner_ipv4_udp_tmplt},
 471         {iavf_pattern_eth_ipv6_gre_ipv4_gtpu_eh_ipv4_tcp,       IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &second_inner_ipv4_tcp_tmplt},
 472         {iavf_pattern_eth_ipv6_gre_ipv6_gtpu_eh_ipv4,           IAVF_RSS_TYPE_GTPU_IPV4,        &second_inner_ipv4_tmplt},
 473         {iavf_pattern_eth_ipv6_gre_ipv6_gtpu_eh_ipv4_udp,       IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &second_inner_ipv4_udp_tmplt},
 474         {iavf_pattern_eth_ipv6_gre_ipv6_gtpu_eh_ipv4_tcp,       IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &second_inner_ipv4_tcp_tmplt},
 475         {iavf_pattern_eth_ipv4_esp,                     IAVF_RSS_TYPE_IPV4_ESP,         &ipv4_esp_tmplt},
 476         {iavf_pattern_eth_ipv4_udp_esp,                 IAVF_RSS_TYPE_IPV4_ESP,         &ipv4_udp_esp_tmplt},
 477         {iavf_pattern_eth_ipv4_ah,                      IAVF_RSS_TYPE_IPV4_AH,          &ipv4_ah_tmplt},
 478         {iavf_pattern_eth_ipv4_l2tpv3,                  IAVF_RSS_TYPE_IPV4_L2TPV3,      &ipv4_l2tpv3_tmplt},
 479         {iavf_pattern_eth_ipv4_pfcp,                    IAVF_RSS_TYPE_IPV4_PFCP,        &ipv4_pfcp_tmplt},
 480         {iavf_pattern_eth_ipv4_gtpc,                    ETH_RSS_IPV4,                   &ipv4_udp_gtpc_tmplt},
 481         {iavf_pattern_eth_ecpri,                        ETH_RSS_ECPRI,                  &eth_ecpri_tmplt},
 482         {iavf_pattern_eth_ipv4_ecpri,                   ETH_RSS_ECPRI,                  &ipv4_ecpri_tmplt},
 483         {iavf_pattern_eth_ipv4_gre_ipv4,                IAVF_RSS_TYPE_INNER_IPV4,       &inner_ipv4_tmplt},
 484         {iavf_pattern_eth_ipv6_gre_ipv4,                IAVF_RSS_TYPE_INNER_IPV4, &inner_ipv4_tmplt},
 485         {iavf_pattern_eth_ipv4_gre_ipv4_tcp,    IAVF_RSS_TYPE_INNER_IPV4_TCP, &inner_ipv4_tcp_tmplt},
 486         {iavf_pattern_eth_ipv6_gre_ipv4_tcp,    IAVF_RSS_TYPE_INNER_IPV4_TCP, &inner_ipv4_tcp_tmplt},
 487         {iavf_pattern_eth_ipv4_gre_ipv4_udp,    IAVF_RSS_TYPE_INNER_IPV4_UDP, &inner_ipv4_udp_tmplt},
 488         {iavf_pattern_eth_ipv6_gre_ipv4_udp,    IAVF_RSS_TYPE_INNER_IPV4_UDP, &inner_ipv4_udp_tmplt},
 489         /* IPv6 */
 490         {iavf_pattern_eth_ipv6,                         IAVF_RSS_TYPE_OUTER_IPV6,       &outer_ipv6_tmplt},
 491         {iavf_pattern_eth_ipv6_frag_ext,                IAVF_RSS_TYPE_OUTER_IPV6_FRAG,  &outer_ipv6_frag_tmplt},
 492         {iavf_pattern_eth_ipv6_udp,                     IAVF_RSS_TYPE_OUTER_IPV6_UDP,   &outer_ipv6_udp_tmplt},
 493         {iavf_pattern_eth_ipv6_tcp,                     IAVF_RSS_TYPE_OUTER_IPV6_TCP,   &outer_ipv6_tcp_tmplt},
 494         {iavf_pattern_eth_ipv6_sctp,                    IAVF_RSS_TYPE_OUTER_IPV6_SCTP,  &outer_ipv6_sctp_tmplt},
 495         {iavf_pattern_eth_vlan_ipv6,                    IAVF_RSS_TYPE_VLAN_IPV6,        &outer_ipv6_tmplt},
 496         {iavf_pattern_eth_vlan_ipv6_frag_ext,           IAVF_RSS_TYPE_OUTER_IPV6_FRAG,  &outer_ipv6_frag_tmplt},
 497         {iavf_pattern_eth_vlan_ipv6_udp,                IAVF_RSS_TYPE_VLAN_IPV6_UDP,    &outer_ipv6_udp_tmplt},
 498         {iavf_pattern_eth_vlan_ipv6_tcp,                IAVF_RSS_TYPE_VLAN_IPV6_TCP,    &outer_ipv6_tcp_tmplt},
 499         {iavf_pattern_eth_vlan_ipv6_sctp,               IAVF_RSS_TYPE_VLAN_IPV6_SCTP,   &outer_ipv6_sctp_tmplt},
 500         {iavf_pattern_eth_ipv6_gtpu,                    ETH_RSS_IPV6,                   &outer_ipv6_udp_tmplt},
 501         {iavf_pattern_eth_ipv4_gtpu_ipv6,               IAVF_RSS_TYPE_GTPU_IPV6,        &inner_ipv6_tmplt},
 502         {iavf_pattern_eth_ipv4_gtpu_ipv6_udp,           IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &inner_ipv6_udp_tmplt},
 503         {iavf_pattern_eth_ipv4_gtpu_ipv6_tcp,           IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &inner_ipv6_tcp_tmplt},
 504         {iavf_pattern_eth_ipv6_gtpu_ipv6,               IAVF_RSS_TYPE_GTPU_IPV6,        &inner_ipv6_tmplt},
 505         {iavf_pattern_eth_ipv6_gtpu_ipv6_udp,           IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &inner_ipv6_udp_tmplt},
 506         {iavf_pattern_eth_ipv6_gtpu_ipv6_tcp,           IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &inner_ipv6_tcp_tmplt},
 507         {iavf_pattern_eth_ipv4_gtpu_eh_ipv6,            IAVF_RSS_TYPE_GTPU_IPV6,        &inner_ipv6_tmplt},
 508         {iavf_pattern_eth_ipv4_gtpu_eh_ipv6_udp,        IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &inner_ipv6_udp_tmplt},
 509         {iavf_pattern_eth_ipv4_gtpu_eh_ipv6_tcp,        IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &inner_ipv6_tcp_tmplt},
 510         {iavf_pattern_eth_ipv6_gtpu_eh_ipv6,            IAVF_RSS_TYPE_GTPU_IPV6,        &inner_ipv6_tmplt},
 511         {iavf_pattern_eth_ipv6_gtpu_eh_ipv6_udp,        IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &inner_ipv6_udp_tmplt},
 512         {iavf_pattern_eth_ipv6_gtpu_eh_ipv6_tcp,        IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &inner_ipv6_tcp_tmplt},
 513         {iavf_pattern_eth_ipv4_gre_ipv4_gtpu_ipv6,              IAVF_RSS_TYPE_GTPU_IPV6,        &second_inner_ipv6_tmplt},
 514         {iavf_pattern_eth_ipv4_gre_ipv4_gtpu_ipv6_udp,          IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &second_inner_ipv6_udp_tmplt},
 515         {iavf_pattern_eth_ipv4_gre_ipv4_gtpu_ipv6_tcp,          IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &second_inner_ipv6_tcp_tmplt},
 516         {iavf_pattern_eth_ipv4_gre_ipv6_gtpu_ipv6,              IAVF_RSS_TYPE_GTPU_IPV6,        &second_inner_ipv6_tmplt},
 517         {iavf_pattern_eth_ipv4_gre_ipv6_gtpu_ipv6_udp,          IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &second_inner_ipv6_udp_tmplt},
 518         {iavf_pattern_eth_ipv4_gre_ipv6_gtpu_ipv6_tcp,          IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &second_inner_ipv6_tcp_tmplt},
 519         {iavf_pattern_eth_ipv6_gre_ipv4_gtpu_ipv6,              IAVF_RSS_TYPE_GTPU_IPV6,        &second_inner_ipv6_tmplt},
 520         {iavf_pattern_eth_ipv6_gre_ipv4_gtpu_ipv6_udp,          IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &second_inner_ipv6_udp_tmplt},
 521         {iavf_pattern_eth_ipv6_gre_ipv4_gtpu_ipv6_tcp,          IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &second_inner_ipv6_tcp_tmplt},
 522         {iavf_pattern_eth_ipv6_gre_ipv6_gtpu_ipv6,              IAVF_RSS_TYPE_GTPU_IPV6,        &second_inner_ipv6_tmplt},
 523         {iavf_pattern_eth_ipv6_gre_ipv6_gtpu_ipv6_udp,          IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &second_inner_ipv6_udp_tmplt},
 524         {iavf_pattern_eth_ipv6_gre_ipv6_gtpu_ipv6_tcp,          IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &second_inner_ipv6_tcp_tmplt},
 525         {iavf_pattern_eth_ipv4_gre_ipv4_gtpu_eh_ipv6,           IAVF_RSS_TYPE_GTPU_IPV6,        &second_inner_ipv6_tmplt},
 526         {iavf_pattern_eth_ipv4_gre_ipv4_gtpu_eh_ipv6_udp,       IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &second_inner_ipv6_udp_tmplt},
 527         {iavf_pattern_eth_ipv4_gre_ipv4_gtpu_eh_ipv6_tcp,       IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &second_inner_ipv6_tcp_tmplt},
 528         {iavf_pattern_eth_ipv4_gre_ipv6_gtpu_eh_ipv6,           IAVF_RSS_TYPE_GTPU_IPV6,        &second_inner_ipv6_tmplt},
 529         {iavf_pattern_eth_ipv4_gre_ipv6_gtpu_eh_ipv6_udp,       IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &second_inner_ipv6_udp_tmplt},
 530         {iavf_pattern_eth_ipv4_gre_ipv6_gtpu_eh_ipv6_tcp,       IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &second_inner_ipv6_tcp_tmplt},
 531         {iavf_pattern_eth_ipv6_gre_ipv4_gtpu_eh_ipv6,           IAVF_RSS_TYPE_GTPU_IPV6,        &second_inner_ipv6_tmplt},
 532         {iavf_pattern_eth_ipv6_gre_ipv4_gtpu_eh_ipv6_udp,       IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &second_inner_ipv6_udp_tmplt},
 533         {iavf_pattern_eth_ipv6_gre_ipv4_gtpu_eh_ipv6_tcp,       IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &second_inner_ipv6_tcp_tmplt},
 534         {iavf_pattern_eth_ipv6_gre_ipv6_gtpu_eh_ipv6,           IAVF_RSS_TYPE_GTPU_IPV6,        &second_inner_ipv6_tmplt},
 535         {iavf_pattern_eth_ipv6_gre_ipv6_gtpu_eh_ipv6_udp,       IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &second_inner_ipv6_udp_tmplt},
 536         {iavf_pattern_eth_ipv6_gre_ipv6_gtpu_eh_ipv6_tcp,       IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &second_inner_ipv6_tcp_tmplt},
 537         {iavf_pattern_eth_ipv6_esp,                     IAVF_RSS_TYPE_IPV6_ESP,         &ipv6_esp_tmplt},
 538         {iavf_pattern_eth_ipv6_udp_esp,                 IAVF_RSS_TYPE_IPV6_ESP,         &ipv6_udp_esp_tmplt},
 539         {iavf_pattern_eth_ipv6_ah,                      IAVF_RSS_TYPE_IPV6_AH,          &ipv6_ah_tmplt},
 540         {iavf_pattern_eth_ipv6_l2tpv3,                  IAVF_RSS_TYPE_IPV6_L2TPV3,      &ipv6_l2tpv3_tmplt},
 541         {iavf_pattern_eth_ipv6_pfcp,                    IAVF_RSS_TYPE_IPV6_PFCP,        &ipv6_pfcp_tmplt},
 542         {iavf_pattern_eth_ipv6_gtpc,                    ETH_RSS_IPV6,                   &ipv6_udp_gtpc_tmplt},
 543         {iavf_pattern_eth_ipv4_gre_ipv6,                IAVF_RSS_TYPE_INNER_IPV6,       &inner_ipv6_tmplt},
 544         {iavf_pattern_eth_ipv6_gre_ipv6,                IAVF_RSS_TYPE_INNER_IPV6, &inner_ipv6_tmplt},
 545         {iavf_pattern_eth_ipv4_gre_ipv6_tcp,    IAVF_RSS_TYPE_INNER_IPV6_TCP, &inner_ipv6_tcp_tmplt},
 546         {iavf_pattern_eth_ipv6_gre_ipv6_tcp,    IAVF_RSS_TYPE_INNER_IPV6_TCP, &inner_ipv6_tcp_tmplt},
 547         {iavf_pattern_eth_ipv4_gre_ipv6_udp,    IAVF_RSS_TYPE_INNER_IPV6_UDP, &inner_ipv6_udp_tmplt},
 548         {iavf_pattern_eth_ipv6_gre_ipv6_udp,    IAVF_RSS_TYPE_INNER_IPV6_UDP, &inner_ipv6_udp_tmplt},
 549 };
 550
 551 static struct iavf_flow_engine iavf_hash_engine = {
 552         .init = iavf_hash_init,
 553         .create = iavf_hash_create,
 554         .destroy = iavf_hash_destroy,
 555         .uninit = iavf_hash_uninit,
 556         .free = iavf_hash_free,
 557         .type = IAVF_FLOW_ENGINE_HASH,
 558 };
 559
 560 /* Register parser for comms package. */
 561 static struct iavf_flow_parser iavf_hash_parser = {
 562         .engine = &iavf_hash_engine,
 563         .array = iavf_hash_pattern_list,
 564         .array_len = RTE_DIM(iavf_hash_pattern_list),
 565         .parse_pattern_action = iavf_hash_parse_pattern_action,
 566         .stage = IAVF_FLOW_STAGE_RSS,
 567 };
 568
 569 int
 570 iavf_rss_hash_set(struct iavf_adapter *ad, uint64_t rss_hf, bool add)
 571 {
 572         struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(ad);
 573         struct virtchnl_rss_cfg rss_cfg;
 574
 575 #define IAVF_RSS_HF_ALL ( \
 576         ETH_RSS_IPV4 | \
 577         ETH_RSS_IPV6 | \
 578         ETH_RSS_NONFRAG_IPV4_UDP | \
 579         ETH_RSS_NONFRAG_IPV6_UDP | \
 580         ETH_RSS_NONFRAG_IPV4_TCP | \
 581         ETH_RSS_NONFRAG_IPV6_TCP | \
 582         ETH_RSS_NONFRAG_IPV4_SCTP | \
 583         ETH_RSS_NONFRAG_IPV6_SCTP)
 584
 585         rss_cfg.rss_algorithm = VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC;
 586         if (rss_hf & ETH_RSS_IPV4) {
 587                 rss_cfg.proto_hdrs = inner_ipv4_tmplt;
 588                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 589         }
 590
 591         if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
 592                 rss_cfg.proto_hdrs = inner_ipv4_udp_tmplt;
 593                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 594         }
 595
 596         if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
 597                 rss_cfg.proto_hdrs = inner_ipv4_tcp_tmplt;
 598                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 599         }
 600
 601         if (rss_hf & ETH_RSS_NONFRAG_IPV4_SCTP) {
 602                 rss_cfg.proto_hdrs = inner_ipv4_sctp_tmplt;
 603                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 604         }
 605
 606         if (rss_hf & ETH_RSS_IPV6) {
 607                 rss_cfg.proto_hdrs = inner_ipv6_tmplt;
 608                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 609         }
 610
 611         if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) {
 612                 rss_cfg.proto_hdrs = inner_ipv6_udp_tmplt;
 613                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 614         }
 615
 616         if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) {
 617                 rss_cfg.proto_hdrs = inner_ipv6_tcp_tmplt;
 618                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 619         }
 620
 621         if (rss_hf & ETH_RSS_NONFRAG_IPV6_SCTP) {
 622                 rss_cfg.proto_hdrs = inner_ipv6_sctp_tmplt;
 623                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 624         }
 625
 626         if (rss_hf & ETH_RSS_FRAG_IPV4) {
 627                 struct virtchnl_proto_hdrs hdr = {
 628                         .tunnel_level = TUNNEL_LEVEL_OUTER,
 629                         .count = 3,
 630                         .proto_hdr = {
 631                                 proto_hdr_eth,
 632                                 proto_hdr_ipv4,
 633                                 {
 634                                         VIRTCHNL_PROTO_HDR_IPV4_FRAG,
 635                                         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_FRAG_PKID),
 636                                         {BUFF_NOUSED},
 637                                 },
 638                         },
 639                 };
 640                 rss_cfg.proto_hdrs = hdr;
 641                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 642         }
 643
 644         if (rss_hf & ETH_RSS_FRAG_IPV6) {
 645                 struct virtchnl_proto_hdrs hdr = {
 646                         .tunnel_level = TUNNEL_LEVEL_OUTER,
 647                         .count = 3,
 648                         .proto_hdr = {
 649                                 proto_hdr_eth,
 650                                 proto_hdr_ipv6,
 651                                 proto_hdr_ipv6_frag,
 652                         },
 653                 };
 654                 rss_cfg.proto_hdrs = hdr;
 655                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 656         }
 657
 658         vf->rss_hf = rss_hf & IAVF_RSS_HF_ALL;
 659         return 0;
 660 }
 661
 662 RTE_INIT(iavf_hash_engine_init)
 663 {
 664         struct iavf_flow_engine *engine = &iavf_hash_engine;
 665
 666         iavf_register_flow_engine(engine);
 667 }
 668
 669 static int
 670 iavf_hash_init(struct iavf_adapter *ad)
 671 {
 672         struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
 673         struct iavf_flow_parser *parser;
 674         int ret;
 675
 676         if (vf->vf_reset)
 677                 return -EIO;
 678
 679         if (!vf->vf_res)
 680                 return -EINVAL;
 681
 682         if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF))
 683                 return -ENOTSUP;
 684
 685         parser = &iavf_hash_parser;
 686
 687         ret = iavf_register_parser(parser, ad);
 688         if (ret) {
 689                 PMD_DRV_LOG(ERR, "fail to register hash parser");
 690                 return ret;
 691         }
 692
 693         return ret;
 694 }
 695
 696 static int
 697 iavf_hash_parse_pattern(const struct rte_flow_item pattern[], uint64_t *phint,
 698                         struct rte_flow_error *error)
 699 {
 700         const struct rte_flow_item *item = pattern;
 701         const struct rte_flow_item_gtp_psc *psc;
 702         const struct rte_flow_item_ecpri *ecpri;
 703         struct rte_ecpri_common_hdr ecpri_common;
 704
 705         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 706                 if (item->last) {
 707                         rte_flow_error_set(error, EINVAL,
 708                                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 709                                            "Not support range");
 710                         return -rte_errno;
 711                 }
 712
 713                 switch (item->type) {
 714                 case RTE_FLOW_ITEM_TYPE_IPV4:
 715                         if (!(*phint & IAVF_PHINT_GTPU_MSK) && !(*phint & IAVF_PHINT_GRE))
 716                                 *phint |= IAVF_PHINT_OUTER_IPV4;
 717                         if ((*phint & IAVF_PHINT_GRE) && !(*phint & IAVF_PHINT_GTPU_MSK))
 718                                 *phint |= IAVF_PHINT_MID_IPV4;
 719                         break;
 720                 case RTE_FLOW_ITEM_TYPE_IPV6:
 721                         if (!(*phint & IAVF_PHINT_GTPU_MSK) && !(*phint & IAVF_PHINT_GRE))
 722                                 *phint |= IAVF_PHINT_OUTER_IPV6;
 723                         if ((*phint & IAVF_PHINT_GRE) && !(*phint & IAVF_PHINT_GTPU_MSK))
 724                                 *phint |= IAVF_PHINT_MID_IPV6;
 725                         break;
 726                 case RTE_FLOW_ITEM_TYPE_GTPU:
 727                         *phint |= IAVF_PHINT_GTPU;
 728                         break;
 729                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
 730                         *phint |= IAVF_PHINT_GTPU_EH;
 731                         psc = item->spec;
 732                         if (!psc)
 733                                 break;
 734                         else if (psc->pdu_type == IAVF_GTPU_EH_UPLINK)
 735                                 *phint |= IAVF_PHINT_GTPU_EH_UP;
 736                         else if (psc->pdu_type == IAVF_GTPU_EH_DWNLINK)
 737                                 *phint |= IAVF_PHINT_GTPU_EH_DWN;
 738                         break;
 739                 case RTE_FLOW_ITEM_TYPE_ECPRI:
 740                         ecpri = item->spec;
 741                         if (!ecpri)
 742                                 break;
 743
 744                         ecpri_common.u32 = rte_be_to_cpu_32(ecpri->hdr.common.u32);
 745
 746                         if (ecpri_common.type !=
 747                                  RTE_ECPRI_MSG_TYPE_IQ_DATA) {
 748                                 rte_flow_error_set(error, EINVAL,
 749                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 750                                         "Unsupported common type.");
 751                                 return -rte_errno;
 752                         }
 753                         break;
 754                 case RTE_FLOW_ITEM_TYPE_GRE:
 755                         *phint |= IAVF_PHINT_GRE;
 756                 default:
 757                         break;
 758                 }
 759         }
 760
 761         return 0;
 762 }
 763
 764 #define REFINE_PROTO_FLD(op, fld) \
 765         VIRTCHNL_##op##_PROTO_HDR_FIELD(hdr, VIRTCHNL_PROTO_HDR_##fld)
 766 #define REPALCE_PROTO_FLD(fld_1, fld_2) \
 767 do { \
 768         REFINE_PROTO_FLD(DEL, fld_1);   \
 769         REFINE_PROTO_FLD(ADD, fld_2);   \
 770 } while (0)
 771
 772 static void
 773 iavf_hash_add_fragment_hdr(struct virtchnl_proto_hdrs *hdrs, int layer)
 774 {
 775         struct virtchnl_proto_hdr *hdr1;
 776         struct virtchnl_proto_hdr *hdr2;
 777         int i;
 778
 779         if (layer < 0 || layer > hdrs->count)
 780                 return;
 781
 782         /* shift headers layer */
 783         for (i = hdrs->count; i >= layer; i--) {
 784                 hdr1 = &hdrs->proto_hdr[i];
 785                 hdr2 = &hdrs->proto_hdr[i - 1];
 786                 *hdr1 = *hdr2;
 787         }
 788
 789         /* adding dummy fragment header */
 790         hdr1 = &hdrs->proto_hdr[layer];
 791         VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, IPV4_FRAG);
 792         hdrs->count = ++layer;
 793 }
 794
 795 /* refine proto hdrs base on l2, l3, l4 rss type */
 796 static void
 797 iavf_refine_proto_hdrs_l234(struct virtchnl_proto_hdrs *proto_hdrs,
 798                             uint64_t rss_type)
 799 {
 800         struct virtchnl_proto_hdr *hdr;
 801         int i;
 802
 803         for (i = 0; i < proto_hdrs->count; i++) {
 804                 hdr = &proto_hdrs->proto_hdr[i];
 805                 switch (hdr->type) {
 806                 case VIRTCHNL_PROTO_HDR_ETH:
 807                         if (!(rss_type & ETH_RSS_ETH))
 808                                 hdr->field_selector = 0;
 809                         else if (rss_type & ETH_RSS_L2_SRC_ONLY)
 810                                 REFINE_PROTO_FLD(DEL, ETH_DST);
 811                         else if (rss_type & ETH_RSS_L2_DST_ONLY)
 812                                 REFINE_PROTO_FLD(DEL, ETH_SRC);
 813                         break;
 814                 case VIRTCHNL_PROTO_HDR_IPV4:
 815                         if (rss_type &
 816                             (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 817                              ETH_RSS_NONFRAG_IPV4_UDP |
 818                              ETH_RSS_NONFRAG_IPV4_TCP |
 819                              ETH_RSS_NONFRAG_IPV4_SCTP)) {
 820                                 if (rss_type & ETH_RSS_FRAG_IPV4) {
 821                                         iavf_hash_add_fragment_hdr(proto_hdrs, i + 1);
 822                                 } else if (rss_type & ETH_RSS_L3_SRC_ONLY) {
 823                                         REFINE_PROTO_FLD(DEL, IPV4_DST);
 824                                 } else if (rss_type & ETH_RSS_L3_DST_ONLY) {
 825                                         REFINE_PROTO_FLD(DEL, IPV4_SRC);
 826                                 } else if (rss_type &
 827                                            (ETH_RSS_L4_SRC_ONLY |
 828                                             ETH_RSS_L4_DST_ONLY)) {
 829                                         REFINE_PROTO_FLD(DEL, IPV4_DST);
 830                                         REFINE_PROTO_FLD(DEL, IPV4_SRC);
 831                                 }
 832                         } else {
 833                                 hdr->field_selector = 0;
 834                         }
 835                         break;
 836                 case VIRTCHNL_PROTO_HDR_IPV4_FRAG:
 837                         if (rss_type &
 838                             (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 839                              ETH_RSS_NONFRAG_IPV4_UDP |
 840                              ETH_RSS_NONFRAG_IPV4_TCP |
 841                              ETH_RSS_NONFRAG_IPV4_SCTP)) {
 842                                 if (rss_type & ETH_RSS_FRAG_IPV4)
 843                                         REFINE_PROTO_FLD(ADD, IPV4_FRAG_PKID);
 844                         } else {
 845                                 hdr->field_selector = 0;
 846                         }
 847                         break;
 848                 case VIRTCHNL_PROTO_HDR_IPV6:
 849                         if (rss_type &
 850                             (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 851                              ETH_RSS_NONFRAG_IPV6_UDP |
 852                              ETH_RSS_NONFRAG_IPV6_TCP |
 853                              ETH_RSS_NONFRAG_IPV6_SCTP)) {
 854                                 if (rss_type & ETH_RSS_L3_SRC_ONLY) {
 855                                         REFINE_PROTO_FLD(DEL, IPV6_DST);
 856                                 } else if (rss_type & ETH_RSS_L3_DST_ONLY) {
 857                                         REFINE_PROTO_FLD(DEL, IPV6_SRC);
 858                                 } else if (rss_type &
 859                                            (ETH_RSS_L4_SRC_ONLY |
 860                                             ETH_RSS_L4_DST_ONLY)) {
 861                                         REFINE_PROTO_FLD(DEL, IPV6_DST);
 862                                         REFINE_PROTO_FLD(DEL, IPV6_SRC);
 863                                 }
 864                         } else {
 865                                 hdr->field_selector = 0;
 866                         }
 867                         if (rss_type & RTE_ETH_RSS_L3_PRE64) {
 868                                 if (REFINE_PROTO_FLD(TEST, IPV6_SRC))
 869                                         REPALCE_PROTO_FLD(IPV6_SRC,
 870                                                           IPV6_PREFIX64_SRC);
 871                                 if (REFINE_PROTO_FLD(TEST, IPV6_DST))
 872                                         REPALCE_PROTO_FLD(IPV6_DST,
 873                                                           IPV6_PREFIX64_DST);
 874                         }
 875                         break;
 876                 case VIRTCHNL_PROTO_HDR_IPV6_EH_FRAG:
 877                         if (rss_type & ETH_RSS_FRAG_IPV6)
 878                                 REFINE_PROTO_FLD(ADD, IPV6_EH_FRAG_PKID);
 879                         else
 880                                 hdr->field_selector = 0;
 881
 882                         break;
 883                 case VIRTCHNL_PROTO_HDR_UDP:
 884                         if (rss_type &
 885                             (ETH_RSS_NONFRAG_IPV4_UDP |
 886                              ETH_RSS_NONFRAG_IPV6_UDP)) {
 887                                 if (rss_type & ETH_RSS_L4_SRC_ONLY)
 888                                         REFINE_PROTO_FLD(DEL, UDP_DST_PORT);
 889                                 else if (rss_type & ETH_RSS_L4_DST_ONLY)
 890                                         REFINE_PROTO_FLD(DEL, UDP_SRC_PORT);
 891                                 else if (rss_type &
 892                                          (ETH_RSS_L3_SRC_ONLY |
 893                                           ETH_RSS_L3_DST_ONLY))
 894                                         hdr->field_selector = 0;
 895                         } else {
 896                                 hdr->field_selector = 0;
 897                         }
 898                         break;
 899                 case VIRTCHNL_PROTO_HDR_TCP:
 900                         if (rss_type &
 901                             (ETH_RSS_NONFRAG_IPV4_TCP |
 902                              ETH_RSS_NONFRAG_IPV6_TCP)) {
 903                                 if (rss_type & ETH_RSS_L4_SRC_ONLY)
 904                                         REFINE_PROTO_FLD(DEL, TCP_DST_PORT);
 905                                 else if (rss_type & ETH_RSS_L4_DST_ONLY)
 906                                         REFINE_PROTO_FLD(DEL, TCP_SRC_PORT);
 907                                 else if (rss_type &
 908                                          (ETH_RSS_L3_SRC_ONLY |
 909                                           ETH_RSS_L3_DST_ONLY))
 910                                         hdr->field_selector = 0;
 911                         } else {
 912                                 hdr->field_selector = 0;
 913                         }
 914                         break;
 915                 case VIRTCHNL_PROTO_HDR_SCTP:
 916                         if (rss_type &
 917                             (ETH_RSS_NONFRAG_IPV4_SCTP |
 918                              ETH_RSS_NONFRAG_IPV6_SCTP)) {
 919                                 if (rss_type & ETH_RSS_L4_SRC_ONLY)
 920                                         REFINE_PROTO_FLD(DEL, SCTP_DST_PORT);
 921                                 else if (rss_type & ETH_RSS_L4_DST_ONLY)
 922                                         REFINE_PROTO_FLD(DEL, SCTP_SRC_PORT);
 923                                 else if (rss_type &
 924                                          (ETH_RSS_L3_SRC_ONLY |
 925                                           ETH_RSS_L3_DST_ONLY))
 926                                         hdr->field_selector = 0;
 927                         } else {
 928                                 hdr->field_selector = 0;
 929                         }
 930                         break;
 931                 case VIRTCHNL_PROTO_HDR_S_VLAN:
 932                         if (!(rss_type & ETH_RSS_S_VLAN))
 933                                 hdr->field_selector = 0;
 934                         break;
 935                 case VIRTCHNL_PROTO_HDR_C_VLAN:
 936                         if (!(rss_type & ETH_RSS_C_VLAN))
 937                                 hdr->field_selector = 0;
 938                         break;
 939                 case VIRTCHNL_PROTO_HDR_L2TPV3:
 940                         if (!(rss_type & ETH_RSS_L2TPV3))
 941                                 hdr->field_selector = 0;
 942                         break;
 943                 case VIRTCHNL_PROTO_HDR_ESP:
 944                         if (!(rss_type & ETH_RSS_ESP))
 945                                 hdr->field_selector = 0;
 946                         break;
 947                 case VIRTCHNL_PROTO_HDR_AH:
 948                         if (!(rss_type & ETH_RSS_AH))
 949                                 hdr->field_selector = 0;
 950                         break;
 951                 case VIRTCHNL_PROTO_HDR_PFCP:
 952                         if (!(rss_type & ETH_RSS_PFCP))
 953                                 hdr->field_selector = 0;
 954                         break;
 955                 case VIRTCHNL_PROTO_HDR_ECPRI:
 956                         if (!(rss_type & ETH_RSS_ECPRI))
 957                                 hdr->field_selector = 0;
 958                         break;
 959                 default:
 960                         break;
 961                 }
 962         }
 963 }
 964
 965 /* refine proto hdrs base on gtpu rss type */
 966 static void
 967 iavf_refine_proto_hdrs_gtpu(struct virtchnl_proto_hdrs *proto_hdrs,
 968                             uint64_t rss_type)
 969 {
 970         struct virtchnl_proto_hdr *hdr;
 971         int i;
 972
 973         if (!(rss_type & ETH_RSS_GTPU))
 974                 return;
 975
 976         for (i = 0; i < proto_hdrs->count; i++) {
 977                 hdr = &proto_hdrs->proto_hdr[i];
 978                 switch (hdr->type) {
 979                 case VIRTCHNL_PROTO_HDR_GTPU_IP:
 980                         REFINE_PROTO_FLD(ADD, GTPU_IP_TEID);
 981                         break;
 982                 default:
 983                         break;
 984                 }
 985         }
 986 }
 987
 988 static void
 989 iavf_refine_proto_hdrs_by_pattern(struct virtchnl_proto_hdrs *proto_hdrs,
 990                                   uint64_t phint)
 991 {
 992         struct virtchnl_proto_hdr *hdr1;
 993         struct virtchnl_proto_hdr *hdr2;
 994         int i, shift_count = 1;
 995         int tun_lvl = proto_hdrs->tunnel_level;
 996
 997         if (!(phint & IAVF_PHINT_GTPU_MSK) && !(phint & IAVF_PHINT_GRE))
 998                 return;
 999
1000         while (tun_lvl) {
1001                 if (phint & IAVF_PHINT_LAYERS_MSK)
1002                         shift_count = 2;
1003
1004                 /* shift headers layer */
1005                 for (i = proto_hdrs->count - 1 + shift_count;
1006                      i > shift_count - 1; i--) {
1007                         hdr1 = &proto_hdrs->proto_hdr[i];
1008                         hdr2 = &proto_hdrs->proto_hdr[i - shift_count];
1009                         *hdr1 = *hdr2;
1010                 }
1011
1012                 if (shift_count == 1) {
1013                         /* adding tunnel header at layer 0 */
1014                         hdr1 = &proto_hdrs->proto_hdr[0];
1015                 } else {
1016                         /* adding tunnel header and outer ip header */
1017                         hdr1 = &proto_hdrs->proto_hdr[1];
1018                         hdr2 = &proto_hdrs->proto_hdr[0];
1019                         hdr2->field_selector = 0;
1020                         proto_hdrs->count++;
1021                         tun_lvl--;
1022
1023                         if (tun_lvl == TUNNEL_LEVEL_OUTER) {
1024                                 if (phint & IAVF_PHINT_OUTER_IPV4)
1025                                         VIRTCHNL_SET_PROTO_HDR_TYPE(hdr2, IPV4);
1026                                 else if (phint & IAVF_PHINT_OUTER_IPV6)
1027                                         VIRTCHNL_SET_PROTO_HDR_TYPE(hdr2, IPV6);
1028                         } else if (tun_lvl == TUNNEL_LEVEL_INNER) {
1029                                 if (phint & IAVF_PHINT_MID_IPV4)
1030                                         VIRTCHNL_SET_PROTO_HDR_TYPE(hdr2, IPV4);
1031                                 else if (phint & IAVF_PHINT_MID_IPV6)
1032                                         VIRTCHNL_SET_PROTO_HDR_TYPE(hdr2, IPV6);
1033                         }
1034                 }
1035
1036                 hdr1->field_selector = 0;
1037                 proto_hdrs->count++;
1038
1039                 if (phint & IAVF_PHINT_GTPU_EH_DWN)
1040                         VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_EH_PDU_DWN);
1041                 else if (phint & IAVF_PHINT_GTPU_EH_UP)
1042                         VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_EH_PDU_UP);
1043                 else if (phint & IAVF_PHINT_GTPU_EH)
1044                         VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_EH);
1045                 else if (phint & IAVF_PHINT_GTPU)
1046                         VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_IP);
1047
1048                 if (phint & IAVF_PHINT_GRE) {
1049                         if (phint & IAVF_PHINT_GTPU) {
1050                                 /* if GTPoGRE, add GRE header at the outer tunnel  */
1051                                 if (tun_lvl == TUNNEL_LEVEL_OUTER)
1052                                         VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GRE);
1053                         } else {
1054                                         VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GRE);
1055                         }
1056                 }
1057         }
1058         proto_hdrs->tunnel_level = tun_lvl;
1059 }
1060
1061 static void iavf_refine_proto_hdrs(struct virtchnl_proto_hdrs *proto_hdrs,
1062                                    uint64_t rss_type, uint64_t phint)
1063 {
1064         iavf_refine_proto_hdrs_l234(proto_hdrs, rss_type);
1065         iavf_refine_proto_hdrs_by_pattern(proto_hdrs, phint);
1066         iavf_refine_proto_hdrs_gtpu(proto_hdrs, rss_type);
1067 }
1068
1069 static uint64_t invalid_rss_comb[] = {
1070         ETH_RSS_IPV4 | ETH_RSS_NONFRAG_IPV4_UDP,
1071         ETH_RSS_IPV4 | ETH_RSS_NONFRAG_IPV4_TCP,
1072         ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_UDP,
1073         ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_TCP,
1074         RTE_ETH_RSS_L3_PRE32 | RTE_ETH_RSS_L3_PRE40 |
1075         RTE_ETH_RSS_L3_PRE48 | RTE_ETH_RSS_L3_PRE56 |
1076         RTE_ETH_RSS_L3_PRE96
1077 };
1078
1079 struct rss_attr_type {
1080         uint64_t attr;
1081         uint64_t type;
1082 };
1083
1084 #define VALID_RSS_IPV4_L4       (ETH_RSS_NONFRAG_IPV4_UDP       | \
1085                                  ETH_RSS_NONFRAG_IPV4_TCP       | \
1086                                  ETH_RSS_NONFRAG_IPV4_SCTP)
1087
1088 #define VALID_RSS_IPV6_L4       (ETH_RSS_NONFRAG_IPV6_UDP       | \
1089                                  ETH_RSS_NONFRAG_IPV6_TCP       | \
1090                                  ETH_RSS_NONFRAG_IPV6_SCTP)
1091
1092 #define VALID_RSS_IPV4          (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | \
1093                                  VALID_RSS_IPV4_L4)
1094 #define VALID_RSS_IPV6          (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | \
1095                                  VALID_RSS_IPV6_L4)
1096 #define VALID_RSS_L3            (VALID_RSS_IPV4 | VALID_RSS_IPV6)
1097 #define VALID_RSS_L4            (VALID_RSS_IPV4_L4 | VALID_RSS_IPV6_L4)
1098
1099 #define VALID_RSS_ATTR          (ETH_RSS_L3_SRC_ONLY    | \
1100                                  ETH_RSS_L3_DST_ONLY    | \
1101                                  ETH_RSS_L4_SRC_ONLY    | \
1102                                  ETH_RSS_L4_DST_ONLY    | \
1103                                  ETH_RSS_L2_SRC_ONLY    | \
1104                                  ETH_RSS_L2_DST_ONLY    | \
1105                                  RTE_ETH_RSS_L3_PRE64)
1106
1107 #define INVALID_RSS_ATTR        (RTE_ETH_RSS_L3_PRE32   | \
1108                                  RTE_ETH_RSS_L3_PRE40   | \
1109                                  RTE_ETH_RSS_L3_PRE48   | \
1110                                  RTE_ETH_RSS_L3_PRE56   | \
1111                                  RTE_ETH_RSS_L3_PRE96)
1112
1113 static struct rss_attr_type rss_attr_to_valid_type[] = {
1114         {ETH_RSS_L2_SRC_ONLY | ETH_RSS_L2_DST_ONLY,     ETH_RSS_ETH},
1115         {ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY,     VALID_RSS_L3},
1116         {ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY,     VALID_RSS_L4},
1117         /* current ipv6 prefix only supports prefix 64 bits*/
1118         {RTE_ETH_RSS_L3_PRE64,                          VALID_RSS_IPV6},
1119         {INVALID_RSS_ATTR,                              0}
1120 };
1121
1122 static bool
1123 iavf_any_invalid_rss_type(enum rte_eth_hash_function rss_func,
1124                           uint64_t rss_type, uint64_t allow_rss_type)
1125 {
1126         uint32_t i;
1127
1128         /**
1129          * Check if l3/l4 SRC/DST_ONLY is set for SYMMETRIC_TOEPLITZ
1130          * hash function.
1131          */
1132         if (rss_func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) {
1133                 if (rss_type & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY |
1134                     ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY))
1135                         return true;
1136
1137                 if (!(rss_type &
1138                    (ETH_RSS_IPV4 | ETH_RSS_IPV6 |
1139                     ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_NONFRAG_IPV6_UDP |
1140                     ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_NONFRAG_IPV6_TCP |
1141                     ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_NONFRAG_IPV6_SCTP)))
1142                         return true;
1143         }
1144
1145         /* check invalid combination */
1146         for (i = 0; i < RTE_DIM(invalid_rss_comb); i++) {
1147                 if (__builtin_popcountll(rss_type & invalid_rss_comb[i]) > 1)
1148                         return true;
1149         }
1150
1151         /* check invalid RSS attribute */
1152         for (i = 0; i < RTE_DIM(rss_attr_to_valid_type); i++) {
1153                 struct rss_attr_type *rat = &rss_attr_to_valid_type[i];
1154
1155                 if (rat->attr & rss_type && !(rat->type & rss_type))
1156                         return true;
1157         }
1158
1159         /* check not allowed RSS type */
1160         rss_type &= ~VALID_RSS_ATTR;
1161
1162         return ((rss_type & allow_rss_type) != rss_type);
1163 }
1164
1165 static int
1166 iavf_hash_parse_action(struct iavf_pattern_match_item *match_item,
1167                        const struct rte_flow_action actions[],
1168                        uint64_t pattern_hint, void **meta,
1169                        struct rte_flow_error *error)
1170 {
1171         struct iavf_rss_meta *rss_meta = (struct iavf_rss_meta *)*meta;
1172         struct virtchnl_proto_hdrs *proto_hdrs;
1173         enum rte_flow_action_type action_type;
1174         const struct rte_flow_action_rss *rss;
1175         const struct rte_flow_action *action;
1176         uint64_t rss_type;
1177
1178         /* Supported action is RSS. */
1179         for (action = actions; action->type !=
1180                 RTE_FLOW_ACTION_TYPE_END; action++) {
1181                 action_type = action->type;
1182                 switch (action_type) {
1183                 case RTE_FLOW_ACTION_TYPE_RSS:
1184                         rss = action->conf;
1185                         rss_type = rss->types;
1186
1187                         if (rss->func ==
1188                             RTE_ETH_HASH_FUNCTION_SIMPLE_XOR){
1189                                 rss_meta->rss_algorithm =
1190                                         VIRTCHNL_RSS_ALG_XOR_ASYMMETRIC;
1191                                 return rte_flow_error_set(error, ENOTSUP,
1192                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
1193                                         "function simple_xor is not supported");
1194                         } else if (rss->func ==
1195                                    RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) {
1196                                 rss_meta->rss_algorithm =
1197                                         VIRTCHNL_RSS_ALG_TOEPLITZ_SYMMETRIC;
1198                         } else {
1199                                 rss_meta->rss_algorithm =
1200                                         VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC;
1201                         }
1202
1203                         if (rss->level)
1204                                 return rte_flow_error_set(error, ENOTSUP,
1205                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
1206                                         "a nonzero RSS encapsulation level is not supported");
1207
1208                         if (rss->key_len)
1209                                 return rte_flow_error_set(error, ENOTSUP,
1210                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
1211                                         "a nonzero RSS key_len is not supported");
1212
1213                         if (rss->queue_num)
1214                                 return rte_flow_error_set(error, ENOTSUP,
1215                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
1216                                         "a non-NULL RSS queue is not supported");
1217
1218                         /**
1219                          * Check simultaneous use of SRC_ONLY and DST_ONLY
1220                          * of the same level.
1221                          */
1222                         rss_type = rte_eth_rss_hf_refine(rss_type);
1223
1224                         if (iavf_any_invalid_rss_type(rss->func, rss_type,
1225                                         match_item->input_set_mask))
1226                                 return rte_flow_error_set(error, ENOTSUP,
1227                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1228                                                 action, "RSS type not supported");
1229                         proto_hdrs = match_item->meta;
1230                         rss_meta->proto_hdrs = *proto_hdrs;
1231                         iavf_refine_proto_hdrs(&rss_meta->proto_hdrs,
1232                                                rss_type, pattern_hint);
1233                         break;
1234
1235                 case RTE_FLOW_ACTION_TYPE_END:
1236                         break;
1237
1238                 default:
1239                         rte_flow_error_set(error, EINVAL,
1240                                            RTE_FLOW_ERROR_TYPE_ACTION, action,
1241                                            "Invalid action.");
1242                         return -rte_errno;
1243                 }
1244         }
1245
1246         return 0;
1247 }
1248
1249 static int
1250 iavf_hash_parse_pattern_action(__rte_unused struct iavf_adapter *ad,
1251                                struct iavf_pattern_match_item *array,
1252                                uint32_t array_len,
1253                                const struct rte_flow_item pattern[],
1254                                const struct rte_flow_action actions[],
1255                                void **meta,
1256                                struct rte_flow_error *error)
1257 {
1258         struct iavf_pattern_match_item *pattern_match_item;
1259         struct iavf_rss_meta *rss_meta_ptr;
1260         uint64_t phint = IAVF_PHINT_NONE;
1261         int ret = 0;
1262
1263         rss_meta_ptr = rte_zmalloc(NULL, sizeof(*rss_meta_ptr), 0);
1264         if (!rss_meta_ptr) {
1265                 rte_flow_error_set(error, EINVAL,
1266                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1267                                    "No memory for rss_meta_ptr");
1268                 return -ENOMEM;
1269         }
1270
1271         /* Check rss supported pattern and find matched pattern. */
1272         pattern_match_item =
1273                 iavf_search_pattern_match_item(pattern, array, array_len,
1274                                                error);
1275         if (!pattern_match_item) {
1276                 ret = -rte_errno;
1277                 goto error;
1278         }
1279
1280         ret = iavf_hash_parse_pattern(pattern, &phint, error);
1281         if (ret)
1282                 goto error;
1283
1284         ret = iavf_hash_parse_action(pattern_match_item, actions, phint,
1285                                      (void **)&rss_meta_ptr, error);
1286
1287 error:
1288         if (!ret && meta)
1289                 *meta = rss_meta_ptr;
1290         else
1291                 rte_free(rss_meta_ptr);
1292
1293         rte_free(pattern_match_item);
1294
1295         return ret;
1296 }
1297
1298 static int
1299 iavf_hash_create(__rte_unused struct iavf_adapter *ad,
1300                  __rte_unused struct rte_flow *flow, void *meta,
1301                  __rte_unused struct rte_flow_error *error)
1302 {
1303         struct iavf_rss_meta *rss_meta = (struct iavf_rss_meta *)meta;
1304         struct virtchnl_rss_cfg *rss_cfg;
1305         int ret = 0;
1306
1307         rss_cfg = rte_zmalloc("iavf rss rule",
1308                               sizeof(struct virtchnl_rss_cfg), 0);
1309         if (!rss_cfg) {
1310                 rte_flow_error_set(error, EINVAL,
1311                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1312                                    "No memory for rss rule");
1313                 return -ENOMEM;
1314         }
1315
1316         rss_cfg->proto_hdrs = rss_meta->proto_hdrs;
1317         rss_cfg->rss_algorithm = rss_meta->rss_algorithm;
1318
1319         ret = iavf_add_del_rss_cfg(ad, rss_cfg, true);
1320         if (!ret) {
1321                 flow->rule = rss_cfg;
1322         } else {
1323                 PMD_DRV_LOG(ERR, "fail to add RSS configure");
1324                 rte_flow_error_set(error, -ret,
1325                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1326                                    "Failed to add rss rule.");
1327                 rte_free(rss_cfg);
1328                 return -rte_errno;
1329         }
1330
1331         rte_free(meta);
1332
1333         return ret;
1334 }
1335
1336 static int
1337 iavf_hash_destroy(__rte_unused struct iavf_adapter *ad,
1338                   struct rte_flow *flow,
1339                   __rte_unused struct rte_flow_error *error)
1340 {
1341         struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
1342         struct virtchnl_rss_cfg *rss_cfg;
1343         int ret = 0;
1344
1345         if (vf->vf_reset)
1346                 return 0;
1347
1348         rss_cfg = (struct virtchnl_rss_cfg *)flow->rule;
1349
1350         ret = iavf_add_del_rss_cfg(ad, rss_cfg, false);
1351         if (ret) {
1352                 PMD_DRV_LOG(ERR, "fail to del RSS configure");
1353                 rte_flow_error_set(error, -ret,
1354                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1355                                    "Failed to delete rss rule.");
1356                 return -rte_errno;
1357         }
1358         return ret;
1359 }
1360
1361 static void
1362 iavf_hash_uninit(struct iavf_adapter *ad)
1363 {
1364         struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
1365         struct rte_eth_rss_conf *rss_conf;
1366
1367         if (vf->vf_reset)
1368                 return;
1369
1370         if (!vf->vf_res)
1371                 return;
1372
1373         if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF))
1374                 return;
1375
1376         rss_conf = &ad->eth_dev->data->dev_conf.rx_adv_conf.rss_conf;
1377         if (iavf_rss_hash_set(ad, rss_conf->rss_hf, false))
1378                 PMD_DRV_LOG(ERR, "fail to delete default RSS");
1379
1380         iavf_unregister_parser(&iavf_hash_parser, ad);
1381 }
1382
1383 static void
1384 iavf_hash_free(struct rte_flow *flow)
1385 {
1386         rte_free(flow->rule);
1387 }