drivers/net/ice/ice_hash.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10 #include <unistd.h>
  11 #include <stdarg.h>
  12
  13 #include <rte_debug.h>
  14 #include <rte_ether.h>
  15 #include <rte_ethdev_driver.h>
  16 #include <rte_log.h>
  17 #include <rte_malloc.h>
  18 #include <rte_eth_ctrl.h>
  19 #include <rte_tailq.h>
  20 #include <rte_flow_driver.h>
  21
  22 #include "ice_logs.h"
  23 #include "base/ice_type.h"
  24 #include "base/ice_flow.h"
  25 #include "ice_ethdev.h"
  26 #include "ice_generic_flow.h"
  27
  28 #define ICE_PHINT_NONE                          0
  29 #define ICE_PHINT_VLAN                          BIT_ULL(0)
  30 #define ICE_PHINT_PPPOE                         BIT_ULL(1)
  31 #define ICE_PHINT_GTPU                          BIT_ULL(2)
  32 #define ICE_PHINT_GTPU_EH                       BIT_ULL(3)
  33 #define ICE_PHINT_GTPU_EH_DWN                   BIT_ULL(4)
  34 #define ICE_PHINT_GTPU_EH_UP                    BIT_ULL(5)
  35
  36 #define ICE_GTPU_EH_DWNLINK     0
  37 #define ICE_GTPU_EH_UPLINK      1
  38
  39 #define ICE_IPV4_PROT BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT)
  40 #define ICE_IPV6_PROT BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT)
  41
  42 #define VALID_RSS_IPV4_L4       (ETH_RSS_NONFRAG_IPV4_UDP       | \
  43                                  ETH_RSS_NONFRAG_IPV4_TCP       | \
  44                                  ETH_RSS_NONFRAG_IPV4_SCTP)
  45
  46 #define VALID_RSS_IPV6_L4       (ETH_RSS_NONFRAG_IPV6_UDP       | \
  47                                  ETH_RSS_NONFRAG_IPV6_TCP       | \
  48                                  ETH_RSS_NONFRAG_IPV6_SCTP)
  49
  50 #define VALID_RSS_IPV4          (ETH_RSS_IPV4 | VALID_RSS_IPV4_L4)
  51 #define VALID_RSS_IPV6          (ETH_RSS_IPV6 | VALID_RSS_IPV6_L4)
  52 #define VALID_RSS_L3            (VALID_RSS_IPV4 | VALID_RSS_IPV6)
  53 #define VALID_RSS_L4            (VALID_RSS_IPV4_L4 | VALID_RSS_IPV6_L4)
  54
  55 #define VALID_RSS_ATTR          (ETH_RSS_L3_SRC_ONLY    | \
  56                                  ETH_RSS_L3_DST_ONLY    | \
  57                                  ETH_RSS_L4_SRC_ONLY    | \
  58                                  ETH_RSS_L4_DST_ONLY    | \
  59                                  ETH_RSS_L2_SRC_ONLY    | \
  60                                  ETH_RSS_L2_DST_ONLY    | \
  61                                  RTE_ETH_RSS_L3_PRE32   | \
  62                                  RTE_ETH_RSS_L3_PRE48   | \
  63                                  RTE_ETH_RSS_L3_PRE64)
  64
  65 #define INVALID_RSS_ATTR        (RTE_ETH_RSS_L3_PRE40   | \
  66                                  RTE_ETH_RSS_L3_PRE56   | \
  67                                  RTE_ETH_RSS_L3_PRE96)
  68
  69 struct ice_rss_meta {
  70         uint8_t hash_function;
  71         struct ice_rss_hash_cfg cfg;
  72 };
  73
  74 struct ice_hash_flow_cfg {
  75         bool simple_xor;
  76         struct ice_rss_cfg rss_cfg;
  77 };
  78
  79 static int
  80 ice_hash_init(struct ice_adapter *ad);
  81
  82 static int
  83 ice_hash_create(struct ice_adapter *ad,
  84                 struct rte_flow *flow,
  85                 void *meta,
  86                 struct rte_flow_error *error);
  87
  88 static int
  89 ice_hash_destroy(struct ice_adapter *ad,
  90                 struct rte_flow *flow,
  91                 struct rte_flow_error *error);
  92
  93 static void
  94 ice_hash_uninit(struct ice_adapter *ad);
  95
  96 static void
  97 ice_hash_free(struct rte_flow *flow);
  98
  99 static int
 100 ice_hash_parse_pattern_action(struct ice_adapter *ad,
 101                         struct ice_pattern_match_item *array,
 102                         uint32_t array_len,
 103                         const struct rte_flow_item pattern[],
 104                         const struct rte_flow_action actions[],
 105                         void **meta,
 106                         struct rte_flow_error *error);
 107
 108 /* Rss configuration template */
 109 struct ice_rss_hash_cfg ipv4_tmplt = {
 110         ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV4 |
 111         ICE_FLOW_SEG_HDR_IPV_OTHER,
 112         ICE_FLOW_HASH_ETH | ICE_FLOW_HASH_IPV4,
 113         ICE_RSS_ANY_HEADERS,
 114         0
 115 };
 116
 117 struct ice_rss_hash_cfg ipv4_udp_tmplt = {
 118         ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV4 |
 119         ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_UDP,
 120         ICE_FLOW_HASH_ETH | ICE_HASH_UDP_IPV4 | ICE_IPV4_PROT,
 121         ICE_RSS_ANY_HEADERS,
 122         0
 123 };
 124
 125 struct ice_rss_hash_cfg ipv4_tcp_tmplt = {
 126         ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV4 |
 127         ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_TCP,
 128         ICE_FLOW_HASH_ETH | ICE_HASH_TCP_IPV4 | ICE_IPV4_PROT,
 129         ICE_RSS_ANY_HEADERS,
 130         0
 131 };
 132
 133 struct ice_rss_hash_cfg ipv4_sctp_tmplt = {
 134         ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV4 |
 135         ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_SCTP,
 136         ICE_FLOW_HASH_ETH | ICE_HASH_SCTP_IPV4 | ICE_IPV4_PROT,
 137         ICE_RSS_ANY_HEADERS,
 138         0
 139 };
 140
 141 struct ice_rss_hash_cfg ipv6_tmplt = {
 142         ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV6 |
 143         ICE_FLOW_SEG_HDR_IPV_OTHER,
 144         ICE_FLOW_HASH_ETH | ICE_FLOW_HASH_IPV6,
 145         ICE_RSS_ANY_HEADERS,
 146         0
 147 };
 148
 149 struct ice_rss_hash_cfg ipv6_udp_tmplt = {
 150         ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV6 |
 151         ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_UDP,
 152         ICE_FLOW_HASH_ETH | ICE_HASH_UDP_IPV6 | ICE_IPV6_PROT,
 153         ICE_RSS_ANY_HEADERS,
 154         0
 155 };
 156
 157 struct ice_rss_hash_cfg ipv6_tcp_tmplt = {
 158         ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV6 |
 159         ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_TCP,
 160         ICE_FLOW_HASH_ETH | ICE_HASH_TCP_IPV6 | ICE_IPV6_PROT,
 161         ICE_RSS_ANY_HEADERS,
 162         0
 163 };
 164
 165 struct ice_rss_hash_cfg ipv6_sctp_tmplt = {
 166         ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV6 |
 167         ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_SCTP,
 168         ICE_FLOW_HASH_ETH | ICE_HASH_SCTP_IPV6 | ICE_IPV6_PROT,
 169         ICE_RSS_ANY_HEADERS,
 170         0
 171 };
 172
 173 struct ice_rss_hash_cfg eth_inner_ipv4_tmplt = {
 174         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER,
 175         ICE_FLOW_HASH_IPV4,
 176         ICE_RSS_INNER_HEADERS_W_OUTER_IPV4,
 177         0
 178 };
 179 struct ice_rss_hash_cfg eth_inner_ipv4_udp_tmplt = {
 180         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 181         ICE_FLOW_SEG_HDR_UDP,
 182         ICE_HASH_UDP_IPV4 | ICE_IPV4_PROT,
 183         ICE_RSS_INNER_HEADERS_W_OUTER_IPV4,
 184         0
 185 };
 186
 187 struct ice_rss_hash_cfg eth_inner_ipv4_tcp_tmplt = {
 188         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 189         ICE_FLOW_SEG_HDR_TCP,
 190         ICE_HASH_TCP_IPV4 | ICE_IPV4_PROT,
 191         ICE_RSS_INNER_HEADERS_W_OUTER_IPV4,
 192         0
 193 };
 194
 195 struct ice_rss_hash_cfg eth_inner_ipv6_tmplt = {
 196         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER,
 197         ICE_FLOW_HASH_IPV6,
 198         ICE_RSS_INNER_HEADERS_W_OUTER_IPV6,
 199         0
 200 };
 201 struct ice_rss_hash_cfg eth_inner_ipv6_udp_tmplt = {
 202         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 203         ICE_FLOW_SEG_HDR_UDP,
 204         ICE_HASH_UDP_IPV6 | ICE_IPV6_PROT,
 205         ICE_RSS_INNER_HEADERS_W_OUTER_IPV6,
 206         0
 207 };
 208
 209 struct ice_rss_hash_cfg eth_inner_ipv6_tcp_tmplt = {
 210         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 211         ICE_FLOW_SEG_HDR_TCP,
 212         ICE_HASH_TCP_IPV6 | ICE_IPV6_PROT,
 213         ICE_RSS_INNER_HEADERS_W_OUTER_IPV6,
 214         0
 215 };
 216
 217 struct ice_rss_hash_cfg eth_ipv4_esp_tmplt = {
 218         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 219         ICE_FLOW_SEG_HDR_ESP,
 220         ICE_FLOW_HASH_ESP_SPI,
 221         ICE_RSS_ANY_HEADERS,
 222         0
 223 };
 224
 225 struct ice_rss_hash_cfg eth_ipv4_udp_esp_tmplt = {
 226         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 227         ICE_FLOW_SEG_HDR_NAT_T_ESP,
 228         ICE_FLOW_HASH_NAT_T_ESP_SPI,
 229         ICE_RSS_ANY_HEADERS,
 230         0
 231 };
 232
 233 struct ice_rss_hash_cfg eth_ipv4_ah_tmplt = {
 234         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 235         ICE_FLOW_SEG_HDR_AH,
 236         ICE_FLOW_HASH_AH_SPI,
 237         ICE_RSS_ANY_HEADERS,
 238         0
 239 };
 240
 241 struct ice_rss_hash_cfg eth_ipv4_l2tpv3_tmplt = {
 242         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 243         ICE_FLOW_SEG_HDR_L2TPV3,
 244         ICE_FLOW_HASH_L2TPV3_SESS_ID,
 245         ICE_RSS_ANY_HEADERS,
 246         0
 247 };
 248
 249 struct ice_rss_hash_cfg eth_ipv4_pfcp_tmplt = {
 250         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 251         ICE_FLOW_SEG_HDR_PFCP_SESSION,
 252         ICE_FLOW_HASH_PFCP_SEID,
 253         ICE_RSS_ANY_HEADERS,
 254         0
 255 };
 256
 257 struct ice_rss_hash_cfg eth_ipv6_esp_tmplt = {
 258         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 259         ICE_FLOW_SEG_HDR_ESP,
 260         ICE_FLOW_HASH_ESP_SPI,
 261         ICE_RSS_ANY_HEADERS,
 262         0
 263 };
 264
 265 struct ice_rss_hash_cfg eth_ipv6_udp_esp_tmplt = {
 266         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 267         ICE_FLOW_SEG_HDR_NAT_T_ESP,
 268         ICE_FLOW_HASH_NAT_T_ESP_SPI,
 269         ICE_RSS_ANY_HEADERS,
 270         0
 271 };
 272
 273 struct ice_rss_hash_cfg eth_ipv6_ah_tmplt = {
 274         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 275         ICE_FLOW_SEG_HDR_AH,
 276         ICE_FLOW_HASH_AH_SPI,
 277         ICE_RSS_ANY_HEADERS,
 278         0
 279 };
 280
 281 struct ice_rss_hash_cfg eth_ipv6_l2tpv3_tmplt = {
 282         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 283         ICE_FLOW_SEG_HDR_L2TPV3,
 284         ICE_FLOW_HASH_L2TPV3_SESS_ID,
 285         ICE_RSS_ANY_HEADERS,
 286         0
 287 };
 288
 289 struct ice_rss_hash_cfg eth_ipv6_pfcp_tmplt = {
 290         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 291         ICE_FLOW_SEG_HDR_PFCP_SESSION,
 292         ICE_FLOW_HASH_PFCP_SEID,
 293         ICE_RSS_ANY_HEADERS,
 294         0
 295 };
 296
 297 struct ice_rss_hash_cfg pppoe_tmplt = {
 298         ICE_FLOW_SEG_HDR_ETH,
 299         ICE_FLOW_HASH_ETH | ICE_FLOW_HASH_PPPOE_SESS_ID,
 300         ICE_RSS_ANY_HEADERS,
 301         0
 302 };
 303
 304 struct ice_rss_hash_cfg empty_tmplt = {
 305         ICE_FLOW_SEG_HDR_NONE,
 306         0,
 307         ICE_RSS_ANY_HEADERS,
 308         0
 309 };
 310
 311 /* rss type super set */
 312 /* Empty rss type to support simple_xor. */
 313 #define ICE_RSS_TYPE_EMPTY      0ULL
 314
 315 /* IPv4 outer*/
 316 #define ICE_RSS_TYPE_OUTER_IPV4         (ETH_RSS_ETH | ETH_RSS_IPV4)
 317 #define ICE_RSS_TYPE_OUTER_IPV4_UDP     (ICE_RSS_TYPE_OUTER_IPV4 | \
 318                                         ETH_RSS_NONFRAG_IPV4_UDP)
 319 #define ICE_RSS_TYPE_OUTER_IPV4_TCP     (ICE_RSS_TYPE_OUTER_IPV4 | \
 320                                         ETH_RSS_NONFRAG_IPV4_TCP)
 321 #define ICE_RSS_TYPE_OUTER_IPV4_SCTP    (ICE_RSS_TYPE_OUTER_IPV4 | \
 322                                         ETH_RSS_NONFRAG_IPV4_SCTP)
 323 /* IPv6 */
 324 #define ICE_RSS_TYPE_OUTER_IPV6         (ETH_RSS_ETH | ETH_RSS_IPV6)
 325 #define ICE_RSS_TYPE_OUTER_IPV6_UDP     (ICE_RSS_TYPE_OUTER_IPV6 | \
 326                                         ETH_RSS_NONFRAG_IPV6_UDP)
 327 #define ICE_RSS_TYPE_OUTER_IPV6_TCP     (ICE_RSS_TYPE_OUTER_IPV6 | \
 328                                         ETH_RSS_NONFRAG_IPV6_TCP)
 329 #define ICE_RSS_TYPE_OUTER_IPV6_SCTP    (ICE_RSS_TYPE_OUTER_IPV6 | \
 330                                         ETH_RSS_NONFRAG_IPV6_SCTP)
 331
 332 /* VLAN IPV4 */
 333 #define ICE_RSS_TYPE_VLAN_IPV4          (ICE_RSS_TYPE_OUTER_IPV4 | \
 334                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 335 #define ICE_RSS_TYPE_VLAN_IPV4_UDP      (ICE_RSS_TYPE_OUTER_IPV4_UDP | \
 336                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 337 #define ICE_RSS_TYPE_VLAN_IPV4_TCP      (ICE_RSS_TYPE_OUTER_IPV4_TCP | \
 338                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 339 #define ICE_RSS_TYPE_VLAN_IPV4_SCTP     (ICE_RSS_TYPE_OUTER_IPV4_SCTP | \
 340                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 341 /* VLAN IPv6 */
 342 #define ICE_RSS_TYPE_VLAN_IPV6          (ICE_RSS_TYPE_OUTER_IPV6 | \
 343                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 344 #define ICE_RSS_TYPE_VLAN_IPV6_UDP      (ICE_RSS_TYPE_OUTER_IPV6_UDP | \
 345                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 346 #define ICE_RSS_TYPE_VLAN_IPV6_TCP      (ICE_RSS_TYPE_OUTER_IPV6_TCP | \
 347                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 348 #define ICE_RSS_TYPE_VLAN_IPV6_SCTP     (ICE_RSS_TYPE_OUTER_IPV6_SCTP | \
 349                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 350
 351 /* IPv4 inner */
 352 #define ICE_RSS_TYPE_INNER_IPV4 ETH_RSS_IPV4
 353 #define ICE_RSS_TYPE_INNER_IPV4_UDP     (ETH_RSS_IPV4 | \
 354                                          ETH_RSS_NONFRAG_IPV4_UDP)
 355 #define ICE_RSS_TYPE_INNER_IPV4_TCP     (ETH_RSS_IPV4 | \
 356                                          ETH_RSS_NONFRAG_IPV4_TCP)
 357 #define ICE_RSS_TYPE_INNER_IPV4_SCTP    (ETH_RSS_IPV4 | \
 358                                          ETH_RSS_NONFRAG_IPV4_SCTP)
 359 /* IPv6 inner */
 360 #define ICE_RSS_TYPE_INNER_IPV6 ETH_RSS_IPV6
 361 #define ICE_RSS_TYPE_INNER_IPV6_UDP     (ETH_RSS_IPV6 | \
 362                                          ETH_RSS_NONFRAG_IPV6_UDP)
 363 #define ICE_RSS_TYPE_INNER_IPV6_TCP     (ETH_RSS_IPV6 | \
 364                                          ETH_RSS_NONFRAG_IPV6_TCP)
 365 #define ICE_RSS_TYPE_INNER_IPV6_SCTP    (ETH_RSS_IPV6 | \
 366                                          ETH_RSS_NONFRAG_IPV6_SCTP)
 367
 368 /* GTPU IPv4 */
 369 #define ICE_RSS_TYPE_GTPU_IPV4          (ICE_RSS_TYPE_INNER_IPV4 | \
 370                                          ETH_RSS_GTPU)
 371 #define ICE_RSS_TYPE_GTPU_IPV4_UDP      (ICE_RSS_TYPE_INNER_IPV4_UDP | \
 372                                          ETH_RSS_GTPU)
 373 #define ICE_RSS_TYPE_GTPU_IPV4_TCP      (ICE_RSS_TYPE_INNER_IPV4_TCP | \
 374                                          ETH_RSS_GTPU)
 375 /* GTPU IPv6 */
 376 #define ICE_RSS_TYPE_GTPU_IPV6          (ICE_RSS_TYPE_INNER_IPV6 | \
 377                                          ETH_RSS_GTPU)
 378 #define ICE_RSS_TYPE_GTPU_IPV6_UDP      (ICE_RSS_TYPE_INNER_IPV6_UDP | \
 379                                          ETH_RSS_GTPU)
 380 #define ICE_RSS_TYPE_GTPU_IPV6_TCP      (ICE_RSS_TYPE_INNER_IPV6_TCP | \
 381                                          ETH_RSS_GTPU)
 382
 383 /* ESP, AH, L2TPV3, PFCP and PPPOE */
 384 #define ICE_RSS_TYPE_IPV4_ESP           (ETH_RSS_ESP | ETH_RSS_IPV4)
 385 #define ICE_RSS_TYPE_IPV4_AH            (ETH_RSS_AH | ETH_RSS_IPV4)
 386 #define ICE_RSS_TYPE_IPV6_ESP           (ETH_RSS_ESP | ETH_RSS_IPV6)
 387 #define ICE_RSS_TYPE_IPV6_AH            (ETH_RSS_AH | ETH_RSS_IPV6)
 388 #define ICE_RSS_TYPE_IPV4_L2TPV3        (ETH_RSS_L2TPV3 | ETH_RSS_IPV4)
 389 #define ICE_RSS_TYPE_IPV6_L2TPV3        (ETH_RSS_L2TPV3 | ETH_RSS_IPV6)
 390 #define ICE_RSS_TYPE_IPV4_PFCP          (ETH_RSS_PFCP | ETH_RSS_IPV4)
 391 #define ICE_RSS_TYPE_IPV6_PFCP          (ETH_RSS_PFCP | ETH_RSS_IPV6)
 392 #define ICE_RSS_TYPE_PPPOE              (ETH_RSS_ETH | ETH_RSS_PPPOE)
 393
 394 /**
 395  * Supported pattern for hash.
 396  * The first member is pattern item type,
 397  * the second member is input set mask,
 398  * the third member is ice_rss_hash_cfg template.
 399  */
 400
 401 /* Supported pattern for os default package. */
 402 static struct ice_pattern_match_item ice_hash_pattern_list_os[] = {
 403         /* IPV4 */
 404         {pattern_eth_ipv4,                      ICE_RSS_TYPE_OUTER_IPV4,        &ipv4_tmplt},
 405         {pattern_eth_ipv4_udp,                  ICE_RSS_TYPE_OUTER_IPV4_UDP,    &ipv4_udp_tmplt},
 406         {pattern_eth_ipv4_tcp,                  ICE_RSS_TYPE_OUTER_IPV4_TCP,    &ipv4_tcp_tmplt},
 407         {pattern_eth_ipv4_sctp,                 ICE_RSS_TYPE_OUTER_IPV4_SCTP,   &ipv4_sctp_tmplt},
 408         /* IPV6 */
 409         {pattern_eth_ipv6,                      ICE_RSS_TYPE_OUTER_IPV6,        &ipv6_tmplt},
 410         {pattern_eth_ipv6_udp,                  ICE_RSS_TYPE_OUTER_IPV6_UDP,    &ipv6_udp_tmplt},
 411         {pattern_eth_ipv6_tcp,                  ICE_RSS_TYPE_OUTER_IPV6_TCP,    &ipv6_tcp_tmplt},
 412         {pattern_eth_ipv6_sctp,                 ICE_RSS_TYPE_OUTER_IPV6_SCTP,   &ipv6_sctp_tmplt},
 413         /* EMPTY */
 414         {pattern_empty,                         ICE_RSS_TYPE_EMPTY,             &empty_tmplt},
 415 };
 416
 417 /* Supported pattern for comms package. */
 418 static struct ice_pattern_match_item ice_hash_pattern_list_comms[] = {
 419         /* IPV4 */
 420         {pattern_eth_ipv4,                      ICE_RSS_TYPE_OUTER_IPV4,        &ipv4_tmplt},
 421         {pattern_eth_ipv4_udp,                  ICE_RSS_TYPE_OUTER_IPV4_UDP,    &ipv4_udp_tmplt},
 422         {pattern_eth_ipv4_tcp,                  ICE_RSS_TYPE_OUTER_IPV4_TCP,    &ipv4_tcp_tmplt},
 423         {pattern_eth_ipv4_sctp,                 ICE_RSS_TYPE_OUTER_IPV4_SCTP,   &ipv4_sctp_tmplt},
 424         {pattern_eth_vlan_ipv4,                 ICE_RSS_TYPE_VLAN_IPV4,         &ipv4_tmplt},
 425         {pattern_eth_vlan_ipv4_udp,             ICE_RSS_TYPE_VLAN_IPV4_UDP,     &ipv4_udp_tmplt},
 426         {pattern_eth_vlan_ipv4_tcp,             ICE_RSS_TYPE_VLAN_IPV4_TCP,     &ipv4_tcp_tmplt},
 427         {pattern_eth_vlan_ipv4_sctp,            ICE_RSS_TYPE_VLAN_IPV4_SCTP,    &ipv4_sctp_tmplt},
 428         {pattern_eth_ipv4_gtpu_ipv4,            ICE_RSS_TYPE_GTPU_IPV4,         &eth_inner_ipv4_tmplt},
 429         {pattern_eth_ipv4_gtpu_ipv4_udp,        ICE_RSS_TYPE_GTPU_IPV4_UDP,     &eth_inner_ipv4_udp_tmplt},
 430         {pattern_eth_ipv4_gtpu_ipv4_tcp,        ICE_RSS_TYPE_GTPU_IPV4_TCP,     &eth_inner_ipv4_tcp_tmplt},
 431         {pattern_eth_ipv6_gtpu_ipv4,            ICE_RSS_TYPE_GTPU_IPV4,         &eth_inner_ipv4_tmplt},
 432         {pattern_eth_ipv6_gtpu_ipv4_udp,        ICE_RSS_TYPE_GTPU_IPV4_UDP,     &eth_inner_ipv4_udp_tmplt},
 433         {pattern_eth_ipv6_gtpu_ipv4_tcp,        ICE_RSS_TYPE_GTPU_IPV4_TCP,     &eth_inner_ipv4_tcp_tmplt},
 434         {pattern_eth_ipv4_gtpu_eh_ipv4,         ICE_RSS_TYPE_GTPU_IPV4,         &eth_inner_ipv4_tmplt},
 435         {pattern_eth_ipv4_gtpu_eh_ipv4_udp,     ICE_RSS_TYPE_GTPU_IPV4_UDP,     &eth_inner_ipv4_udp_tmplt},
 436         {pattern_eth_ipv4_gtpu_eh_ipv4_tcp,     ICE_RSS_TYPE_GTPU_IPV4_TCP,     &eth_inner_ipv4_tcp_tmplt},
 437         {pattern_eth_ipv6_gtpu_eh_ipv4,         ICE_RSS_TYPE_GTPU_IPV4,         &eth_inner_ipv4_tmplt},
 438         {pattern_eth_ipv6_gtpu_eh_ipv4_udp,     ICE_RSS_TYPE_GTPU_IPV4_UDP,     &eth_inner_ipv4_udp_tmplt},
 439         {pattern_eth_ipv6_gtpu_eh_ipv4_tcp,     ICE_RSS_TYPE_GTPU_IPV4_TCP,     &eth_inner_ipv4_tcp_tmplt},
 440         {pattern_eth_pppoes_ipv4,               ICE_RSS_TYPE_OUTER_IPV4,        &ipv4_tmplt},
 441         {pattern_eth_pppoes_ipv4_udp,           ICE_RSS_TYPE_OUTER_IPV4_UDP,    &ipv4_udp_tmplt},
 442         {pattern_eth_pppoes_ipv4_tcp,           ICE_RSS_TYPE_OUTER_IPV4_TCP,    &ipv4_tcp_tmplt},
 443         {pattern_eth_ipv4_esp,                  ICE_RSS_TYPE_IPV4_ESP,          &eth_ipv4_esp_tmplt},
 444         {pattern_eth_ipv4_udp_esp,              ICE_RSS_TYPE_IPV4_ESP,          &eth_ipv4_udp_esp_tmplt},
 445         {pattern_eth_ipv4_ah,                   ICE_RSS_TYPE_IPV4_AH,           &eth_ipv4_ah_tmplt},
 446         {pattern_eth_ipv4_l2tp,                 ICE_RSS_TYPE_IPV4_L2TPV3,       &eth_ipv4_l2tpv3_tmplt},
 447         {pattern_eth_ipv4_pfcp,                 ICE_RSS_TYPE_IPV4_PFCP,         &eth_ipv4_pfcp_tmplt},
 448         /* IPV6 */
 449         {pattern_eth_ipv6,                      ICE_RSS_TYPE_OUTER_IPV6,        &ipv6_tmplt},
 450         {pattern_eth_ipv6_udp,                  ICE_RSS_TYPE_OUTER_IPV6_UDP,    &ipv6_udp_tmplt},
 451         {pattern_eth_ipv6_tcp,                  ICE_RSS_TYPE_OUTER_IPV6_TCP,    &ipv6_tcp_tmplt},
 452         {pattern_eth_ipv6_sctp,                 ICE_RSS_TYPE_OUTER_IPV6_SCTP,   &ipv6_sctp_tmplt},
 453         {pattern_eth_vlan_ipv6,                 ICE_RSS_TYPE_VLAN_IPV6,         &ipv6_tmplt},
 454         {pattern_eth_vlan_ipv6_udp,             ICE_RSS_TYPE_VLAN_IPV6_UDP,     &ipv6_udp_tmplt},
 455         {pattern_eth_vlan_ipv6_tcp,             ICE_RSS_TYPE_VLAN_IPV6_TCP,     &ipv6_tcp_tmplt},
 456         {pattern_eth_vlan_ipv6_sctp,            ICE_RSS_TYPE_VLAN_IPV6_SCTP,    &ipv6_sctp_tmplt},
 457         {pattern_eth_ipv4_gtpu_ipv6,            ICE_RSS_TYPE_GTPU_IPV6,         &eth_inner_ipv6_tmplt},
 458         {pattern_eth_ipv4_gtpu_ipv6_udp,        ICE_RSS_TYPE_GTPU_IPV6_UDP,     &eth_inner_ipv6_udp_tmplt},
 459         {pattern_eth_ipv4_gtpu_ipv6_tcp,        ICE_RSS_TYPE_GTPU_IPV6_TCP,     &eth_inner_ipv6_tcp_tmplt},
 460         {pattern_eth_ipv6_gtpu_ipv6,            ICE_RSS_TYPE_GTPU_IPV6,         &eth_inner_ipv6_tmplt},
 461         {pattern_eth_ipv6_gtpu_ipv6_udp,        ICE_RSS_TYPE_GTPU_IPV6_UDP,     &eth_inner_ipv6_udp_tmplt},
 462         {pattern_eth_ipv6_gtpu_ipv6_tcp,        ICE_RSS_TYPE_GTPU_IPV6_TCP,     &eth_inner_ipv6_tcp_tmplt},
 463         {pattern_eth_ipv4_gtpu_eh_ipv6,         ICE_RSS_TYPE_GTPU_IPV6,         &eth_inner_ipv6_tmplt},
 464         {pattern_eth_ipv4_gtpu_eh_ipv6_udp,     ICE_RSS_TYPE_GTPU_IPV6_UDP,     &eth_inner_ipv6_udp_tmplt},
 465         {pattern_eth_ipv4_gtpu_eh_ipv6_tcp,     ICE_RSS_TYPE_GTPU_IPV6_TCP,     &eth_inner_ipv6_tcp_tmplt},
 466         {pattern_eth_ipv6_gtpu_eh_ipv6,         ICE_RSS_TYPE_GTPU_IPV6,         &eth_inner_ipv6_tmplt},
 467         {pattern_eth_ipv6_gtpu_eh_ipv6_udp,     ICE_RSS_TYPE_GTPU_IPV6_UDP,     &eth_inner_ipv6_udp_tmplt},
 468         {pattern_eth_ipv6_gtpu_eh_ipv6_tcp,     ICE_RSS_TYPE_GTPU_IPV6_TCP,     &eth_inner_ipv6_tcp_tmplt},
 469         {pattern_eth_pppoes_ipv6,               ICE_RSS_TYPE_OUTER_IPV6,        &ipv6_tmplt},
 470         {pattern_eth_pppoes_ipv6_udp,           ICE_RSS_TYPE_OUTER_IPV6_UDP,    &ipv6_udp_tmplt},
 471         {pattern_eth_pppoes_ipv6_tcp,           ICE_RSS_TYPE_OUTER_IPV6_TCP,    &ipv6_tcp_tmplt},
 472         {pattern_eth_ipv6_esp,                  ICE_RSS_TYPE_IPV6_ESP,          &eth_ipv6_esp_tmplt},
 473         {pattern_eth_ipv6_udp_esp,              ICE_RSS_TYPE_IPV6_ESP,          &eth_ipv6_udp_esp_tmplt},
 474         {pattern_eth_ipv6_ah,                   ICE_RSS_TYPE_IPV6_AH,           &eth_ipv6_ah_tmplt},
 475         {pattern_eth_ipv6_l2tp,                 ICE_RSS_TYPE_IPV6_L2TPV3,       &eth_ipv6_l2tpv3_tmplt},
 476         {pattern_eth_ipv6_pfcp,                 ICE_RSS_TYPE_IPV6_PFCP,         &eth_ipv6_pfcp_tmplt},
 477         /* PPPOE */
 478         {pattern_eth_pppoes,                    ICE_RSS_TYPE_PPPOE,             &pppoe_tmplt},
 479         /* EMPTY */
 480         {pattern_empty,                         ICE_RSS_TYPE_EMPTY,             &empty_tmplt},
 481 };
 482
 483 static struct ice_flow_engine ice_hash_engine = {
 484         .init = ice_hash_init,
 485         .create = ice_hash_create,
 486         .destroy = ice_hash_destroy,
 487         .uninit = ice_hash_uninit,
 488         .free = ice_hash_free,
 489         .type = ICE_FLOW_ENGINE_HASH,
 490 };
 491
 492 /* Register parser for os package. */
 493 static struct ice_flow_parser ice_hash_parser_os = {
 494         .engine = &ice_hash_engine,
 495         .array = ice_hash_pattern_list_os,
 496         .array_len = RTE_DIM(ice_hash_pattern_list_os),
 497         .parse_pattern_action = ice_hash_parse_pattern_action,
 498         .stage = ICE_FLOW_STAGE_RSS,
 499 };
 500
 501 /* Register parser for comms package. */
 502 static struct ice_flow_parser ice_hash_parser_comms = {
 503         .engine = &ice_hash_engine,
 504         .array = ice_hash_pattern_list_comms,
 505         .array_len = RTE_DIM(ice_hash_pattern_list_comms),
 506         .parse_pattern_action = ice_hash_parse_pattern_action,
 507         .stage = ICE_FLOW_STAGE_RSS,
 508 };
 509
 510 RTE_INIT(ice_hash_engine_init)
 511 {
 512         struct ice_flow_engine *engine = &ice_hash_engine;
 513         ice_register_flow_engine(engine);
 514 }
 515
 516 static int
 517 ice_hash_init(struct ice_adapter *ad)
 518 {
 519         struct ice_flow_parser *parser = NULL;
 520
 521         if (ad->hw.dcf_enabled)
 522                 return 0;
 523
 524         if (ad->active_pkg_type == ICE_PKG_TYPE_OS_DEFAULT)
 525                 parser = &ice_hash_parser_os;
 526         else if (ad->active_pkg_type == ICE_PKG_TYPE_COMMS)
 527                 parser = &ice_hash_parser_comms;
 528         else
 529                 return -EINVAL;
 530
 531         return ice_register_parser(parser, ad);
 532 }
 533
 534 static int
 535 ice_hash_parse_pattern(const struct rte_flow_item pattern[], uint64_t *phint,
 536                        struct rte_flow_error *error)
 537 {
 538         const struct rte_flow_item *item = pattern;
 539         const struct rte_flow_item_gtp_psc *psc;
 540
 541         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 542                 if (item->last) {
 543                         rte_flow_error_set(error, EINVAL,
 544                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 545                                         "Not support range");
 546                         return -rte_errno;
 547                 }
 548
 549                 switch (item->type) {
 550                 case RTE_FLOW_ITEM_TYPE_VLAN:
 551                         *phint |= ICE_PHINT_VLAN;
 552                         break;
 553                 case RTE_FLOW_ITEM_TYPE_PPPOES:
 554                         *phint |= ICE_PHINT_PPPOE;
 555                         break;
 556                 case RTE_FLOW_ITEM_TYPE_GTPU:
 557                         *phint |= ICE_PHINT_GTPU;
 558                         break;
 559                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
 560                         *phint |= ICE_PHINT_GTPU_EH;
 561                         psc = item->spec;
 562                         if (!psc)
 563                                 break;
 564                         else if (psc->pdu_type == ICE_GTPU_EH_UPLINK)
 565                                 *phint |= ICE_PHINT_GTPU_EH_UP;
 566                         else if (psc->pdu_type == ICE_GTPU_EH_DWNLINK)
 567                                 *phint |= ICE_PHINT_GTPU_EH_DWN;
 568                         break;
 569                 default:
 570                         break;
 571                 }
 572         }
 573
 574         return 0;
 575 }
 576
 577 static void
 578 ice_refine_hash_cfg_l234(struct ice_rss_hash_cfg *hash_cfg,
 579                          uint64_t rss_type)
 580 {
 581         uint32_t *addl_hdrs = &hash_cfg->addl_hdrs;
 582         uint64_t *hash_flds = &hash_cfg->hash_flds;
 583
 584         if (*addl_hdrs & ICE_FLOW_SEG_HDR_ETH) {
 585                 if (!(rss_type & ETH_RSS_ETH))
 586                         *hash_flds &= ~ICE_FLOW_HASH_ETH;
 587                 if (rss_type & ETH_RSS_L2_SRC_ONLY)
 588                         *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_ETH_DA));
 589                 else if (rss_type & ETH_RSS_L2_DST_ONLY)
 590                         *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_ETH_SA));
 591                 *addl_hdrs &= ~ICE_FLOW_SEG_HDR_ETH;
 592         }
 593
 594         if (*addl_hdrs & ICE_FLOW_SEG_HDR_VLAN) {
 595                 if (rss_type & ETH_RSS_C_VLAN)
 596                         *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_C_VLAN);
 597                 else if (rss_type & ETH_RSS_S_VLAN)
 598                         *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_S_VLAN);
 599         }
 600
 601         if (*addl_hdrs & ICE_FLOW_SEG_HDR_PPPOE) {
 602                 if (!(rss_type & ETH_RSS_PPPOE))
 603                         *hash_flds &= ~ICE_FLOW_HASH_PPPOE_SESS_ID;
 604         }
 605
 606         if (*addl_hdrs & ICE_FLOW_SEG_HDR_IPV4) {
 607                 if (rss_type &
 608                    (ETH_RSS_IPV4 |
 609                     ETH_RSS_NONFRAG_IPV4_UDP |
 610                     ETH_RSS_NONFRAG_IPV4_TCP |
 611                     ETH_RSS_NONFRAG_IPV4_SCTP)) {
 612                         if (rss_type & ETH_RSS_L3_SRC_ONLY)
 613                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA));
 614                         else if (rss_type & ETH_RSS_L3_DST_ONLY)
 615                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA));
 616                         else if (rss_type &
 617                                 (ETH_RSS_L4_SRC_ONLY |
 618                                 ETH_RSS_L4_DST_ONLY))
 619                                 *hash_flds &= ~ICE_FLOW_HASH_IPV4;
 620                 } else {
 621                         *hash_flds &= ~ICE_FLOW_HASH_IPV4;
 622                 }
 623         }
 624
 625         if (*addl_hdrs & ICE_FLOW_SEG_HDR_IPV6) {
 626                 if (rss_type &
 627                    (ETH_RSS_IPV6 |
 628                     ETH_RSS_NONFRAG_IPV6_UDP |
 629                     ETH_RSS_NONFRAG_IPV6_TCP |
 630                     ETH_RSS_NONFRAG_IPV6_SCTP)) {
 631                         if (rss_type & ETH_RSS_L3_SRC_ONLY)
 632                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA));
 633                         else if (rss_type & ETH_RSS_L3_DST_ONLY)
 634                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA));
 635                         else if (rss_type &
 636                                 (ETH_RSS_L4_SRC_ONLY |
 637                                 ETH_RSS_L4_DST_ONLY))
 638                                 *hash_flds &= ~ICE_FLOW_HASH_IPV6;
 639                 } else {
 640                         *hash_flds &= ~ICE_FLOW_HASH_IPV6;
 641                 }
 642
 643                 if (rss_type & RTE_ETH_RSS_L3_PRE32) {
 644                         if (rss_type & ETH_RSS_L3_SRC_ONLY) {
 645                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA));
 646                                 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE32_SA));
 647                         } else if (rss_type & ETH_RSS_L3_DST_ONLY) {
 648                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA));
 649                                 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE32_DA));
 650                         } else {
 651                                 *hash_flds &= ~ICE_FLOW_HASH_IPV6;
 652                                 *hash_flds |= ICE_FLOW_HASH_IPV6_PRE32;
 653                         }
 654                 }
 655                 if (rss_type & RTE_ETH_RSS_L3_PRE48) {
 656                         if (rss_type & ETH_RSS_L3_SRC_ONLY) {
 657                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA));
 658                                 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE48_SA));
 659                         } else if (rss_type & ETH_RSS_L3_DST_ONLY) {
 660                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA));
 661                                 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE48_DA));
 662                         } else {
 663                                 *hash_flds &= ~ICE_FLOW_HASH_IPV6;
 664                                 *hash_flds |= ICE_FLOW_HASH_IPV6_PRE48;
 665                         }
 666                 }
 667                 if (rss_type & RTE_ETH_RSS_L3_PRE64) {
 668                         if (rss_type & ETH_RSS_L3_SRC_ONLY) {
 669                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA));
 670                                 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE64_SA));
 671                         } else if (rss_type & ETH_RSS_L3_DST_ONLY) {
 672                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA));
 673                                 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE64_DA));
 674                         } else {
 675                                 *hash_flds &= ~ICE_FLOW_HASH_IPV6;
 676                                 *hash_flds |= ICE_FLOW_HASH_IPV6_PRE64;
 677                         }
 678                 }
 679         }
 680
 681         if (*addl_hdrs & ICE_FLOW_SEG_HDR_UDP) {
 682                 if (rss_type &
 683                    (ETH_RSS_NONFRAG_IPV4_UDP |
 684                     ETH_RSS_NONFRAG_IPV6_UDP)) {
 685                         if (rss_type & ETH_RSS_L4_SRC_ONLY)
 686                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT));
 687                         else if (rss_type & ETH_RSS_L4_DST_ONLY)
 688                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT));
 689                         else if (rss_type &
 690                                 (ETH_RSS_L3_SRC_ONLY |
 691                                   ETH_RSS_L3_DST_ONLY))
 692                                 *hash_flds &= ~ICE_FLOW_HASH_UDP_PORT;
 693                 } else {
 694                         *hash_flds &= ~ICE_FLOW_HASH_UDP_PORT;
 695                 }
 696         }
 697
 698         if (*addl_hdrs & ICE_FLOW_SEG_HDR_TCP) {
 699                 if (rss_type &
 700                    (ETH_RSS_NONFRAG_IPV4_TCP |
 701                     ETH_RSS_NONFRAG_IPV6_TCP)) {
 702                         if (rss_type & ETH_RSS_L4_SRC_ONLY)
 703                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT));
 704                         else if (rss_type & ETH_RSS_L4_DST_ONLY)
 705                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT));
 706                         else if (rss_type &
 707                                 (ETH_RSS_L3_SRC_ONLY |
 708                                   ETH_RSS_L3_DST_ONLY))
 709                                 *hash_flds &= ~ICE_FLOW_HASH_TCP_PORT;
 710                 } else {
 711                         *hash_flds &= ~ICE_FLOW_HASH_TCP_PORT;
 712                 }
 713         }
 714
 715         if (*addl_hdrs & ICE_FLOW_SEG_HDR_SCTP) {
 716                 if (rss_type &
 717                    (ETH_RSS_NONFRAG_IPV4_SCTP |
 718                     ETH_RSS_NONFRAG_IPV6_SCTP)) {
 719                         if (rss_type & ETH_RSS_L4_SRC_ONLY)
 720                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT));
 721                         else if (rss_type & ETH_RSS_L4_DST_ONLY)
 722                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT));
 723                         else if (rss_type &
 724                                 (ETH_RSS_L3_SRC_ONLY |
 725                                   ETH_RSS_L3_DST_ONLY))
 726                                 *hash_flds &= ~ICE_FLOW_HASH_SCTP_PORT;
 727                 } else {
 728                         *hash_flds &= ~ICE_FLOW_HASH_SCTP_PORT;
 729                 }
 730         }
 731
 732         if (*addl_hdrs & ICE_FLOW_SEG_HDR_L2TPV3) {
 733                 if (!(rss_type & ETH_RSS_L2TPV3))
 734                         *hash_flds &= ~ICE_FLOW_HASH_L2TPV3_SESS_ID;
 735         }
 736
 737         if (*addl_hdrs & ICE_FLOW_SEG_HDR_ESP) {
 738                 if (!(rss_type & ETH_RSS_ESP))
 739                         *hash_flds &= ~ICE_FLOW_HASH_ESP_SPI;
 740         }
 741
 742         if (*addl_hdrs & ICE_FLOW_SEG_HDR_AH) {
 743                 if (!(rss_type & ETH_RSS_AH))
 744                         *hash_flds &= ~ICE_FLOW_HASH_AH_SPI;
 745         }
 746
 747         if (*addl_hdrs & ICE_FLOW_SEG_HDR_PFCP_SESSION) {
 748                 if (!(rss_type & ETH_RSS_PFCP))
 749                         *hash_flds &= ~ICE_FLOW_HASH_PFCP_SEID;
 750         }
 751 }
 752
 753 static void
 754 ice_refine_proto_hdrs_by_pattern(struct ice_rss_hash_cfg *hash_cfg,
 755                                  uint64_t phint)
 756 {
 757         uint32_t *addl_hdrs = &hash_cfg->addl_hdrs;
 758         if (phint & ICE_PHINT_VLAN)
 759                 *addl_hdrs |= ICE_FLOW_SEG_HDR_VLAN;
 760
 761         if (phint & ICE_PHINT_PPPOE)
 762                 *addl_hdrs |= ICE_FLOW_SEG_HDR_PPPOE;
 763
 764         if (phint & ICE_PHINT_GTPU_EH_DWN)
 765                 *addl_hdrs |= ICE_FLOW_SEG_HDR_GTPU_DWN;
 766         else if (phint & ICE_PHINT_GTPU_EH_UP)
 767                 *addl_hdrs |= ICE_FLOW_SEG_HDR_GTPU_UP;
 768         else if (phint & ICE_PHINT_GTPU_EH)
 769                 *addl_hdrs |= ICE_FLOW_SEG_HDR_GTPU_EH;
 770         else if (phint & ICE_PHINT_GTPU)
 771                 *addl_hdrs |= ICE_FLOW_SEG_HDR_GTPU_IP;
 772 }
 773
 774 static void
 775 ice_refine_hash_cfg_gtpu(struct ice_rss_hash_cfg *hash_cfg,
 776                          uint64_t rss_type)
 777 {
 778         uint32_t *addl_hdrs = &hash_cfg->addl_hdrs;
 779         uint64_t *hash_flds = &hash_cfg->hash_flds;
 780
 781         /* update hash field for gtpu eh/gtpu dwn/gtpu up. */
 782         if (!(rss_type & ETH_RSS_GTPU))
 783                 return;
 784
 785         if (*addl_hdrs & ICE_FLOW_SEG_HDR_GTPU_DWN)
 786                 *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_GTPU_DWN_TEID);
 787         else if (*addl_hdrs & ICE_FLOW_SEG_HDR_GTPU_UP)
 788                 *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_GTPU_UP_TEID);
 789         else if (*addl_hdrs & ICE_FLOW_SEG_HDR_GTPU_EH)
 790                 *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_GTPU_EH_TEID);
 791         else if (*addl_hdrs & ICE_FLOW_SEG_HDR_GTPU_IP)
 792                 *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_GTPU_IP_TEID);
 793 }
 794
 795 static void ice_refine_hash_cfg(struct ice_rss_hash_cfg *hash_cfg,
 796                                 uint64_t rss_type, uint64_t phint)
 797 {
 798         ice_refine_proto_hdrs_by_pattern(hash_cfg, phint);
 799         ice_refine_hash_cfg_l234(hash_cfg, rss_type);
 800         ice_refine_hash_cfg_gtpu(hash_cfg, rss_type);
 801 }
 802
 803 static uint64_t invalid_rss_comb[] = {
 804         ETH_RSS_IPV4 | ETH_RSS_NONFRAG_IPV4_UDP,
 805         ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_UDP,
 806         RTE_ETH_RSS_L3_PRE40 |
 807         RTE_ETH_RSS_L3_PRE56 |
 808         RTE_ETH_RSS_L3_PRE96
 809 };
 810
 811 struct rss_attr_type {
 812         uint64_t attr;
 813         uint64_t type;
 814 };
 815
 816 static struct rss_attr_type rss_attr_to_valid_type[] = {
 817         {ETH_RSS_L2_SRC_ONLY | ETH_RSS_L2_DST_ONLY,     ETH_RSS_ETH},
 818         {ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY,     VALID_RSS_L3},
 819         {ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY,     VALID_RSS_L4},
 820         /* current ipv6 prefix only supports prefix 64 bits*/
 821         {RTE_ETH_RSS_L3_PRE32,                          VALID_RSS_IPV6},
 822         {RTE_ETH_RSS_L3_PRE48,                          VALID_RSS_IPV6},
 823         {RTE_ETH_RSS_L3_PRE64,                          VALID_RSS_IPV6},
 824         {INVALID_RSS_ATTR,                              0}
 825 };
 826
 827 static bool
 828 ice_any_invalid_rss_type(enum rte_eth_hash_function rss_func,
 829                          uint64_t rss_type, uint64_t allow_rss_type)
 830 {
 831         uint32_t i;
 832
 833         /**
 834          * Check if l3/l4 SRC/DST_ONLY is set for SYMMETRIC_TOEPLITZ
 835          * hash function.
 836          */
 837         if (rss_func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) {
 838                 if (rss_type & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY |
 839                     ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY))
 840                         return true;
 841         }
 842
 843         /* check invalid combination */
 844         for (i = 0; i < RTE_DIM(invalid_rss_comb); i++) {
 845                 if (__builtin_popcountll(rss_type & invalid_rss_comb[i]) > 1)
 846                         return true;
 847         }
 848
 849         /* check invalid RSS attribute */
 850         for (i = 0; i < RTE_DIM(rss_attr_to_valid_type); i++) {
 851                 struct rss_attr_type *rat = &rss_attr_to_valid_type[i];
 852
 853                 if (rat->attr & rss_type && !(rat->type & rss_type))
 854                         return true;
 855         }
 856
 857         /* check not allowed RSS type */
 858         rss_type &= ~VALID_RSS_ATTR;
 859
 860         return ((rss_type & allow_rss_type) != rss_type);
 861 }
 862
 863 static int
 864 ice_hash_parse_action(struct ice_pattern_match_item *pattern_match_item,
 865                 const struct rte_flow_action actions[],
 866                 uint64_t pattern_hint, void **meta,
 867                 struct rte_flow_error *error)
 868 {
 869         struct ice_rss_meta *rss_meta = (struct ice_rss_meta *)*meta;
 870         struct ice_rss_hash_cfg *cfg = pattern_match_item->meta;
 871         enum rte_flow_action_type action_type;
 872         const struct rte_flow_action_rss *rss;
 873         const struct rte_flow_action *action;
 874         uint64_t rss_type;
 875
 876         /* Supported action is RSS. */
 877         for (action = actions; action->type !=
 878                 RTE_FLOW_ACTION_TYPE_END; action++) {
 879                 action_type = action->type;
 880                 switch (action_type) {
 881                 case RTE_FLOW_ACTION_TYPE_RSS:
 882                         rss = action->conf;
 883                         rss_type = rss->types;
 884
 885                         /* Check hash function and save it to rss_meta. */
 886                         if (pattern_match_item->pattern_list !=
 887                             pattern_empty && rss->func ==
 888                             RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) {
 889                                 return rte_flow_error_set(error, ENOTSUP,
 890                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 891                                         "Not supported flow");
 892                         } else if (rss->func ==
 893                                    RTE_ETH_HASH_FUNCTION_SIMPLE_XOR){
 894                                 rss_meta->hash_function =
 895                                 RTE_ETH_HASH_FUNCTION_SIMPLE_XOR;
 896                                 return 0;
 897                         } else if (rss->func ==
 898                                    RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) {
 899                                 rss_meta->hash_function =
 900                                 RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ;
 901                                 cfg->symm = true;
 902                         }
 903
 904                         if (rss->level)
 905                                 return rte_flow_error_set(error, ENOTSUP,
 906                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 907                                         "a nonzero RSS encapsulation level is not supported");
 908
 909                         if (rss->key_len)
 910                                 return rte_flow_error_set(error, ENOTSUP,
 911                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 912                                         "a nonzero RSS key_len is not supported");
 913
 914                         if (rss->queue)
 915                                 return rte_flow_error_set(error, ENOTSUP,
 916                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 917                                         "a non-NULL RSS queue is not supported");
 918
 919                         /**
 920                          * Check simultaneous use of SRC_ONLY and DST_ONLY
 921                          * of the same level.
 922                          */
 923                         rss_type = rte_eth_rss_hf_refine(rss_type);
 924
 925                         if (ice_any_invalid_rss_type(rss->func, rss_type,
 926                                         pattern_match_item->input_set_mask))
 927                                 return rte_flow_error_set(error, ENOTSUP,
 928                                         RTE_FLOW_ERROR_TYPE_ACTION,
 929                                         action, "RSS type not supported");
 930
 931                         rss_meta->cfg = *cfg;
 932                         ice_refine_hash_cfg(&rss_meta->cfg,
 933                                             rss_type, pattern_hint);
 934
 935                         break;
 936                 case RTE_FLOW_ACTION_TYPE_END:
 937                         break;
 938
 939                 default:
 940                         rte_flow_error_set(error, EINVAL,
 941                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 942                                         "Invalid action.");
 943                         return -rte_errno;
 944                 }
 945         }
 946
 947         return 0;
 948 }
 949
 950 static int
 951 ice_hash_parse_pattern_action(__rte_unused struct ice_adapter *ad,
 952                         struct ice_pattern_match_item *array,
 953                         uint32_t array_len,
 954                         const struct rte_flow_item pattern[],
 955                         const struct rte_flow_action actions[],
 956                         void **meta,
 957                         struct rte_flow_error *error)
 958 {
 959         int ret = 0;
 960         struct ice_pattern_match_item *pattern_match_item;
 961         struct ice_rss_meta *rss_meta_ptr;
 962         uint64_t phint = ICE_PHINT_NONE;
 963
 964         rss_meta_ptr = rte_zmalloc(NULL, sizeof(*rss_meta_ptr), 0);
 965         if (!rss_meta_ptr) {
 966                 rte_flow_error_set(error, EINVAL,
 967                                 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 968                                 "No memory for rss_meta_ptr");
 969                 return -ENOMEM;
 970         }
 971
 972         /* Check rss supported pattern and find matched pattern. */
 973         pattern_match_item = ice_search_pattern_match_item(pattern,
 974                                         array, array_len, error);
 975         if (!pattern_match_item) {
 976                 ret = -rte_errno;
 977                 goto error;
 978         }
 979
 980         ret = ice_hash_parse_pattern(pattern, &phint, error);
 981         if (ret)
 982                 goto error;
 983
 984         /* Check rss action. */
 985         ret = ice_hash_parse_action(pattern_match_item, actions, phint,
 986                                     (void **)&rss_meta_ptr, error);
 987
 988 error:
 989         if (!ret && meta)
 990                 *meta = rss_meta_ptr;
 991         else
 992                 rte_free(rss_meta_ptr);
 993         rte_free(pattern_match_item);
 994
 995         return ret;
 996 }
 997
 998 static int
 999 ice_hash_create(struct ice_adapter *ad,
1000                 struct rte_flow *flow,
1001                 void *meta,
1002                 struct rte_flow_error *error)
1003 {
1004         struct ice_pf *pf = &ad->pf;
1005         struct ice_hw *hw = ICE_PF_TO_HW(pf);
1006         struct ice_vsi *vsi = pf->main_vsi;
1007         int ret;
1008         uint32_t reg;
1009         struct ice_hash_flow_cfg *filter_ptr;
1010         struct ice_rss_meta *rss_meta = (struct ice_rss_meta *)meta;
1011         uint8_t hash_function = rss_meta->hash_function;
1012
1013         filter_ptr = rte_zmalloc("ice_rss_filter",
1014                                 sizeof(struct ice_hash_flow_cfg), 0);
1015         if (!filter_ptr) {
1016                 rte_flow_error_set(error, EINVAL,
1017                                 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1018                                 "No memory for filter_ptr");
1019                 return -ENOMEM;
1020         }
1021
1022         if (hash_function == RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) {
1023                 /* Enable registers for simple_xor hash function. */
1024                 reg = ICE_READ_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id));
1025                 reg = (reg & (~VSIQF_HASH_CTL_HASH_SCHEME_M)) |
1026                         (2 << VSIQF_HASH_CTL_HASH_SCHEME_S);
1027                 ICE_WRITE_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id), reg);
1028
1029                 filter_ptr->simple_xor = 1;
1030
1031                 goto out;
1032         } else {
1033                 memcpy(&filter_ptr->rss_cfg.hash, &rss_meta->cfg,
1034                        sizeof(struct ice_rss_hash_cfg));
1035                 ret = ice_add_rss_cfg_wrap(pf, vsi->idx,
1036                                            &filter_ptr->rss_cfg.hash);
1037                 if (ret) {
1038                         rte_flow_error_set(error, EINVAL,
1039                                         RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1040                                         "rss flow create fail");
1041                         goto error;
1042                 }
1043         }
1044
1045 out:
1046         flow->rule = filter_ptr;
1047         rte_free(meta);
1048         return 0;
1049
1050 error:
1051         rte_free(filter_ptr);
1052         rte_free(meta);
1053         return -rte_errno;
1054 }
1055
1056 static int
1057 ice_hash_destroy(struct ice_adapter *ad,
1058                 struct rte_flow *flow,
1059                 struct rte_flow_error *error)
1060 {
1061         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(ad);
1062         struct ice_hw *hw = ICE_PF_TO_HW(pf);
1063         struct ice_vsi *vsi = pf->main_vsi;
1064         int ret;
1065         uint32_t reg;
1066         struct ice_hash_flow_cfg *filter_ptr;
1067
1068         filter_ptr = (struct ice_hash_flow_cfg *)flow->rule;
1069
1070         if (filter_ptr->simple_xor == 1) {
1071                 /* Return to symmetric_toeplitz state. */
1072                 reg = ICE_READ_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id));
1073                 reg = (reg & (~VSIQF_HASH_CTL_HASH_SCHEME_M)) |
1074                         (1 << VSIQF_HASH_CTL_HASH_SCHEME_S);
1075                 ICE_WRITE_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id), reg);
1076         } else {
1077                 ret = ice_rem_rss_cfg_wrap(pf, vsi->idx,
1078                                            &filter_ptr->rss_cfg.hash);
1079                 /* Fixme: Ignore the error if a rule does not exist.
1080                  * Currently a rule for inputset change or symm turn on/off
1081                  * will overwrite an exist rule, while application still
1082                  * have 2 rte_flow handles.
1083                  **/
1084                 if (ret && ret != ICE_ERR_DOES_NOT_EXIST) {
1085                         rte_flow_error_set(error, EINVAL,
1086                                         RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1087                                         "rss flow destroy fail");
1088                         goto error;
1089                 }
1090         }
1091
1092         rte_free(filter_ptr);
1093         return 0;
1094
1095 error:
1096         rte_free(filter_ptr);
1097         return -rte_errno;
1098 }
1099
1100 static void
1101 ice_hash_uninit(struct ice_adapter *ad)
1102 {
1103         if (ad->hw.dcf_enabled)
1104                 return;
1105
1106         if (ad->active_pkg_type == ICE_PKG_TYPE_OS_DEFAULT)
1107                 ice_unregister_parser(&ice_hash_parser_os, ad);
1108         else if (ad->active_pkg_type == ICE_PKG_TYPE_COMMS)
1109                 ice_unregister_parser(&ice_hash_parser_comms, ad);
1110 }
1111
1112 static void
1113 ice_hash_free(struct rte_flow *flow)
1114 {
1115         rte_free(flow->rule);
1116 }