drivers/net/ice/ice_hash.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10 #include <unistd.h>
  11 #include <stdarg.h>
  12
  13 #include <rte_debug.h>
  14 #include <rte_ether.h>
  15 #include <ethdev_driver.h>
  16 #include <rte_log.h>
  17 #include <rte_malloc.h>
  18 #include <rte_eth_ctrl.h>
  19 #include <rte_tailq.h>
  20 #include <rte_flow_driver.h>
  21
  22 #include "ice_logs.h"
  23 #include "base/ice_type.h"
  24 #include "base/ice_flow.h"
  25 #include "ice_ethdev.h"
  26 #include "ice_generic_flow.h"
  27
  28 #define ICE_PHINT_NONE                          0
  29 #define ICE_PHINT_VLAN                          BIT_ULL(0)
  30 #define ICE_PHINT_PPPOE                         BIT_ULL(1)
  31 #define ICE_PHINT_GTPU                          BIT_ULL(2)
  32 #define ICE_PHINT_GTPU_EH                       BIT_ULL(3)
  33 #define ICE_PHINT_GTPU_EH_DWN                   BIT_ULL(4)
  34 #define ICE_PHINT_GTPU_EH_UP                    BIT_ULL(5)
  35
  36 #define ICE_GTPU_EH_DWNLINK     0
  37 #define ICE_GTPU_EH_UPLINK      1
  38
  39 #define ICE_IPV4_PROT           BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT)
  40 #define ICE_IPV6_PROT           BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT)
  41
  42 #define VALID_RSS_IPV4_L4       (ETH_RSS_NONFRAG_IPV4_UDP       | \
  43                                  ETH_RSS_NONFRAG_IPV4_TCP       | \
  44                                  ETH_RSS_NONFRAG_IPV4_SCTP)
  45
  46 #define VALID_RSS_IPV6_L4       (ETH_RSS_NONFRAG_IPV6_UDP       | \
  47                                  ETH_RSS_NONFRAG_IPV6_TCP       | \
  48                                  ETH_RSS_NONFRAG_IPV6_SCTP)
  49
  50 #define VALID_RSS_IPV4          (ETH_RSS_IPV4 | VALID_RSS_IPV4_L4)
  51 #define VALID_RSS_IPV6          (ETH_RSS_IPV6 | VALID_RSS_IPV6_L4)
  52 #define VALID_RSS_L3            (VALID_RSS_IPV4 | VALID_RSS_IPV6)
  53 #define VALID_RSS_L4            (VALID_RSS_IPV4_L4 | VALID_RSS_IPV6_L4)
  54
  55 #define VALID_RSS_ATTR          (ETH_RSS_L3_SRC_ONLY    | \
  56                                  ETH_RSS_L3_DST_ONLY    | \
  57                                  ETH_RSS_L4_SRC_ONLY    | \
  58                                  ETH_RSS_L4_DST_ONLY    | \
  59                                  ETH_RSS_L2_SRC_ONLY    | \
  60                                  ETH_RSS_L2_DST_ONLY    | \
  61                                  RTE_ETH_RSS_L3_PRE32   | \
  62                                  RTE_ETH_RSS_L3_PRE48   | \
  63                                  RTE_ETH_RSS_L3_PRE64)
  64
  65 #define INVALID_RSS_ATTR        (RTE_ETH_RSS_L3_PRE40   | \
  66                                  RTE_ETH_RSS_L3_PRE56   | \
  67                                  RTE_ETH_RSS_L3_PRE96)
  68
  69 struct ice_rss_meta {
  70         uint8_t hash_function;
  71         struct ice_rss_hash_cfg cfg;
  72 };
  73
  74 struct ice_hash_flow_cfg {
  75         bool simple_xor;
  76         struct ice_rss_cfg rss_cfg;
  77 };
  78
  79 static int
  80 ice_hash_init(struct ice_adapter *ad);
  81
  82 static int
  83 ice_hash_create(struct ice_adapter *ad,
  84                 struct rte_flow *flow,
  85                 void *meta,
  86                 struct rte_flow_error *error);
  87
  88 static int
  89 ice_hash_destroy(struct ice_adapter *ad,
  90                 struct rte_flow *flow,
  91                 struct rte_flow_error *error);
  92
  93 static void
  94 ice_hash_uninit(struct ice_adapter *ad);
  95
  96 static void
  97 ice_hash_free(struct rte_flow *flow);
  98
  99 static int
 100 ice_hash_parse_pattern_action(struct ice_adapter *ad,
 101                         struct ice_pattern_match_item *array,
 102                         uint32_t array_len,
 103                         const struct rte_flow_item pattern[],
 104                         const struct rte_flow_action actions[],
 105                         void **meta,
 106                         struct rte_flow_error *error);
 107
 108 /* Rss configuration template */
 109 struct ice_rss_hash_cfg ipv4_tmplt = {
 110         ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV4 |
 111         ICE_FLOW_SEG_HDR_IPV_OTHER,
 112         ICE_FLOW_HASH_ETH | ICE_FLOW_HASH_IPV4,
 113         ICE_RSS_OUTER_HEADERS,
 114         0
 115 };
 116
 117 struct ice_rss_hash_cfg ipv4_udp_tmplt = {
 118         ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV4 |
 119         ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_UDP,
 120         ICE_FLOW_HASH_ETH | ICE_HASH_UDP_IPV4 | ICE_IPV4_PROT,
 121         ICE_RSS_OUTER_HEADERS,
 122         0
 123 };
 124
 125 struct ice_rss_hash_cfg ipv4_tcp_tmplt = {
 126         ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV4 |
 127         ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_TCP,
 128         ICE_FLOW_HASH_ETH | ICE_HASH_TCP_IPV4 | ICE_IPV4_PROT,
 129         ICE_RSS_OUTER_HEADERS,
 130         0
 131 };
 132
 133 struct ice_rss_hash_cfg ipv4_sctp_tmplt = {
 134         ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV4 |
 135         ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_SCTP,
 136         ICE_FLOW_HASH_ETH | ICE_HASH_SCTP_IPV4 | ICE_IPV4_PROT,
 137         ICE_RSS_OUTER_HEADERS,
 138         0
 139 };
 140
 141 struct ice_rss_hash_cfg ipv6_tmplt = {
 142         ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV6 |
 143         ICE_FLOW_SEG_HDR_IPV_OTHER,
 144         ICE_FLOW_HASH_ETH | ICE_FLOW_HASH_IPV6,
 145         ICE_RSS_OUTER_HEADERS,
 146         0
 147 };
 148
 149 struct ice_rss_hash_cfg ipv6_udp_tmplt = {
 150         ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV6 |
 151         ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_UDP,
 152         ICE_FLOW_HASH_ETH | ICE_HASH_UDP_IPV6 | ICE_IPV6_PROT,
 153         ICE_RSS_OUTER_HEADERS,
 154         0
 155 };
 156
 157 struct ice_rss_hash_cfg ipv6_tcp_tmplt = {
 158         ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV6 |
 159         ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_TCP,
 160         ICE_FLOW_HASH_ETH | ICE_HASH_TCP_IPV6 | ICE_IPV6_PROT,
 161         ICE_RSS_OUTER_HEADERS,
 162         0
 163 };
 164
 165 struct ice_rss_hash_cfg ipv6_sctp_tmplt = {
 166         ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV6 |
 167         ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_SCTP,
 168         ICE_FLOW_HASH_ETH | ICE_HASH_SCTP_IPV6 | ICE_IPV6_PROT,
 169         ICE_RSS_OUTER_HEADERS,
 170         0
 171 };
 172
 173 struct ice_rss_hash_cfg outer_ipv4_inner_ipv4_tmplt = {
 174         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER,
 175         ICE_FLOW_HASH_IPV4,
 176         ICE_RSS_INNER_HEADERS_W_OUTER_IPV4,
 177         0
 178 };
 179 struct ice_rss_hash_cfg outer_ipv4_inner_ipv4_udp_tmplt = {
 180         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 181         ICE_FLOW_SEG_HDR_UDP,
 182         ICE_HASH_UDP_IPV4 | ICE_IPV4_PROT,
 183         ICE_RSS_INNER_HEADERS_W_OUTER_IPV4,
 184         0
 185 };
 186
 187 struct ice_rss_hash_cfg outer_ipv4_inner_ipv4_tcp_tmplt = {
 188         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 189         ICE_FLOW_SEG_HDR_TCP,
 190         ICE_HASH_TCP_IPV4 | ICE_IPV4_PROT,
 191         ICE_RSS_INNER_HEADERS_W_OUTER_IPV4,
 192         0
 193 };
 194
 195 struct ice_rss_hash_cfg outer_ipv6_inner_ipv4_tmplt = {
 196         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER,
 197         ICE_FLOW_HASH_IPV4,
 198         ICE_RSS_INNER_HEADERS_W_OUTER_IPV6,
 199         0
 200 };
 201 struct ice_rss_hash_cfg outer_ipv6_inner_ipv4_udp_tmplt = {
 202         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 203         ICE_FLOW_SEG_HDR_UDP,
 204         ICE_HASH_UDP_IPV4 | ICE_IPV4_PROT,
 205         ICE_RSS_INNER_HEADERS_W_OUTER_IPV6,
 206         0
 207 };
 208
 209 struct ice_rss_hash_cfg outer_ipv6_inner_ipv4_tcp_tmplt = {
 210         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 211         ICE_FLOW_SEG_HDR_TCP,
 212         ICE_HASH_TCP_IPV4 | ICE_IPV4_PROT,
 213         ICE_RSS_INNER_HEADERS_W_OUTER_IPV6,
 214         0
 215 };
 216
 217 struct ice_rss_hash_cfg outer_ipv4_inner_ipv6_tmplt = {
 218         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER,
 219         ICE_FLOW_HASH_IPV6,
 220         ICE_RSS_INNER_HEADERS_W_OUTER_IPV4,
 221         0
 222 };
 223 struct ice_rss_hash_cfg outer_ipv4_inner_ipv6_udp_tmplt = {
 224         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 225         ICE_FLOW_SEG_HDR_UDP,
 226         ICE_HASH_UDP_IPV6 | ICE_IPV6_PROT,
 227         ICE_RSS_INNER_HEADERS_W_OUTER_IPV4,
 228         0
 229 };
 230
 231 struct ice_rss_hash_cfg outer_ipv4_inner_ipv6_tcp_tmplt = {
 232         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 233         ICE_FLOW_SEG_HDR_TCP,
 234         ICE_HASH_TCP_IPV6 | ICE_IPV6_PROT,
 235         ICE_RSS_INNER_HEADERS_W_OUTER_IPV4,
 236         0
 237 };
 238
 239 struct ice_rss_hash_cfg outer_ipv6_inner_ipv6_tmplt = {
 240         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER,
 241         ICE_FLOW_HASH_IPV6,
 242         ICE_RSS_INNER_HEADERS_W_OUTER_IPV6,
 243         0
 244 };
 245 struct ice_rss_hash_cfg outer_ipv6_inner_ipv6_udp_tmplt = {
 246         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 247         ICE_FLOW_SEG_HDR_UDP,
 248         ICE_HASH_UDP_IPV6 | ICE_IPV6_PROT,
 249         ICE_RSS_INNER_HEADERS_W_OUTER_IPV6,
 250         0
 251 };
 252
 253 struct ice_rss_hash_cfg outer_ipv6_inner_ipv6_tcp_tmplt = {
 254         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 255         ICE_FLOW_SEG_HDR_TCP,
 256         ICE_HASH_TCP_IPV6 | ICE_IPV6_PROT,
 257         ICE_RSS_INNER_HEADERS_W_OUTER_IPV6,
 258         0
 259 };
 260
 261 struct ice_rss_hash_cfg eth_ipv4_esp_tmplt = {
 262         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 263         ICE_FLOW_SEG_HDR_ESP,
 264         ICE_FLOW_HASH_ESP_SPI,
 265         ICE_RSS_OUTER_HEADERS,
 266         0
 267 };
 268
 269 struct ice_rss_hash_cfg eth_ipv4_udp_esp_tmplt = {
 270         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 271         ICE_FLOW_SEG_HDR_NAT_T_ESP,
 272         ICE_FLOW_HASH_NAT_T_ESP_SPI,
 273         ICE_RSS_OUTER_HEADERS,
 274         0
 275 };
 276
 277 struct ice_rss_hash_cfg eth_ipv4_ah_tmplt = {
 278         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 279         ICE_FLOW_SEG_HDR_AH,
 280         ICE_FLOW_HASH_AH_SPI,
 281         ICE_RSS_OUTER_HEADERS,
 282         0
 283 };
 284
 285 struct ice_rss_hash_cfg eth_ipv4_l2tpv3_tmplt = {
 286         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 287         ICE_FLOW_SEG_HDR_L2TPV3,
 288         ICE_FLOW_HASH_L2TPV3_SESS_ID,
 289         ICE_RSS_OUTER_HEADERS,
 290         0
 291 };
 292
 293 struct ice_rss_hash_cfg eth_ipv4_pfcp_tmplt = {
 294         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 295         ICE_FLOW_SEG_HDR_PFCP_SESSION,
 296         ICE_FLOW_HASH_PFCP_SEID,
 297         ICE_RSS_OUTER_HEADERS,
 298         0
 299 };
 300
 301 struct ice_rss_hash_cfg eth_ipv6_esp_tmplt = {
 302         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 303         ICE_FLOW_SEG_HDR_ESP,
 304         ICE_FLOW_HASH_ESP_SPI,
 305         ICE_RSS_OUTER_HEADERS,
 306         0
 307 };
 308
 309 struct ice_rss_hash_cfg eth_ipv6_udp_esp_tmplt = {
 310         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 311         ICE_FLOW_SEG_HDR_NAT_T_ESP,
 312         ICE_FLOW_HASH_NAT_T_ESP_SPI,
 313         ICE_RSS_OUTER_HEADERS,
 314         0
 315 };
 316
 317 struct ice_rss_hash_cfg eth_ipv6_ah_tmplt = {
 318         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 319         ICE_FLOW_SEG_HDR_AH,
 320         ICE_FLOW_HASH_AH_SPI,
 321         ICE_RSS_OUTER_HEADERS,
 322         0
 323 };
 324
 325 struct ice_rss_hash_cfg eth_ipv6_l2tpv3_tmplt = {
 326         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 327         ICE_FLOW_SEG_HDR_L2TPV3,
 328         ICE_FLOW_HASH_L2TPV3_SESS_ID,
 329         ICE_RSS_OUTER_HEADERS,
 330         0
 331 };
 332
 333 struct ice_rss_hash_cfg eth_ipv6_pfcp_tmplt = {
 334         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 335         ICE_FLOW_SEG_HDR_PFCP_SESSION,
 336         ICE_FLOW_HASH_PFCP_SEID,
 337         ICE_RSS_OUTER_HEADERS,
 338         0
 339 };
 340
 341 struct ice_rss_hash_cfg pppoe_tmplt = {
 342         ICE_FLOW_SEG_HDR_ETH,
 343         ICE_FLOW_HASH_ETH | ICE_FLOW_HASH_PPPOE_SESS_ID,
 344         ICE_RSS_OUTER_HEADERS,
 345         0
 346 };
 347
 348 struct ice_rss_hash_cfg empty_tmplt = {
 349         ICE_FLOW_SEG_HDR_NONE,
 350         0,
 351         ICE_RSS_ANY_HEADERS,
 352         0
 353 };
 354
 355 /* IPv4 */
 356 #define ICE_RSS_TYPE_ETH_IPV4           (ETH_RSS_ETH | ETH_RSS_IPV4)
 357 #define ICE_RSS_TYPE_ETH_IPV4_UDP       (ICE_RSS_TYPE_ETH_IPV4 | \
 358                                          ETH_RSS_NONFRAG_IPV4_UDP)
 359 #define ICE_RSS_TYPE_ETH_IPV4_TCP       (ICE_RSS_TYPE_ETH_IPV4 | \
 360                                          ETH_RSS_NONFRAG_IPV4_TCP)
 361 #define ICE_RSS_TYPE_ETH_IPV4_SCTP      (ICE_RSS_TYPE_ETH_IPV4 | \
 362                                          ETH_RSS_NONFRAG_IPV4_SCTP)
 363 #define ICE_RSS_TYPE_IPV4               ETH_RSS_IPV4
 364 #define ICE_RSS_TYPE_IPV4_UDP           (ETH_RSS_IPV4 | \
 365                                          ETH_RSS_NONFRAG_IPV4_UDP)
 366 #define ICE_RSS_TYPE_IPV4_TCP           (ETH_RSS_IPV4 | \
 367                                          ETH_RSS_NONFRAG_IPV4_TCP)
 368 #define ICE_RSS_TYPE_IPV4_SCTP          (ETH_RSS_IPV4 | \
 369                                          ETH_RSS_NONFRAG_IPV4_SCTP)
 370
 371 /* IPv6 */
 372 #define ICE_RSS_TYPE_ETH_IPV6           (ETH_RSS_ETH | ETH_RSS_IPV6)
 373 #define ICE_RSS_TYPE_ETH_IPV6_UDP       (ICE_RSS_TYPE_ETH_IPV6 | \
 374                                          ETH_RSS_NONFRAG_IPV6_UDP)
 375 #define ICE_RSS_TYPE_ETH_IPV6_TCP       (ICE_RSS_TYPE_ETH_IPV6 | \
 376                                          ETH_RSS_NONFRAG_IPV6_TCP)
 377 #define ICE_RSS_TYPE_ETH_IPV6_SCTP      (ICE_RSS_TYPE_ETH_IPV6 | \
 378                                          ETH_RSS_NONFRAG_IPV6_SCTP)
 379 #define ICE_RSS_TYPE_IPV6               ETH_RSS_IPV6
 380 #define ICE_RSS_TYPE_IPV6_UDP           (ETH_RSS_IPV6 | \
 381                                          ETH_RSS_NONFRAG_IPV6_UDP)
 382 #define ICE_RSS_TYPE_IPV6_TCP           (ETH_RSS_IPV6 | \
 383                                          ETH_RSS_NONFRAG_IPV6_TCP)
 384 #define ICE_RSS_TYPE_IPV6_SCTP          (ETH_RSS_IPV6 | \
 385                                          ETH_RSS_NONFRAG_IPV6_SCTP)
 386
 387 /* VLAN IPV4 */
 388 #define ICE_RSS_TYPE_VLAN_IPV4          (ICE_RSS_TYPE_IPV4 | \
 389                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 390 #define ICE_RSS_TYPE_VLAN_IPV4_UDP      (ICE_RSS_TYPE_IPV4_UDP | \
 391                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 392 #define ICE_RSS_TYPE_VLAN_IPV4_TCP      (ICE_RSS_TYPE_IPV4_TCP | \
 393                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 394 #define ICE_RSS_TYPE_VLAN_IPV4_SCTP     (ICE_RSS_TYPE_IPV4_SCTP | \
 395                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 396 /* VLAN IPv6 */
 397 #define ICE_RSS_TYPE_VLAN_IPV6          (ICE_RSS_TYPE_IPV6 | \
 398                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 399 #define ICE_RSS_TYPE_VLAN_IPV6_UDP      (ICE_RSS_TYPE_IPV6_UDP | \
 400                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 401 #define ICE_RSS_TYPE_VLAN_IPV6_TCP      (ICE_RSS_TYPE_IPV6_TCP | \
 402                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 403 #define ICE_RSS_TYPE_VLAN_IPV6_SCTP     (ICE_RSS_TYPE_IPV6_SCTP | \
 404                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 405
 406 /* GTPU IPv4 */
 407 #define ICE_RSS_TYPE_GTPU_IPV4          (ICE_RSS_TYPE_IPV4 | \
 408                                          ETH_RSS_GTPU)
 409 #define ICE_RSS_TYPE_GTPU_IPV4_UDP      (ICE_RSS_TYPE_IPV4_UDP | \
 410                                          ETH_RSS_GTPU)
 411 #define ICE_RSS_TYPE_GTPU_IPV4_TCP      (ICE_RSS_TYPE_IPV4_TCP | \
 412                                          ETH_RSS_GTPU)
 413 /* GTPU IPv6 */
 414 #define ICE_RSS_TYPE_GTPU_IPV6          (ICE_RSS_TYPE_IPV6 | \
 415                                          ETH_RSS_GTPU)
 416 #define ICE_RSS_TYPE_GTPU_IPV6_UDP      (ICE_RSS_TYPE_IPV6_UDP | \
 417                                          ETH_RSS_GTPU)
 418 #define ICE_RSS_TYPE_GTPU_IPV6_TCP      (ICE_RSS_TYPE_IPV6_TCP | \
 419                                          ETH_RSS_GTPU)
 420
 421 /* PPPOE */
 422 #define ICE_RSS_TYPE_PPPOE              (ETH_RSS_ETH | ETH_RSS_PPPOE)
 423
 424 /* PPPOE IPv4 */
 425 #define ICE_RSS_TYPE_PPPOE_IPV4         (ICE_RSS_TYPE_IPV4 | \
 426                                          ICE_RSS_TYPE_PPPOE)
 427 #define ICE_RSS_TYPE_PPPOE_IPV4_UDP     (ICE_RSS_TYPE_IPV4_UDP | \
 428                                          ICE_RSS_TYPE_PPPOE)
 429 #define ICE_RSS_TYPE_PPPOE_IPV4_TCP     (ICE_RSS_TYPE_IPV4_TCP | \
 430                                          ICE_RSS_TYPE_PPPOE)
 431
 432 /* PPPOE IPv6 */
 433 #define ICE_RSS_TYPE_PPPOE_IPV6         (ICE_RSS_TYPE_IPV6 | \
 434                                          ICE_RSS_TYPE_PPPOE)
 435 #define ICE_RSS_TYPE_PPPOE_IPV6_UDP     (ICE_RSS_TYPE_IPV6_UDP | \
 436                                          ICE_RSS_TYPE_PPPOE)
 437 #define ICE_RSS_TYPE_PPPOE_IPV6_TCP     (ICE_RSS_TYPE_IPV6_TCP | \
 438                                          ICE_RSS_TYPE_PPPOE)
 439
 440 /* ESP, AH, L2TPV3 and PFCP */
 441 #define ICE_RSS_TYPE_IPV4_ESP           (ETH_RSS_ESP | ETH_RSS_IPV4)
 442 #define ICE_RSS_TYPE_IPV6_ESP           (ETH_RSS_ESP | ETH_RSS_IPV6)
 443 #define ICE_RSS_TYPE_IPV4_AH            (ETH_RSS_AH | ETH_RSS_IPV4)
 444 #define ICE_RSS_TYPE_IPV6_AH            (ETH_RSS_AH | ETH_RSS_IPV6)
 445 #define ICE_RSS_TYPE_IPV4_L2TPV3        (ETH_RSS_L2TPV3 | ETH_RSS_IPV4)
 446 #define ICE_RSS_TYPE_IPV6_L2TPV3        (ETH_RSS_L2TPV3 | ETH_RSS_IPV6)
 447 #define ICE_RSS_TYPE_IPV4_PFCP          (ETH_RSS_PFCP | ETH_RSS_IPV4)
 448 #define ICE_RSS_TYPE_IPV6_PFCP          (ETH_RSS_PFCP | ETH_RSS_IPV6)
 449
 450 /**
 451  * Supported pattern for hash.
 452  * The first member is pattern item type,
 453  * the second member is input set mask,
 454  * the third member is ice_rss_hash_cfg template.
 455  */
 456 static struct ice_pattern_match_item ice_hash_pattern_list[] = {
 457         /* IPV4 */
 458         {pattern_eth_ipv4,                      ICE_RSS_TYPE_ETH_IPV4,          ICE_INSET_NONE, &ipv4_tmplt},
 459         {pattern_eth_ipv4_udp,                  ICE_RSS_TYPE_ETH_IPV4_UDP,      ICE_INSET_NONE, &ipv4_udp_tmplt},
 460         {pattern_eth_ipv4_tcp,                  ICE_RSS_TYPE_ETH_IPV4_TCP,      ICE_INSET_NONE, &ipv4_tcp_tmplt},
 461         {pattern_eth_ipv4_sctp,                 ICE_RSS_TYPE_ETH_IPV4_SCTP,     ICE_INSET_NONE, &ipv4_sctp_tmplt},
 462         {pattern_eth_vlan_ipv4,                 ICE_RSS_TYPE_VLAN_IPV4,         ICE_INSET_NONE, &ipv4_tmplt},
 463         {pattern_eth_vlan_ipv4_udp,             ICE_RSS_TYPE_VLAN_IPV4_UDP,     ICE_INSET_NONE, &ipv4_udp_tmplt},
 464         {pattern_eth_vlan_ipv4_tcp,             ICE_RSS_TYPE_VLAN_IPV4_TCP,     ICE_INSET_NONE, &ipv4_tcp_tmplt},
 465         {pattern_eth_vlan_ipv4_sctp,            ICE_RSS_TYPE_VLAN_IPV4_SCTP,    ICE_INSET_NONE, &ipv4_sctp_tmplt},
 466         {pattern_eth_ipv4_gtpu_ipv4,            ICE_RSS_TYPE_GTPU_IPV4,         ICE_INSET_NONE, &outer_ipv4_inner_ipv4_tmplt},
 467         {pattern_eth_ipv4_gtpu_ipv4_udp,        ICE_RSS_TYPE_GTPU_IPV4_UDP,     ICE_INSET_NONE, &outer_ipv4_inner_ipv4_udp_tmplt},
 468         {pattern_eth_ipv4_gtpu_ipv4_tcp,        ICE_RSS_TYPE_GTPU_IPV4_TCP,     ICE_INSET_NONE, &outer_ipv4_inner_ipv4_tcp_tmplt},
 469         {pattern_eth_ipv6_gtpu_ipv4,            ICE_RSS_TYPE_GTPU_IPV4,         ICE_INSET_NONE, &outer_ipv6_inner_ipv4_tmplt},
 470         {pattern_eth_ipv6_gtpu_ipv4_udp,        ICE_RSS_TYPE_GTPU_IPV4_UDP,     ICE_INSET_NONE, &outer_ipv6_inner_ipv4_udp_tmplt},
 471         {pattern_eth_ipv6_gtpu_ipv4_tcp,        ICE_RSS_TYPE_GTPU_IPV4_TCP,     ICE_INSET_NONE, &outer_ipv6_inner_ipv4_tcp_tmplt},
 472         {pattern_eth_ipv4_gtpu_eh_ipv4,         ICE_RSS_TYPE_GTPU_IPV4,         ICE_INSET_NONE, &outer_ipv4_inner_ipv4_tmplt},
 473         {pattern_eth_ipv4_gtpu_eh_ipv4_udp,     ICE_RSS_TYPE_GTPU_IPV4_UDP,     ICE_INSET_NONE, &outer_ipv4_inner_ipv4_udp_tmplt},
 474         {pattern_eth_ipv4_gtpu_eh_ipv4_tcp,     ICE_RSS_TYPE_GTPU_IPV4_TCP,     ICE_INSET_NONE, &outer_ipv4_inner_ipv4_tcp_tmplt},
 475         {pattern_eth_ipv6_gtpu_eh_ipv4,         ICE_RSS_TYPE_GTPU_IPV4,         ICE_INSET_NONE, &outer_ipv6_inner_ipv4_tmplt},
 476         {pattern_eth_ipv6_gtpu_eh_ipv4_udp,     ICE_RSS_TYPE_GTPU_IPV4_UDP,     ICE_INSET_NONE, &outer_ipv6_inner_ipv4_udp_tmplt},
 477         {pattern_eth_ipv6_gtpu_eh_ipv4_tcp,     ICE_RSS_TYPE_GTPU_IPV4_TCP,     ICE_INSET_NONE, &outer_ipv6_inner_ipv4_tcp_tmplt},
 478         {pattern_eth_pppoes_ipv4,               ICE_RSS_TYPE_PPPOE_IPV4,        ICE_INSET_NONE, &ipv4_tmplt},
 479         {pattern_eth_pppoes_ipv4_udp,           ICE_RSS_TYPE_PPPOE_IPV4_UDP,    ICE_INSET_NONE, &ipv4_udp_tmplt},
 480         {pattern_eth_pppoes_ipv4_tcp,           ICE_RSS_TYPE_PPPOE_IPV4_TCP,    ICE_INSET_NONE, &ipv4_tcp_tmplt},
 481         {pattern_eth_ipv4_esp,                  ICE_RSS_TYPE_IPV4_ESP,          ICE_INSET_NONE, &eth_ipv4_esp_tmplt},
 482         {pattern_eth_ipv4_udp_esp,              ICE_RSS_TYPE_IPV4_ESP,          ICE_INSET_NONE, &eth_ipv4_udp_esp_tmplt},
 483         {pattern_eth_ipv4_ah,                   ICE_RSS_TYPE_IPV4_AH,           ICE_INSET_NONE, &eth_ipv4_ah_tmplt},
 484         {pattern_eth_ipv4_l2tp,                 ICE_RSS_TYPE_IPV4_L2TPV3,       ICE_INSET_NONE, &eth_ipv4_l2tpv3_tmplt},
 485         {pattern_eth_ipv4_pfcp,                 ICE_RSS_TYPE_IPV4_PFCP,         ICE_INSET_NONE, &eth_ipv4_pfcp_tmplt},
 486         /* IPV6 */
 487         {pattern_eth_ipv6,                      ICE_RSS_TYPE_ETH_IPV6,          ICE_INSET_NONE, &ipv6_tmplt},
 488         {pattern_eth_ipv6_udp,                  ICE_RSS_TYPE_ETH_IPV6_UDP,      ICE_INSET_NONE, &ipv6_udp_tmplt},
 489         {pattern_eth_ipv6_tcp,                  ICE_RSS_TYPE_ETH_IPV6_TCP,      ICE_INSET_NONE, &ipv6_tcp_tmplt},
 490         {pattern_eth_ipv6_sctp,                 ICE_RSS_TYPE_ETH_IPV6_SCTP,     ICE_INSET_NONE, &ipv6_sctp_tmplt},
 491         {pattern_eth_vlan_ipv6,                 ICE_RSS_TYPE_VLAN_IPV6,         ICE_INSET_NONE, &ipv6_tmplt},
 492         {pattern_eth_vlan_ipv6_udp,             ICE_RSS_TYPE_VLAN_IPV6_UDP,     ICE_INSET_NONE, &ipv6_udp_tmplt},
 493         {pattern_eth_vlan_ipv6_tcp,             ICE_RSS_TYPE_VLAN_IPV6_TCP,     ICE_INSET_NONE, &ipv6_tcp_tmplt},
 494         {pattern_eth_vlan_ipv6_sctp,            ICE_RSS_TYPE_VLAN_IPV6_SCTP,    ICE_INSET_NONE, &ipv6_sctp_tmplt},
 495         {pattern_eth_ipv4_gtpu_ipv6,            ICE_RSS_TYPE_GTPU_IPV6,         ICE_INSET_NONE, &outer_ipv4_inner_ipv6_tmplt},
 496         {pattern_eth_ipv4_gtpu_ipv6_udp,        ICE_RSS_TYPE_GTPU_IPV6_UDP,     ICE_INSET_NONE, &outer_ipv4_inner_ipv6_udp_tmplt},
 497         {pattern_eth_ipv4_gtpu_ipv6_tcp,        ICE_RSS_TYPE_GTPU_IPV6_TCP,     ICE_INSET_NONE, &outer_ipv4_inner_ipv6_tcp_tmplt},
 498         {pattern_eth_ipv6_gtpu_ipv6,            ICE_RSS_TYPE_GTPU_IPV6,         ICE_INSET_NONE, &outer_ipv6_inner_ipv6_tmplt},
 499         {pattern_eth_ipv6_gtpu_ipv6_udp,        ICE_RSS_TYPE_GTPU_IPV6_UDP,     ICE_INSET_NONE, &outer_ipv6_inner_ipv6_udp_tmplt},
 500         {pattern_eth_ipv6_gtpu_ipv6_tcp,        ICE_RSS_TYPE_GTPU_IPV6_TCP,     ICE_INSET_NONE, &outer_ipv6_inner_ipv6_tcp_tmplt},
 501         {pattern_eth_ipv4_gtpu_eh_ipv6,         ICE_RSS_TYPE_GTPU_IPV6,         ICE_INSET_NONE, &outer_ipv4_inner_ipv6_tmplt},
 502         {pattern_eth_ipv4_gtpu_eh_ipv6_udp,     ICE_RSS_TYPE_GTPU_IPV6_UDP,     ICE_INSET_NONE, &outer_ipv4_inner_ipv6_udp_tmplt},
 503         {pattern_eth_ipv4_gtpu_eh_ipv6_tcp,     ICE_RSS_TYPE_GTPU_IPV6_TCP,     ICE_INSET_NONE, &outer_ipv4_inner_ipv6_tcp_tmplt},
 504         {pattern_eth_ipv6_gtpu_eh_ipv6,         ICE_RSS_TYPE_GTPU_IPV6,         ICE_INSET_NONE, &outer_ipv6_inner_ipv6_tmplt},
 505         {pattern_eth_ipv6_gtpu_eh_ipv6_udp,     ICE_RSS_TYPE_GTPU_IPV6_UDP,     ICE_INSET_NONE, &outer_ipv6_inner_ipv6_udp_tmplt},
 506         {pattern_eth_ipv6_gtpu_eh_ipv6_tcp,     ICE_RSS_TYPE_GTPU_IPV6_TCP,     ICE_INSET_NONE, &outer_ipv6_inner_ipv6_tcp_tmplt},
 507         {pattern_eth_pppoes_ipv6,               ICE_RSS_TYPE_PPPOE_IPV6,        ICE_INSET_NONE, &ipv6_tmplt},
 508         {pattern_eth_pppoes_ipv6_udp,           ICE_RSS_TYPE_PPPOE_IPV6_UDP,    ICE_INSET_NONE, &ipv6_udp_tmplt},
 509         {pattern_eth_pppoes_ipv6_tcp,           ICE_RSS_TYPE_PPPOE_IPV6_TCP,    ICE_INSET_NONE, &ipv6_tcp_tmplt},
 510         {pattern_eth_ipv6_esp,                  ICE_RSS_TYPE_IPV6_ESP,          ICE_INSET_NONE, &eth_ipv6_esp_tmplt},
 511         {pattern_eth_ipv6_udp_esp,              ICE_RSS_TYPE_IPV6_ESP,          ICE_INSET_NONE, &eth_ipv6_udp_esp_tmplt},
 512         {pattern_eth_ipv6_ah,                   ICE_RSS_TYPE_IPV6_AH,           ICE_INSET_NONE, &eth_ipv6_ah_tmplt},
 513         {pattern_eth_ipv6_l2tp,                 ICE_RSS_TYPE_IPV6_L2TPV3,       ICE_INSET_NONE, &eth_ipv6_l2tpv3_tmplt},
 514         {pattern_eth_ipv6_pfcp,                 ICE_RSS_TYPE_IPV6_PFCP,         ICE_INSET_NONE, &eth_ipv6_pfcp_tmplt},
 515         /* PPPOE */
 516         {pattern_eth_pppoes,                    ICE_RSS_TYPE_PPPOE,             ICE_INSET_NONE, &pppoe_tmplt},
 517         /* EMPTY */
 518         {pattern_empty,                         ICE_INSET_NONE,                 ICE_INSET_NONE, &empty_tmplt},
 519 };
 520
 521 static struct ice_flow_engine ice_hash_engine = {
 522         .init = ice_hash_init,
 523         .create = ice_hash_create,
 524         .destroy = ice_hash_destroy,
 525         .uninit = ice_hash_uninit,
 526         .free = ice_hash_free,
 527         .type = ICE_FLOW_ENGINE_HASH,
 528 };
 529
 530 /* Register parser for os package. */
 531 static struct ice_flow_parser ice_hash_parser = {
 532         .engine = &ice_hash_engine,
 533         .array = ice_hash_pattern_list,
 534         .array_len = RTE_DIM(ice_hash_pattern_list),
 535         .parse_pattern_action = ice_hash_parse_pattern_action,
 536         .stage = ICE_FLOW_STAGE_RSS,
 537 };
 538
 539 RTE_INIT(ice_hash_engine_init)
 540 {
 541         struct ice_flow_engine *engine = &ice_hash_engine;
 542         ice_register_flow_engine(engine);
 543 }
 544
 545 static int
 546 ice_hash_init(struct ice_adapter *ad)
 547 {
 548         struct ice_flow_parser *parser = NULL;
 549
 550         if (ad->hw.dcf_enabled)
 551                 return 0;
 552
 553         parser = &ice_hash_parser;
 554
 555         return ice_register_parser(parser, ad);
 556 }
 557
 558 static int
 559 ice_hash_parse_pattern(const struct rte_flow_item pattern[], uint64_t *phint,
 560                        struct rte_flow_error *error)
 561 {
 562         const struct rte_flow_item *item = pattern;
 563         const struct rte_flow_item_gtp_psc *psc;
 564
 565         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 566                 if (item->last) {
 567                         rte_flow_error_set(error, EINVAL,
 568                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 569                                         "Not support range");
 570                         return -rte_errno;
 571                 }
 572
 573                 switch (item->type) {
 574                 case RTE_FLOW_ITEM_TYPE_VLAN:
 575                         *phint |= ICE_PHINT_VLAN;
 576                         break;
 577                 case RTE_FLOW_ITEM_TYPE_PPPOES:
 578                         *phint |= ICE_PHINT_PPPOE;
 579                         break;
 580                 case RTE_FLOW_ITEM_TYPE_GTPU:
 581                         *phint |= ICE_PHINT_GTPU;
 582                         break;
 583                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
 584                         *phint |= ICE_PHINT_GTPU_EH;
 585                         psc = item->spec;
 586                         if (!psc)
 587                                 break;
 588                         else if (psc->pdu_type == ICE_GTPU_EH_UPLINK)
 589                                 *phint |= ICE_PHINT_GTPU_EH_UP;
 590                         else if (psc->pdu_type == ICE_GTPU_EH_DWNLINK)
 591                                 *phint |= ICE_PHINT_GTPU_EH_DWN;
 592                         break;
 593                 default:
 594                         break;
 595                 }
 596         }
 597
 598         return 0;
 599 }
 600
 601 static void
 602 ice_refine_hash_cfg_l234(struct ice_rss_hash_cfg *hash_cfg,
 603                          uint64_t rss_type)
 604 {
 605         uint32_t *addl_hdrs = &hash_cfg->addl_hdrs;
 606         uint64_t *hash_flds = &hash_cfg->hash_flds;
 607
 608         if (*addl_hdrs & ICE_FLOW_SEG_HDR_ETH) {
 609                 if (!(rss_type & ETH_RSS_ETH))
 610                         *hash_flds &= ~ICE_FLOW_HASH_ETH;
 611                 if (rss_type & ETH_RSS_L2_SRC_ONLY)
 612                         *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_ETH_DA));
 613                 else if (rss_type & ETH_RSS_L2_DST_ONLY)
 614                         *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_ETH_SA));
 615                 *addl_hdrs &= ~ICE_FLOW_SEG_HDR_ETH;
 616         }
 617
 618         if (*addl_hdrs & ICE_FLOW_SEG_HDR_VLAN) {
 619                 if (rss_type & ETH_RSS_C_VLAN)
 620                         *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_C_VLAN);
 621                 else if (rss_type & ETH_RSS_S_VLAN)
 622                         *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_S_VLAN);
 623         }
 624
 625         if (*addl_hdrs & ICE_FLOW_SEG_HDR_PPPOE) {
 626                 if (!(rss_type & ETH_RSS_PPPOE))
 627                         *hash_flds &= ~ICE_FLOW_HASH_PPPOE_SESS_ID;
 628         }
 629
 630         if (*addl_hdrs & ICE_FLOW_SEG_HDR_IPV4) {
 631                 if (rss_type &
 632                    (ETH_RSS_IPV4 |
 633                     ETH_RSS_NONFRAG_IPV4_UDP |
 634                     ETH_RSS_NONFRAG_IPV4_TCP |
 635                     ETH_RSS_NONFRAG_IPV4_SCTP)) {
 636                         if (rss_type & ETH_RSS_L3_SRC_ONLY)
 637                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA));
 638                         else if (rss_type & ETH_RSS_L3_DST_ONLY)
 639                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA));
 640                         else if (rss_type &
 641                                 (ETH_RSS_L4_SRC_ONLY |
 642                                 ETH_RSS_L4_DST_ONLY))
 643                                 *hash_flds &= ~ICE_FLOW_HASH_IPV4;
 644                 } else {
 645                         *hash_flds &= ~ICE_FLOW_HASH_IPV4;
 646                 }
 647         }
 648
 649         if (*addl_hdrs & ICE_FLOW_SEG_HDR_IPV6) {
 650                 if (rss_type &
 651                    (ETH_RSS_IPV6 |
 652                     ETH_RSS_NONFRAG_IPV6_UDP |
 653                     ETH_RSS_NONFRAG_IPV6_TCP |
 654                     ETH_RSS_NONFRAG_IPV6_SCTP)) {
 655                         if (rss_type & ETH_RSS_L3_SRC_ONLY)
 656                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA));
 657                         else if (rss_type & ETH_RSS_L3_DST_ONLY)
 658                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA));
 659                         else if (rss_type &
 660                                 (ETH_RSS_L4_SRC_ONLY |
 661                                 ETH_RSS_L4_DST_ONLY))
 662                                 *hash_flds &= ~ICE_FLOW_HASH_IPV6;
 663                 } else {
 664                         *hash_flds &= ~ICE_FLOW_HASH_IPV6;
 665                 }
 666
 667                 if (rss_type & RTE_ETH_RSS_L3_PRE32) {
 668                         if (rss_type & ETH_RSS_L3_SRC_ONLY) {
 669                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA));
 670                                 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE32_SA));
 671                         } else if (rss_type & ETH_RSS_L3_DST_ONLY) {
 672                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA));
 673                                 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE32_DA));
 674                         } else {
 675                                 *hash_flds &= ~ICE_FLOW_HASH_IPV6;
 676                                 *hash_flds |= ICE_FLOW_HASH_IPV6_PRE32;
 677                         }
 678                 }
 679                 if (rss_type & RTE_ETH_RSS_L3_PRE48) {
 680                         if (rss_type & ETH_RSS_L3_SRC_ONLY) {
 681                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA));
 682                                 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE48_SA));
 683                         } else if (rss_type & ETH_RSS_L3_DST_ONLY) {
 684                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA));
 685                                 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE48_DA));
 686                         } else {
 687                                 *hash_flds &= ~ICE_FLOW_HASH_IPV6;
 688                                 *hash_flds |= ICE_FLOW_HASH_IPV6_PRE48;
 689                         }
 690                 }
 691                 if (rss_type & RTE_ETH_RSS_L3_PRE64) {
 692                         if (rss_type & ETH_RSS_L3_SRC_ONLY) {
 693                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA));
 694                                 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE64_SA));
 695                         } else if (rss_type & ETH_RSS_L3_DST_ONLY) {
 696                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA));
 697                                 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE64_DA));
 698                         } else {
 699                                 *hash_flds &= ~ICE_FLOW_HASH_IPV6;
 700                                 *hash_flds |= ICE_FLOW_HASH_IPV6_PRE64;
 701                         }
 702                 }
 703         }
 704
 705         if (*addl_hdrs & ICE_FLOW_SEG_HDR_UDP) {
 706                 if (rss_type &
 707                    (ETH_RSS_NONFRAG_IPV4_UDP |
 708                     ETH_RSS_NONFRAG_IPV6_UDP)) {
 709                         if (rss_type & ETH_RSS_L4_SRC_ONLY)
 710                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT));
 711                         else if (rss_type & ETH_RSS_L4_DST_ONLY)
 712                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT));
 713                         else if (rss_type &
 714                                 (ETH_RSS_L3_SRC_ONLY |
 715                                   ETH_RSS_L3_DST_ONLY))
 716                                 *hash_flds &= ~ICE_FLOW_HASH_UDP_PORT;
 717                 } else {
 718                         *hash_flds &= ~ICE_FLOW_HASH_UDP_PORT;
 719                 }
 720         }
 721
 722         if (*addl_hdrs & ICE_FLOW_SEG_HDR_TCP) {
 723                 if (rss_type &
 724                    (ETH_RSS_NONFRAG_IPV4_TCP |
 725                     ETH_RSS_NONFRAG_IPV6_TCP)) {
 726                         if (rss_type & ETH_RSS_L4_SRC_ONLY)
 727                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT));
 728                         else if (rss_type & ETH_RSS_L4_DST_ONLY)
 729                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT));
 730                         else if (rss_type &
 731                                 (ETH_RSS_L3_SRC_ONLY |
 732                                   ETH_RSS_L3_DST_ONLY))
 733                                 *hash_flds &= ~ICE_FLOW_HASH_TCP_PORT;
 734                 } else {
 735                         *hash_flds &= ~ICE_FLOW_HASH_TCP_PORT;
 736                 }
 737         }
 738
 739         if (*addl_hdrs & ICE_FLOW_SEG_HDR_SCTP) {
 740                 if (rss_type &
 741                    (ETH_RSS_NONFRAG_IPV4_SCTP |
 742                     ETH_RSS_NONFRAG_IPV6_SCTP)) {
 743                         if (rss_type & ETH_RSS_L4_SRC_ONLY)
 744                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT));
 745                         else if (rss_type & ETH_RSS_L4_DST_ONLY)
 746                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT));
 747                         else if (rss_type &
 748                                 (ETH_RSS_L3_SRC_ONLY |
 749                                   ETH_RSS_L3_DST_ONLY))
 750                                 *hash_flds &= ~ICE_FLOW_HASH_SCTP_PORT;
 751                 } else {
 752                         *hash_flds &= ~ICE_FLOW_HASH_SCTP_PORT;
 753                 }
 754         }
 755
 756         if (*addl_hdrs & ICE_FLOW_SEG_HDR_L2TPV3) {
 757                 if (!(rss_type & ETH_RSS_L2TPV3))
 758                         *hash_flds &= ~ICE_FLOW_HASH_L2TPV3_SESS_ID;
 759         }
 760
 761         if (*addl_hdrs & ICE_FLOW_SEG_HDR_ESP) {
 762                 if (!(rss_type & ETH_RSS_ESP))
 763                         *hash_flds &= ~ICE_FLOW_HASH_ESP_SPI;
 764         }
 765
 766         if (*addl_hdrs & ICE_FLOW_SEG_HDR_AH) {
 767                 if (!(rss_type & ETH_RSS_AH))
 768                         *hash_flds &= ~ICE_FLOW_HASH_AH_SPI;
 769         }
 770
 771         if (*addl_hdrs & ICE_FLOW_SEG_HDR_PFCP_SESSION) {
 772                 if (!(rss_type & ETH_RSS_PFCP))
 773                         *hash_flds &= ~ICE_FLOW_HASH_PFCP_SEID;
 774         }
 775 }
 776
 777 static void
 778 ice_refine_proto_hdrs_by_pattern(struct ice_rss_hash_cfg *hash_cfg,
 779                                  uint64_t phint)
 780 {
 781         uint32_t *addl_hdrs = &hash_cfg->addl_hdrs;
 782         if (phint & ICE_PHINT_VLAN)
 783                 *addl_hdrs |= ICE_FLOW_SEG_HDR_VLAN;
 784
 785         if (phint & ICE_PHINT_PPPOE)
 786                 *addl_hdrs |= ICE_FLOW_SEG_HDR_PPPOE;
 787
 788         if (phint & ICE_PHINT_GTPU_EH_DWN)
 789                 *addl_hdrs |= ICE_FLOW_SEG_HDR_GTPU_DWN;
 790         else if (phint & ICE_PHINT_GTPU_EH_UP)
 791                 *addl_hdrs |= ICE_FLOW_SEG_HDR_GTPU_UP;
 792         else if (phint & ICE_PHINT_GTPU_EH)
 793                 *addl_hdrs |= ICE_FLOW_SEG_HDR_GTPU_EH;
 794         else if (phint & ICE_PHINT_GTPU)
 795                 *addl_hdrs |= ICE_FLOW_SEG_HDR_GTPU_IP;
 796 }
 797
 798 static void
 799 ice_refine_hash_cfg_gtpu(struct ice_rss_hash_cfg *hash_cfg,
 800                          uint64_t rss_type)
 801 {
 802         uint32_t *addl_hdrs = &hash_cfg->addl_hdrs;
 803         uint64_t *hash_flds = &hash_cfg->hash_flds;
 804
 805         /* update hash field for gtpu eh/gtpu dwn/gtpu up. */
 806         if (!(rss_type & ETH_RSS_GTPU))
 807                 return;
 808
 809         if (*addl_hdrs & ICE_FLOW_SEG_HDR_GTPU_DWN)
 810                 *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_GTPU_DWN_TEID);
 811         else if (*addl_hdrs & ICE_FLOW_SEG_HDR_GTPU_UP)
 812                 *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_GTPU_UP_TEID);
 813         else if (*addl_hdrs & ICE_FLOW_SEG_HDR_GTPU_EH)
 814                 *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_GTPU_EH_TEID);
 815         else if (*addl_hdrs & ICE_FLOW_SEG_HDR_GTPU_IP)
 816                 *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_GTPU_IP_TEID);
 817 }
 818
 819 static void ice_refine_hash_cfg(struct ice_rss_hash_cfg *hash_cfg,
 820                                 uint64_t rss_type, uint64_t phint)
 821 {
 822         ice_refine_proto_hdrs_by_pattern(hash_cfg, phint);
 823         ice_refine_hash_cfg_l234(hash_cfg, rss_type);
 824         ice_refine_hash_cfg_gtpu(hash_cfg, rss_type);
 825 }
 826
 827 static uint64_t invalid_rss_comb[] = {
 828         ETH_RSS_IPV4 | ETH_RSS_NONFRAG_IPV4_UDP,
 829         ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_UDP,
 830         RTE_ETH_RSS_L3_PRE40 |
 831         RTE_ETH_RSS_L3_PRE56 |
 832         RTE_ETH_RSS_L3_PRE96
 833 };
 834
 835 struct rss_attr_type {
 836         uint64_t attr;
 837         uint64_t type;
 838 };
 839
 840 static struct rss_attr_type rss_attr_to_valid_type[] = {
 841         {ETH_RSS_L2_SRC_ONLY | ETH_RSS_L2_DST_ONLY,     ETH_RSS_ETH},
 842         {ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY,     VALID_RSS_L3},
 843         {ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY,     VALID_RSS_L4},
 844         /* current ipv6 prefix only supports prefix 64 bits*/
 845         {RTE_ETH_RSS_L3_PRE32,                          VALID_RSS_IPV6},
 846         {RTE_ETH_RSS_L3_PRE48,                          VALID_RSS_IPV6},
 847         {RTE_ETH_RSS_L3_PRE64,                          VALID_RSS_IPV6},
 848         {INVALID_RSS_ATTR,                              0}
 849 };
 850
 851 static bool
 852 ice_any_invalid_rss_type(enum rte_eth_hash_function rss_func,
 853                          uint64_t rss_type, uint64_t allow_rss_type)
 854 {
 855         uint32_t i;
 856
 857         /**
 858          * Check if l3/l4 SRC/DST_ONLY is set for SYMMETRIC_TOEPLITZ
 859          * hash function.
 860          */
 861         if (rss_func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) {
 862                 if (rss_type & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY |
 863                     ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY))
 864                         return true;
 865
 866                 if (!(rss_type &
 867                    (ETH_RSS_IPV4 | ETH_RSS_IPV6 |
 868                     ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_NONFRAG_IPV6_UDP |
 869                     ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_NONFRAG_IPV6_TCP |
 870                     ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_NONFRAG_IPV6_SCTP)))
 871                         return true;
 872         }
 873
 874         /* check invalid combination */
 875         for (i = 0; i < RTE_DIM(invalid_rss_comb); i++) {
 876                 if (__builtin_popcountll(rss_type & invalid_rss_comb[i]) > 1)
 877                         return true;
 878         }
 879
 880         /* check invalid RSS attribute */
 881         for (i = 0; i < RTE_DIM(rss_attr_to_valid_type); i++) {
 882                 struct rss_attr_type *rat = &rss_attr_to_valid_type[i];
 883
 884                 if (rat->attr & rss_type && !(rat->type & rss_type))
 885                         return true;
 886         }
 887
 888         /* check not allowed RSS type */
 889         rss_type &= ~VALID_RSS_ATTR;
 890
 891         return ((rss_type & allow_rss_type) != rss_type);
 892 }
 893
 894 static int
 895 ice_hash_parse_action(struct ice_pattern_match_item *pattern_match_item,
 896                 const struct rte_flow_action actions[],
 897                 uint64_t pattern_hint, void **meta,
 898                 struct rte_flow_error *error)
 899 {
 900         struct ice_rss_meta *rss_meta = (struct ice_rss_meta *)*meta;
 901         struct ice_rss_hash_cfg *cfg = pattern_match_item->meta;
 902         enum rte_flow_action_type action_type;
 903         const struct rte_flow_action_rss *rss;
 904         const struct rte_flow_action *action;
 905         uint64_t rss_type;
 906
 907         /* Supported action is RSS. */
 908         for (action = actions; action->type !=
 909                 RTE_FLOW_ACTION_TYPE_END; action++) {
 910                 action_type = action->type;
 911                 switch (action_type) {
 912                 case RTE_FLOW_ACTION_TYPE_RSS:
 913                         rss = action->conf;
 914                         rss_type = rss->types;
 915
 916                         /* Check hash function and save it to rss_meta. */
 917                         if (pattern_match_item->pattern_list !=
 918                             pattern_empty && rss->func ==
 919                             RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) {
 920                                 return rte_flow_error_set(error, ENOTSUP,
 921                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 922                                         "Not supported flow");
 923                         } else if (rss->func ==
 924                                    RTE_ETH_HASH_FUNCTION_SIMPLE_XOR){
 925                                 rss_meta->hash_function =
 926                                 RTE_ETH_HASH_FUNCTION_SIMPLE_XOR;
 927                                 return 0;
 928                         } else if (rss->func ==
 929                                    RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) {
 930                                 rss_meta->hash_function =
 931                                 RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ;
 932                                 cfg->symm = true;
 933                         }
 934
 935                         if (rss->level)
 936                                 return rte_flow_error_set(error, ENOTSUP,
 937                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 938                                         "a nonzero RSS encapsulation level is not supported");
 939
 940                         if (rss->key_len)
 941                                 return rte_flow_error_set(error, ENOTSUP,
 942                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 943                                         "a nonzero RSS key_len is not supported");
 944
 945                         if (rss->queue)
 946                                 return rte_flow_error_set(error, ENOTSUP,
 947                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 948                                         "a non-NULL RSS queue is not supported");
 949
 950                         /**
 951                          * Check simultaneous use of SRC_ONLY and DST_ONLY
 952                          * of the same level.
 953                          */
 954                         rss_type = rte_eth_rss_hf_refine(rss_type);
 955
 956                         if (ice_any_invalid_rss_type(rss->func, rss_type,
 957                                         pattern_match_item->input_set_mask_o))
 958                                 return rte_flow_error_set(error, ENOTSUP,
 959                                         RTE_FLOW_ERROR_TYPE_ACTION,
 960                                         action, "RSS type not supported");
 961
 962                         rss_meta->cfg = *cfg;
 963                         ice_refine_hash_cfg(&rss_meta->cfg,
 964                                             rss_type, pattern_hint);
 965                         break;
 966                 case RTE_FLOW_ACTION_TYPE_END:
 967                         break;
 968
 969                 default:
 970                         rte_flow_error_set(error, EINVAL,
 971                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 972                                         "Invalid action.");
 973                         return -rte_errno;
 974                 }
 975         }
 976
 977         return 0;
 978 }
 979
 980 static int
 981 ice_hash_parse_pattern_action(__rte_unused struct ice_adapter *ad,
 982                         struct ice_pattern_match_item *array,
 983                         uint32_t array_len,
 984                         const struct rte_flow_item pattern[],
 985                         const struct rte_flow_action actions[],
 986                         void **meta,
 987                         struct rte_flow_error *error)
 988 {
 989         int ret = 0;
 990         struct ice_pattern_match_item *pattern_match_item;
 991         struct ice_rss_meta *rss_meta_ptr;
 992         uint64_t phint = ICE_PHINT_NONE;
 993
 994         rss_meta_ptr = rte_zmalloc(NULL, sizeof(*rss_meta_ptr), 0);
 995         if (!rss_meta_ptr) {
 996                 rte_flow_error_set(error, EINVAL,
 997                                 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 998                                 "No memory for rss_meta_ptr");
 999                 return -ENOMEM;
1000         }
1001
1002         /* Check rss supported pattern and find matched pattern. */
1003         pattern_match_item = ice_search_pattern_match_item(ad, pattern, array,
1004                                                            array_len, error);
1005         if (!pattern_match_item) {
1006                 ret = -rte_errno;
1007                 goto error;
1008         }
1009
1010         ret = ice_hash_parse_pattern(pattern, &phint, error);
1011         if (ret)
1012                 goto error;
1013
1014         /* Check rss action. */
1015         ret = ice_hash_parse_action(pattern_match_item, actions, phint,
1016                                     (void **)&rss_meta_ptr, error);
1017
1018 error:
1019         if (!ret && meta)
1020                 *meta = rss_meta_ptr;
1021         else
1022                 rte_free(rss_meta_ptr);
1023         rte_free(pattern_match_item);
1024
1025         return ret;
1026 }
1027
1028 static int
1029 ice_hash_create(struct ice_adapter *ad,
1030                 struct rte_flow *flow,
1031                 void *meta,
1032                 struct rte_flow_error *error)
1033 {
1034         struct ice_pf *pf = &ad->pf;
1035         struct ice_hw *hw = ICE_PF_TO_HW(pf);
1036         struct ice_vsi *vsi = pf->main_vsi;
1037         int ret;
1038         uint32_t reg;
1039         struct ice_hash_flow_cfg *filter_ptr;
1040         struct ice_rss_meta *rss_meta = (struct ice_rss_meta *)meta;
1041         uint8_t hash_function = rss_meta->hash_function;
1042
1043         filter_ptr = rte_zmalloc("ice_rss_filter",
1044                                 sizeof(struct ice_hash_flow_cfg), 0);
1045         if (!filter_ptr) {
1046                 rte_flow_error_set(error, EINVAL,
1047                                 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1048                                 "No memory for filter_ptr");
1049                 return -ENOMEM;
1050         }
1051
1052         if (hash_function == RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) {
1053                 /* Enable registers for simple_xor hash function. */
1054                 reg = ICE_READ_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id));
1055                 reg = (reg & (~VSIQF_HASH_CTL_HASH_SCHEME_M)) |
1056                         (2 << VSIQF_HASH_CTL_HASH_SCHEME_S);
1057                 ICE_WRITE_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id), reg);
1058
1059                 filter_ptr->simple_xor = 1;
1060
1061                 goto out;
1062         } else {
1063                 memcpy(&filter_ptr->rss_cfg.hash, &rss_meta->cfg,
1064                        sizeof(struct ice_rss_hash_cfg));
1065                 ret = ice_add_rss_cfg_wrap(pf, vsi->idx,
1066                                            &filter_ptr->rss_cfg.hash);
1067                 if (ret) {
1068                         rte_flow_error_set(error, EINVAL,
1069                                         RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1070                                         "rss flow create fail");
1071                         goto error;
1072                 }
1073         }
1074
1075 out:
1076         flow->rule = filter_ptr;
1077         rte_free(meta);
1078         return 0;
1079
1080 error:
1081         rte_free(filter_ptr);
1082         rte_free(meta);
1083         return -rte_errno;
1084 }
1085
1086 static int
1087 ice_hash_destroy(struct ice_adapter *ad,
1088                 struct rte_flow *flow,
1089                 struct rte_flow_error *error)
1090 {
1091         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(ad);
1092         struct ice_hw *hw = ICE_PF_TO_HW(pf);
1093         struct ice_vsi *vsi = pf->main_vsi;
1094         int ret;
1095         uint32_t reg;
1096         struct ice_hash_flow_cfg *filter_ptr;
1097
1098         filter_ptr = (struct ice_hash_flow_cfg *)flow->rule;
1099
1100         if (filter_ptr->simple_xor == 1) {
1101                 /* Return to symmetric_toeplitz state. */
1102                 reg = ICE_READ_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id));
1103                 reg = (reg & (~VSIQF_HASH_CTL_HASH_SCHEME_M)) |
1104                         (1 << VSIQF_HASH_CTL_HASH_SCHEME_S);
1105                 ICE_WRITE_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id), reg);
1106         } else {
1107                 ret = ice_rem_rss_cfg_wrap(pf, vsi->idx,
1108                                            &filter_ptr->rss_cfg.hash);
1109                 /* Fixme: Ignore the error if a rule does not exist.
1110                  * Currently a rule for inputset change or symm turn on/off
1111                  * will overwrite an exist rule, while application still
1112                  * have 2 rte_flow handles.
1113                  **/
1114                 if (ret && ret != ICE_ERR_DOES_NOT_EXIST) {
1115                         rte_flow_error_set(error, EINVAL,
1116                                         RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1117                                         "rss flow destroy fail");
1118                         goto error;
1119                 }
1120         }
1121
1122         rte_free(filter_ptr);
1123         return 0;
1124
1125 error:
1126         rte_free(filter_ptr);
1127         return -rte_errno;
1128 }
1129
1130 static void
1131 ice_hash_uninit(struct ice_adapter *ad)
1132 {
1133         if (ad->hw.dcf_enabled)
1134                 return;
1135
1136         ice_unregister_parser(&ice_hash_parser, ad);
1137 }
1138
1139 static void
1140 ice_hash_free(struct rte_flow *flow)
1141 {
1142         rte_free(flow->rule);
1143 }