drivers/net/ice/ice_hash.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10 #include <unistd.h>
  11 #include <stdarg.h>
  12
  13 #include <rte_debug.h>
  14 #include <rte_ether.h>
  15 #include <ethdev_driver.h>
  16 #include <rte_log.h>
  17 #include <rte_malloc.h>
  18 #include <rte_eth_ctrl.h>
  19 #include <rte_tailq.h>
  20 #include <rte_flow_driver.h>
  21
  22 #include "ice_logs.h"
  23 #include "base/ice_type.h"
  24 #include "base/ice_flow.h"
  25 #include "ice_ethdev.h"
  26 #include "ice_generic_flow.h"
  27
  28 #define ICE_PHINT_NONE                          0
  29 #define ICE_PHINT_VLAN                          BIT_ULL(0)
  30 #define ICE_PHINT_PPPOE                         BIT_ULL(1)
  31 #define ICE_PHINT_GTPU                          BIT_ULL(2)
  32 #define ICE_PHINT_GTPU_EH                       BIT_ULL(3)
  33 #define ICE_PHINT_GTPU_EH_DWN                   BIT_ULL(4)
  34 #define ICE_PHINT_GTPU_EH_UP                    BIT_ULL(5)
  35
  36 #define ICE_GTPU_EH_DWNLINK     0
  37 #define ICE_GTPU_EH_UPLINK      1
  38
  39 #define ICE_IPV4_PROT           BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT)
  40 #define ICE_IPV6_PROT           BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT)
  41
  42 #define VALID_RSS_IPV4_L4       (ETH_RSS_NONFRAG_IPV4_UDP       | \
  43                                  ETH_RSS_NONFRAG_IPV4_TCP       | \
  44                                  ETH_RSS_NONFRAG_IPV4_SCTP)
  45
  46 #define VALID_RSS_IPV6_L4       (ETH_RSS_NONFRAG_IPV6_UDP       | \
  47                                  ETH_RSS_NONFRAG_IPV6_TCP       | \
  48                                  ETH_RSS_NONFRAG_IPV6_SCTP)
  49
  50 #define VALID_RSS_IPV4          (ETH_RSS_IPV4 | VALID_RSS_IPV4_L4)
  51 #define VALID_RSS_IPV6          (ETH_RSS_IPV6 | VALID_RSS_IPV6_L4)
  52 #define VALID_RSS_L3            (VALID_RSS_IPV4 | VALID_RSS_IPV6)
  53 #define VALID_RSS_L4            (VALID_RSS_IPV4_L4 | VALID_RSS_IPV6_L4)
  54
  55 #define VALID_RSS_ATTR          (ETH_RSS_L3_SRC_ONLY    | \
  56                                  ETH_RSS_L3_DST_ONLY    | \
  57                                  ETH_RSS_L4_SRC_ONLY    | \
  58                                  ETH_RSS_L4_DST_ONLY    | \
  59                                  ETH_RSS_L2_SRC_ONLY    | \
  60                                  ETH_RSS_L2_DST_ONLY    | \
  61                                  RTE_ETH_RSS_L3_PRE32   | \
  62                                  RTE_ETH_RSS_L3_PRE48   | \
  63                                  RTE_ETH_RSS_L3_PRE64)
  64
  65 #define INVALID_RSS_ATTR        (RTE_ETH_RSS_L3_PRE40   | \
  66                                  RTE_ETH_RSS_L3_PRE56   | \
  67                                  RTE_ETH_RSS_L3_PRE96)
  68
  69 struct ice_rss_meta {
  70         uint8_t hash_function;
  71         struct ice_rss_hash_cfg cfg;
  72 };
  73
  74 struct ice_hash_flow_cfg {
  75         bool simple_xor;
  76         struct ice_rss_cfg rss_cfg;
  77 };
  78
  79 static int
  80 ice_hash_init(struct ice_adapter *ad);
  81
  82 static int
  83 ice_hash_create(struct ice_adapter *ad,
  84                 struct rte_flow *flow,
  85                 void *meta,
  86                 struct rte_flow_error *error);
  87
  88 static int
  89 ice_hash_destroy(struct ice_adapter *ad,
  90                 struct rte_flow *flow,
  91                 struct rte_flow_error *error);
  92
  93 static void
  94 ice_hash_uninit(struct ice_adapter *ad);
  95
  96 static void
  97 ice_hash_free(struct rte_flow *flow);
  98
  99 static int
 100 ice_hash_parse_pattern_action(struct ice_adapter *ad,
 101                         struct ice_pattern_match_item *array,
 102                         uint32_t array_len,
 103                         const struct rte_flow_item pattern[],
 104                         const struct rte_flow_action actions[],
 105                         uint32_t priority,
 106                         void **meta,
 107                         struct rte_flow_error *error);
 108
 109 /* Rss configuration template */
 110 struct ice_rss_hash_cfg ipv4_tmplt = {
 111         ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV4 |
 112         ICE_FLOW_SEG_HDR_IPV_OTHER,
 113         ICE_FLOW_HASH_ETH | ICE_FLOW_HASH_IPV4,
 114         ICE_RSS_OUTER_HEADERS,
 115         0
 116 };
 117
 118 struct ice_rss_hash_cfg ipv4_udp_tmplt = {
 119         ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV4 |
 120         ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_UDP,
 121         ICE_FLOW_HASH_ETH | ICE_HASH_UDP_IPV4 | ICE_IPV4_PROT,
 122         ICE_RSS_OUTER_HEADERS,
 123         0
 124 };
 125
 126 struct ice_rss_hash_cfg ipv4_tcp_tmplt = {
 127         ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV4 |
 128         ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_TCP,
 129         ICE_FLOW_HASH_ETH | ICE_HASH_TCP_IPV4 | ICE_IPV4_PROT,
 130         ICE_RSS_OUTER_HEADERS,
 131         0
 132 };
 133
 134 struct ice_rss_hash_cfg ipv4_sctp_tmplt = {
 135         ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV4 |
 136         ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_SCTP,
 137         ICE_FLOW_HASH_ETH | ICE_HASH_SCTP_IPV4 | ICE_IPV4_PROT,
 138         ICE_RSS_OUTER_HEADERS,
 139         0
 140 };
 141
 142 struct ice_rss_hash_cfg ipv6_tmplt = {
 143         ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV6 |
 144         ICE_FLOW_SEG_HDR_IPV_OTHER,
 145         ICE_FLOW_HASH_ETH | ICE_FLOW_HASH_IPV6,
 146         ICE_RSS_OUTER_HEADERS,
 147         0
 148 };
 149
 150 struct ice_rss_hash_cfg ipv6_udp_tmplt = {
 151         ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV6 |
 152         ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_UDP,
 153         ICE_FLOW_HASH_ETH | ICE_HASH_UDP_IPV6 | ICE_IPV6_PROT,
 154         ICE_RSS_OUTER_HEADERS,
 155         0
 156 };
 157
 158 struct ice_rss_hash_cfg ipv6_tcp_tmplt = {
 159         ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV6 |
 160         ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_TCP,
 161         ICE_FLOW_HASH_ETH | ICE_HASH_TCP_IPV6 | ICE_IPV6_PROT,
 162         ICE_RSS_OUTER_HEADERS,
 163         0
 164 };
 165
 166 struct ice_rss_hash_cfg ipv6_sctp_tmplt = {
 167         ICE_FLOW_SEG_HDR_ETH | ICE_FLOW_SEG_HDR_IPV6 |
 168         ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_SCTP,
 169         ICE_FLOW_HASH_ETH | ICE_HASH_SCTP_IPV6 | ICE_IPV6_PROT,
 170         ICE_RSS_OUTER_HEADERS,
 171         0
 172 };
 173
 174 struct ice_rss_hash_cfg outer_ipv4_inner_ipv4_tmplt = {
 175         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER,
 176         ICE_FLOW_HASH_IPV4,
 177         ICE_RSS_INNER_HEADERS_W_OUTER_IPV4,
 178         0
 179 };
 180 struct ice_rss_hash_cfg outer_ipv4_inner_ipv4_udp_tmplt = {
 181         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 182         ICE_FLOW_SEG_HDR_UDP,
 183         ICE_HASH_UDP_IPV4 | ICE_IPV4_PROT,
 184         ICE_RSS_INNER_HEADERS_W_OUTER_IPV4,
 185         0
 186 };
 187
 188 struct ice_rss_hash_cfg outer_ipv4_inner_ipv4_tcp_tmplt = {
 189         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 190         ICE_FLOW_SEG_HDR_TCP,
 191         ICE_HASH_TCP_IPV4 | ICE_IPV4_PROT,
 192         ICE_RSS_INNER_HEADERS_W_OUTER_IPV4,
 193         0
 194 };
 195
 196 struct ice_rss_hash_cfg outer_ipv6_inner_ipv4_tmplt = {
 197         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER,
 198         ICE_FLOW_HASH_IPV4,
 199         ICE_RSS_INNER_HEADERS_W_OUTER_IPV6,
 200         0
 201 };
 202 struct ice_rss_hash_cfg outer_ipv6_inner_ipv4_udp_tmplt = {
 203         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 204         ICE_FLOW_SEG_HDR_UDP,
 205         ICE_HASH_UDP_IPV4 | ICE_IPV4_PROT,
 206         ICE_RSS_INNER_HEADERS_W_OUTER_IPV6,
 207         0
 208 };
 209
 210 struct ice_rss_hash_cfg outer_ipv6_inner_ipv4_tcp_tmplt = {
 211         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 212         ICE_FLOW_SEG_HDR_TCP,
 213         ICE_HASH_TCP_IPV4 | ICE_IPV4_PROT,
 214         ICE_RSS_INNER_HEADERS_W_OUTER_IPV6,
 215         0
 216 };
 217
 218 struct ice_rss_hash_cfg outer_ipv4_inner_ipv6_tmplt = {
 219         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER,
 220         ICE_FLOW_HASH_IPV6,
 221         ICE_RSS_INNER_HEADERS_W_OUTER_IPV4,
 222         0
 223 };
 224 struct ice_rss_hash_cfg outer_ipv4_inner_ipv6_udp_tmplt = {
 225         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 226         ICE_FLOW_SEG_HDR_UDP,
 227         ICE_HASH_UDP_IPV6 | ICE_IPV6_PROT,
 228         ICE_RSS_INNER_HEADERS_W_OUTER_IPV4,
 229         0
 230 };
 231
 232 struct ice_rss_hash_cfg outer_ipv4_inner_ipv6_tcp_tmplt = {
 233         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 234         ICE_FLOW_SEG_HDR_TCP,
 235         ICE_HASH_TCP_IPV6 | ICE_IPV6_PROT,
 236         ICE_RSS_INNER_HEADERS_W_OUTER_IPV4,
 237         0
 238 };
 239
 240 struct ice_rss_hash_cfg outer_ipv6_inner_ipv6_tmplt = {
 241         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER,
 242         ICE_FLOW_HASH_IPV6,
 243         ICE_RSS_INNER_HEADERS_W_OUTER_IPV6,
 244         0
 245 };
 246 struct ice_rss_hash_cfg outer_ipv6_inner_ipv6_udp_tmplt = {
 247         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 248         ICE_FLOW_SEG_HDR_UDP,
 249         ICE_HASH_UDP_IPV6 | ICE_IPV6_PROT,
 250         ICE_RSS_INNER_HEADERS_W_OUTER_IPV6,
 251         0
 252 };
 253
 254 struct ice_rss_hash_cfg outer_ipv6_inner_ipv6_tcp_tmplt = {
 255         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 256         ICE_FLOW_SEG_HDR_TCP,
 257         ICE_HASH_TCP_IPV6 | ICE_IPV6_PROT,
 258         ICE_RSS_INNER_HEADERS_W_OUTER_IPV6,
 259         0
 260 };
 261
 262 struct ice_rss_hash_cfg eth_ipv4_esp_tmplt = {
 263         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 264         ICE_FLOW_SEG_HDR_ESP,
 265         ICE_FLOW_HASH_ESP_SPI,
 266         ICE_RSS_OUTER_HEADERS,
 267         0
 268 };
 269
 270 struct ice_rss_hash_cfg eth_ipv4_udp_esp_tmplt = {
 271         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 272         ICE_FLOW_SEG_HDR_NAT_T_ESP,
 273         ICE_FLOW_HASH_NAT_T_ESP_SPI,
 274         ICE_RSS_OUTER_HEADERS,
 275         0
 276 };
 277
 278 struct ice_rss_hash_cfg eth_ipv4_ah_tmplt = {
 279         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 280         ICE_FLOW_SEG_HDR_AH,
 281         ICE_FLOW_HASH_AH_SPI,
 282         ICE_RSS_OUTER_HEADERS,
 283         0
 284 };
 285
 286 struct ice_rss_hash_cfg eth_ipv4_l2tpv3_tmplt = {
 287         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 288         ICE_FLOW_SEG_HDR_L2TPV3,
 289         ICE_FLOW_HASH_L2TPV3_SESS_ID,
 290         ICE_RSS_OUTER_HEADERS,
 291         0
 292 };
 293
 294 struct ice_rss_hash_cfg eth_ipv4_pfcp_tmplt = {
 295         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 296         ICE_FLOW_SEG_HDR_PFCP_SESSION,
 297         ICE_FLOW_HASH_PFCP_SEID,
 298         ICE_RSS_OUTER_HEADERS,
 299         0
 300 };
 301
 302 struct ice_rss_hash_cfg eth_ipv6_esp_tmplt = {
 303         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 304         ICE_FLOW_SEG_HDR_ESP,
 305         ICE_FLOW_HASH_ESP_SPI,
 306         ICE_RSS_OUTER_HEADERS,
 307         0
 308 };
 309
 310 struct ice_rss_hash_cfg eth_ipv6_udp_esp_tmplt = {
 311         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 312         ICE_FLOW_SEG_HDR_NAT_T_ESP,
 313         ICE_FLOW_HASH_NAT_T_ESP_SPI,
 314         ICE_RSS_OUTER_HEADERS,
 315         0
 316 };
 317
 318 struct ice_rss_hash_cfg eth_ipv6_ah_tmplt = {
 319         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 320         ICE_FLOW_SEG_HDR_AH,
 321         ICE_FLOW_HASH_AH_SPI,
 322         ICE_RSS_OUTER_HEADERS,
 323         0
 324 };
 325
 326 struct ice_rss_hash_cfg eth_ipv6_l2tpv3_tmplt = {
 327         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 328         ICE_FLOW_SEG_HDR_L2TPV3,
 329         ICE_FLOW_HASH_L2TPV3_SESS_ID,
 330         ICE_RSS_OUTER_HEADERS,
 331         0
 332 };
 333
 334 struct ice_rss_hash_cfg eth_ipv6_pfcp_tmplt = {
 335         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 336         ICE_FLOW_SEG_HDR_PFCP_SESSION,
 337         ICE_FLOW_HASH_PFCP_SEID,
 338         ICE_RSS_OUTER_HEADERS,
 339         0
 340 };
 341
 342 struct ice_rss_hash_cfg pppoe_tmplt = {
 343         ICE_FLOW_SEG_HDR_ETH,
 344         ICE_FLOW_HASH_ETH | ICE_FLOW_HASH_PPPOE_SESS_ID,
 345         ICE_RSS_OUTER_HEADERS,
 346         0
 347 };
 348
 349 struct ice_rss_hash_cfg empty_tmplt = {
 350         ICE_FLOW_SEG_HDR_NONE,
 351         0,
 352         ICE_RSS_ANY_HEADERS,
 353         0
 354 };
 355
 356 /* IPv4 */
 357 #define ICE_RSS_TYPE_ETH_IPV4           (ETH_RSS_ETH | ETH_RSS_IPV4)
 358 #define ICE_RSS_TYPE_ETH_IPV4_UDP       (ICE_RSS_TYPE_ETH_IPV4 | \
 359                                          ETH_RSS_NONFRAG_IPV4_UDP)
 360 #define ICE_RSS_TYPE_ETH_IPV4_TCP       (ICE_RSS_TYPE_ETH_IPV4 | \
 361                                          ETH_RSS_NONFRAG_IPV4_TCP)
 362 #define ICE_RSS_TYPE_ETH_IPV4_SCTP      (ICE_RSS_TYPE_ETH_IPV4 | \
 363                                          ETH_RSS_NONFRAG_IPV4_SCTP)
 364 #define ICE_RSS_TYPE_IPV4               ETH_RSS_IPV4
 365 #define ICE_RSS_TYPE_IPV4_UDP           (ETH_RSS_IPV4 | \
 366                                          ETH_RSS_NONFRAG_IPV4_UDP)
 367 #define ICE_RSS_TYPE_IPV4_TCP           (ETH_RSS_IPV4 | \
 368                                          ETH_RSS_NONFRAG_IPV4_TCP)
 369 #define ICE_RSS_TYPE_IPV4_SCTP          (ETH_RSS_IPV4 | \
 370                                          ETH_RSS_NONFRAG_IPV4_SCTP)
 371
 372 /* IPv6 */
 373 #define ICE_RSS_TYPE_ETH_IPV6           (ETH_RSS_ETH | ETH_RSS_IPV6)
 374 #define ICE_RSS_TYPE_ETH_IPV6_UDP       (ICE_RSS_TYPE_ETH_IPV6 | \
 375                                          ETH_RSS_NONFRAG_IPV6_UDP)
 376 #define ICE_RSS_TYPE_ETH_IPV6_TCP       (ICE_RSS_TYPE_ETH_IPV6 | \
 377                                          ETH_RSS_NONFRAG_IPV6_TCP)
 378 #define ICE_RSS_TYPE_ETH_IPV6_SCTP      (ICE_RSS_TYPE_ETH_IPV6 | \
 379                                          ETH_RSS_NONFRAG_IPV6_SCTP)
 380 #define ICE_RSS_TYPE_IPV6               ETH_RSS_IPV6
 381 #define ICE_RSS_TYPE_IPV6_UDP           (ETH_RSS_IPV6 | \
 382                                          ETH_RSS_NONFRAG_IPV6_UDP)
 383 #define ICE_RSS_TYPE_IPV6_TCP           (ETH_RSS_IPV6 | \
 384                                          ETH_RSS_NONFRAG_IPV6_TCP)
 385 #define ICE_RSS_TYPE_IPV6_SCTP          (ETH_RSS_IPV6 | \
 386                                          ETH_RSS_NONFRAG_IPV6_SCTP)
 387
 388 /* VLAN IPV4 */
 389 #define ICE_RSS_TYPE_VLAN_IPV4          (ICE_RSS_TYPE_IPV4 | \
 390                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 391 #define ICE_RSS_TYPE_VLAN_IPV4_UDP      (ICE_RSS_TYPE_IPV4_UDP | \
 392                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 393 #define ICE_RSS_TYPE_VLAN_IPV4_TCP      (ICE_RSS_TYPE_IPV4_TCP | \
 394                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 395 #define ICE_RSS_TYPE_VLAN_IPV4_SCTP     (ICE_RSS_TYPE_IPV4_SCTP | \
 396                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 397 /* VLAN IPv6 */
 398 #define ICE_RSS_TYPE_VLAN_IPV6          (ICE_RSS_TYPE_IPV6 | \
 399                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 400 #define ICE_RSS_TYPE_VLAN_IPV6_UDP      (ICE_RSS_TYPE_IPV6_UDP | \
 401                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 402 #define ICE_RSS_TYPE_VLAN_IPV6_TCP      (ICE_RSS_TYPE_IPV6_TCP | \
 403                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 404 #define ICE_RSS_TYPE_VLAN_IPV6_SCTP     (ICE_RSS_TYPE_IPV6_SCTP | \
 405                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 406
 407 /* GTPU IPv4 */
 408 #define ICE_RSS_TYPE_GTPU_IPV4          (ICE_RSS_TYPE_IPV4 | \
 409                                          ETH_RSS_GTPU)
 410 #define ICE_RSS_TYPE_GTPU_IPV4_UDP      (ICE_RSS_TYPE_IPV4_UDP | \
 411                                          ETH_RSS_GTPU)
 412 #define ICE_RSS_TYPE_GTPU_IPV4_TCP      (ICE_RSS_TYPE_IPV4_TCP | \
 413                                          ETH_RSS_GTPU)
 414 /* GTPU IPv6 */
 415 #define ICE_RSS_TYPE_GTPU_IPV6          (ICE_RSS_TYPE_IPV6 | \
 416                                          ETH_RSS_GTPU)
 417 #define ICE_RSS_TYPE_GTPU_IPV6_UDP      (ICE_RSS_TYPE_IPV6_UDP | \
 418                                          ETH_RSS_GTPU)
 419 #define ICE_RSS_TYPE_GTPU_IPV6_TCP      (ICE_RSS_TYPE_IPV6_TCP | \
 420                                          ETH_RSS_GTPU)
 421
 422 /* PPPOE */
 423 #define ICE_RSS_TYPE_PPPOE              (ETH_RSS_ETH | ETH_RSS_PPPOE)
 424
 425 /* PPPOE IPv4 */
 426 #define ICE_RSS_TYPE_PPPOE_IPV4         (ICE_RSS_TYPE_IPV4 | \
 427                                          ICE_RSS_TYPE_PPPOE)
 428 #define ICE_RSS_TYPE_PPPOE_IPV4_UDP     (ICE_RSS_TYPE_IPV4_UDP | \
 429                                          ICE_RSS_TYPE_PPPOE)
 430 #define ICE_RSS_TYPE_PPPOE_IPV4_TCP     (ICE_RSS_TYPE_IPV4_TCP | \
 431                                          ICE_RSS_TYPE_PPPOE)
 432
 433 /* PPPOE IPv6 */
 434 #define ICE_RSS_TYPE_PPPOE_IPV6         (ICE_RSS_TYPE_IPV6 | \
 435                                          ICE_RSS_TYPE_PPPOE)
 436 #define ICE_RSS_TYPE_PPPOE_IPV6_UDP     (ICE_RSS_TYPE_IPV6_UDP | \
 437                                          ICE_RSS_TYPE_PPPOE)
 438 #define ICE_RSS_TYPE_PPPOE_IPV6_TCP     (ICE_RSS_TYPE_IPV6_TCP | \
 439                                          ICE_RSS_TYPE_PPPOE)
 440
 441 /* ESP, AH, L2TPV3 and PFCP */
 442 #define ICE_RSS_TYPE_IPV4_ESP           (ETH_RSS_ESP | ETH_RSS_IPV4)
 443 #define ICE_RSS_TYPE_IPV6_ESP           (ETH_RSS_ESP | ETH_RSS_IPV6)
 444 #define ICE_RSS_TYPE_IPV4_AH            (ETH_RSS_AH | ETH_RSS_IPV4)
 445 #define ICE_RSS_TYPE_IPV6_AH            (ETH_RSS_AH | ETH_RSS_IPV6)
 446 #define ICE_RSS_TYPE_IPV4_L2TPV3        (ETH_RSS_L2TPV3 | ETH_RSS_IPV4)
 447 #define ICE_RSS_TYPE_IPV6_L2TPV3        (ETH_RSS_L2TPV3 | ETH_RSS_IPV6)
 448 #define ICE_RSS_TYPE_IPV4_PFCP          (ETH_RSS_PFCP | ETH_RSS_IPV4)
 449 #define ICE_RSS_TYPE_IPV6_PFCP          (ETH_RSS_PFCP | ETH_RSS_IPV6)
 450
 451 /**
 452  * Supported pattern for hash.
 453  * The first member is pattern item type,
 454  * the second member is input set mask,
 455  * the third member is ice_rss_hash_cfg template.
 456  */
 457 static struct ice_pattern_match_item ice_hash_pattern_list[] = {
 458         /* IPV4 */
 459         {pattern_eth_ipv4,                      ICE_RSS_TYPE_ETH_IPV4,          ICE_INSET_NONE, &ipv4_tmplt},
 460         {pattern_eth_ipv4_udp,                  ICE_RSS_TYPE_ETH_IPV4_UDP,      ICE_INSET_NONE, &ipv4_udp_tmplt},
 461         {pattern_eth_ipv4_tcp,                  ICE_RSS_TYPE_ETH_IPV4_TCP,      ICE_INSET_NONE, &ipv4_tcp_tmplt},
 462         {pattern_eth_ipv4_sctp,                 ICE_RSS_TYPE_ETH_IPV4_SCTP,     ICE_INSET_NONE, &ipv4_sctp_tmplt},
 463         {pattern_eth_vlan_ipv4,                 ICE_RSS_TYPE_VLAN_IPV4,         ICE_INSET_NONE, &ipv4_tmplt},
 464         {pattern_eth_vlan_ipv4_udp,             ICE_RSS_TYPE_VLAN_IPV4_UDP,     ICE_INSET_NONE, &ipv4_udp_tmplt},
 465         {pattern_eth_vlan_ipv4_tcp,             ICE_RSS_TYPE_VLAN_IPV4_TCP,     ICE_INSET_NONE, &ipv4_tcp_tmplt},
 466         {pattern_eth_vlan_ipv4_sctp,            ICE_RSS_TYPE_VLAN_IPV4_SCTP,    ICE_INSET_NONE, &ipv4_sctp_tmplt},
 467         {pattern_eth_ipv4_gtpu_ipv4,            ICE_RSS_TYPE_GTPU_IPV4,         ICE_INSET_NONE, &outer_ipv4_inner_ipv4_tmplt},
 468         {pattern_eth_ipv4_gtpu_ipv4_udp,        ICE_RSS_TYPE_GTPU_IPV4_UDP,     ICE_INSET_NONE, &outer_ipv4_inner_ipv4_udp_tmplt},
 469         {pattern_eth_ipv4_gtpu_ipv4_tcp,        ICE_RSS_TYPE_GTPU_IPV4_TCP,     ICE_INSET_NONE, &outer_ipv4_inner_ipv4_tcp_tmplt},
 470         {pattern_eth_ipv6_gtpu_ipv4,            ICE_RSS_TYPE_GTPU_IPV4,         ICE_INSET_NONE, &outer_ipv6_inner_ipv4_tmplt},
 471         {pattern_eth_ipv6_gtpu_ipv4_udp,        ICE_RSS_TYPE_GTPU_IPV4_UDP,     ICE_INSET_NONE, &outer_ipv6_inner_ipv4_udp_tmplt},
 472         {pattern_eth_ipv6_gtpu_ipv4_tcp,        ICE_RSS_TYPE_GTPU_IPV4_TCP,     ICE_INSET_NONE, &outer_ipv6_inner_ipv4_tcp_tmplt},
 473         {pattern_eth_ipv4_gtpu_eh_ipv4,         ICE_RSS_TYPE_GTPU_IPV4,         ICE_INSET_NONE, &outer_ipv4_inner_ipv4_tmplt},
 474         {pattern_eth_ipv4_gtpu_eh_ipv4_udp,     ICE_RSS_TYPE_GTPU_IPV4_UDP,     ICE_INSET_NONE, &outer_ipv4_inner_ipv4_udp_tmplt},
 475         {pattern_eth_ipv4_gtpu_eh_ipv4_tcp,     ICE_RSS_TYPE_GTPU_IPV4_TCP,     ICE_INSET_NONE, &outer_ipv4_inner_ipv4_tcp_tmplt},
 476         {pattern_eth_ipv6_gtpu_eh_ipv4,         ICE_RSS_TYPE_GTPU_IPV4,         ICE_INSET_NONE, &outer_ipv6_inner_ipv4_tmplt},
 477         {pattern_eth_ipv6_gtpu_eh_ipv4_udp,     ICE_RSS_TYPE_GTPU_IPV4_UDP,     ICE_INSET_NONE, &outer_ipv6_inner_ipv4_udp_tmplt},
 478         {pattern_eth_ipv6_gtpu_eh_ipv4_tcp,     ICE_RSS_TYPE_GTPU_IPV4_TCP,     ICE_INSET_NONE, &outer_ipv6_inner_ipv4_tcp_tmplt},
 479         {pattern_eth_pppoes_ipv4,               ICE_RSS_TYPE_PPPOE_IPV4,        ICE_INSET_NONE, &ipv4_tmplt},
 480         {pattern_eth_pppoes_ipv4_udp,           ICE_RSS_TYPE_PPPOE_IPV4_UDP,    ICE_INSET_NONE, &ipv4_udp_tmplt},
 481         {pattern_eth_pppoes_ipv4_tcp,           ICE_RSS_TYPE_PPPOE_IPV4_TCP,    ICE_INSET_NONE, &ipv4_tcp_tmplt},
 482         {pattern_eth_ipv4_esp,                  ICE_RSS_TYPE_IPV4_ESP,          ICE_INSET_NONE, &eth_ipv4_esp_tmplt},
 483         {pattern_eth_ipv4_udp_esp,              ICE_RSS_TYPE_IPV4_ESP,          ICE_INSET_NONE, &eth_ipv4_udp_esp_tmplt},
 484         {pattern_eth_ipv4_ah,                   ICE_RSS_TYPE_IPV4_AH,           ICE_INSET_NONE, &eth_ipv4_ah_tmplt},
 485         {pattern_eth_ipv4_l2tp,                 ICE_RSS_TYPE_IPV4_L2TPV3,       ICE_INSET_NONE, &eth_ipv4_l2tpv3_tmplt},
 486         {pattern_eth_ipv4_pfcp,                 ICE_RSS_TYPE_IPV4_PFCP,         ICE_INSET_NONE, &eth_ipv4_pfcp_tmplt},
 487         /* IPV6 */
 488         {pattern_eth_ipv6,                      ICE_RSS_TYPE_ETH_IPV6,          ICE_INSET_NONE, &ipv6_tmplt},
 489         {pattern_eth_ipv6_udp,                  ICE_RSS_TYPE_ETH_IPV6_UDP,      ICE_INSET_NONE, &ipv6_udp_tmplt},
 490         {pattern_eth_ipv6_tcp,                  ICE_RSS_TYPE_ETH_IPV6_TCP,      ICE_INSET_NONE, &ipv6_tcp_tmplt},
 491         {pattern_eth_ipv6_sctp,                 ICE_RSS_TYPE_ETH_IPV6_SCTP,     ICE_INSET_NONE, &ipv6_sctp_tmplt},
 492         {pattern_eth_vlan_ipv6,                 ICE_RSS_TYPE_VLAN_IPV6,         ICE_INSET_NONE, &ipv6_tmplt},
 493         {pattern_eth_vlan_ipv6_udp,             ICE_RSS_TYPE_VLAN_IPV6_UDP,     ICE_INSET_NONE, &ipv6_udp_tmplt},
 494         {pattern_eth_vlan_ipv6_tcp,             ICE_RSS_TYPE_VLAN_IPV6_TCP,     ICE_INSET_NONE, &ipv6_tcp_tmplt},
 495         {pattern_eth_vlan_ipv6_sctp,            ICE_RSS_TYPE_VLAN_IPV6_SCTP,    ICE_INSET_NONE, &ipv6_sctp_tmplt},
 496         {pattern_eth_ipv4_gtpu_ipv6,            ICE_RSS_TYPE_GTPU_IPV6,         ICE_INSET_NONE, &outer_ipv4_inner_ipv6_tmplt},
 497         {pattern_eth_ipv4_gtpu_ipv6_udp,        ICE_RSS_TYPE_GTPU_IPV6_UDP,     ICE_INSET_NONE, &outer_ipv4_inner_ipv6_udp_tmplt},
 498         {pattern_eth_ipv4_gtpu_ipv6_tcp,        ICE_RSS_TYPE_GTPU_IPV6_TCP,     ICE_INSET_NONE, &outer_ipv4_inner_ipv6_tcp_tmplt},
 499         {pattern_eth_ipv6_gtpu_ipv6,            ICE_RSS_TYPE_GTPU_IPV6,         ICE_INSET_NONE, &outer_ipv6_inner_ipv6_tmplt},
 500         {pattern_eth_ipv6_gtpu_ipv6_udp,        ICE_RSS_TYPE_GTPU_IPV6_UDP,     ICE_INSET_NONE, &outer_ipv6_inner_ipv6_udp_tmplt},
 501         {pattern_eth_ipv6_gtpu_ipv6_tcp,        ICE_RSS_TYPE_GTPU_IPV6_TCP,     ICE_INSET_NONE, &outer_ipv6_inner_ipv6_tcp_tmplt},
 502         {pattern_eth_ipv4_gtpu_eh_ipv6,         ICE_RSS_TYPE_GTPU_IPV6,         ICE_INSET_NONE, &outer_ipv4_inner_ipv6_tmplt},
 503         {pattern_eth_ipv4_gtpu_eh_ipv6_udp,     ICE_RSS_TYPE_GTPU_IPV6_UDP,     ICE_INSET_NONE, &outer_ipv4_inner_ipv6_udp_tmplt},
 504         {pattern_eth_ipv4_gtpu_eh_ipv6_tcp,     ICE_RSS_TYPE_GTPU_IPV6_TCP,     ICE_INSET_NONE, &outer_ipv4_inner_ipv6_tcp_tmplt},
 505         {pattern_eth_ipv6_gtpu_eh_ipv6,         ICE_RSS_TYPE_GTPU_IPV6,         ICE_INSET_NONE, &outer_ipv6_inner_ipv6_tmplt},
 506         {pattern_eth_ipv6_gtpu_eh_ipv6_udp,     ICE_RSS_TYPE_GTPU_IPV6_UDP,     ICE_INSET_NONE, &outer_ipv6_inner_ipv6_udp_tmplt},
 507         {pattern_eth_ipv6_gtpu_eh_ipv6_tcp,     ICE_RSS_TYPE_GTPU_IPV6_TCP,     ICE_INSET_NONE, &outer_ipv6_inner_ipv6_tcp_tmplt},
 508         {pattern_eth_pppoes_ipv6,               ICE_RSS_TYPE_PPPOE_IPV6,        ICE_INSET_NONE, &ipv6_tmplt},
 509         {pattern_eth_pppoes_ipv6_udp,           ICE_RSS_TYPE_PPPOE_IPV6_UDP,    ICE_INSET_NONE, &ipv6_udp_tmplt},
 510         {pattern_eth_pppoes_ipv6_tcp,           ICE_RSS_TYPE_PPPOE_IPV6_TCP,    ICE_INSET_NONE, &ipv6_tcp_tmplt},
 511         {pattern_eth_ipv6_esp,                  ICE_RSS_TYPE_IPV6_ESP,          ICE_INSET_NONE, &eth_ipv6_esp_tmplt},
 512         {pattern_eth_ipv6_udp_esp,              ICE_RSS_TYPE_IPV6_ESP,          ICE_INSET_NONE, &eth_ipv6_udp_esp_tmplt},
 513         {pattern_eth_ipv6_ah,                   ICE_RSS_TYPE_IPV6_AH,           ICE_INSET_NONE, &eth_ipv6_ah_tmplt},
 514         {pattern_eth_ipv6_l2tp,                 ICE_RSS_TYPE_IPV6_L2TPV3,       ICE_INSET_NONE, &eth_ipv6_l2tpv3_tmplt},
 515         {pattern_eth_ipv6_pfcp,                 ICE_RSS_TYPE_IPV6_PFCP,         ICE_INSET_NONE, &eth_ipv6_pfcp_tmplt},
 516         /* PPPOE */
 517         {pattern_eth_pppoes,                    ICE_RSS_TYPE_PPPOE,             ICE_INSET_NONE, &pppoe_tmplt},
 518         /* EMPTY */
 519         {pattern_empty,                         ICE_INSET_NONE,                 ICE_INSET_NONE, &empty_tmplt},
 520 };
 521
 522 static struct ice_flow_engine ice_hash_engine = {
 523         .init = ice_hash_init,
 524         .create = ice_hash_create,
 525         .destroy = ice_hash_destroy,
 526         .uninit = ice_hash_uninit,
 527         .free = ice_hash_free,
 528         .type = ICE_FLOW_ENGINE_HASH,
 529 };
 530
 531 /* Register parser for os package. */
 532 static struct ice_flow_parser ice_hash_parser = {
 533         .engine = &ice_hash_engine,
 534         .array = ice_hash_pattern_list,
 535         .array_len = RTE_DIM(ice_hash_pattern_list),
 536         .parse_pattern_action = ice_hash_parse_pattern_action,
 537         .stage = ICE_FLOW_STAGE_RSS,
 538 };
 539
 540 RTE_INIT(ice_hash_engine_init)
 541 {
 542         struct ice_flow_engine *engine = &ice_hash_engine;
 543         ice_register_flow_engine(engine);
 544 }
 545
 546 static int
 547 ice_hash_init(struct ice_adapter *ad)
 548 {
 549         struct ice_flow_parser *parser = NULL;
 550
 551         if (ad->hw.dcf_enabled)
 552                 return 0;
 553
 554         parser = &ice_hash_parser;
 555
 556         return ice_register_parser(parser, ad);
 557 }
 558
 559 static int
 560 ice_hash_parse_pattern(const struct rte_flow_item pattern[], uint64_t *phint,
 561                        struct rte_flow_error *error)
 562 {
 563         const struct rte_flow_item *item = pattern;
 564         const struct rte_flow_item_gtp_psc *psc;
 565
 566         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 567                 if (item->last) {
 568                         rte_flow_error_set(error, EINVAL,
 569                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 570                                         "Not support range");
 571                         return -rte_errno;
 572                 }
 573
 574                 switch (item->type) {
 575                 case RTE_FLOW_ITEM_TYPE_VLAN:
 576                         *phint |= ICE_PHINT_VLAN;
 577                         break;
 578                 case RTE_FLOW_ITEM_TYPE_PPPOES:
 579                         *phint |= ICE_PHINT_PPPOE;
 580                         break;
 581                 case RTE_FLOW_ITEM_TYPE_GTPU:
 582                         *phint |= ICE_PHINT_GTPU;
 583                         break;
 584                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
 585                         *phint |= ICE_PHINT_GTPU_EH;
 586                         psc = item->spec;
 587                         if (!psc)
 588                                 break;
 589                         else if (psc->pdu_type == ICE_GTPU_EH_UPLINK)
 590                                 *phint |= ICE_PHINT_GTPU_EH_UP;
 591                         else if (psc->pdu_type == ICE_GTPU_EH_DWNLINK)
 592                                 *phint |= ICE_PHINT_GTPU_EH_DWN;
 593                         break;
 594                 default:
 595                         break;
 596                 }
 597         }
 598
 599         return 0;
 600 }
 601
 602 static void
 603 ice_refine_hash_cfg_l234(struct ice_rss_hash_cfg *hash_cfg,
 604                          uint64_t rss_type)
 605 {
 606         uint32_t *addl_hdrs = &hash_cfg->addl_hdrs;
 607         uint64_t *hash_flds = &hash_cfg->hash_flds;
 608
 609         if (*addl_hdrs & ICE_FLOW_SEG_HDR_ETH) {
 610                 if (!(rss_type & ETH_RSS_ETH))
 611                         *hash_flds &= ~ICE_FLOW_HASH_ETH;
 612                 if (rss_type & ETH_RSS_L2_SRC_ONLY)
 613                         *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_ETH_DA));
 614                 else if (rss_type & ETH_RSS_L2_DST_ONLY)
 615                         *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_ETH_SA));
 616                 *addl_hdrs &= ~ICE_FLOW_SEG_HDR_ETH;
 617         }
 618
 619         if (*addl_hdrs & ICE_FLOW_SEG_HDR_VLAN) {
 620                 if (rss_type & ETH_RSS_C_VLAN)
 621                         *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_C_VLAN);
 622                 else if (rss_type & ETH_RSS_S_VLAN)
 623                         *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_S_VLAN);
 624         }
 625
 626         if (*addl_hdrs & ICE_FLOW_SEG_HDR_PPPOE) {
 627                 if (!(rss_type & ETH_RSS_PPPOE))
 628                         *hash_flds &= ~ICE_FLOW_HASH_PPPOE_SESS_ID;
 629         }
 630
 631         if (*addl_hdrs & ICE_FLOW_SEG_HDR_IPV4) {
 632                 if (rss_type &
 633                    (ETH_RSS_IPV4 |
 634                     ETH_RSS_NONFRAG_IPV4_UDP |
 635                     ETH_RSS_NONFRAG_IPV4_TCP |
 636                     ETH_RSS_NONFRAG_IPV4_SCTP)) {
 637                         if (rss_type & ETH_RSS_L3_SRC_ONLY)
 638                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA));
 639                         else if (rss_type & ETH_RSS_L3_DST_ONLY)
 640                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA));
 641                         else if (rss_type &
 642                                 (ETH_RSS_L4_SRC_ONLY |
 643                                 ETH_RSS_L4_DST_ONLY))
 644                                 *hash_flds &= ~ICE_FLOW_HASH_IPV4;
 645                 } else {
 646                         *hash_flds &= ~ICE_FLOW_HASH_IPV4;
 647                 }
 648         }
 649
 650         if (*addl_hdrs & ICE_FLOW_SEG_HDR_IPV6) {
 651                 if (rss_type &
 652                    (ETH_RSS_IPV6 |
 653                     ETH_RSS_NONFRAG_IPV6_UDP |
 654                     ETH_RSS_NONFRAG_IPV6_TCP |
 655                     ETH_RSS_NONFRAG_IPV6_SCTP)) {
 656                         if (rss_type & ETH_RSS_L3_SRC_ONLY)
 657                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA));
 658                         else if (rss_type & ETH_RSS_L3_DST_ONLY)
 659                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA));
 660                         else if (rss_type &
 661                                 (ETH_RSS_L4_SRC_ONLY |
 662                                 ETH_RSS_L4_DST_ONLY))
 663                                 *hash_flds &= ~ICE_FLOW_HASH_IPV6;
 664                 } else {
 665                         *hash_flds &= ~ICE_FLOW_HASH_IPV6;
 666                 }
 667
 668                 if (rss_type & RTE_ETH_RSS_L3_PRE32) {
 669                         if (rss_type & ETH_RSS_L3_SRC_ONLY) {
 670                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA));
 671                                 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE32_SA));
 672                         } else if (rss_type & ETH_RSS_L3_DST_ONLY) {
 673                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA));
 674                                 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE32_DA));
 675                         } else {
 676                                 *hash_flds &= ~ICE_FLOW_HASH_IPV6;
 677                                 *hash_flds |= ICE_FLOW_HASH_IPV6_PRE32;
 678                         }
 679                 }
 680                 if (rss_type & RTE_ETH_RSS_L3_PRE48) {
 681                         if (rss_type & ETH_RSS_L3_SRC_ONLY) {
 682                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA));
 683                                 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE48_SA));
 684                         } else if (rss_type & ETH_RSS_L3_DST_ONLY) {
 685                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA));
 686                                 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE48_DA));
 687                         } else {
 688                                 *hash_flds &= ~ICE_FLOW_HASH_IPV6;
 689                                 *hash_flds |= ICE_FLOW_HASH_IPV6_PRE48;
 690                         }
 691                 }
 692                 if (rss_type & RTE_ETH_RSS_L3_PRE64) {
 693                         if (rss_type & ETH_RSS_L3_SRC_ONLY) {
 694                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA));
 695                                 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE64_SA));
 696                         } else if (rss_type & ETH_RSS_L3_DST_ONLY) {
 697                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA));
 698                                 *hash_flds |= (BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PRE64_DA));
 699                         } else {
 700                                 *hash_flds &= ~ICE_FLOW_HASH_IPV6;
 701                                 *hash_flds |= ICE_FLOW_HASH_IPV6_PRE64;
 702                         }
 703                 }
 704         }
 705
 706         if (*addl_hdrs & ICE_FLOW_SEG_HDR_UDP) {
 707                 if (rss_type &
 708                    (ETH_RSS_NONFRAG_IPV4_UDP |
 709                     ETH_RSS_NONFRAG_IPV6_UDP)) {
 710                         if (rss_type & ETH_RSS_L4_SRC_ONLY)
 711                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT));
 712                         else if (rss_type & ETH_RSS_L4_DST_ONLY)
 713                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT));
 714                         else if (rss_type &
 715                                 (ETH_RSS_L3_SRC_ONLY |
 716                                   ETH_RSS_L3_DST_ONLY))
 717                                 *hash_flds &= ~ICE_FLOW_HASH_UDP_PORT;
 718                 } else {
 719                         *hash_flds &= ~ICE_FLOW_HASH_UDP_PORT;
 720                 }
 721         }
 722
 723         if (*addl_hdrs & ICE_FLOW_SEG_HDR_TCP) {
 724                 if (rss_type &
 725                    (ETH_RSS_NONFRAG_IPV4_TCP |
 726                     ETH_RSS_NONFRAG_IPV6_TCP)) {
 727                         if (rss_type & ETH_RSS_L4_SRC_ONLY)
 728                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT));
 729                         else if (rss_type & ETH_RSS_L4_DST_ONLY)
 730                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT));
 731                         else if (rss_type &
 732                                 (ETH_RSS_L3_SRC_ONLY |
 733                                   ETH_RSS_L3_DST_ONLY))
 734                                 *hash_flds &= ~ICE_FLOW_HASH_TCP_PORT;
 735                 } else {
 736                         *hash_flds &= ~ICE_FLOW_HASH_TCP_PORT;
 737                 }
 738         }
 739
 740         if (*addl_hdrs & ICE_FLOW_SEG_HDR_SCTP) {
 741                 if (rss_type &
 742                    (ETH_RSS_NONFRAG_IPV4_SCTP |
 743                     ETH_RSS_NONFRAG_IPV6_SCTP)) {
 744                         if (rss_type & ETH_RSS_L4_SRC_ONLY)
 745                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT));
 746                         else if (rss_type & ETH_RSS_L4_DST_ONLY)
 747                                 *hash_flds &= ~(BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT));
 748                         else if (rss_type &
 749                                 (ETH_RSS_L3_SRC_ONLY |
 750                                   ETH_RSS_L3_DST_ONLY))
 751                                 *hash_flds &= ~ICE_FLOW_HASH_SCTP_PORT;
 752                 } else {
 753                         *hash_flds &= ~ICE_FLOW_HASH_SCTP_PORT;
 754                 }
 755         }
 756
 757         if (*addl_hdrs & ICE_FLOW_SEG_HDR_L2TPV3) {
 758                 if (!(rss_type & ETH_RSS_L2TPV3))
 759                         *hash_flds &= ~ICE_FLOW_HASH_L2TPV3_SESS_ID;
 760         }
 761
 762         if (*addl_hdrs & ICE_FLOW_SEG_HDR_ESP) {
 763                 if (!(rss_type & ETH_RSS_ESP))
 764                         *hash_flds &= ~ICE_FLOW_HASH_ESP_SPI;
 765         }
 766
 767         if (*addl_hdrs & ICE_FLOW_SEG_HDR_AH) {
 768                 if (!(rss_type & ETH_RSS_AH))
 769                         *hash_flds &= ~ICE_FLOW_HASH_AH_SPI;
 770         }
 771
 772         if (*addl_hdrs & ICE_FLOW_SEG_HDR_PFCP_SESSION) {
 773                 if (!(rss_type & ETH_RSS_PFCP))
 774                         *hash_flds &= ~ICE_FLOW_HASH_PFCP_SEID;
 775         }
 776 }
 777
 778 static void
 779 ice_refine_proto_hdrs_by_pattern(struct ice_rss_hash_cfg *hash_cfg,
 780                                  uint64_t phint)
 781 {
 782         uint32_t *addl_hdrs = &hash_cfg->addl_hdrs;
 783         if (phint & ICE_PHINT_VLAN)
 784                 *addl_hdrs |= ICE_FLOW_SEG_HDR_VLAN;
 785
 786         if (phint & ICE_PHINT_PPPOE)
 787                 *addl_hdrs |= ICE_FLOW_SEG_HDR_PPPOE;
 788
 789         if (phint & ICE_PHINT_GTPU_EH_DWN)
 790                 *addl_hdrs |= ICE_FLOW_SEG_HDR_GTPU_DWN;
 791         else if (phint & ICE_PHINT_GTPU_EH_UP)
 792                 *addl_hdrs |= ICE_FLOW_SEG_HDR_GTPU_UP;
 793         else if (phint & ICE_PHINT_GTPU_EH)
 794                 *addl_hdrs |= ICE_FLOW_SEG_HDR_GTPU_EH;
 795         else if (phint & ICE_PHINT_GTPU)
 796                 *addl_hdrs |= ICE_FLOW_SEG_HDR_GTPU_IP;
 797 }
 798
 799 static void
 800 ice_refine_hash_cfg_gtpu(struct ice_rss_hash_cfg *hash_cfg,
 801                          uint64_t rss_type)
 802 {
 803         uint32_t *addl_hdrs = &hash_cfg->addl_hdrs;
 804         uint64_t *hash_flds = &hash_cfg->hash_flds;
 805
 806         /* update hash field for gtpu eh/gtpu dwn/gtpu up. */
 807         if (!(rss_type & ETH_RSS_GTPU))
 808                 return;
 809
 810         if (*addl_hdrs & ICE_FLOW_SEG_HDR_GTPU_DWN)
 811                 *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_GTPU_DWN_TEID);
 812         else if (*addl_hdrs & ICE_FLOW_SEG_HDR_GTPU_UP)
 813                 *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_GTPU_UP_TEID);
 814         else if (*addl_hdrs & ICE_FLOW_SEG_HDR_GTPU_EH)
 815                 *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_GTPU_EH_TEID);
 816         else if (*addl_hdrs & ICE_FLOW_SEG_HDR_GTPU_IP)
 817                 *hash_flds |= BIT_ULL(ICE_FLOW_FIELD_IDX_GTPU_IP_TEID);
 818 }
 819
 820 static void ice_refine_hash_cfg(struct ice_rss_hash_cfg *hash_cfg,
 821                                 uint64_t rss_type, uint64_t phint)
 822 {
 823         ice_refine_proto_hdrs_by_pattern(hash_cfg, phint);
 824         ice_refine_hash_cfg_l234(hash_cfg, rss_type);
 825         ice_refine_hash_cfg_gtpu(hash_cfg, rss_type);
 826 }
 827
 828 static uint64_t invalid_rss_comb[] = {
 829         ETH_RSS_IPV4 | ETH_RSS_NONFRAG_IPV4_UDP,
 830         ETH_RSS_IPV4 | ETH_RSS_NONFRAG_IPV4_TCP,
 831         ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_UDP,
 832         ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_TCP,
 833         RTE_ETH_RSS_L3_PRE40 |
 834         RTE_ETH_RSS_L3_PRE56 |
 835         RTE_ETH_RSS_L3_PRE96
 836 };
 837
 838 struct rss_attr_type {
 839         uint64_t attr;
 840         uint64_t type;
 841 };
 842
 843 static struct rss_attr_type rss_attr_to_valid_type[] = {
 844         {ETH_RSS_L2_SRC_ONLY | ETH_RSS_L2_DST_ONLY,     ETH_RSS_ETH},
 845         {ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY,     VALID_RSS_L3},
 846         {ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY,     VALID_RSS_L4},
 847         /* current ipv6 prefix only supports prefix 64 bits*/
 848         {RTE_ETH_RSS_L3_PRE32,                          VALID_RSS_IPV6},
 849         {RTE_ETH_RSS_L3_PRE48,                          VALID_RSS_IPV6},
 850         {RTE_ETH_RSS_L3_PRE64,                          VALID_RSS_IPV6},
 851         {INVALID_RSS_ATTR,                              0}
 852 };
 853
 854 static bool
 855 ice_any_invalid_rss_type(enum rte_eth_hash_function rss_func,
 856                          uint64_t rss_type, uint64_t allow_rss_type)
 857 {
 858         uint32_t i;
 859
 860         /**
 861          * Check if l3/l4 SRC/DST_ONLY is set for SYMMETRIC_TOEPLITZ
 862          * hash function.
 863          */
 864         if (rss_func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) {
 865                 if (rss_type & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY |
 866                     ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY))
 867                         return true;
 868
 869                 if (!(rss_type &
 870                    (ETH_RSS_IPV4 | ETH_RSS_IPV6 |
 871                     ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_NONFRAG_IPV6_UDP |
 872                     ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_NONFRAG_IPV6_TCP |
 873                     ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_NONFRAG_IPV6_SCTP)))
 874                         return true;
 875         }
 876
 877         /* check invalid combination */
 878         for (i = 0; i < RTE_DIM(invalid_rss_comb); i++) {
 879                 if (__builtin_popcountll(rss_type & invalid_rss_comb[i]) > 1)
 880                         return true;
 881         }
 882
 883         /* check invalid RSS attribute */
 884         for (i = 0; i < RTE_DIM(rss_attr_to_valid_type); i++) {
 885                 struct rss_attr_type *rat = &rss_attr_to_valid_type[i];
 886
 887                 if (rat->attr & rss_type && !(rat->type & rss_type))
 888                         return true;
 889         }
 890
 891         /* check not allowed RSS type */
 892         rss_type &= ~VALID_RSS_ATTR;
 893
 894         return ((rss_type & allow_rss_type) != rss_type);
 895 }
 896
 897 static int
 898 ice_hash_parse_action(struct ice_pattern_match_item *pattern_match_item,
 899                 const struct rte_flow_action actions[],
 900                 uint64_t pattern_hint, void **meta,
 901                 struct rte_flow_error *error)
 902 {
 903         struct ice_rss_meta *rss_meta = (struct ice_rss_meta *)*meta;
 904         struct ice_rss_hash_cfg *cfg = pattern_match_item->meta;
 905         enum rte_flow_action_type action_type;
 906         const struct rte_flow_action_rss *rss;
 907         const struct rte_flow_action *action;
 908         uint64_t rss_type;
 909
 910         /* Supported action is RSS. */
 911         for (action = actions; action->type !=
 912                 RTE_FLOW_ACTION_TYPE_END; action++) {
 913                 action_type = action->type;
 914                 switch (action_type) {
 915                 case RTE_FLOW_ACTION_TYPE_RSS:
 916                         rss = action->conf;
 917                         rss_type = rss->types;
 918
 919                         /* Check hash function and save it to rss_meta. */
 920                         if (pattern_match_item->pattern_list !=
 921                             pattern_empty && rss->func ==
 922                             RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) {
 923                                 return rte_flow_error_set(error, ENOTSUP,
 924                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 925                                         "Not supported flow");
 926                         } else if (rss->func ==
 927                                    RTE_ETH_HASH_FUNCTION_SIMPLE_XOR){
 928                                 rss_meta->hash_function =
 929                                 RTE_ETH_HASH_FUNCTION_SIMPLE_XOR;
 930                                 return 0;
 931                         } else if (rss->func ==
 932                                    RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) {
 933                                 rss_meta->hash_function =
 934                                 RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ;
 935                                 cfg->symm = true;
 936                         }
 937
 938                         if (rss->level)
 939                                 return rte_flow_error_set(error, ENOTSUP,
 940                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 941                                         "a nonzero RSS encapsulation level is not supported");
 942
 943                         if (rss->key_len)
 944                                 return rte_flow_error_set(error, ENOTSUP,
 945                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 946                                         "a nonzero RSS key_len is not supported");
 947
 948                         if (rss->queue)
 949                                 return rte_flow_error_set(error, ENOTSUP,
 950                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 951                                         "a non-NULL RSS queue is not supported");
 952
 953                         /**
 954                          * Check simultaneous use of SRC_ONLY and DST_ONLY
 955                          * of the same level.
 956                          */
 957                         rss_type = rte_eth_rss_hf_refine(rss_type);
 958
 959                         if (ice_any_invalid_rss_type(rss->func, rss_type,
 960                                         pattern_match_item->input_set_mask_o))
 961                                 return rte_flow_error_set(error, ENOTSUP,
 962                                         RTE_FLOW_ERROR_TYPE_ACTION,
 963                                         action, "RSS type not supported");
 964
 965                         rss_meta->cfg = *cfg;
 966                         ice_refine_hash_cfg(&rss_meta->cfg,
 967                                             rss_type, pattern_hint);
 968                         break;
 969                 case RTE_FLOW_ACTION_TYPE_END:
 970                         break;
 971
 972                 default:
 973                         rte_flow_error_set(error, EINVAL,
 974                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 975                                         "Invalid action.");
 976                         return -rte_errno;
 977                 }
 978         }
 979
 980         return 0;
 981 }
 982
 983 static int
 984 ice_hash_parse_pattern_action(__rte_unused struct ice_adapter *ad,
 985                         struct ice_pattern_match_item *array,
 986                         uint32_t array_len,
 987                         const struct rte_flow_item pattern[],
 988                         const struct rte_flow_action actions[],
 989                         uint32_t priority __rte_unused,
 990                         void **meta,
 991                         struct rte_flow_error *error)
 992 {
 993         int ret = 0;
 994         struct ice_pattern_match_item *pattern_match_item;
 995         struct ice_rss_meta *rss_meta_ptr;
 996         uint64_t phint = ICE_PHINT_NONE;
 997
 998         rss_meta_ptr = rte_zmalloc(NULL, sizeof(*rss_meta_ptr), 0);
 999         if (!rss_meta_ptr) {
1000                 rte_flow_error_set(error, EINVAL,
1001                                 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1002                                 "No memory for rss_meta_ptr");
1003                 return -ENOMEM;
1004         }
1005
1006         /* Check rss supported pattern and find matched pattern. */
1007         pattern_match_item = ice_search_pattern_match_item(ad, pattern, array,
1008                                                            array_len, error);
1009         if (!pattern_match_item) {
1010                 ret = -rte_errno;
1011                 goto error;
1012         }
1013
1014         ret = ice_hash_parse_pattern(pattern, &phint, error);
1015         if (ret)
1016                 goto error;
1017
1018         /* Check rss action. */
1019         ret = ice_hash_parse_action(pattern_match_item, actions, phint,
1020                                     (void **)&rss_meta_ptr, error);
1021
1022 error:
1023         if (!ret && meta)
1024                 *meta = rss_meta_ptr;
1025         else
1026                 rte_free(rss_meta_ptr);
1027         rte_free(pattern_match_item);
1028
1029         return ret;
1030 }
1031
1032 static int
1033 ice_hash_create(struct ice_adapter *ad,
1034                 struct rte_flow *flow,
1035                 void *meta,
1036                 struct rte_flow_error *error)
1037 {
1038         struct ice_pf *pf = &ad->pf;
1039         struct ice_hw *hw = ICE_PF_TO_HW(pf);
1040         struct ice_vsi *vsi = pf->main_vsi;
1041         int ret;
1042         uint32_t reg;
1043         struct ice_hash_flow_cfg *filter_ptr;
1044         struct ice_rss_meta *rss_meta = (struct ice_rss_meta *)meta;
1045         uint8_t hash_function = rss_meta->hash_function;
1046
1047         filter_ptr = rte_zmalloc("ice_rss_filter",
1048                                 sizeof(struct ice_hash_flow_cfg), 0);
1049         if (!filter_ptr) {
1050                 rte_flow_error_set(error, EINVAL,
1051                                 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1052                                 "No memory for filter_ptr");
1053                 return -ENOMEM;
1054         }
1055
1056         if (hash_function == RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) {
1057                 /* Enable registers for simple_xor hash function. */
1058                 reg = ICE_READ_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id));
1059                 reg = (reg & (~VSIQF_HASH_CTL_HASH_SCHEME_M)) |
1060                         (2 << VSIQF_HASH_CTL_HASH_SCHEME_S);
1061                 ICE_WRITE_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id), reg);
1062
1063                 filter_ptr->simple_xor = 1;
1064
1065                 goto out;
1066         } else {
1067                 memcpy(&filter_ptr->rss_cfg.hash, &rss_meta->cfg,
1068                        sizeof(struct ice_rss_hash_cfg));
1069                 ret = ice_add_rss_cfg_wrap(pf, vsi->idx,
1070                                            &filter_ptr->rss_cfg.hash);
1071                 if (ret) {
1072                         rte_flow_error_set(error, EINVAL,
1073                                         RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1074                                         "rss flow create fail");
1075                         goto error;
1076                 }
1077         }
1078
1079 out:
1080         flow->rule = filter_ptr;
1081         rte_free(meta);
1082         return 0;
1083
1084 error:
1085         rte_free(filter_ptr);
1086         rte_free(meta);
1087         return -rte_errno;
1088 }
1089
1090 static int
1091 ice_hash_destroy(struct ice_adapter *ad,
1092                 struct rte_flow *flow,
1093                 struct rte_flow_error *error)
1094 {
1095         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(ad);
1096         struct ice_hw *hw = ICE_PF_TO_HW(pf);
1097         struct ice_vsi *vsi = pf->main_vsi;
1098         int ret;
1099         uint32_t reg;
1100         struct ice_hash_flow_cfg *filter_ptr;
1101
1102         filter_ptr = (struct ice_hash_flow_cfg *)flow->rule;
1103
1104         if (filter_ptr->simple_xor == 1) {
1105                 /* Return to symmetric_toeplitz state. */
1106                 reg = ICE_READ_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id));
1107                 reg = (reg & (~VSIQF_HASH_CTL_HASH_SCHEME_M)) |
1108                         (1 << VSIQF_HASH_CTL_HASH_SCHEME_S);
1109                 ICE_WRITE_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id), reg);
1110         } else {
1111                 ret = ice_rem_rss_cfg_wrap(pf, vsi->idx,
1112                                            &filter_ptr->rss_cfg.hash);
1113                 /* Fixme: Ignore the error if a rule does not exist.
1114                  * Currently a rule for inputset change or symm turn on/off
1115                  * will overwrite an exist rule, while application still
1116                  * have 2 rte_flow handles.
1117                  **/
1118                 if (ret && ret != ICE_ERR_DOES_NOT_EXIST) {
1119                         rte_flow_error_set(error, EINVAL,
1120                                         RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1121                                         "rss flow destroy fail");
1122                         goto error;
1123                 }
1124         }
1125
1126         rte_free(filter_ptr);
1127         return 0;
1128
1129 error:
1130         rte_free(filter_ptr);
1131         return -rte_errno;
1132 }
1133
1134 static void
1135 ice_hash_uninit(struct ice_adapter *ad)
1136 {
1137         if (ad->hw.dcf_enabled)
1138                 return;
1139
1140         ice_unregister_parser(&ice_hash_parser, ad);
1141 }
1142
1143 static void
1144 ice_hash_free(struct rte_flow *flow)
1145 {
1146         rte_free(flow->rule);
1147 }