drivers/net/i40e/i40e_hash.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10 #include <assert.h>
  11
  12 #include <rte_malloc.h>
  13 #include <rte_tailq.h>
  14 #include "base/i40e_prototype.h"
  15 #include "i40e_logs.h"
  16 #include "i40e_ethdev.h"
  17 #include "i40e_hash.h"
  18
  19 #ifndef BIT
  20 #define BIT(n)                          (1UL << (n))
  21 #endif
  22
  23 #ifndef BIT_ULL
  24 #define BIT_ULL(n)                      (1ULL << (n))
  25 #endif
  26
  27 /* Pattern item headers */
  28 #define I40E_HASH_HDR_ETH               0x01ULL
  29 #define I40E_HASH_HDR_IPV4              0x10ULL
  30 #define I40E_HASH_HDR_IPV6              0x20ULL
  31 #define I40E_HASH_HDR_TCP               0x100ULL
  32 #define I40E_HASH_HDR_UDP               0x200ULL
  33 #define I40E_HASH_HDR_SCTP              0x400ULL
  34 #define I40E_HASH_HDR_ESP               0x10000ULL
  35 #define I40E_HASH_HDR_L2TPV3            0x20000ULL
  36 #define I40E_HASH_HDR_AH                0x40000ULL
  37 #define I40E_HASH_HDR_GTPC              0x100000ULL
  38 #define I40E_HASH_HDR_GTPU              0x200000ULL
  39
  40 #define I40E_HASH_HDR_INNER_SHIFT       32
  41 #define I40E_HASH_HDR_IPV4_INNER        (I40E_HASH_HDR_IPV4 << \
  42                                         I40E_HASH_HDR_INNER_SHIFT)
  43 #define I40E_HASH_HDR_IPV6_INNER        (I40E_HASH_HDR_IPV6 << \
  44                                         I40E_HASH_HDR_INNER_SHIFT)
  45
  46 /* ETH */
  47 #define I40E_PHINT_ETH                  I40E_HASH_HDR_ETH
  48
  49 /* IPv4 */
  50 #define I40E_PHINT_IPV4                 (I40E_HASH_HDR_ETH | I40E_HASH_HDR_IPV4)
  51 #define I40E_PHINT_IPV4_TCP             (I40E_PHINT_IPV4 | I40E_HASH_HDR_TCP)
  52 #define I40E_PHINT_IPV4_UDP             (I40E_PHINT_IPV4 | I40E_HASH_HDR_UDP)
  53 #define I40E_PHINT_IPV4_SCTP            (I40E_PHINT_IPV4 | I40E_HASH_HDR_SCTP)
  54
  55 /* IPv6 */
  56 #define I40E_PHINT_IPV6                 (I40E_HASH_HDR_ETH | I40E_HASH_HDR_IPV6)
  57 #define I40E_PHINT_IPV6_TCP             (I40E_PHINT_IPV6 | I40E_HASH_HDR_TCP)
  58 #define I40E_PHINT_IPV6_UDP             (I40E_PHINT_IPV6 | I40E_HASH_HDR_UDP)
  59 #define I40E_PHINT_IPV6_SCTP            (I40E_PHINT_IPV6 | I40E_HASH_HDR_SCTP)
  60
  61 /* ESP */
  62 #define I40E_PHINT_IPV4_ESP             (I40E_PHINT_IPV4 | I40E_HASH_HDR_ESP)
  63 #define I40E_PHINT_IPV6_ESP             (I40E_PHINT_IPV6 | I40E_HASH_HDR_ESP)
  64 #define I40E_PHINT_IPV4_UDP_ESP         (I40E_PHINT_IPV4_UDP | \
  65                                         I40E_HASH_HDR_ESP)
  66 #define I40E_PHINT_IPV6_UDP_ESP         (I40E_PHINT_IPV6_UDP | \
  67                                         I40E_HASH_HDR_ESP)
  68
  69 /* GTPC */
  70 #define I40E_PHINT_IPV4_GTPC            (I40E_PHINT_IPV4_UDP | \
  71                                         I40E_HASH_HDR_GTPC)
  72 #define I40E_PHINT_IPV6_GTPC            (I40E_PHINT_IPV6_UDP | \
  73                                         I40E_HASH_HDR_GTPC)
  74
  75 /* GTPU */
  76 #define I40E_PHINT_IPV4_GTPU            (I40E_PHINT_IPV4_UDP | \
  77                                         I40E_HASH_HDR_GTPU)
  78 #define I40E_PHINT_IPV4_GTPU_IPV4       (I40E_PHINT_IPV4_GTPU | \
  79                                         I40E_HASH_HDR_IPV4_INNER)
  80 #define I40E_PHINT_IPV4_GTPU_IPV6       (I40E_PHINT_IPV4_GTPU | \
  81                                         I40E_HASH_HDR_IPV6_INNER)
  82 #define I40E_PHINT_IPV6_GTPU            (I40E_PHINT_IPV6_UDP | \
  83                                         I40E_HASH_HDR_GTPU)
  84 #define I40E_PHINT_IPV6_GTPU_IPV4       (I40E_PHINT_IPV6_GTPU | \
  85                                         I40E_HASH_HDR_IPV4_INNER)
  86 #define I40E_PHINT_IPV6_GTPU_IPV6       (I40E_PHINT_IPV6_GTPU | \
  87                                         I40E_HASH_HDR_IPV6_INNER)
  88
  89 /* L2TPV3 */
  90 #define I40E_PHINT_IPV4_L2TPV3          (I40E_PHINT_IPV4 | I40E_HASH_HDR_L2TPV3)
  91 #define I40E_PHINT_IPV6_L2TPV3          (I40E_PHINT_IPV6 | I40E_HASH_HDR_L2TPV3)
  92
  93 /* AH */
  94 #define I40E_PHINT_IPV4_AH              (I40E_PHINT_IPV4 | I40E_HASH_HDR_AH)
  95 #define I40E_PHINT_IPV6_AH              (I40E_PHINT_IPV6 | I40E_HASH_HDR_AH)
  96
  97 /* Structure of mapping RSS type to input set */
  98 struct i40e_hash_map_rss_inset {
  99         uint64_t rss_type;
 100         uint64_t inset;
 101 };
 102
 103 const struct i40e_hash_map_rss_inset i40e_hash_rss_inset[] = {
 104         /* IPv4 */
 105         { ETH_RSS_IPV4, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST },
 106         { ETH_RSS_FRAG_IPV4, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST },
 107
 108         { ETH_RSS_NONFRAG_IPV4_OTHER,
 109           I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST },
 110
 111         { ETH_RSS_NONFRAG_IPV4_TCP, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
 112           I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT },
 113
 114         { ETH_RSS_NONFRAG_IPV4_UDP, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
 115           I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT },
 116
 117         { ETH_RSS_NONFRAG_IPV4_SCTP, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
 118           I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | I40E_INSET_SCTP_VT },
 119
 120         /* IPv6 */
 121         { ETH_RSS_IPV6, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST },
 122         { ETH_RSS_FRAG_IPV6, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST },
 123
 124         { ETH_RSS_NONFRAG_IPV6_OTHER,
 125           I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST },
 126
 127         { ETH_RSS_NONFRAG_IPV6_TCP, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
 128           I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT },
 129
 130         { ETH_RSS_NONFRAG_IPV6_UDP, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
 131           I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT },
 132
 133         { ETH_RSS_NONFRAG_IPV6_SCTP, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
 134           I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | I40E_INSET_SCTP_VT },
 135
 136         /* Port */
 137         { ETH_RSS_PORT, I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT },
 138
 139         /* Ether */
 140         { ETH_RSS_L2_PAYLOAD, I40E_INSET_LAST_ETHER_TYPE },
 141         { ETH_RSS_ETH, I40E_INSET_DMAC | I40E_INSET_SMAC },
 142
 143         /* VLAN */
 144         { ETH_RSS_S_VLAN, I40E_INSET_VLAN_OUTER },
 145         { ETH_RSS_C_VLAN, I40E_INSET_VLAN_INNER },
 146 };
 147
 148 #define I40E_HASH_VOID_NEXT_ALLOW       BIT_ULL(RTE_FLOW_ITEM_TYPE_ETH)
 149
 150 #define I40E_HASH_ETH_NEXT_ALLOW        (BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV4) | \
 151                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV6) | \
 152                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_VLAN))
 153
 154 #define I40E_HASH_IP_NEXT_ALLOW         (BIT_ULL(RTE_FLOW_ITEM_TYPE_TCP) | \
 155                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_UDP) | \
 156                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_SCTP) | \
 157                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_ESP) | \
 158                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_L2TPV3OIP) |\
 159                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_AH))
 160
 161 #define I40E_HASH_UDP_NEXT_ALLOW        (BIT_ULL(RTE_FLOW_ITEM_TYPE_GTPU) | \
 162                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_GTPC))
 163
 164 #define I40E_HASH_GTPU_NEXT_ALLOW       (BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV4) | \
 165                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV6))
 166
 167 static const uint64_t pattern_next_allow_items[] = {
 168         [RTE_FLOW_ITEM_TYPE_VOID] = I40E_HASH_VOID_NEXT_ALLOW,
 169         [RTE_FLOW_ITEM_TYPE_ETH] = I40E_HASH_ETH_NEXT_ALLOW,
 170         [RTE_FLOW_ITEM_TYPE_IPV4] = I40E_HASH_IP_NEXT_ALLOW,
 171         [RTE_FLOW_ITEM_TYPE_IPV6] = I40E_HASH_IP_NEXT_ALLOW,
 172         [RTE_FLOW_ITEM_TYPE_UDP] = I40E_HASH_UDP_NEXT_ALLOW,
 173         [RTE_FLOW_ITEM_TYPE_GTPU] = I40E_HASH_GTPU_NEXT_ALLOW,
 174 };
 175
 176 static const uint64_t pattern_item_header[] = {
 177         [RTE_FLOW_ITEM_TYPE_ETH] = I40E_HASH_HDR_ETH,
 178         [RTE_FLOW_ITEM_TYPE_IPV4] = I40E_HASH_HDR_IPV4,
 179         [RTE_FLOW_ITEM_TYPE_IPV6] = I40E_HASH_HDR_IPV6,
 180         [RTE_FLOW_ITEM_TYPE_TCP] = I40E_HASH_HDR_TCP,
 181         [RTE_FLOW_ITEM_TYPE_UDP] = I40E_HASH_HDR_UDP,
 182         [RTE_FLOW_ITEM_TYPE_SCTP] = I40E_HASH_HDR_SCTP,
 183         [RTE_FLOW_ITEM_TYPE_ESP] = I40E_HASH_HDR_ESP,
 184         [RTE_FLOW_ITEM_TYPE_GTPC] = I40E_HASH_HDR_GTPC,
 185         [RTE_FLOW_ITEM_TYPE_GTPU] = I40E_HASH_HDR_GTPU,
 186         [RTE_FLOW_ITEM_TYPE_L2TPV3OIP] = I40E_HASH_HDR_L2TPV3,
 187         [RTE_FLOW_ITEM_TYPE_AH] = I40E_HASH_HDR_AH,
 188 };
 189
 190 /* Structure of matched pattern */
 191 struct i40e_hash_match_pattern {
 192         uint64_t pattern_type;
 193         uint64_t rss_mask;      /* Supported RSS type for this pattern */
 194         bool custom_pctype_flag;/* true for custom packet type */
 195         uint8_t pctype;
 196 };
 197
 198 #define I40E_HASH_MAP_PATTERN(pattern, rss_mask, pctype) { \
 199         pattern, rss_mask, false, pctype  }
 200
 201 #define I40E_HASH_MAP_CUS_PATTERN(pattern, rss_mask, cus_pctype) { \
 202         pattern, rss_mask, true, cus_pctype }
 203
 204 #define I40E_HASH_L2_RSS_MASK           (ETH_RSS_VLAN | ETH_RSS_ETH | \
 205                                         ETH_RSS_L2_SRC_ONLY | \
 206                                         ETH_RSS_L2_DST_ONLY)
 207
 208 #define I40E_HASH_L23_RSS_MASK          (I40E_HASH_L2_RSS_MASK | \
 209                                         ETH_RSS_L3_SRC_ONLY | \
 210                                         ETH_RSS_L3_DST_ONLY)
 211
 212 #define I40E_HASH_IPV4_L23_RSS_MASK     (ETH_RSS_IPV4 | I40E_HASH_L23_RSS_MASK)
 213 #define I40E_HASH_IPV6_L23_RSS_MASK     (ETH_RSS_IPV6 | I40E_HASH_L23_RSS_MASK)
 214
 215 #define I40E_HASH_L234_RSS_MASK         (I40E_HASH_L23_RSS_MASK | \
 216                                         ETH_RSS_PORT | ETH_RSS_L4_SRC_ONLY | \
 217                                         ETH_RSS_L4_DST_ONLY)
 218
 219 #define I40E_HASH_IPV4_L234_RSS_MASK    (I40E_HASH_L234_RSS_MASK | ETH_RSS_IPV4)
 220 #define I40E_HASH_IPV6_L234_RSS_MASK    (I40E_HASH_L234_RSS_MASK | ETH_RSS_IPV6)
 221
 222 #define I40E_HASH_L4_TYPES              (ETH_RSS_NONFRAG_IPV4_TCP | \
 223                                         ETH_RSS_NONFRAG_IPV4_UDP | \
 224                                         ETH_RSS_NONFRAG_IPV4_SCTP | \
 225                                         ETH_RSS_NONFRAG_IPV6_TCP | \
 226                                         ETH_RSS_NONFRAG_IPV6_UDP | \
 227                                         ETH_RSS_NONFRAG_IPV6_SCTP)
 228
 229 /* Current supported patterns and RSS types.
 230  * All items that have the same pattern types are together.
 231  */
 232 static const struct i40e_hash_match_pattern match_patterns[] = {
 233         /* Ether */
 234         I40E_HASH_MAP_PATTERN(I40E_PHINT_ETH,
 235                               ETH_RSS_L2_PAYLOAD | I40E_HASH_L2_RSS_MASK,
 236                               I40E_FILTER_PCTYPE_L2_PAYLOAD),
 237
 238         /* IPv4 */
 239         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4,
 240                               ETH_RSS_FRAG_IPV4 | I40E_HASH_IPV4_L23_RSS_MASK,
 241                               I40E_FILTER_PCTYPE_FRAG_IPV4),
 242
 243         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4,
 244                               ETH_RSS_NONFRAG_IPV4_OTHER |
 245                               I40E_HASH_IPV4_L23_RSS_MASK,
 246                               I40E_FILTER_PCTYPE_NONF_IPV4_OTHER),
 247
 248         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4_TCP,
 249                               ETH_RSS_NONFRAG_IPV4_TCP |
 250                               I40E_HASH_IPV4_L234_RSS_MASK,
 251                               I40E_FILTER_PCTYPE_NONF_IPV4_TCP),
 252
 253         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4_UDP,
 254                               ETH_RSS_NONFRAG_IPV4_UDP |
 255                               I40E_HASH_IPV4_L234_RSS_MASK,
 256                               I40E_FILTER_PCTYPE_NONF_IPV4_UDP),
 257
 258         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4_SCTP,
 259                               ETH_RSS_NONFRAG_IPV4_SCTP |
 260                               I40E_HASH_IPV4_L234_RSS_MASK,
 261                               I40E_FILTER_PCTYPE_NONF_IPV4_SCTP),
 262
 263         /* IPv6 */
 264         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6,
 265                               ETH_RSS_FRAG_IPV6 | I40E_HASH_IPV6_L23_RSS_MASK,
 266                               I40E_FILTER_PCTYPE_FRAG_IPV6),
 267
 268         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6,
 269                               ETH_RSS_NONFRAG_IPV6_OTHER |
 270                               I40E_HASH_IPV6_L23_RSS_MASK,
 271                               I40E_FILTER_PCTYPE_NONF_IPV6_OTHER),
 272
 273         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6_TCP,
 274                               ETH_RSS_NONFRAG_IPV6_TCP |
 275                               I40E_HASH_IPV6_L234_RSS_MASK,
 276                               I40E_FILTER_PCTYPE_NONF_IPV6_TCP),
 277
 278         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6_UDP,
 279                               ETH_RSS_NONFRAG_IPV6_UDP |
 280                               I40E_HASH_IPV6_L234_RSS_MASK,
 281                               I40E_FILTER_PCTYPE_NONF_IPV6_UDP),
 282
 283         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6_SCTP,
 284                               ETH_RSS_NONFRAG_IPV6_SCTP |
 285                               I40E_HASH_IPV6_L234_RSS_MASK,
 286                               I40E_FILTER_PCTYPE_NONF_IPV6_SCTP),
 287
 288         /* ESP */
 289         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_ESP,
 290                                   ETH_RSS_ESP, I40E_CUSTOMIZED_ESP_IPV4),
 291         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_ESP,
 292                                   ETH_RSS_ESP, I40E_CUSTOMIZED_ESP_IPV6),
 293         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_UDP_ESP,
 294                                   ETH_RSS_ESP, I40E_CUSTOMIZED_ESP_IPV4_UDP),
 295         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_UDP_ESP,
 296                                   ETH_RSS_ESP, I40E_CUSTOMIZED_ESP_IPV6_UDP),
 297
 298         /* GTPC */
 299         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_GTPC,
 300                                   I40E_HASH_IPV4_L234_RSS_MASK,
 301                                   I40E_CUSTOMIZED_GTPC),
 302         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_GTPC,
 303                                   I40E_HASH_IPV6_L234_RSS_MASK,
 304                                   I40E_CUSTOMIZED_GTPC),
 305
 306         /* GTPU */
 307         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_GTPU,
 308                                   I40E_HASH_IPV4_L234_RSS_MASK,
 309                                   I40E_CUSTOMIZED_GTPU),
 310         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_GTPU_IPV4,
 311                                   ETH_RSS_GTPU, I40E_CUSTOMIZED_GTPU_IPV4),
 312         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_GTPU_IPV6,
 313                                   ETH_RSS_GTPU, I40E_CUSTOMIZED_GTPU_IPV6),
 314         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_GTPU,
 315                                   I40E_HASH_IPV6_L234_RSS_MASK,
 316                                   I40E_CUSTOMIZED_GTPU),
 317         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_GTPU_IPV4,
 318                                   ETH_RSS_GTPU, I40E_CUSTOMIZED_GTPU_IPV4),
 319         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_GTPU_IPV6,
 320                                   ETH_RSS_GTPU, I40E_CUSTOMIZED_GTPU_IPV6),
 321
 322         /* L2TPV3 */
 323         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_L2TPV3,
 324                                   ETH_RSS_L2TPV3, I40E_CUSTOMIZED_IPV4_L2TPV3),
 325         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_L2TPV3,
 326                                   ETH_RSS_L2TPV3, I40E_CUSTOMIZED_IPV6_L2TPV3),
 327
 328         /* AH */
 329         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_AH, ETH_RSS_AH,
 330                                   I40E_CUSTOMIZED_AH_IPV4),
 331         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_AH, ETH_RSS_AH,
 332                                   I40E_CUSTOMIZED_AH_IPV6),
 333 };
 334
 335 static int
 336 i40e_hash_get_pattern_type(const struct rte_flow_item pattern[],
 337                            uint64_t *pattern_types,
 338                            struct rte_flow_error *error)
 339 {
 340         const char *message = "Pattern not supported";
 341         enum rte_flow_item_type prev_item_type = RTE_FLOW_ITEM_TYPE_VOID;
 342         enum rte_flow_item_type last_item_type = prev_item_type;
 343         uint64_t item_hdr, pattern_hdrs = 0;
 344         bool inner_flag = false;
 345         int vlan_count = 0;
 346
 347         for (; pattern->type != RTE_FLOW_ITEM_TYPE_END; pattern++) {
 348                 if (pattern->type == RTE_FLOW_ITEM_TYPE_VOID)
 349                         continue;
 350
 351                 if (pattern->mask || pattern->spec || pattern->last) {
 352                         message = "Header info should not be specified";
 353                         goto not_sup;
 354                 }
 355
 356                 /* Check the previous item allows this sub-item. */
 357                 if (prev_item_type >= (enum rte_flow_item_type)
 358                                 RTE_DIM(pattern_next_allow_items) ||
 359                     !(pattern_next_allow_items[prev_item_type] &
 360                                 BIT_ULL(pattern->type)))
 361                         goto not_sup;
 362
 363                 /* For VLAN item, it does no matter about to pattern type
 364                  * recognition. So just count the number of VLAN and do not
 365                  * change the value of variable `prev_item_type`.
 366                  */
 367                 last_item_type = pattern->type;
 368                 if (last_item_type == RTE_FLOW_ITEM_TYPE_VLAN) {
 369                         if (vlan_count >= 2)
 370                                 goto not_sup;
 371                         vlan_count++;
 372                         continue;
 373                 }
 374
 375                 prev_item_type = last_item_type;
 376                 assert(last_item_type < (enum rte_flow_item_type)
 377                                 RTE_DIM(pattern_item_header));
 378                 item_hdr = pattern_item_header[last_item_type];
 379                 assert(item_hdr);
 380
 381                 if (inner_flag) {
 382                         item_hdr <<= I40E_HASH_HDR_INNER_SHIFT;
 383
 384                         /* Inner layer should not have GTPU item */
 385                         if (last_item_type == RTE_FLOW_ITEM_TYPE_GTPU)
 386                                 goto not_sup;
 387                 } else {
 388                         if (last_item_type == RTE_FLOW_ITEM_TYPE_GTPU) {
 389                                 inner_flag = true;
 390                                 vlan_count = 0;
 391                         }
 392                 }
 393
 394                 if (item_hdr & pattern_hdrs)
 395                         goto not_sup;
 396
 397                 pattern_hdrs |= item_hdr;
 398         }
 399
 400         if (pattern_hdrs && last_item_type != RTE_FLOW_ITEM_TYPE_VLAN) {
 401                 *pattern_types = pattern_hdrs;
 402                 return 0;
 403         }
 404
 405 not_sup:
 406         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 407                                   pattern, message);
 408 }
 409
 410 static uint64_t
 411 i40e_hash_get_x722_ext_pctypes(uint8_t match_pctype)
 412 {
 413         uint64_t pctypes = 0;
 414
 415         switch (match_pctype) {
 416         case I40E_FILTER_PCTYPE_NONF_IPV4_TCP:
 417                 pctypes = BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK);
 418                 break;
 419
 420         case I40E_FILTER_PCTYPE_NONF_IPV4_UDP:
 421                 pctypes = BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) |
 422                           BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP);
 423                 break;
 424
 425         case I40E_FILTER_PCTYPE_NONF_IPV6_TCP:
 426                 pctypes = BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK);
 427                 break;
 428
 429         case I40E_FILTER_PCTYPE_NONF_IPV6_UDP:
 430                 pctypes = BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) |
 431                           BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP);
 432                 break;
 433         }
 434
 435         return pctypes;
 436 }
 437
 438 static int
 439 i40e_hash_translate_gtp_inset(struct i40e_rte_flow_rss_conf *rss_conf,
 440                               struct rte_flow_error *error)
 441 {
 442         if (rss_conf->inset &
 443             (I40E_INSET_IPV4_SRC | I40E_INSET_IPV6_SRC |
 444             I40E_INSET_DST_PORT | I40E_INSET_SRC_PORT))
 445                 return rte_flow_error_set(error, ENOTSUP,
 446                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 447                                           NULL,
 448                                           "Only support external destination IP");
 449
 450         if (rss_conf->inset & I40E_INSET_IPV4_DST)
 451                 rss_conf->inset = (rss_conf->inset & ~I40E_INSET_IPV4_DST) |
 452                                   I40E_INSET_TUNNEL_IPV4_DST;
 453
 454         if (rss_conf->inset & I40E_INSET_IPV6_DST)
 455                 rss_conf->inset = (rss_conf->inset & ~I40E_INSET_IPV6_DST) |
 456                                   I40E_INSET_TUNNEL_IPV6_DST;
 457
 458         return 0;
 459 }
 460
 461 static int
 462 i40e_hash_get_pctypes(const struct rte_eth_dev *dev,
 463                       const struct i40e_hash_match_pattern *match,
 464                       struct i40e_rte_flow_rss_conf *rss_conf,
 465                       struct rte_flow_error *error)
 466 {
 467         if (match->custom_pctype_flag) {
 468                 struct i40e_pf *pf;
 469                 struct i40e_customized_pctype *custom_type;
 470
 471                 pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 472                 custom_type = i40e_find_customized_pctype(pf, match->pctype);
 473                 if (!custom_type || !custom_type->valid)
 474                         return rte_flow_error_set(error, ENOTSUP,
 475                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 476                                                   NULL, "PCTYPE not supported");
 477
 478                 rss_conf->config_pctypes |= BIT_ULL(custom_type->pctype);
 479
 480                 if (match->pctype == I40E_CUSTOMIZED_GTPU ||
 481                     match->pctype == I40E_CUSTOMIZED_GTPC)
 482                         return i40e_hash_translate_gtp_inset(rss_conf, error);
 483         } else {
 484                 struct i40e_hw *hw =
 485                                 I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 486                 uint64_t types;
 487
 488                 rss_conf->config_pctypes |= BIT_ULL(match->pctype);
 489                 if (hw->mac.type == I40E_MAC_X722) {
 490                         types = i40e_hash_get_x722_ext_pctypes(match->pctype);
 491                         rss_conf->config_pctypes |= types;
 492                 }
 493         }
 494
 495         return 0;
 496 }
 497
 498 static int
 499 i40e_hash_get_pattern_pctypes(const struct rte_eth_dev *dev,
 500                               const struct rte_flow_item pattern[],
 501                               const struct rte_flow_action_rss *rss_act,
 502                               struct i40e_rte_flow_rss_conf *rss_conf,
 503                               struct rte_flow_error *error)
 504 {
 505         uint64_t pattern_types = 0;
 506         bool match_flag = false;
 507         int i, ret;
 508
 509         ret = i40e_hash_get_pattern_type(pattern, &pattern_types, error);
 510         if (ret)
 511                 return ret;
 512
 513         for (i = 0; i < (int)RTE_DIM(match_patterns); i++) {
 514                 const struct i40e_hash_match_pattern *match =
 515                                                         &match_patterns[i];
 516
 517                 /* Check pattern types match. All items that have the same
 518                  * pattern types are together, so if the pattern types match
 519                  * previous item but they doesn't match current item, it means
 520                  * the pattern types do not match all remain items.
 521                  */
 522                 if (pattern_types != match->pattern_type) {
 523                         if (match_flag)
 524                                 break;
 525                         continue;
 526                 }
 527                 match_flag = true;
 528
 529                 /* Check RSS types match */
 530                 if (!(rss_act->types & ~match->rss_mask)) {
 531                         ret = i40e_hash_get_pctypes(dev, match,
 532                                                     rss_conf, error);
 533                         if (ret)
 534                                 return ret;
 535                 }
 536         }
 537
 538         if (rss_conf->config_pctypes)
 539                 return 0;
 540
 541         if (match_flag)
 542                 return rte_flow_error_set(error, ENOTSUP,
 543                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 544                                           NULL, "RSS types not supported");
 545
 546         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 547                                   NULL, "Pattern not supported");
 548 }
 549
 550 static uint64_t
 551 i40e_hash_get_inset(uint64_t rss_types)
 552 {
 553         uint64_t mask, inset = 0;
 554         int i;
 555
 556         for (i = 0; i < (int)RTE_DIM(i40e_hash_rss_inset); i++) {
 557                 if (rss_types & i40e_hash_rss_inset[i].rss_type)
 558                         inset |= i40e_hash_rss_inset[i].inset;
 559         }
 560
 561         if (!inset)
 562                 return 0;
 563
 564         /* If SRC_ONLY and DST_ONLY of the same level are used simultaneously,
 565          * it is the same case as none of them are added.
 566          */
 567         mask = rss_types & (ETH_RSS_L2_SRC_ONLY | ETH_RSS_L2_DST_ONLY);
 568         if (mask == ETH_RSS_L2_SRC_ONLY)
 569                 inset &= ~I40E_INSET_DMAC;
 570         else if (mask == ETH_RSS_L2_DST_ONLY)
 571                 inset &= ~I40E_INSET_SMAC;
 572
 573         mask = rss_types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY);
 574         if (mask == ETH_RSS_L3_SRC_ONLY)
 575                 inset &= ~(I40E_INSET_IPV4_DST | I40E_INSET_IPV6_DST);
 576         else if (mask == ETH_RSS_L3_DST_ONLY)
 577                 inset &= ~(I40E_INSET_IPV4_SRC | I40E_INSET_IPV6_SRC);
 578
 579         mask = rss_types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY);
 580         if (mask == ETH_RSS_L4_SRC_ONLY)
 581                 inset &= ~I40E_INSET_DST_PORT;
 582         else if (mask == ETH_RSS_L4_DST_ONLY)
 583                 inset &= ~I40E_INSET_SRC_PORT;
 584
 585         if (rss_types & I40E_HASH_L4_TYPES) {
 586                 uint64_t l3_mask = rss_types &
 587                                    (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY);
 588                 uint64_t l4_mask = rss_types &
 589                                    (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY);
 590
 591                 if (l3_mask && !l4_mask)
 592                         inset &= ~(I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT);
 593                 else if (!l3_mask && l4_mask)
 594                         inset &= ~(I40E_INSET_IPV4_DST | I40E_INSET_IPV6_DST |
 595                                  I40E_INSET_IPV4_SRC | I40E_INSET_IPV6_SRC);
 596         }
 597
 598         return inset;
 599 }
 600
 601 static int
 602 i40e_hash_config_func(struct i40e_hw *hw, enum rte_eth_hash_function func)
 603 {
 604         struct i40e_pf *pf;
 605         uint32_t reg;
 606         uint8_t symmetric = 0;
 607
 608         reg = i40e_read_rx_ctl(hw, I40E_GLQF_CTL);
 609
 610         if (func == RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) {
 611                 if (!(reg & I40E_GLQF_CTL_HTOEP_MASK))
 612                         goto set_symmetric;
 613
 614                 reg &= ~I40E_GLQF_CTL_HTOEP_MASK;
 615         } else {
 616                 if (func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ)
 617                         symmetric = 1;
 618
 619                 if (reg & I40E_GLQF_CTL_HTOEP_MASK)
 620                         goto set_symmetric;
 621
 622                 reg |= I40E_GLQF_CTL_HTOEP_MASK;
 623         }
 624
 625         pf = &((struct i40e_adapter *)hw->back)->pf;
 626         if (pf->support_multi_driver) {
 627                 PMD_DRV_LOG(ERR,
 628                             "Modify hash function is not permitted when multi-driver enabled");
 629                 return -EPERM;
 630         }
 631
 632         PMD_DRV_LOG(INFO, "NIC hash function is setting to %d", func);
 633         i40e_write_rx_ctl(hw, I40E_GLQF_CTL, reg);
 634         I40E_WRITE_FLUSH(hw);
 635
 636 set_symmetric:
 637         i40e_set_symmetric_hash_enable_per_port(hw, symmetric);
 638         return 0;
 639 }
 640
 641 static int
 642 i40e_hash_config_pctype_symmetric(struct i40e_hw *hw,
 643                                   uint32_t pctype,
 644                                   bool symmetric)
 645 {
 646         struct i40e_pf *pf = &((struct i40e_adapter *)hw->back)->pf;
 647         uint32_t reg;
 648
 649         /* For X722, get translated pctype in fd pctype register */
 650         if (hw->mac.type == I40E_MAC_X722)
 651                 pctype = i40e_read_rx_ctl(hw, I40E_GLQF_FD_PCTYPES(pctype));
 652
 653         reg = i40e_read_rx_ctl(hw, I40E_GLQF_HSYM(pctype));
 654         if (symmetric) {
 655                 if (reg & I40E_GLQF_HSYM_SYMH_ENA_MASK)
 656                         return 0;
 657                 reg |= I40E_GLQF_HSYM_SYMH_ENA_MASK;
 658         } else {
 659                 if (!(reg & I40E_GLQF_HSYM_SYMH_ENA_MASK))
 660                         return 0;
 661                 reg &= ~I40E_GLQF_HSYM_SYMH_ENA_MASK;
 662         }
 663
 664         if (pf->support_multi_driver) {
 665                 PMD_DRV_LOG(ERR,
 666                             "Enable/Disable symmetric hash is not permitted when multi-driver enabled");
 667                 return -EPERM;
 668         }
 669
 670         i40e_write_rx_ctl(hw, I40E_GLQF_HSYM(pctype), reg);
 671         I40E_WRITE_FLUSH(hw);
 672         return 0;
 673 }
 674
 675 static void
 676 i40e_hash_enable_pctype(struct i40e_hw *hw,
 677                         uint32_t pctype, bool enable)
 678 {
 679         uint32_t reg, reg_val, mask;
 680
 681         if (pctype < 32) {
 682                 mask = BIT(pctype);
 683                 reg = I40E_PFQF_HENA(0);
 684         } else {
 685                 mask = BIT(pctype - 32);
 686                 reg = I40E_PFQF_HENA(1);
 687         }
 688
 689         reg_val = i40e_read_rx_ctl(hw, reg);
 690
 691         if (enable) {
 692                 if (reg_val & mask)
 693                         return;
 694
 695                 reg_val |= mask;
 696         } else {
 697                 if (!(reg_val & mask))
 698                         return;
 699
 700                 reg_val &= ~mask;
 701         }
 702
 703         i40e_write_rx_ctl(hw, reg, reg_val);
 704         I40E_WRITE_FLUSH(hw);
 705 }
 706
 707 static int
 708 i40e_hash_config_pctype(struct i40e_hw *hw,
 709                         struct i40e_rte_flow_rss_conf *rss_conf,
 710                         uint32_t pctype)
 711 {
 712         uint64_t rss_types = rss_conf->conf.types;
 713         int ret;
 714
 715         if (rss_types == 0) {
 716                 i40e_hash_enable_pctype(hw, pctype, false);
 717                 return 0;
 718         }
 719
 720         if (rss_conf->inset) {
 721                 ret = i40e_set_hash_inset(hw, rss_conf->inset, pctype, false);
 722                 if (ret)
 723                         return ret;
 724         }
 725
 726         i40e_hash_enable_pctype(hw, pctype, true);
 727         return 0;
 728 }
 729
 730 static int
 731 i40e_hash_config_region(struct i40e_pf *pf,
 732                         const struct i40e_rte_flow_rss_conf *rss_conf)
 733 {
 734         struct i40e_hw *hw = &pf->adapter->hw;
 735         struct rte_eth_dev *dev = pf->adapter->eth_dev;
 736         struct i40e_queue_region_info *regions = pf->queue_region.region;
 737         uint32_t num = pf->queue_region.queue_region_number;
 738         uint32_t i, region_id_mask = 0;
 739
 740         /* Use a 32 bit variable to represent all regions */
 741         RTE_BUILD_BUG_ON(I40E_REGION_MAX_INDEX > 31);
 742
 743         /* Re-configure the region if it existed */
 744         for (i = 0; i < num; i++) {
 745                 if (rss_conf->region_queue_start ==
 746                     regions[i].queue_start_index &&
 747                     rss_conf->region_queue_num == regions[i].queue_num) {
 748                         uint32_t j;
 749
 750                         for (j = 0; j < regions[i].user_priority_num; j++) {
 751                                 if (regions[i].user_priority[j] ==
 752                                     rss_conf->region_priority)
 753                                         return 0;
 754                         }
 755
 756                         if (j >= I40E_MAX_USER_PRIORITY) {
 757                                 PMD_DRV_LOG(ERR,
 758                                             "Priority number exceed the maximum %d",
 759                                             I40E_MAX_USER_PRIORITY);
 760                                 return -ENOSPC;
 761                         }
 762
 763                         regions[i].user_priority[j] = rss_conf->region_priority;
 764                         regions[i].user_priority_num++;
 765                         return i40e_flush_queue_region_all_conf(dev, hw, pf, 1);
 766                 }
 767
 768                 region_id_mask |= BIT(regions[i].region_id);
 769         }
 770
 771         if (num > I40E_REGION_MAX_INDEX) {
 772                 PMD_DRV_LOG(ERR, "Queue region resource used up");
 773                 return -ENOSPC;
 774         }
 775
 776         /* Add a new region */
 777
 778         pf->queue_region.queue_region_number++;
 779         memset(&regions[num], 0, sizeof(regions[0]));
 780
 781         regions[num].region_id = rte_bsf32(~region_id_mask);
 782         regions[num].queue_num = rss_conf->region_queue_num;
 783         regions[num].queue_start_index = rss_conf->region_queue_start;
 784         regions[num].user_priority[0] = rss_conf->region_priority;
 785         regions[num].user_priority_num = 1;
 786
 787         return i40e_flush_queue_region_all_conf(dev, hw, pf, 1);
 788 }
 789
 790 static int
 791 i40e_hash_config(struct i40e_pf *pf,
 792                  struct i40e_rte_flow_rss_conf *rss_conf)
 793 {
 794         struct rte_flow_action_rss *rss_info = &rss_conf->conf;
 795         struct i40e_hw *hw = &pf->adapter->hw;
 796         uint64_t pctypes;
 797         int ret;
 798
 799         if (rss_info->func != RTE_ETH_HASH_FUNCTION_DEFAULT) {
 800                 ret = i40e_hash_config_func(hw, rss_info->func);
 801                 if (ret)
 802                         return ret;
 803
 804                 if (rss_info->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
 805                         rss_conf->misc_reset_flags |=
 806                                         I40E_HASH_FLOW_RESET_FLAG_FUNC;
 807         }
 808
 809         if (rss_conf->region_queue_num > 0) {
 810                 ret = i40e_hash_config_region(pf, rss_conf);
 811                 if (ret)
 812                         return ret;
 813
 814                 rss_conf->misc_reset_flags |= I40E_HASH_FLOW_RESET_FLAG_REGION;
 815         }
 816
 817         if (rss_info->key_len > 0) {
 818                 ret = i40e_set_rss_key(pf->main_vsi, rss_conf->key,
 819                                        rss_info->key_len);
 820                 if (ret)
 821                         return ret;
 822
 823                 rss_conf->misc_reset_flags |= I40E_HASH_FLOW_RESET_FLAG_KEY;
 824         }
 825
 826         /* Update lookup table */
 827         if (rss_info->queue_num > 0) {
 828                 uint8_t lut[ETH_RSS_RETA_SIZE_512];
 829                 uint32_t i, j = 0;
 830
 831                 for (i = 0; i < hw->func_caps.rss_table_size; i++) {
 832                         lut[i] = (uint8_t)rss_info->queue[j];
 833                         j = (j == rss_info->queue_num - 1) ? 0 : (j + 1);
 834                 }
 835
 836                 ret = i40e_set_rss_lut(pf->main_vsi, lut, (uint16_t)i);
 837                 if (ret)
 838                         return ret;
 839
 840                 pf->hash_enabled_queues = 0;
 841                 for (i = 0; i < rss_info->queue_num; i++)
 842                         pf->hash_enabled_queues |= BIT_ULL(lut[i]);
 843
 844                 pf->adapter->rss_reta_updated = 0;
 845                 rss_conf->misc_reset_flags |= I40E_HASH_FLOW_RESET_FLAG_QUEUE;
 846         }
 847
 848         /* The codes behind configure the input sets and symmetric hash
 849          * function of the packet types and enable hash on them.
 850          */
 851         pctypes = rss_conf->config_pctypes;
 852         if (!pctypes)
 853                 return 0;
 854
 855         /* For first flow that will enable hash on any packet type, we clean
 856          * the RSS sets that by legacy configuration commands and parameters.
 857          */
 858         if (!pf->hash_filter_enabled) {
 859                 i40e_pf_disable_rss(pf);
 860                 pf->hash_filter_enabled = true;
 861         }
 862
 863         do {
 864                 uint32_t idx = rte_bsf64(pctypes);
 865                 uint64_t bit = BIT_ULL(idx);
 866
 867                 if (rss_conf->symmetric_enable) {
 868                         ret = i40e_hash_config_pctype_symmetric(hw, idx, true);
 869                         if (ret)
 870                                 return ret;
 871
 872                         rss_conf->reset_symmetric_pctypes |= bit;
 873                 }
 874
 875                 ret = i40e_hash_config_pctype(hw, rss_conf, idx);
 876                 if (ret)
 877                         return ret;
 878
 879                 rss_conf->reset_config_pctypes |= bit;
 880                 pctypes &= ~bit;
 881         } while (pctypes);
 882
 883         return 0;
 884 }
 885
 886 static void
 887 i40e_hash_parse_key(const struct rte_flow_action_rss *rss_act,
 888                     struct i40e_rte_flow_rss_conf *rss_conf)
 889 {
 890         const uint8_t *key = rss_act->key;
 891
 892         if (!key || rss_act->key_len != sizeof(rss_conf->key)) {
 893                 const uint32_t rss_key_default[] = {0x6b793944,
 894                         0x23504cb5, 0x5bea75b6, 0x309f4f12, 0x3dc0a2b8,
 895                         0x024ddcdf, 0x339b8ca0, 0x4c4af64a, 0x34fac605,
 896                         0x55d85839, 0x3a58997d, 0x2ec938e1, 0x66031581};
 897
 898                 if (rss_act->key_len != sizeof(rss_conf->key))
 899                         PMD_DRV_LOG(WARNING,
 900                                     "RSS key length invalid, must be %u bytes, now set key to default",
 901                                     (uint32_t)sizeof(rss_conf->key));
 902
 903                 memcpy(rss_conf->key, rss_key_default, sizeof(rss_conf->key));
 904         } else {
 905                 memcpy(rss_conf->key, key, sizeof(rss_conf->key));
 906         }
 907
 908         rss_conf->conf.key = rss_conf->key;
 909         rss_conf->conf.key_len = sizeof(rss_conf->key);
 910 }
 911
 912 static int
 913 i40e_hash_parse_queues(const struct rte_eth_dev *dev,
 914                        const struct rte_flow_action_rss *rss_act,
 915                        struct i40e_rte_flow_rss_conf *rss_conf,
 916                        struct rte_flow_error *error)
 917 {
 918         struct i40e_pf *pf;
 919         struct i40e_hw *hw;
 920         uint16_t i;
 921         int max_queue;
 922
 923         hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 924         if (!rss_act->queue_num ||
 925             rss_act->queue_num > hw->func_caps.rss_table_size)
 926                 return rte_flow_error_set(error, EINVAL,
 927                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 928                                           NULL, "Invalid RSS queue number");
 929
 930         if (rss_act->key_len)
 931                 PMD_DRV_LOG(WARNING,
 932                             "RSS key is ignored when queues specified");
 933
 934         pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 935         if (pf->dev_data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 936                 max_queue = i40e_pf_calc_configured_queues_num(pf);
 937         else
 938                 max_queue = pf->dev_data->nb_rx_queues;
 939
 940         max_queue = RTE_MIN(max_queue, I40E_MAX_Q_PER_TC);
 941
 942         for (i = 0; i < rss_act->queue_num; i++) {
 943                 if ((int)rss_act->queue[i] >= max_queue)
 944                         break;
 945         }
 946
 947         if (i < rss_act->queue_num)
 948                 return rte_flow_error_set(error, EINVAL,
 949                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 950                                           NULL, "Invalid RSS queues");
 951
 952         memcpy(rss_conf->queue, rss_act->queue,
 953                rss_act->queue_num * sizeof(rss_conf->queue[0]));
 954         rss_conf->conf.queue = rss_conf->queue;
 955         rss_conf->conf.queue_num = rss_act->queue_num;
 956         return 0;
 957 }
 958
 959 static int
 960 i40e_hash_parse_queue_region(const struct rte_eth_dev *dev,
 961                              const struct rte_flow_item pattern[],
 962                              const struct rte_flow_action_rss *rss_act,
 963                              struct i40e_rte_flow_rss_conf *rss_conf,
 964                              struct rte_flow_error *error)
 965 {
 966         struct i40e_pf *pf;
 967         const struct rte_flow_item_vlan *vlan_spec, *vlan_mask;
 968         uint64_t hash_queues;
 969         uint32_t i;
 970
 971         if (pattern[1].type != RTE_FLOW_ITEM_TYPE_END)
 972                 return rte_flow_error_set(error, ENOTSUP,
 973                                           RTE_FLOW_ERROR_TYPE_ITEM_NUM,
 974                                           &pattern[1],
 975                                           "Pattern not supported.");
 976
 977         vlan_spec = pattern->spec;
 978         vlan_mask = pattern->mask;
 979         if (!vlan_spec || !vlan_mask ||
 980             (rte_be_to_cpu_16(vlan_mask->tci) >> 13) != 7)
 981                 return rte_flow_error_set(error, EINVAL,
 982                                           RTE_FLOW_ERROR_TYPE_ITEM, pattern,
 983                                           "Pattern error.");
 984
 985         if (!rss_act->queue)
 986                 return rte_flow_error_set(error, EINVAL,
 987                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 988                                           NULL, "Queues not specified");
 989
 990         if (rss_act->key_len)
 991                 PMD_DRV_LOG(WARNING,
 992                             "RSS key is ignored when configure queue region");
 993
 994         /* Use a 64 bit variable to represent all queues in a region. */
 995         RTE_BUILD_BUG_ON(I40E_MAX_Q_PER_TC > 64);
 996
 997         if (!rss_act->queue_num ||
 998             !rte_is_power_of_2(rss_act->queue_num) ||
 999             rss_act->queue_num + rss_act->queue[0] > I40E_MAX_Q_PER_TC)
1000                 return rte_flow_error_set(error, EINVAL,
1001                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1002                                           NULL, "Queue number error");
1003
1004         for (i = 1; i < rss_act->queue_num; i++) {
1005                 if (rss_act->queue[i - 1] + 1 != rss_act->queue[i])
1006                         break;
1007         }
1008
1009         if (i < rss_act->queue_num)
1010                 return rte_flow_error_set(error, EINVAL,
1011                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1012                                           NULL,
1013                                           "Queues must be incremented continuously");
1014
1015         /* Map all queues to bits of uint64_t */
1016         hash_queues = (BIT_ULL(rss_act->queue[0] + rss_act->queue_num) - 1) &
1017                       ~(BIT_ULL(rss_act->queue[0]) - 1);
1018
1019         pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1020         if (hash_queues & ~pf->hash_enabled_queues)
1021                 return rte_flow_error_set(error, EINVAL,
1022                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1023                                           NULL, "Some queues are not in LUT");
1024
1025         rss_conf->region_queue_num = (uint8_t)rss_act->queue_num;
1026         rss_conf->region_queue_start = rss_act->queue[0];
1027         rss_conf->region_priority = rte_be_to_cpu_16(vlan_spec->tci) >> 13;
1028         return 0;
1029 }
1030
1031 static int
1032 i40e_hash_parse_global_conf(const struct rte_eth_dev *dev,
1033                             const struct rte_flow_item pattern[],
1034                             const struct rte_flow_action_rss *rss_act,
1035                             struct i40e_rte_flow_rss_conf *rss_conf,
1036                             struct rte_flow_error *error)
1037 {
1038         if (rss_act->func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ)
1039                 return rte_flow_error_set(error, EINVAL,
1040                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1041                                           NULL,
1042                                           "Symmetric function should be set with pattern types");
1043
1044         rss_conf->conf.func = rss_act->func;
1045
1046         if (rss_act->types)
1047                 PMD_DRV_LOG(WARNING,
1048                             "RSS types are ignored when no pattern specified");
1049
1050         if (pattern[0].type == RTE_FLOW_ITEM_TYPE_VLAN)
1051                 return i40e_hash_parse_queue_region(dev, pattern, rss_act,
1052                                                     rss_conf, error);
1053
1054         if (rss_act->queue)
1055                 return i40e_hash_parse_queues(dev, rss_act, rss_conf, error);
1056
1057         if (rss_act->key_len) {
1058                 i40e_hash_parse_key(rss_act, rss_conf);
1059                 return 0;
1060         }
1061
1062         if (rss_act->func == RTE_ETH_HASH_FUNCTION_DEFAULT)
1063                 PMD_DRV_LOG(WARNING, "Nothing change");
1064         return 0;
1065 }
1066
1067 static bool
1068 i40e_hash_validate_rss_types(uint64_t rss_types)
1069 {
1070         uint64_t type, mask;
1071
1072         /* Validate L2 */
1073         type = ETH_RSS_ETH & rss_types;
1074         mask = (ETH_RSS_L2_SRC_ONLY | ETH_RSS_L2_DST_ONLY) & rss_types;
1075         if (!type && mask)
1076                 return false;
1077
1078         /* Validate L3 */
1079         type = (I40E_HASH_L4_TYPES | ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
1080                ETH_RSS_NONFRAG_IPV4_OTHER | ETH_RSS_IPV6 |
1081                ETH_RSS_FRAG_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER) & rss_types;
1082         mask = (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY) & rss_types;
1083         if (!type && mask)
1084                 return false;
1085
1086         /* Validate L4 */
1087         type = (I40E_HASH_L4_TYPES | ETH_RSS_PORT) & rss_types;
1088         mask = (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY) & rss_types;
1089         if (!type && mask)
1090                 return false;
1091
1092         return true;
1093 }
1094
1095 static int
1096 i40e_hash_parse_pattern_act(const struct rte_eth_dev *dev,
1097                             const struct rte_flow_item pattern[],
1098                             const struct rte_flow_action_rss *rss_act,
1099                             struct i40e_rte_flow_rss_conf *rss_conf,
1100                             struct rte_flow_error *error)
1101 {
1102         if (rss_act->queue)
1103                 return rte_flow_error_set(error, EINVAL,
1104                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1105                                           NULL,
1106                                           "RSS Queues not supported when pattern specified");
1107
1108         switch (rss_act->func) {
1109         case RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ:
1110                 rss_conf->symmetric_enable = true;
1111                 break;
1112         case RTE_ETH_HASH_FUNCTION_DEFAULT:
1113         case RTE_ETH_HASH_FUNCTION_TOEPLITZ:
1114         case RTE_ETH_HASH_FUNCTION_SIMPLE_XOR:
1115                 break;
1116         default:
1117                 return rte_flow_error_set(error, EINVAL,
1118                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1119                                 NULL,
1120                                 "RSS hash function not supported "
1121                                 "when pattern specified");
1122         }
1123
1124         if (!i40e_hash_validate_rss_types(rss_act->types))
1125                 return rte_flow_error_set(error, EINVAL,
1126                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1127                                           NULL, "RSS types are invalid");
1128
1129         if (rss_act->key_len)
1130                 i40e_hash_parse_key(rss_act, rss_conf);
1131
1132         rss_conf->conf.func = rss_act->func;
1133         rss_conf->conf.types = rss_act->types;
1134         rss_conf->inset = i40e_hash_get_inset(rss_act->types);
1135
1136         return i40e_hash_get_pattern_pctypes(dev, pattern, rss_act,
1137                                              rss_conf, error);
1138 }
1139
1140 int
1141 i40e_hash_parse(const struct rte_eth_dev *dev,
1142                 const struct rte_flow_item pattern[],
1143                 const struct rte_flow_action actions[],
1144                 struct i40e_rte_flow_rss_conf *rss_conf,
1145                 struct rte_flow_error *error)
1146 {
1147         const struct rte_flow_action_rss *rss_act;
1148
1149         if (actions[1].type != RTE_FLOW_ACTION_TYPE_END)
1150                 return rte_flow_error_set(error, EINVAL,
1151                                           RTE_FLOW_ERROR_TYPE_ACTION,
1152                                           &actions[1],
1153                                           "Only support one action for RSS.");
1154
1155         rss_act = (const struct rte_flow_action_rss *)actions[0].conf;
1156         if (rss_act->level)
1157                 return rte_flow_error_set(error, ENOTSUP,
1158                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1159                                           actions,
1160                                           "RSS level is not supported");
1161
1162         while (pattern->type == RTE_FLOW_ITEM_TYPE_VOID)
1163                 pattern++;
1164
1165         if (pattern[0].type == RTE_FLOW_ITEM_TYPE_END ||
1166             pattern[0].type == RTE_FLOW_ITEM_TYPE_VLAN)
1167                 return i40e_hash_parse_global_conf(dev, pattern, rss_act,
1168                                                    rss_conf, error);
1169
1170         return i40e_hash_parse_pattern_act(dev, pattern, rss_act,
1171                                            rss_conf, error);
1172 }
1173
1174 static void
1175 i40e_invalid_rss_filter(const struct i40e_rte_flow_rss_conf *ref_conf,
1176                         struct i40e_rte_flow_rss_conf *conf)
1177 {
1178         uint32_t reset_flags = conf->misc_reset_flags;
1179
1180         conf->misc_reset_flags &= ~ref_conf->misc_reset_flags;
1181
1182         if ((reset_flags & I40E_HASH_FLOW_RESET_FLAG_REGION) &&
1183             (ref_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_REGION) &&
1184             (conf->region_queue_start != ref_conf->region_queue_start ||
1185              conf->region_queue_num != ref_conf->region_queue_num))
1186                 conf->misc_reset_flags |= I40E_HASH_FLOW_RESET_FLAG_REGION;
1187
1188         conf->reset_config_pctypes &= ~ref_conf->reset_config_pctypes;
1189         conf->reset_symmetric_pctypes &= ~ref_conf->reset_symmetric_pctypes;
1190 }
1191
1192 int
1193 i40e_hash_filter_restore(struct i40e_pf *pf)
1194 {
1195         struct i40e_rss_filter *filter;
1196         int ret;
1197
1198         TAILQ_FOREACH(filter, &pf->rss_config_list, next) {
1199                 struct i40e_rte_flow_rss_conf *rss_conf =
1200                                                 &filter->rss_filter_info;
1201                 struct i40e_rss_filter *prev;
1202
1203                 rss_conf->misc_reset_flags = 0;
1204                 rss_conf->reset_config_pctypes = 0;
1205                 rss_conf->reset_symmetric_pctypes = 0;
1206
1207                 ret = i40e_hash_config(pf, rss_conf);
1208                 if (ret) {
1209                         pf->hash_filter_enabled = 0;
1210                         i40e_pf_disable_rss(pf);
1211                         PMD_DRV_LOG(ERR,
1212                                     "Re-configure RSS failed, RSS has been disabled");
1213                         return ret;
1214                 }
1215
1216                 /* Invalid previous RSS filter */
1217                 TAILQ_FOREACH(prev, &pf->rss_config_list, next) {
1218                         if (prev == filter)
1219                                 break;
1220                         i40e_invalid_rss_filter(rss_conf,
1221                                                 &prev->rss_filter_info);
1222                 }
1223         }
1224
1225         return 0;
1226 }
1227
1228 int
1229 i40e_hash_filter_create(struct i40e_pf *pf,
1230                         struct i40e_rte_flow_rss_conf *rss_conf)
1231 {
1232         struct i40e_rss_filter *filter, *prev;
1233         struct i40e_rte_flow_rss_conf *new_conf;
1234         int ret;
1235
1236         filter = rte_zmalloc("i40e_rss_filter", sizeof(*filter), 0);
1237         if (!filter) {
1238                 PMD_DRV_LOG(ERR, "Failed to allocate memory.");
1239                 return -ENOMEM;
1240         }
1241
1242         new_conf = &filter->rss_filter_info;
1243
1244         memcpy(new_conf, rss_conf, sizeof(*new_conf));
1245         if (new_conf->conf.queue_num)
1246                 new_conf->conf.queue = new_conf->queue;
1247         if (new_conf->conf.key_len)
1248                 new_conf->conf.key = new_conf->key;
1249
1250         ret = i40e_hash_config(pf, new_conf);
1251         if (ret) {
1252                 rte_free(filter);
1253                 if (i40e_pf_config_rss(pf))
1254                         return ret;
1255
1256                 (void)i40e_hash_filter_restore(pf);
1257                 return ret;
1258         }
1259
1260         /* Invalid previous RSS filter */
1261         TAILQ_FOREACH(prev, &pf->rss_config_list, next)
1262                 i40e_invalid_rss_filter(new_conf, &prev->rss_filter_info);
1263
1264         TAILQ_INSERT_TAIL(&pf->rss_config_list, filter, next);
1265         return 0;
1266 }
1267
1268 static int
1269 i40e_hash_reset_conf(struct i40e_pf *pf,
1270                      struct i40e_rte_flow_rss_conf *rss_conf)
1271 {
1272         struct i40e_hw *hw = &pf->adapter->hw;
1273         uint64_t inset;
1274         uint32_t idx;
1275         int ret;
1276
1277         if (rss_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_FUNC) {
1278                 ret = i40e_hash_config_func(hw, RTE_ETH_HASH_FUNCTION_TOEPLITZ);
1279                 if (ret)
1280                         return ret;
1281
1282                 rss_conf->misc_reset_flags &= ~I40E_HASH_FLOW_RESET_FLAG_FUNC;
1283         }
1284
1285         if (rss_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_REGION) {
1286                 ret = i40e_flush_queue_region_all_conf(pf->adapter->eth_dev,
1287                                                        hw, pf, 0);
1288                 if (ret)
1289                         return ret;
1290
1291                 rss_conf->misc_reset_flags &= ~I40E_HASH_FLOW_RESET_FLAG_REGION;
1292         }
1293
1294         if (rss_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_KEY) {
1295                 ret = i40e_pf_reset_rss_key(pf);
1296                 if (ret)
1297                         return ret;
1298
1299                 rss_conf->misc_reset_flags &= ~I40E_HASH_FLOW_RESET_FLAG_KEY;
1300         }
1301
1302         if (rss_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_QUEUE) {
1303                 if (!pf->adapter->rss_reta_updated) {
1304                         ret = i40e_pf_reset_rss_reta(pf);
1305                         if (ret)
1306                                 return ret;
1307                 }
1308
1309                 pf->hash_enabled_queues = 0;
1310                 rss_conf->misc_reset_flags &= ~I40E_HASH_FLOW_RESET_FLAG_QUEUE;
1311         }
1312
1313         while (rss_conf->reset_config_pctypes) {
1314                 idx = rte_bsf64(rss_conf->reset_config_pctypes);
1315
1316                 i40e_hash_enable_pctype(hw, idx, false);
1317                 inset = i40e_get_default_input_set(idx);
1318                 if (inset) {
1319                         ret = i40e_set_hash_inset(hw, inset, idx, false);
1320                         if (ret)
1321                                 return ret;
1322                 }
1323
1324                 rss_conf->reset_config_pctypes &= ~BIT_ULL(idx);
1325         }
1326
1327         while (rss_conf->reset_symmetric_pctypes) {
1328                 idx = rte_bsf64(rss_conf->reset_symmetric_pctypes);
1329
1330                 ret = i40e_hash_config_pctype_symmetric(hw, idx, false);
1331                 if (ret)
1332                         return ret;
1333
1334                 rss_conf->reset_symmetric_pctypes &= ~BIT_ULL(idx);
1335         }
1336
1337         return 0;
1338 }
1339
1340 int
1341 i40e_hash_filter_destroy(struct i40e_pf *pf,
1342                          const struct i40e_rss_filter *rss_filter)
1343 {
1344         struct i40e_rss_filter *filter;
1345         int ret;
1346
1347         TAILQ_FOREACH(filter, &pf->rss_config_list, next) {
1348                 if (rss_filter == filter) {
1349                         ret = i40e_hash_reset_conf(pf,
1350                                                    &filter->rss_filter_info);
1351                         if (ret)
1352                                 return ret;
1353
1354                         TAILQ_REMOVE(&pf->rss_config_list, filter, next);
1355                         rte_free(filter);
1356                         return 0;
1357                 }
1358         }
1359
1360         return -ENOENT;
1361 }
1362
1363 int
1364 i40e_hash_filter_flush(struct i40e_pf *pf)
1365 {
1366         struct rte_flow *flow, *next;
1367
1368         TAILQ_FOREACH_SAFE(flow, &pf->flow_list, node, next) {
1369                 if (flow->filter_type != RTE_ETH_FILTER_HASH)
1370                         continue;
1371
1372                 if (flow->rule) {
1373                         struct i40e_rss_filter *filter = flow->rule;
1374                         int ret;
1375
1376                         ret = i40e_hash_reset_conf(pf,
1377                                                    &filter->rss_filter_info);
1378                         if (ret)
1379                                 return ret;
1380
1381                         TAILQ_REMOVE(&pf->rss_config_list, filter, next);
1382                         rte_free(filter);
1383                 }
1384
1385                 TAILQ_REMOVE(&pf->flow_list, flow, node);
1386                 rte_free(flow);
1387         }
1388
1389         assert(!pf->rss_config_list.tqh_first);
1390         return 0;
1391 }