drivers/net/i40e/i40e_hash.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10 #include <assert.h>
  11
  12 #include <rte_malloc.h>
  13 #include <rte_tailq.h>
  14 #include "base/i40e_prototype.h"
  15 #include "i40e_logs.h"
  16 #include "i40e_ethdev.h"
  17 #include "i40e_hash.h"
  18
  19 #ifndef BIT
  20 #define BIT(n)                          (1UL << (n))
  21 #endif
  22
  23 #ifndef BIT_ULL
  24 #define BIT_ULL(n)                      (1ULL << (n))
  25 #endif
  26
  27 /* Pattern item headers */
  28 #define I40E_HASH_HDR_ETH               0x01ULL
  29 #define I40E_HASH_HDR_IPV4              0x10ULL
  30 #define I40E_HASH_HDR_IPV6              0x20ULL
  31 #define I40E_HASH_HDR_TCP               0x100ULL
  32 #define I40E_HASH_HDR_UDP               0x200ULL
  33 #define I40E_HASH_HDR_SCTP              0x400ULL
  34 #define I40E_HASH_HDR_ESP               0x10000ULL
  35 #define I40E_HASH_HDR_L2TPV3            0x20000ULL
  36 #define I40E_HASH_HDR_AH                0x40000ULL
  37 #define I40E_HASH_HDR_GTPC              0x100000ULL
  38 #define I40E_HASH_HDR_GTPU              0x200000ULL
  39
  40 #define I40E_HASH_HDR_INNER_SHIFT       32
  41 #define I40E_HASH_HDR_IPV4_INNER        (I40E_HASH_HDR_IPV4 << \
  42                                         I40E_HASH_HDR_INNER_SHIFT)
  43 #define I40E_HASH_HDR_IPV6_INNER        (I40E_HASH_HDR_IPV6 << \
  44                                         I40E_HASH_HDR_INNER_SHIFT)
  45
  46 /* ETH */
  47 #define I40E_PHINT_ETH                  I40E_HASH_HDR_ETH
  48
  49 /* IPv4 */
  50 #define I40E_PHINT_IPV4                 (I40E_HASH_HDR_ETH | I40E_HASH_HDR_IPV4)
  51 #define I40E_PHINT_IPV4_TCP             (I40E_PHINT_IPV4 | I40E_HASH_HDR_TCP)
  52 #define I40E_PHINT_IPV4_UDP             (I40E_PHINT_IPV4 | I40E_HASH_HDR_UDP)
  53 #define I40E_PHINT_IPV4_SCTP            (I40E_PHINT_IPV4 | I40E_HASH_HDR_SCTP)
  54
  55 /* IPv6 */
  56 #define I40E_PHINT_IPV6                 (I40E_HASH_HDR_ETH | I40E_HASH_HDR_IPV6)
  57 #define I40E_PHINT_IPV6_TCP             (I40E_PHINT_IPV6 | I40E_HASH_HDR_TCP)
  58 #define I40E_PHINT_IPV6_UDP             (I40E_PHINT_IPV6 | I40E_HASH_HDR_UDP)
  59 #define I40E_PHINT_IPV6_SCTP            (I40E_PHINT_IPV6 | I40E_HASH_HDR_SCTP)
  60
  61 /* ESP */
  62 #define I40E_PHINT_IPV4_ESP             (I40E_PHINT_IPV4 | I40E_HASH_HDR_ESP)
  63 #define I40E_PHINT_IPV6_ESP             (I40E_PHINT_IPV6 | I40E_HASH_HDR_ESP)
  64 #define I40E_PHINT_IPV4_UDP_ESP         (I40E_PHINT_IPV4_UDP | \
  65                                         I40E_HASH_HDR_ESP)
  66 #define I40E_PHINT_IPV6_UDP_ESP         (I40E_PHINT_IPV6_UDP | \
  67                                         I40E_HASH_HDR_ESP)
  68
  69 /* GTPC */
  70 #define I40E_PHINT_IPV4_GTPC            (I40E_PHINT_IPV4_UDP | \
  71                                         I40E_HASH_HDR_GTPC)
  72 #define I40E_PHINT_IPV6_GTPC            (I40E_PHINT_IPV6_UDP | \
  73                                         I40E_HASH_HDR_GTPC)
  74
  75 /* GTPU */
  76 #define I40E_PHINT_IPV4_GTPU            (I40E_PHINT_IPV4_UDP | \
  77                                         I40E_HASH_HDR_GTPU)
  78 #define I40E_PHINT_IPV4_GTPU_IPV4       (I40E_PHINT_IPV4_GTPU | \
  79                                         I40E_HASH_HDR_IPV4_INNER)
  80 #define I40E_PHINT_IPV4_GTPU_IPV6       (I40E_PHINT_IPV4_GTPU | \
  81                                         I40E_HASH_HDR_IPV6_INNER)
  82 #define I40E_PHINT_IPV6_GTPU            (I40E_PHINT_IPV6_UDP | \
  83                                         I40E_HASH_HDR_GTPU)
  84 #define I40E_PHINT_IPV6_GTPU_IPV4       (I40E_PHINT_IPV6_GTPU | \
  85                                         I40E_HASH_HDR_IPV4_INNER)
  86 #define I40E_PHINT_IPV6_GTPU_IPV6       (I40E_PHINT_IPV6_GTPU | \
  87                                         I40E_HASH_HDR_IPV6_INNER)
  88
  89 /* L2TPV3 */
  90 #define I40E_PHINT_IPV4_L2TPV3          (I40E_PHINT_IPV4 | I40E_HASH_HDR_L2TPV3)
  91 #define I40E_PHINT_IPV6_L2TPV3          (I40E_PHINT_IPV6 | I40E_HASH_HDR_L2TPV3)
  92
  93 /* AH */
  94 #define I40E_PHINT_IPV4_AH              (I40E_PHINT_IPV4 | I40E_HASH_HDR_AH)
  95 #define I40E_PHINT_IPV6_AH              (I40E_PHINT_IPV6 | I40E_HASH_HDR_AH)
  96
  97 /* Structure of mapping RSS type to input set */
  98 struct i40e_hash_map_rss_inset {
  99         uint64_t rss_type;
 100         uint64_t inset;
 101 };
 102
 103 const struct i40e_hash_map_rss_inset i40e_hash_rss_inset[] = {
 104         /* IPv4 */
 105         { ETH_RSS_IPV4, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST },
 106         { ETH_RSS_FRAG_IPV4, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST },
 107
 108         { ETH_RSS_NONFRAG_IPV4_OTHER,
 109           I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST },
 110
 111         { ETH_RSS_NONFRAG_IPV4_TCP, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
 112           I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT },
 113
 114         { ETH_RSS_NONFRAG_IPV4_UDP, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
 115           I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT },
 116
 117         { ETH_RSS_NONFRAG_IPV4_SCTP, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
 118           I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | I40E_INSET_SCTP_VT },
 119
 120         /* IPv6 */
 121         { ETH_RSS_IPV6, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST },
 122         { ETH_RSS_FRAG_IPV6, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST },
 123
 124         { ETH_RSS_NONFRAG_IPV6_OTHER,
 125           I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST },
 126
 127         { ETH_RSS_NONFRAG_IPV6_TCP, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
 128           I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT },
 129
 130         { ETH_RSS_NONFRAG_IPV6_UDP, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
 131           I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT },
 132
 133         { ETH_RSS_NONFRAG_IPV6_SCTP, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
 134           I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | I40E_INSET_SCTP_VT },
 135
 136         /* Port */
 137         { ETH_RSS_PORT, I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT },
 138
 139         /* Ether */
 140         { ETH_RSS_L2_PAYLOAD, I40E_INSET_LAST_ETHER_TYPE },
 141         { ETH_RSS_ETH, I40E_INSET_DMAC | I40E_INSET_SMAC },
 142
 143         /* VLAN */
 144         { ETH_RSS_S_VLAN, I40E_INSET_VLAN_OUTER },
 145         { ETH_RSS_C_VLAN, I40E_INSET_VLAN_INNER },
 146 };
 147
 148 #define I40E_HASH_VOID_NEXT_ALLOW       BIT_ULL(RTE_FLOW_ITEM_TYPE_ETH)
 149
 150 #define I40E_HASH_ETH_NEXT_ALLOW        (BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV4) | \
 151                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV6) | \
 152                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_VLAN))
 153
 154 #define I40E_HASH_IP_NEXT_ALLOW         (BIT_ULL(RTE_FLOW_ITEM_TYPE_TCP) | \
 155                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_UDP) | \
 156                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_SCTP) | \
 157                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_ESP) | \
 158                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_L2TPV3OIP) |\
 159                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_AH))
 160
 161 #define I40E_HASH_UDP_NEXT_ALLOW        (BIT_ULL(RTE_FLOW_ITEM_TYPE_GTPU) | \
 162                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_GTPC))
 163
 164 #define I40E_HASH_GTPU_NEXT_ALLOW       (BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV4) | \
 165                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV6))
 166
 167 static const uint64_t pattern_next_allow_items[] = {
 168         [RTE_FLOW_ITEM_TYPE_VOID] = I40E_HASH_VOID_NEXT_ALLOW,
 169         [RTE_FLOW_ITEM_TYPE_ETH] = I40E_HASH_ETH_NEXT_ALLOW,
 170         [RTE_FLOW_ITEM_TYPE_IPV4] = I40E_HASH_IP_NEXT_ALLOW,
 171         [RTE_FLOW_ITEM_TYPE_IPV6] = I40E_HASH_IP_NEXT_ALLOW,
 172         [RTE_FLOW_ITEM_TYPE_UDP] = I40E_HASH_UDP_NEXT_ALLOW,
 173         [RTE_FLOW_ITEM_TYPE_GTPU] = I40E_HASH_GTPU_NEXT_ALLOW,
 174 };
 175
 176 static const uint64_t pattern_item_header[] = {
 177         [RTE_FLOW_ITEM_TYPE_ETH] = I40E_HASH_HDR_ETH,
 178         [RTE_FLOW_ITEM_TYPE_IPV4] = I40E_HASH_HDR_IPV4,
 179         [RTE_FLOW_ITEM_TYPE_IPV6] = I40E_HASH_HDR_IPV6,
 180         [RTE_FLOW_ITEM_TYPE_TCP] = I40E_HASH_HDR_TCP,
 181         [RTE_FLOW_ITEM_TYPE_UDP] = I40E_HASH_HDR_UDP,
 182         [RTE_FLOW_ITEM_TYPE_SCTP] = I40E_HASH_HDR_SCTP,
 183         [RTE_FLOW_ITEM_TYPE_ESP] = I40E_HASH_HDR_ESP,
 184         [RTE_FLOW_ITEM_TYPE_GTPC] = I40E_HASH_HDR_GTPC,
 185         [RTE_FLOW_ITEM_TYPE_GTPU] = I40E_HASH_HDR_GTPU,
 186         [RTE_FLOW_ITEM_TYPE_L2TPV3OIP] = I40E_HASH_HDR_L2TPV3,
 187         [RTE_FLOW_ITEM_TYPE_AH] = I40E_HASH_HDR_AH,
 188 };
 189
 190 /* Structure of matched pattern */
 191 struct i40e_hash_match_pattern {
 192         uint64_t pattern_type;
 193         uint64_t rss_mask;      /* Supported RSS type for this pattern */
 194         bool custom_pctype_flag;/* true for custom packet type */
 195         uint8_t pctype;
 196 };
 197
 198 #define I40E_HASH_MAP_PATTERN(pattern, rss_mask, pctype) { \
 199         pattern, rss_mask, false, pctype  }
 200
 201 #define I40E_HASH_MAP_CUS_PATTERN(pattern, rss_mask, cus_pctype) { \
 202         pattern, rss_mask, true, cus_pctype }
 203
 204 #define I40E_HASH_L2_RSS_MASK           (ETH_RSS_ETH | ETH_RSS_L2_SRC_ONLY | \
 205                                         ETH_RSS_L2_DST_ONLY)
 206
 207 #define I40E_HASH_L23_RSS_MASK          (I40E_HASH_L2_RSS_MASK | \
 208                                         ETH_RSS_VLAN | \
 209                                         ETH_RSS_L3_SRC_ONLY | \
 210                                         ETH_RSS_L3_DST_ONLY)
 211
 212 #define I40E_HASH_IPV4_L23_RSS_MASK     (ETH_RSS_IPV4 | I40E_HASH_L23_RSS_MASK)
 213 #define I40E_HASH_IPV6_L23_RSS_MASK     (ETH_RSS_IPV6 | I40E_HASH_L23_RSS_MASK)
 214
 215 #define I40E_HASH_L234_RSS_MASK         (I40E_HASH_L23_RSS_MASK | \
 216                                         ETH_RSS_PORT | ETH_RSS_L4_SRC_ONLY | \
 217                                         ETH_RSS_L4_DST_ONLY)
 218
 219 #define I40E_HASH_IPV4_L234_RSS_MASK    (I40E_HASH_L234_RSS_MASK | ETH_RSS_IPV4)
 220 #define I40E_HASH_IPV6_L234_RSS_MASK    (I40E_HASH_L234_RSS_MASK | ETH_RSS_IPV6)
 221
 222 #define I40E_HASH_L4_TYPES              (ETH_RSS_NONFRAG_IPV4_TCP | \
 223                                         ETH_RSS_NONFRAG_IPV4_UDP | \
 224                                         ETH_RSS_NONFRAG_IPV4_SCTP | \
 225                                         ETH_RSS_NONFRAG_IPV6_TCP | \
 226                                         ETH_RSS_NONFRAG_IPV6_UDP | \
 227                                         ETH_RSS_NONFRAG_IPV6_SCTP)
 228
 229 /* Current supported patterns and RSS types.
 230  * All items that have the same pattern types are together.
 231  */
 232 static const struct i40e_hash_match_pattern match_patterns[] = {
 233         /* Ether */
 234         I40E_HASH_MAP_PATTERN(I40E_PHINT_ETH,
 235                               ETH_RSS_L2_PAYLOAD | I40E_HASH_L2_RSS_MASK,
 236                               I40E_FILTER_PCTYPE_L2_PAYLOAD),
 237
 238         /* IPv4 */
 239         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4,
 240                               ETH_RSS_FRAG_IPV4 | I40E_HASH_IPV4_L23_RSS_MASK,
 241                               I40E_FILTER_PCTYPE_FRAG_IPV4),
 242
 243         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4,
 244                               ETH_RSS_NONFRAG_IPV4_OTHER |
 245                               I40E_HASH_IPV4_L23_RSS_MASK,
 246                               I40E_FILTER_PCTYPE_NONF_IPV4_OTHER),
 247
 248         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4_TCP,
 249                               ETH_RSS_NONFRAG_IPV4_TCP |
 250                               I40E_HASH_IPV4_L234_RSS_MASK,
 251                               I40E_FILTER_PCTYPE_NONF_IPV4_TCP),
 252
 253         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4_UDP,
 254                               ETH_RSS_NONFRAG_IPV4_UDP |
 255                               I40E_HASH_IPV4_L234_RSS_MASK,
 256                               I40E_FILTER_PCTYPE_NONF_IPV4_UDP),
 257
 258         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4_SCTP,
 259                               ETH_RSS_NONFRAG_IPV4_SCTP |
 260                               I40E_HASH_IPV4_L234_RSS_MASK,
 261                               I40E_FILTER_PCTYPE_NONF_IPV4_SCTP),
 262
 263         /* IPv6 */
 264         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6,
 265                               ETH_RSS_FRAG_IPV6 | I40E_HASH_IPV6_L23_RSS_MASK,
 266                               I40E_FILTER_PCTYPE_FRAG_IPV6),
 267
 268         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6,
 269                               ETH_RSS_NONFRAG_IPV6_OTHER |
 270                               I40E_HASH_IPV6_L23_RSS_MASK,
 271                               I40E_FILTER_PCTYPE_NONF_IPV6_OTHER),
 272
 273         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6_TCP,
 274                               ETH_RSS_NONFRAG_IPV6_TCP |
 275                               I40E_HASH_IPV6_L234_RSS_MASK,
 276                               I40E_FILTER_PCTYPE_NONF_IPV6_TCP),
 277
 278         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6_UDP,
 279                               ETH_RSS_NONFRAG_IPV6_UDP |
 280                               I40E_HASH_IPV6_L234_RSS_MASK,
 281                               I40E_FILTER_PCTYPE_NONF_IPV6_UDP),
 282
 283         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6_SCTP,
 284                               ETH_RSS_NONFRAG_IPV6_SCTP |
 285                               I40E_HASH_IPV6_L234_RSS_MASK,
 286                               I40E_FILTER_PCTYPE_NONF_IPV6_SCTP),
 287
 288         /* ESP */
 289         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_ESP,
 290                                   ETH_RSS_ESP, I40E_CUSTOMIZED_ESP_IPV4),
 291         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_ESP,
 292                                   ETH_RSS_ESP, I40E_CUSTOMIZED_ESP_IPV6),
 293         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_UDP_ESP,
 294                                   ETH_RSS_ESP, I40E_CUSTOMIZED_ESP_IPV4_UDP),
 295         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_UDP_ESP,
 296                                   ETH_RSS_ESP, I40E_CUSTOMIZED_ESP_IPV6_UDP),
 297
 298         /* GTPC */
 299         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_GTPC,
 300                                   I40E_HASH_IPV4_L234_RSS_MASK,
 301                                   I40E_CUSTOMIZED_GTPC),
 302         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_GTPC,
 303                                   I40E_HASH_IPV6_L234_RSS_MASK,
 304                                   I40E_CUSTOMIZED_GTPC),
 305
 306         /* GTPU */
 307         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_GTPU,
 308                                   I40E_HASH_IPV4_L234_RSS_MASK,
 309                                   I40E_CUSTOMIZED_GTPU),
 310         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_GTPU_IPV4,
 311                                   ETH_RSS_GTPU, I40E_CUSTOMIZED_GTPU_IPV4),
 312         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_GTPU_IPV6,
 313                                   ETH_RSS_GTPU, I40E_CUSTOMIZED_GTPU_IPV6),
 314         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_GTPU,
 315                                   I40E_HASH_IPV6_L234_RSS_MASK,
 316                                   I40E_CUSTOMIZED_GTPU),
 317         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_GTPU_IPV4,
 318                                   ETH_RSS_GTPU, I40E_CUSTOMIZED_GTPU_IPV4),
 319         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_GTPU_IPV6,
 320                                   ETH_RSS_GTPU, I40E_CUSTOMIZED_GTPU_IPV6),
 321
 322         /* L2TPV3 */
 323         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_L2TPV3,
 324                                   ETH_RSS_L2TPV3, I40E_CUSTOMIZED_IPV4_L2TPV3),
 325         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_L2TPV3,
 326                                   ETH_RSS_L2TPV3, I40E_CUSTOMIZED_IPV6_L2TPV3),
 327
 328         /* AH */
 329         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_AH, ETH_RSS_AH,
 330                                   I40E_CUSTOMIZED_AH_IPV4),
 331         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_AH, ETH_RSS_AH,
 332                                   I40E_CUSTOMIZED_AH_IPV6),
 333 };
 334
 335 static int
 336 i40e_hash_get_pattern_type(const struct rte_flow_item pattern[],
 337                            uint64_t *pattern_types,
 338                            struct rte_flow_error *error)
 339 {
 340         const char *message = "Pattern not supported";
 341         enum rte_flow_item_type prev_item_type = RTE_FLOW_ITEM_TYPE_VOID;
 342         enum rte_flow_item_type last_item_type = prev_item_type;
 343         uint64_t item_hdr, pattern_hdrs = 0;
 344         bool inner_flag = false;
 345         int vlan_count = 0;
 346
 347         for (; pattern->type != RTE_FLOW_ITEM_TYPE_END; pattern++) {
 348                 if (pattern->type == RTE_FLOW_ITEM_TYPE_VOID)
 349                         continue;
 350
 351                 if (pattern->mask || pattern->spec || pattern->last) {
 352                         message = "Header info should not be specified";
 353                         goto not_sup;
 354                 }
 355
 356                 /* Check the previous item allows this sub-item. */
 357                 if (prev_item_type >= RTE_DIM(pattern_next_allow_items) ||
 358                     !(pattern_next_allow_items[prev_item_type] &
 359                     BIT_ULL(pattern->type)))
 360                         goto not_sup;
 361
 362                 /* For VLAN item, it does no matter about to pattern type
 363                  * recognition. So just count the number of VLAN and do not
 364                  * change the value of variable `prev_item_type`.
 365                  */
 366                 last_item_type = pattern->type;
 367                 if (last_item_type == RTE_FLOW_ITEM_TYPE_VLAN) {
 368                         if (vlan_count >= 2)
 369                                 goto not_sup;
 370                         vlan_count++;
 371                         continue;
 372                 }
 373
 374                 prev_item_type = last_item_type;
 375                 assert(last_item_type < RTE_DIM(pattern_item_header));
 376                 item_hdr = pattern_item_header[last_item_type];
 377                 assert(item_hdr);
 378
 379                 if (inner_flag) {
 380                         item_hdr <<= I40E_HASH_HDR_INNER_SHIFT;
 381
 382                         /* Inner layer should not have GTPU item */
 383                         if (last_item_type == RTE_FLOW_ITEM_TYPE_GTPU)
 384                                 goto not_sup;
 385                 } else {
 386                         if (last_item_type == RTE_FLOW_ITEM_TYPE_GTPU) {
 387                                 inner_flag = true;
 388                                 vlan_count = 0;
 389                         }
 390                 }
 391
 392                 if (item_hdr & pattern_hdrs)
 393                         goto not_sup;
 394
 395                 pattern_hdrs |= item_hdr;
 396         }
 397
 398         if (pattern_hdrs && last_item_type != RTE_FLOW_ITEM_TYPE_VLAN) {
 399                 *pattern_types = pattern_hdrs;
 400                 return 0;
 401         }
 402
 403 not_sup:
 404         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 405                                   pattern, message);
 406 }
 407
 408 static uint64_t
 409 i40e_hash_get_x722_ext_pctypes(uint8_t match_pctype)
 410 {
 411         uint64_t pctypes = 0;
 412
 413         switch (match_pctype) {
 414         case I40E_FILTER_PCTYPE_NONF_IPV4_TCP:
 415                 pctypes = BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK);
 416                 break;
 417
 418         case I40E_FILTER_PCTYPE_NONF_IPV4_UDP:
 419                 pctypes = BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) |
 420                           BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP);
 421                 break;
 422
 423         case I40E_FILTER_PCTYPE_NONF_IPV6_TCP:
 424                 pctypes = BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK);
 425                 break;
 426
 427         case I40E_FILTER_PCTYPE_NONF_IPV6_UDP:
 428                 pctypes = BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) |
 429                           BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP);
 430                 break;
 431         }
 432
 433         return pctypes;
 434 }
 435
 436 static int
 437 i40e_hash_translate_gtp_inset(struct i40e_rte_flow_rss_conf *rss_conf,
 438                               struct rte_flow_error *error)
 439 {
 440         if (rss_conf->inset &
 441             (I40E_INSET_IPV4_SRC | I40E_INSET_IPV6_SRC |
 442             I40E_INSET_DST_PORT | I40E_INSET_SRC_PORT))
 443                 return rte_flow_error_set(error, ENOTSUP,
 444                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 445                                           NULL,
 446                                           "Only support external destination IP");
 447
 448         if (rss_conf->inset & I40E_INSET_IPV4_DST)
 449                 rss_conf->inset = (rss_conf->inset & ~I40E_INSET_IPV4_DST) |
 450                                   I40E_INSET_TUNNEL_IPV4_DST;
 451
 452         if (rss_conf->inset & I40E_INSET_IPV6_DST)
 453                 rss_conf->inset = (rss_conf->inset & ~I40E_INSET_IPV6_DST) |
 454                                   I40E_INSET_TUNNEL_IPV6_DST;
 455
 456         return 0;
 457 }
 458
 459 static int
 460 i40e_hash_get_pctypes(const struct rte_eth_dev *dev,
 461                       const struct i40e_hash_match_pattern *match,
 462                       struct i40e_rte_flow_rss_conf *rss_conf,
 463                       struct rte_flow_error *error)
 464 {
 465         if (match->custom_pctype_flag) {
 466                 struct i40e_pf *pf;
 467                 struct i40e_customized_pctype *custom_type;
 468
 469                 pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 470                 custom_type = i40e_find_customized_pctype(pf, match->pctype);
 471                 if (!custom_type || !custom_type->valid)
 472                         return rte_flow_error_set(error, ENOTSUP,
 473                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 474                                                   NULL, "PCTYPE not supported");
 475
 476                 rss_conf->config_pctypes |= BIT_ULL(custom_type->pctype);
 477
 478                 if (match->pctype == I40E_CUSTOMIZED_GTPU ||
 479                     match->pctype == I40E_CUSTOMIZED_GTPC)
 480                         return i40e_hash_translate_gtp_inset(rss_conf, error);
 481         } else {
 482                 struct i40e_hw *hw =
 483                                 I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 484                 uint64_t types;
 485
 486                 rss_conf->config_pctypes |= BIT_ULL(match->pctype);
 487                 if (hw->mac.type == I40E_MAC_X722) {
 488                         types = i40e_hash_get_x722_ext_pctypes(match->pctype);
 489                         rss_conf->config_pctypes |= types;
 490                 }
 491         }
 492
 493         return 0;
 494 }
 495
 496 static int
 497 i40e_hash_get_pattern_pctypes(const struct rte_eth_dev *dev,
 498                               const struct rte_flow_item pattern[],
 499                               const struct rte_flow_action_rss *rss_act,
 500                               struct i40e_rte_flow_rss_conf *rss_conf,
 501                               struct rte_flow_error *error)
 502 {
 503         uint64_t pattern_types = 0;
 504         bool match_flag = false;
 505         int i, ret;
 506
 507         ret = i40e_hash_get_pattern_type(pattern, &pattern_types, error);
 508         if (ret)
 509                 return ret;
 510
 511         for (i = 0; i < (int)RTE_DIM(match_patterns); i++) {
 512                 const struct i40e_hash_match_pattern *match =
 513                                                         &match_patterns[i];
 514
 515                 /* Check pattern types match. All items that have the same
 516                  * pattern types are together, so if the pattern types match
 517                  * previous item but they doesn't match current item, it means
 518                  * the pattern types do not match all remain items.
 519                  */
 520                 if (pattern_types != match->pattern_type) {
 521                         if (match_flag)
 522                                 break;
 523                         continue;
 524                 }
 525                 match_flag = true;
 526
 527                 /* Check RSS types match */
 528                 if (!(rss_act->types & ~match->rss_mask)) {
 529                         ret = i40e_hash_get_pctypes(dev, match,
 530                                                     rss_conf, error);
 531                         if (ret)
 532                                 return ret;
 533                 }
 534         }
 535
 536         if (rss_conf->config_pctypes)
 537                 return 0;
 538
 539         if (match_flag)
 540                 return rte_flow_error_set(error, ENOTSUP,
 541                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 542                                           NULL, "RSS types not supported");
 543
 544         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 545                                   NULL, "Pattern not supported");
 546 }
 547
 548 static uint64_t
 549 i40e_hash_get_inset(uint64_t rss_types)
 550 {
 551         uint64_t mask, inset = 0;
 552         int i;
 553
 554         for (i = 0; i < (int)RTE_DIM(i40e_hash_rss_inset); i++) {
 555                 if (rss_types & i40e_hash_rss_inset[i].rss_type)
 556                         inset |= i40e_hash_rss_inset[i].inset;
 557         }
 558
 559         if (!inset)
 560                 return 0;
 561
 562         /* If SRC_ONLY and DST_ONLY of the same level are used simultaneously,
 563          * it is the same case as none of them are added.
 564          */
 565         mask = rss_types & (ETH_RSS_L2_SRC_ONLY | ETH_RSS_L2_DST_ONLY);
 566         if (mask == ETH_RSS_L2_SRC_ONLY)
 567                 inset &= ~I40E_INSET_DMAC;
 568         else if (mask == ETH_RSS_L2_DST_ONLY)
 569                 inset &= ~I40E_INSET_SMAC;
 570
 571         mask = rss_types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY);
 572         if (mask == ETH_RSS_L3_SRC_ONLY)
 573                 inset &= ~(I40E_INSET_IPV4_DST | I40E_INSET_IPV6_DST);
 574         else if (mask == ETH_RSS_L3_DST_ONLY)
 575                 inset &= ~(I40E_INSET_IPV4_SRC | I40E_INSET_IPV6_SRC);
 576
 577         mask = rss_types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY);
 578         if (mask == ETH_RSS_L4_SRC_ONLY)
 579                 inset &= ~I40E_INSET_DST_PORT;
 580         else if (mask == ETH_RSS_L4_DST_ONLY)
 581                 inset &= ~I40E_INSET_SRC_PORT;
 582
 583         if (rss_types & I40E_HASH_L4_TYPES) {
 584                 uint64_t l3_mask = rss_types &
 585                                    (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY);
 586                 uint64_t l4_mask = rss_types &
 587                                    (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY);
 588
 589                 if (l3_mask && !l4_mask)
 590                         inset &= ~(I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT);
 591                 else if (!l3_mask && l4_mask)
 592                         inset &= ~(I40E_INSET_IPV4_DST | I40E_INSET_IPV6_DST |
 593                                  I40E_INSET_IPV4_SRC | I40E_INSET_IPV6_SRC);
 594         }
 595
 596         return inset;
 597 }
 598
 599 static int
 600 i40e_hash_config_func(struct i40e_hw *hw, enum rte_eth_hash_function func)
 601 {
 602         struct i40e_pf *pf;
 603         uint32_t reg;
 604         uint8_t symmetric = 0;
 605
 606         reg = i40e_read_rx_ctl(hw, I40E_GLQF_CTL);
 607
 608         if (func == RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) {
 609                 if (!(reg & I40E_GLQF_CTL_HTOEP_MASK))
 610                         goto set_symmetric;
 611
 612                 reg &= ~I40E_GLQF_CTL_HTOEP_MASK;
 613         } else {
 614                 if (func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ)
 615                         symmetric = 1;
 616
 617                 if (reg & I40E_GLQF_CTL_HTOEP_MASK)
 618                         goto set_symmetric;
 619
 620                 reg |= I40E_GLQF_CTL_HTOEP_MASK;
 621         }
 622
 623         pf = &((struct i40e_adapter *)hw->back)->pf;
 624         if (pf->support_multi_driver) {
 625                 PMD_DRV_LOG(ERR,
 626                             "Modify hash function is not permitted when multi-driver enabled");
 627                 return -EPERM;
 628         }
 629
 630         PMD_DRV_LOG(INFO, "NIC hash function is setting to %d", func);
 631         i40e_write_rx_ctl(hw, I40E_GLQF_CTL, reg);
 632         I40E_WRITE_FLUSH(hw);
 633
 634 set_symmetric:
 635         i40e_set_symmetric_hash_enable_per_port(hw, symmetric);
 636         return 0;
 637 }
 638
 639 static int
 640 i40e_hash_config_pctype_symmetric(struct i40e_hw *hw,
 641                                   uint32_t pctype,
 642                                   bool symmetric)
 643 {
 644         struct i40e_pf *pf = &((struct i40e_adapter *)hw->back)->pf;
 645         uint32_t reg;
 646
 647         /* For X722, get translated pctype in fd pctype register */
 648         if (hw->mac.type == I40E_MAC_X722)
 649                 pctype = i40e_read_rx_ctl(hw, I40E_GLQF_FD_PCTYPES(pctype));
 650
 651         reg = i40e_read_rx_ctl(hw, I40E_GLQF_HSYM(pctype));
 652         if (symmetric) {
 653                 if (reg & I40E_GLQF_HSYM_SYMH_ENA_MASK)
 654                         return 0;
 655                 reg |= I40E_GLQF_HSYM_SYMH_ENA_MASK;
 656         } else {
 657                 if (!(reg & I40E_GLQF_HSYM_SYMH_ENA_MASK))
 658                         return 0;
 659                 reg &= ~I40E_GLQF_HSYM_SYMH_ENA_MASK;
 660         }
 661
 662         if (pf->support_multi_driver) {
 663                 PMD_DRV_LOG(ERR,
 664                             "Enable/Disable symmetric hash is not permitted when multi-driver enabled");
 665                 return -EPERM;
 666         }
 667
 668         i40e_write_rx_ctl(hw, I40E_GLQF_HSYM(pctype), reg);
 669         I40E_WRITE_FLUSH(hw);
 670         return 0;
 671 }
 672
 673 static void
 674 i40e_hash_enable_pctype(struct i40e_hw *hw,
 675                         uint32_t pctype, bool enable)
 676 {
 677         uint32_t reg, reg_val, mask;
 678
 679         /* For X722, get translated pctype in fd pctype register */
 680         if (hw->mac.type == I40E_MAC_X722)
 681                 pctype = i40e_read_rx_ctl(hw, I40E_GLQF_FD_PCTYPES(pctype));
 682
 683         if (pctype < 32) {
 684                 mask = BIT(pctype);
 685                 reg = I40E_PFQF_HENA(0);
 686         } else {
 687                 mask = BIT(pctype - 32);
 688                 reg = I40E_PFQF_HENA(1);
 689         }
 690
 691         reg_val = i40e_read_rx_ctl(hw, reg);
 692
 693         if (enable) {
 694                 if (reg_val & mask)
 695                         return;
 696
 697                 reg_val |= mask;
 698         } else {
 699                 if (!(reg_val & mask))
 700                         return;
 701
 702                 reg_val &= ~mask;
 703         }
 704
 705         i40e_write_rx_ctl(hw, reg, reg_val);
 706         I40E_WRITE_FLUSH(hw);
 707 }
 708
 709 static int
 710 i40e_hash_config_pctype(struct i40e_hw *hw,
 711                         struct i40e_rte_flow_rss_conf *rss_conf,
 712                         uint32_t pctype)
 713 {
 714         uint64_t rss_types = rss_conf->conf.types;
 715         int ret;
 716
 717         if (rss_types == 0) {
 718                 i40e_hash_enable_pctype(hw, pctype, false);
 719                 return 0;
 720         }
 721
 722         if (rss_conf->inset) {
 723                 ret = i40e_set_hash_inset(hw, rss_conf->inset, pctype, false);
 724                 if (ret)
 725                         return ret;
 726         }
 727
 728         i40e_hash_enable_pctype(hw, pctype, true);
 729         return 0;
 730 }
 731
 732 static int
 733 i40e_hash_config_region(struct i40e_pf *pf,
 734                         const struct i40e_rte_flow_rss_conf *rss_conf)
 735 {
 736         struct i40e_hw *hw = &pf->adapter->hw;
 737         struct rte_eth_dev *dev = pf->adapter->eth_dev;
 738         struct i40e_queue_region_info *regions = pf->queue_region.region;
 739         uint32_t num = pf->queue_region.queue_region_number;
 740         uint32_t i, region_id_mask = 0;
 741
 742         /* Use a 32 bit variable to represent all regions */
 743         RTE_BUILD_BUG_ON(I40E_REGION_MAX_INDEX > 31);
 744
 745         /* Re-configure the region if it existed */
 746         for (i = 0; i < num; i++) {
 747                 if (rss_conf->region_queue_start ==
 748                     regions[i].queue_start_index &&
 749                     rss_conf->region_queue_num == regions[i].queue_num) {
 750                         uint32_t j;
 751
 752                         for (j = 0; j < regions[i].user_priority_num; j++) {
 753                                 if (regions[i].user_priority[j] ==
 754                                     rss_conf->region_priority)
 755                                         return 0;
 756                         }
 757
 758                         if (j >= I40E_MAX_USER_PRIORITY) {
 759                                 PMD_DRV_LOG(ERR,
 760                                             "Priority number exceed the maximum %d",
 761                                             I40E_MAX_USER_PRIORITY);
 762                                 return -ENOSPC;
 763                         }
 764
 765                         regions[i].user_priority[j] = rss_conf->region_priority;
 766                         regions[i].user_priority_num++;
 767                         return i40e_flush_queue_region_all_conf(dev, hw, pf, 1);
 768                 }
 769
 770                 region_id_mask |= BIT(regions[i].region_id);
 771         }
 772
 773         if (num > I40E_REGION_MAX_INDEX) {
 774                 PMD_DRV_LOG(ERR, "Queue region resource used up");
 775                 return -ENOSPC;
 776         }
 777
 778         /* Add a new region */
 779
 780         pf->queue_region.queue_region_number++;
 781         memset(&regions[num], 0, sizeof(regions[0]));
 782
 783         regions[num].region_id = rte_bsf32(~region_id_mask);
 784         regions[num].queue_num = rss_conf->region_queue_num;
 785         regions[num].queue_start_index = rss_conf->region_queue_start;
 786         regions[num].user_priority[0] = rss_conf->region_priority;
 787         regions[num].user_priority_num = 1;
 788
 789         return i40e_flush_queue_region_all_conf(dev, hw, pf, 1);
 790 }
 791
 792 static int
 793 i40e_hash_config(struct i40e_pf *pf,
 794                  struct i40e_rte_flow_rss_conf *rss_conf)
 795 {
 796         struct rte_flow_action_rss *rss_info = &rss_conf->conf;
 797         struct i40e_hw *hw = &pf->adapter->hw;
 798         uint64_t pctypes;
 799         int ret;
 800
 801         if (rss_info->func != RTE_ETH_HASH_FUNCTION_DEFAULT) {
 802                 ret = i40e_hash_config_func(hw, rss_info->func);
 803                 if (ret)
 804                         return ret;
 805
 806                 if (rss_info->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
 807                         rss_conf->misc_reset_flags |=
 808                                         I40E_HASH_FLOW_RESET_FLAG_FUNC;
 809         }
 810
 811         if (rss_conf->region_queue_num > 0) {
 812                 ret = i40e_hash_config_region(pf, rss_conf);
 813                 if (ret)
 814                         return ret;
 815
 816                 rss_conf->misc_reset_flags |= I40E_HASH_FLOW_RESET_FLAG_REGION;
 817         }
 818
 819         if (rss_info->key_len > 0) {
 820                 ret = i40e_set_rss_key(pf->main_vsi, rss_conf->key,
 821                                        rss_info->key_len);
 822                 if (ret)
 823                         return ret;
 824
 825                 rss_conf->misc_reset_flags |= I40E_HASH_FLOW_RESET_FLAG_KEY;
 826         }
 827
 828         /* Update lookup table */
 829         if (rss_info->queue_num > 0) {
 830                 uint8_t lut[ETH_RSS_RETA_SIZE_512];
 831                 uint32_t i, j = 0;
 832
 833                 for (i = 0; i < hw->func_caps.rss_table_size; i++) {
 834                         lut[i] = (uint8_t)rss_info->queue[j];
 835                         j = (j == rss_info->queue_num - 1) ? 0 : (j + 1);
 836                 }
 837
 838                 ret = i40e_set_rss_lut(pf->main_vsi, lut, (uint16_t)i);
 839                 if (ret)
 840                         return ret;
 841
 842                 pf->hash_enabled_queues = 0;
 843                 for (i = 0; i < rss_info->queue_num; i++)
 844                         pf->hash_enabled_queues |= BIT_ULL(lut[i]);
 845
 846                 pf->adapter->rss_reta_updated = 0;
 847                 rss_conf->misc_reset_flags |= I40E_HASH_FLOW_RESET_FLAG_QUEUE;
 848         }
 849
 850         /* The codes behind configure the input sets and symmetric hash
 851          * function of the packet types and enable hash on them.
 852          */
 853         pctypes = rss_conf->config_pctypes;
 854         if (!pctypes)
 855                 return 0;
 856
 857         /* For first flow that will enable hash on any packet type, we clean
 858          * the RSS sets that by legacy configuration commands and parameters.
 859          */
 860         if (!pf->hash_filter_enabled) {
 861                 i40e_pf_disable_rss(pf);
 862                 pf->hash_filter_enabled = true;
 863         }
 864
 865         do {
 866                 uint32_t idx = rte_bsf64(pctypes);
 867                 uint64_t bit = BIT_ULL(idx);
 868
 869                 if (rss_conf->symmetric_enable) {
 870                         ret = i40e_hash_config_pctype_symmetric(hw, idx, true);
 871                         if (ret)
 872                                 return ret;
 873
 874                         rss_conf->reset_symmetric_pctypes |= bit;
 875                 }
 876
 877                 ret = i40e_hash_config_pctype(hw, rss_conf, idx);
 878                 if (ret)
 879                         return ret;
 880
 881                 rss_conf->reset_config_pctypes |= bit;
 882                 pctypes &= ~bit;
 883         } while (pctypes);
 884
 885         return 0;
 886 }
 887
 888 static void
 889 i40e_hash_parse_key(const struct rte_flow_action_rss *rss_act,
 890                     struct i40e_rte_flow_rss_conf *rss_conf)
 891 {
 892         const uint8_t *key = rss_act->key;
 893
 894         if (!key || rss_act->key_len != sizeof(rss_conf->key)) {
 895                 const uint32_t rss_key_default[] = {0x6b793944,
 896                         0x23504cb5, 0x5bea75b6, 0x309f4f12, 0x3dc0a2b8,
 897                         0x024ddcdf, 0x339b8ca0, 0x4c4af64a, 0x34fac605,
 898                         0x55d85839, 0x3a58997d, 0x2ec938e1, 0x66031581};
 899
 900                 if (rss_act->key_len != sizeof(rss_conf->key))
 901                         PMD_DRV_LOG(WARNING,
 902                                     "RSS key length invalid, must be %u bytes, now set key to default",
 903                                     (uint32_t)sizeof(rss_conf->key));
 904                 key = (const uint8_t *)rss_key_default;
 905         }
 906
 907         memcpy(rss_conf->key, key, sizeof(rss_conf->key));
 908         rss_conf->conf.key = rss_conf->key;
 909         rss_conf->conf.key_len = sizeof(rss_conf->key);
 910 }
 911
 912 static int
 913 i40e_hash_parse_queues(const struct rte_eth_dev *dev,
 914                        const struct rte_flow_action_rss *rss_act,
 915                        struct i40e_rte_flow_rss_conf *rss_conf,
 916                        struct rte_flow_error *error)
 917 {
 918         struct i40e_pf *pf;
 919         struct i40e_hw *hw;
 920         uint16_t i;
 921         int max_queue;
 922
 923         hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 924         if (!rss_act->queue_num ||
 925             rss_act->queue_num > hw->func_caps.rss_table_size)
 926                 return rte_flow_error_set(error, EINVAL,
 927                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 928                                           NULL, "Invalid RSS queue number");
 929
 930         if (rss_act->key_len)
 931                 PMD_DRV_LOG(WARNING,
 932                             "RSS key is ignored when queues specified");
 933
 934         pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 935         if (pf->dev_data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 936                 max_queue = i40e_pf_calc_configured_queues_num(pf);
 937         else
 938                 max_queue = pf->dev_data->nb_rx_queues;
 939
 940         max_queue = RTE_MIN(max_queue, I40E_MAX_Q_PER_TC);
 941
 942         for (i = 0; i < rss_act->queue_num; i++) {
 943                 if ((int)rss_act->queue[i] >= max_queue)
 944                         break;
 945         }
 946
 947         if (i < rss_act->queue_num)
 948                 return rte_flow_error_set(error, EINVAL,
 949                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 950                                           NULL, "Invalid RSS queues");
 951
 952         memcpy(rss_conf->queue, rss_act->queue,
 953                rss_act->queue_num * sizeof(rss_conf->queue[0]));
 954         rss_conf->conf.queue = rss_conf->queue;
 955         rss_conf->conf.queue_num = rss_act->queue_num;
 956         return 0;
 957 }
 958
 959 static int
 960 i40e_hash_parse_queue_region(const struct rte_eth_dev *dev,
 961                              const struct rte_flow_item pattern[],
 962                              const struct rte_flow_action_rss *rss_act,
 963                              struct i40e_rte_flow_rss_conf *rss_conf,
 964                              struct rte_flow_error *error)
 965 {
 966         struct i40e_pf *pf;
 967         const struct rte_flow_item_vlan *vlan_spec, *vlan_mask;
 968         uint64_t hash_queues;
 969         uint32_t i;
 970
 971         if (pattern[1].type != RTE_FLOW_ITEM_TYPE_END)
 972                 return rte_flow_error_set(error, ENOTSUP,
 973                                           RTE_FLOW_ERROR_TYPE_ITEM_NUM,
 974                                           &pattern[1],
 975                                           "Pattern not supported.");
 976
 977         vlan_spec = pattern->spec;
 978         vlan_mask = pattern->mask;
 979         if (!vlan_spec || !vlan_mask ||
 980             (rte_be_to_cpu_16(vlan_mask->tci) >> 13) != 7)
 981                 return rte_flow_error_set(error, EINVAL,
 982                                           RTE_FLOW_ERROR_TYPE_ITEM, pattern,
 983                                           "Pattern error.");
 984
 985         if (!rss_act->queue)
 986                 return rte_flow_error_set(error, EINVAL,
 987                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 988                                           NULL, "Queues not specified");
 989
 990         if (rss_act->key_len)
 991                 PMD_DRV_LOG(WARNING,
 992                             "RSS key is ignored when configure queue region");
 993
 994         /* Use a 64 bit variable to represent all queues in a region. */
 995         RTE_BUILD_BUG_ON(I40E_MAX_Q_PER_TC > 64);
 996
 997         if (!rss_act->queue_num ||
 998             !rte_is_power_of_2(rss_act->queue_num) ||
 999             rss_act->queue_num + rss_act->queue[0] > I40E_MAX_Q_PER_TC)
1000                 return rte_flow_error_set(error, EINVAL,
1001                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1002                                           NULL, "Queue number error");
1003
1004         for (i = 1; i < rss_act->queue_num; i++) {
1005                 if (rss_act->queue[i - 1] + 1 != rss_act->queue[i])
1006                         break;
1007         }
1008
1009         if (i < rss_act->queue_num)
1010                 return rte_flow_error_set(error, EINVAL,
1011                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1012                                           NULL,
1013                                           "Queues must be incremented continuously");
1014
1015         /* Map all queues to bits of uint64_t */
1016         hash_queues = (BIT_ULL(rss_act->queue[0] + rss_act->queue_num) - 1) &
1017                       ~(BIT_ULL(rss_act->queue[0]) - 1);
1018
1019         pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1020         if (hash_queues & ~pf->hash_enabled_queues)
1021                 return rte_flow_error_set(error, EINVAL,
1022                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1023                                           NULL, "Some queues are not in LUT");
1024
1025         rss_conf->region_queue_num = (uint8_t)rss_act->queue_num;
1026         rss_conf->region_queue_start = rss_act->queue[0];
1027         rss_conf->region_priority = rte_be_to_cpu_16(vlan_spec->tci) >> 13;
1028         return 0;
1029 }
1030
1031 static int
1032 i40e_hash_parse_global_conf(const struct rte_eth_dev *dev,
1033                             const struct rte_flow_item pattern[],
1034                             const struct rte_flow_action_rss *rss_act,
1035                             struct i40e_rte_flow_rss_conf *rss_conf,
1036                             struct rte_flow_error *error)
1037 {
1038         if (rss_act->func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ)
1039                 return rte_flow_error_set(error, EINVAL,
1040                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1041                                           NULL,
1042                                           "Symmetric function should be set with pattern types");
1043
1044         rss_conf->conf.func = rss_act->func;
1045
1046         if (rss_act->types)
1047                 PMD_DRV_LOG(WARNING,
1048                             "RSS types are ignored when no pattern specified");
1049
1050         if (pattern[0].type == RTE_FLOW_ITEM_TYPE_VLAN)
1051                 return i40e_hash_parse_queue_region(dev, pattern, rss_act,
1052                                                     rss_conf, error);
1053
1054         if (rss_act->queue)
1055                 return i40e_hash_parse_queues(dev, rss_act, rss_conf, error);
1056
1057         if (rss_act->key_len) {
1058                 i40e_hash_parse_key(rss_act, rss_conf);
1059                 return 0;
1060         }
1061
1062         if (rss_act->func == RTE_ETH_HASH_FUNCTION_DEFAULT)
1063                 PMD_DRV_LOG(WARNING, "Nothing change");
1064         return 0;
1065 }
1066
1067 static bool
1068 i40e_hash_validate_rss_types(uint64_t rss_types)
1069 {
1070         uint64_t type, mask;
1071
1072         /* Validate L2 */
1073         type = ETH_RSS_ETH & rss_types;
1074         mask = (ETH_RSS_L2_SRC_ONLY | ETH_RSS_L2_DST_ONLY) & rss_types;
1075         if (!type && mask)
1076                 return false;
1077
1078         /* Validate L3 */
1079         type = (I40E_HASH_L4_TYPES | ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
1080                ETH_RSS_NONFRAG_IPV4_OTHER | ETH_RSS_IPV6 |
1081                ETH_RSS_FRAG_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER) & rss_types;
1082         mask = (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY) & rss_types;
1083         if (!type && mask)
1084                 return false;
1085
1086         /* Validate L4 */
1087         type = (I40E_HASH_L4_TYPES | ETH_RSS_PORT) & rss_types;
1088         mask = (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY) & rss_types;
1089         if (!type && mask)
1090                 return false;
1091
1092         return true;
1093 }
1094
1095 static int
1096 i40e_hash_parse_pattern_act(const struct rte_eth_dev *dev,
1097                             const struct rte_flow_item pattern[],
1098                             const struct rte_flow_action_rss *rss_act,
1099                             struct i40e_rte_flow_rss_conf *rss_conf,
1100                             struct rte_flow_error *error)
1101 {
1102         if (rss_act->queue)
1103                 return rte_flow_error_set(error, EINVAL,
1104                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1105                                           NULL,
1106                                           "RSS Queues not supported when pattern specified");
1107
1108         if (rss_act->func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ)
1109                 rss_conf->symmetric_enable = true;
1110         else if (rss_act->func != RTE_ETH_HASH_FUNCTION_DEFAULT)
1111                 return rte_flow_error_set(error, -EINVAL,
1112                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1113                                           NULL,
1114                                           "Only symmetric TOEPLITZ supported when pattern specified");
1115
1116         if (!i40e_hash_validate_rss_types(rss_act->types))
1117                 return rte_flow_error_set(error, EINVAL,
1118                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1119                                           NULL, "RSS types are invalid");
1120
1121         if (rss_act->key_len)
1122                 i40e_hash_parse_key(rss_act, rss_conf);
1123
1124         rss_conf->conf.func = rss_act->func;
1125         rss_conf->conf.types = rss_act->types;
1126         rss_conf->inset = i40e_hash_get_inset(rss_act->types);
1127
1128         return i40e_hash_get_pattern_pctypes(dev, pattern, rss_act,
1129                                              rss_conf, error);
1130 }
1131
1132 int
1133 i40e_hash_parse(const struct rte_eth_dev *dev,
1134                 const struct rte_flow_item pattern[],
1135                 const struct rte_flow_action actions[],
1136                 struct i40e_rte_flow_rss_conf *rss_conf,
1137                 struct rte_flow_error *error)
1138 {
1139         const struct rte_flow_action_rss *rss_act;
1140
1141         if (actions[1].type != RTE_FLOW_ACTION_TYPE_END)
1142                 return rte_flow_error_set(error, EINVAL,
1143                                           RTE_FLOW_ERROR_TYPE_ACTION,
1144                                           &actions[1],
1145                                           "Only support one action for RSS.");
1146
1147         rss_act = (const struct rte_flow_action_rss *)actions[0].conf;
1148         if (rss_act->level)
1149                 return rte_flow_error_set(error, ENOTSUP,
1150                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1151                                           actions,
1152                                           "RSS level is not supported");
1153
1154         while (pattern->type == RTE_FLOW_ITEM_TYPE_VOID)
1155                 pattern++;
1156
1157         if (pattern[0].type == RTE_FLOW_ITEM_TYPE_END ||
1158             pattern[0].type == RTE_FLOW_ITEM_TYPE_VLAN)
1159                 return i40e_hash_parse_global_conf(dev, pattern, rss_act,
1160                                                    rss_conf, error);
1161
1162         return i40e_hash_parse_pattern_act(dev, pattern, rss_act,
1163                                            rss_conf, error);
1164 }
1165
1166 static void
1167 i40e_invalid_rss_filter(const struct i40e_rte_flow_rss_conf *ref_conf,
1168                         struct i40e_rte_flow_rss_conf *conf)
1169 {
1170         uint32_t reset_flags = conf->misc_reset_flags;
1171
1172         conf->misc_reset_flags &= ~ref_conf->misc_reset_flags;
1173
1174         if ((reset_flags & I40E_HASH_FLOW_RESET_FLAG_REGION) &&
1175             (ref_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_REGION) &&
1176             (conf->region_queue_start != ref_conf->region_queue_start ||
1177              conf->region_queue_num != ref_conf->region_queue_num))
1178                 conf->misc_reset_flags |= I40E_HASH_FLOW_RESET_FLAG_REGION;
1179
1180         conf->reset_config_pctypes &= ~ref_conf->reset_config_pctypes;
1181         conf->reset_symmetric_pctypes &= ~ref_conf->reset_symmetric_pctypes;
1182 }
1183
1184 int
1185 i40e_hash_filter_restore(struct i40e_pf *pf)
1186 {
1187         struct i40e_rss_filter *filter;
1188         int ret;
1189
1190         TAILQ_FOREACH(filter, &pf->rss_config_list, next) {
1191                 struct i40e_rte_flow_rss_conf *rss_conf =
1192                                                 &filter->rss_filter_info;
1193                 struct i40e_rss_filter *prev;
1194
1195                 rss_conf->misc_reset_flags = 0;
1196                 rss_conf->reset_config_pctypes = 0;
1197                 rss_conf->reset_symmetric_pctypes = 0;
1198
1199                 ret = i40e_hash_config(pf, rss_conf);
1200                 if (ret) {
1201                         pf->hash_filter_enabled = 0;
1202                         i40e_pf_disable_rss(pf);
1203                         PMD_DRV_LOG(ERR,
1204                                     "Re-configure RSS failed, RSS has been disabled");
1205                         return ret;
1206                 }
1207
1208                 /* Invalid previous RSS filter */
1209                 TAILQ_FOREACH(prev, &pf->rss_config_list, next) {
1210                         if (prev == filter)
1211                                 break;
1212                         i40e_invalid_rss_filter(rss_conf,
1213                                                 &prev->rss_filter_info);
1214                 }
1215         }
1216
1217         return 0;
1218 }
1219
1220 int
1221 i40e_hash_filter_create(struct i40e_pf *pf,
1222                         struct i40e_rte_flow_rss_conf *rss_conf)
1223 {
1224         struct i40e_rss_filter *filter, *prev;
1225         struct i40e_rte_flow_rss_conf *new_conf;
1226         int ret;
1227
1228         filter = rte_zmalloc("i40e_rss_filter", sizeof(*filter), 0);
1229         if (!filter) {
1230                 PMD_DRV_LOG(ERR, "Failed to allocate memory.");
1231                 return -ENOMEM;
1232         }
1233
1234         new_conf = &filter->rss_filter_info;
1235
1236         memcpy(new_conf, rss_conf, sizeof(*new_conf));
1237         if (new_conf->conf.queue_num)
1238                 new_conf->conf.queue = new_conf->queue;
1239         if (new_conf->conf.key_len)
1240                 new_conf->conf.key = new_conf->key;
1241
1242         ret = i40e_hash_config(pf, new_conf);
1243         if (ret) {
1244                 rte_free(filter);
1245                 if (i40e_pf_config_rss(pf))
1246                         return ret;
1247
1248                 (void)i40e_hash_filter_restore(pf);
1249                 return ret;
1250         }
1251
1252         /* Invalid previous RSS filter */
1253         TAILQ_FOREACH(prev, &pf->rss_config_list, next)
1254                 i40e_invalid_rss_filter(new_conf, &prev->rss_filter_info);
1255
1256         TAILQ_INSERT_TAIL(&pf->rss_config_list, filter, next);
1257         return 0;
1258 }
1259
1260 static int
1261 i40e_hash_reset_conf(struct i40e_pf *pf,
1262                      struct i40e_rte_flow_rss_conf *rss_conf)
1263 {
1264         struct i40e_hw *hw = &pf->adapter->hw;
1265         uint64_t inset;
1266         uint32_t idx;
1267         int ret;
1268
1269         if (rss_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_FUNC) {
1270                 ret = i40e_hash_config_func(hw, RTE_ETH_HASH_FUNCTION_TOEPLITZ);
1271                 if (ret)
1272                         return ret;
1273
1274                 rss_conf->misc_reset_flags &= ~I40E_HASH_FLOW_RESET_FLAG_FUNC;
1275         }
1276
1277         if (rss_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_REGION) {
1278                 ret = i40e_flush_queue_region_all_conf(pf->adapter->eth_dev,
1279                                                        hw, pf, 0);
1280                 if (ret)
1281                         return ret;
1282
1283                 rss_conf->misc_reset_flags &= ~I40E_HASH_FLOW_RESET_FLAG_REGION;
1284         }
1285
1286         if (rss_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_KEY) {
1287                 ret = i40e_pf_reset_rss_key(pf);
1288                 if (ret)
1289                         return ret;
1290
1291                 rss_conf->misc_reset_flags &= ~I40E_HASH_FLOW_RESET_FLAG_KEY;
1292         }
1293
1294         if (rss_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_QUEUE) {
1295                 if (!pf->adapter->rss_reta_updated) {
1296                         ret = i40e_pf_reset_rss_reta(pf);
1297                         if (ret)
1298                                 return ret;
1299                 }
1300
1301                 pf->hash_enabled_queues = 0;
1302                 rss_conf->misc_reset_flags &= ~I40E_HASH_FLOW_RESET_FLAG_QUEUE;
1303         }
1304
1305         while (rss_conf->reset_config_pctypes) {
1306                 idx = rte_bsf64(rss_conf->reset_config_pctypes);
1307
1308                 i40e_hash_enable_pctype(hw, idx, false);
1309                 inset = i40e_get_default_input_set(idx);
1310                 if (inset) {
1311                         ret = i40e_set_hash_inset(hw, inset, idx, false);
1312                         if (ret)
1313                                 return ret;
1314                 }
1315
1316                 rss_conf->reset_config_pctypes &= ~BIT_ULL(idx);
1317         }
1318
1319         while (rss_conf->reset_symmetric_pctypes) {
1320                 idx = rte_bsf64(rss_conf->reset_symmetric_pctypes);
1321
1322                 ret = i40e_hash_config_pctype_symmetric(hw, idx, false);
1323                 if (ret)
1324                         return ret;
1325
1326                 rss_conf->reset_symmetric_pctypes &= ~BIT_ULL(idx);
1327         }
1328
1329         return 0;
1330 }
1331
1332 int
1333 i40e_hash_filter_destroy(struct i40e_pf *pf,
1334                          const struct i40e_rss_filter *rss_filter)
1335 {
1336         struct i40e_rss_filter *filter;
1337         int ret;
1338
1339         TAILQ_FOREACH(filter, &pf->rss_config_list, next) {
1340                 if (rss_filter == filter) {
1341                         ret = i40e_hash_reset_conf(pf,
1342                                                    &filter->rss_filter_info);
1343                         if (ret)
1344                                 return ret;
1345
1346                         TAILQ_REMOVE(&pf->rss_config_list, filter, next);
1347                         rte_free(filter);
1348                         return 0;
1349                 }
1350         }
1351
1352         return -ENOENT;
1353 }
1354
1355 int
1356 i40e_hash_filter_flush(struct i40e_pf *pf)
1357 {
1358         struct rte_flow *flow, *next;
1359
1360         TAILQ_FOREACH_SAFE(flow, &pf->flow_list, node, next) {
1361                 if (flow->filter_type != RTE_ETH_FILTER_HASH)
1362                         continue;
1363
1364                 if (flow->rule) {
1365                         struct i40e_rss_filter *filter = flow->rule;
1366                         int ret;
1367
1368                         ret = i40e_hash_reset_conf(pf,
1369                                                    &filter->rss_filter_info);
1370                         if (ret)
1371                                 return ret;
1372
1373                         TAILQ_REMOVE(&pf->rss_config_list, filter, next);
1374                         rte_free(filter);
1375                 }
1376
1377                 TAILQ_REMOVE(&pf->flow_list, flow, node);
1378                 rte_free(flow);
1379         }
1380
1381         assert(!pf->rss_config_list.tqh_first);
1382         return 0;
1383 }