drivers/net/i40e/i40e_hash.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10 #include <assert.h>
  11
  12 #include <rte_malloc.h>
  13 #include <rte_tailq.h>
  14 #include "base/i40e_prototype.h"
  15 #include "i40e_logs.h"
  16 #include "i40e_ethdev.h"
  17 #include "i40e_hash.h"
  18
  19 #ifndef BIT
  20 #define BIT(n)                          (1UL << (n))
  21 #endif
  22
  23 #ifndef BIT_ULL
  24 #define BIT_ULL(n)                      (1ULL << (n))
  25 #endif
  26
  27 /* Pattern item headers */
  28 #define I40E_HASH_HDR_ETH               0x01ULL
  29 #define I40E_HASH_HDR_IPV4              0x10ULL
  30 #define I40E_HASH_HDR_IPV6              0x20ULL
  31 #define I40E_HASH_HDR_TCP               0x100ULL
  32 #define I40E_HASH_HDR_UDP               0x200ULL
  33 #define I40E_HASH_HDR_SCTP              0x400ULL
  34 #define I40E_HASH_HDR_ESP               0x10000ULL
  35 #define I40E_HASH_HDR_L2TPV3            0x20000ULL
  36 #define I40E_HASH_HDR_AH                0x40000ULL
  37 #define I40E_HASH_HDR_GTPC              0x100000ULL
  38 #define I40E_HASH_HDR_GTPU              0x200000ULL
  39
  40 #define I40E_HASH_HDR_INNER_SHIFT       32
  41 #define I40E_HASH_HDR_IPV4_INNER        (I40E_HASH_HDR_IPV4 << \
  42                                         I40E_HASH_HDR_INNER_SHIFT)
  43 #define I40E_HASH_HDR_IPV6_INNER        (I40E_HASH_HDR_IPV6 << \
  44                                         I40E_HASH_HDR_INNER_SHIFT)
  45
  46 /* ETH */
  47 #define I40E_PHINT_ETH                  I40E_HASH_HDR_ETH
  48
  49 /* IPv4 */
  50 #define I40E_PHINT_IPV4                 (I40E_HASH_HDR_ETH | I40E_HASH_HDR_IPV4)
  51 #define I40E_PHINT_IPV4_TCP             (I40E_PHINT_IPV4 | I40E_HASH_HDR_TCP)
  52 #define I40E_PHINT_IPV4_UDP             (I40E_PHINT_IPV4 | I40E_HASH_HDR_UDP)
  53 #define I40E_PHINT_IPV4_SCTP            (I40E_PHINT_IPV4 | I40E_HASH_HDR_SCTP)
  54
  55 /* IPv6 */
  56 #define I40E_PHINT_IPV6                 (I40E_HASH_HDR_ETH | I40E_HASH_HDR_IPV6)
  57 #define I40E_PHINT_IPV6_TCP             (I40E_PHINT_IPV6 | I40E_HASH_HDR_TCP)
  58 #define I40E_PHINT_IPV6_UDP             (I40E_PHINT_IPV6 | I40E_HASH_HDR_UDP)
  59 #define I40E_PHINT_IPV6_SCTP            (I40E_PHINT_IPV6 | I40E_HASH_HDR_SCTP)
  60
  61 /* ESP */
  62 #define I40E_PHINT_IPV4_ESP             (I40E_PHINT_IPV4 | I40E_HASH_HDR_ESP)
  63 #define I40E_PHINT_IPV6_ESP             (I40E_PHINT_IPV6 | I40E_HASH_HDR_ESP)
  64 #define I40E_PHINT_IPV4_UDP_ESP         (I40E_PHINT_IPV4_UDP | \
  65                                         I40E_HASH_HDR_ESP)
  66 #define I40E_PHINT_IPV6_UDP_ESP         (I40E_PHINT_IPV6_UDP | \
  67                                         I40E_HASH_HDR_ESP)
  68
  69 /* GTPC */
  70 #define I40E_PHINT_IPV4_GTPC            (I40E_PHINT_IPV4_UDP | \
  71                                         I40E_HASH_HDR_GTPC)
  72 #define I40E_PHINT_IPV6_GTPC            (I40E_PHINT_IPV6_UDP | \
  73                                         I40E_HASH_HDR_GTPC)
  74
  75 /* GTPU */
  76 #define I40E_PHINT_IPV4_GTPU            (I40E_PHINT_IPV4_UDP | \
  77                                         I40E_HASH_HDR_GTPU)
  78 #define I40E_PHINT_IPV4_GTPU_IPV4       (I40E_PHINT_IPV4_GTPU | \
  79                                         I40E_HASH_HDR_IPV4_INNER)
  80 #define I40E_PHINT_IPV4_GTPU_IPV6       (I40E_PHINT_IPV4_GTPU | \
  81                                         I40E_HASH_HDR_IPV6_INNER)
  82 #define I40E_PHINT_IPV6_GTPU            (I40E_PHINT_IPV6_UDP | \
  83                                         I40E_HASH_HDR_GTPU)
  84 #define I40E_PHINT_IPV6_GTPU_IPV4       (I40E_PHINT_IPV6_GTPU | \
  85                                         I40E_HASH_HDR_IPV4_INNER)
  86 #define I40E_PHINT_IPV6_GTPU_IPV6       (I40E_PHINT_IPV6_GTPU | \
  87                                         I40E_HASH_HDR_IPV6_INNER)
  88
  89 /* L2TPV3 */
  90 #define I40E_PHINT_IPV4_L2TPV3          (I40E_PHINT_IPV4 | I40E_HASH_HDR_L2TPV3)
  91 #define I40E_PHINT_IPV6_L2TPV3          (I40E_PHINT_IPV6 | I40E_HASH_HDR_L2TPV3)
  92
  93 /* AH */
  94 #define I40E_PHINT_IPV4_AH              (I40E_PHINT_IPV4 | I40E_HASH_HDR_AH)
  95 #define I40E_PHINT_IPV6_AH              (I40E_PHINT_IPV6 | I40E_HASH_HDR_AH)
  96
  97 /* Structure of mapping RSS type to input set */
  98 struct i40e_hash_map_rss_inset {
  99         uint64_t rss_type;
 100         uint64_t inset;
 101 };
 102
 103 const struct i40e_hash_map_rss_inset i40e_hash_rss_inset[] = {
 104         /* IPv4 */
 105         { ETH_RSS_IPV4, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST },
 106         { ETH_RSS_FRAG_IPV4, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST },
 107
 108         { ETH_RSS_NONFRAG_IPV4_OTHER,
 109           I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST },
 110
 111         { ETH_RSS_NONFRAG_IPV4_TCP, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
 112           I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT },
 113
 114         { ETH_RSS_NONFRAG_IPV4_UDP, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
 115           I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT },
 116
 117         { ETH_RSS_NONFRAG_IPV4_SCTP, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
 118           I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | I40E_INSET_SCTP_VT },
 119
 120         /* IPv6 */
 121         { ETH_RSS_IPV6, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST },
 122         { ETH_RSS_FRAG_IPV6, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST },
 123
 124         { ETH_RSS_NONFRAG_IPV6_OTHER,
 125           I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST },
 126
 127         { ETH_RSS_NONFRAG_IPV6_TCP, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
 128           I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT },
 129
 130         { ETH_RSS_NONFRAG_IPV6_UDP, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
 131           I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT },
 132
 133         { ETH_RSS_NONFRAG_IPV6_SCTP, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
 134           I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | I40E_INSET_SCTP_VT },
 135
 136         /* Port */
 137         { ETH_RSS_PORT, I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT },
 138
 139         /* Ether */
 140         { ETH_RSS_L2_PAYLOAD, I40E_INSET_LAST_ETHER_TYPE },
 141         { ETH_RSS_ETH, I40E_INSET_DMAC | I40E_INSET_SMAC },
 142
 143         /* VLAN */
 144         { ETH_RSS_S_VLAN, I40E_INSET_VLAN_OUTER },
 145         { ETH_RSS_C_VLAN, I40E_INSET_VLAN_INNER },
 146 };
 147
 148 #define I40E_HASH_VOID_NEXT_ALLOW       BIT_ULL(RTE_FLOW_ITEM_TYPE_ETH)
 149
 150 #define I40E_HASH_ETH_NEXT_ALLOW        (BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV4) | \
 151                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV6) | \
 152                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_VLAN))
 153
 154 #define I40E_HASH_IP_NEXT_ALLOW         (BIT_ULL(RTE_FLOW_ITEM_TYPE_TCP) | \
 155                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_UDP) | \
 156                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_SCTP) | \
 157                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_ESP) | \
 158                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_L2TPV3OIP) |\
 159                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_AH))
 160
 161 #define I40E_HASH_UDP_NEXT_ALLOW        (BIT_ULL(RTE_FLOW_ITEM_TYPE_GTPU) | \
 162                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_GTPC))
 163
 164 #define I40E_HASH_GTPU_NEXT_ALLOW       (BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV4) | \
 165                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV6))
 166
 167 static const uint64_t pattern_next_allow_items[] = {
 168         [RTE_FLOW_ITEM_TYPE_VOID] = I40E_HASH_VOID_NEXT_ALLOW,
 169         [RTE_FLOW_ITEM_TYPE_ETH] = I40E_HASH_ETH_NEXT_ALLOW,
 170         [RTE_FLOW_ITEM_TYPE_IPV4] = I40E_HASH_IP_NEXT_ALLOW,
 171         [RTE_FLOW_ITEM_TYPE_IPV6] = I40E_HASH_IP_NEXT_ALLOW,
 172         [RTE_FLOW_ITEM_TYPE_UDP] = I40E_HASH_UDP_NEXT_ALLOW,
 173         [RTE_FLOW_ITEM_TYPE_GTPU] = I40E_HASH_GTPU_NEXT_ALLOW,
 174 };
 175
 176 static const uint64_t pattern_item_header[] = {
 177         [RTE_FLOW_ITEM_TYPE_ETH] = I40E_HASH_HDR_ETH,
 178         [RTE_FLOW_ITEM_TYPE_IPV4] = I40E_HASH_HDR_IPV4,
 179         [RTE_FLOW_ITEM_TYPE_IPV6] = I40E_HASH_HDR_IPV6,
 180         [RTE_FLOW_ITEM_TYPE_TCP] = I40E_HASH_HDR_TCP,
 181         [RTE_FLOW_ITEM_TYPE_UDP] = I40E_HASH_HDR_UDP,
 182         [RTE_FLOW_ITEM_TYPE_SCTP] = I40E_HASH_HDR_SCTP,
 183         [RTE_FLOW_ITEM_TYPE_ESP] = I40E_HASH_HDR_ESP,
 184         [RTE_FLOW_ITEM_TYPE_GTPC] = I40E_HASH_HDR_GTPC,
 185         [RTE_FLOW_ITEM_TYPE_GTPU] = I40E_HASH_HDR_GTPU,
 186         [RTE_FLOW_ITEM_TYPE_L2TPV3OIP] = I40E_HASH_HDR_L2TPV3,
 187         [RTE_FLOW_ITEM_TYPE_AH] = I40E_HASH_HDR_AH,
 188 };
 189
 190 /* Structure of matched pattern */
 191 struct i40e_hash_match_pattern {
 192         uint64_t pattern_type;
 193         uint64_t rss_mask;      /* Supported RSS type for this pattern */
 194         bool custom_pctype_flag;/* true for custom packet type */
 195         uint8_t pctype;
 196 };
 197
 198 #define I40E_HASH_MAP_PATTERN(pattern, rss_mask, pctype) { \
 199         pattern, rss_mask, false, pctype  }
 200
 201 #define I40E_HASH_MAP_CUS_PATTERN(pattern, rss_mask, cus_pctype) { \
 202         pattern, rss_mask, true, cus_pctype }
 203
 204 #define I40E_HASH_L2_RSS_MASK           (ETH_RSS_ETH | ETH_RSS_L2_SRC_ONLY | \
 205                                         ETH_RSS_L2_DST_ONLY)
 206
 207 #define I40E_HASH_L23_RSS_MASK          (I40E_HASH_L2_RSS_MASK | \
 208                                         ETH_RSS_VLAN | \
 209                                         ETH_RSS_L3_SRC_ONLY | \
 210                                         ETH_RSS_L3_DST_ONLY)
 211
 212 #define I40E_HASH_IPV4_L23_RSS_MASK     (ETH_RSS_IPV4 | I40E_HASH_L23_RSS_MASK)
 213 #define I40E_HASH_IPV6_L23_RSS_MASK     (ETH_RSS_IPV6 | I40E_HASH_L23_RSS_MASK)
 214
 215 #define I40E_HASH_L234_RSS_MASK         (I40E_HASH_L23_RSS_MASK | \
 216                                         ETH_RSS_PORT | ETH_RSS_L4_SRC_ONLY | \
 217                                         ETH_RSS_L4_DST_ONLY)
 218
 219 #define I40E_HASH_IPV4_L234_RSS_MASK    (I40E_HASH_L234_RSS_MASK | ETH_RSS_IPV4)
 220 #define I40E_HASH_IPV6_L234_RSS_MASK    (I40E_HASH_L234_RSS_MASK | ETH_RSS_IPV6)
 221
 222 #define I40E_HASH_L4_TYPES              (ETH_RSS_NONFRAG_IPV4_TCP | \
 223                                         ETH_RSS_NONFRAG_IPV4_UDP | \
 224                                         ETH_RSS_NONFRAG_IPV4_SCTP | \
 225                                         ETH_RSS_NONFRAG_IPV6_TCP | \
 226                                         ETH_RSS_NONFRAG_IPV6_UDP | \
 227                                         ETH_RSS_NONFRAG_IPV6_SCTP)
 228
 229 /* Current supported patterns and RSS types.
 230  * All items that have the same pattern types are together.
 231  */
 232 static const struct i40e_hash_match_pattern match_patterns[] = {
 233         /* Ether */
 234         I40E_HASH_MAP_PATTERN(I40E_PHINT_ETH,
 235                               ETH_RSS_L2_PAYLOAD | I40E_HASH_L2_RSS_MASK,
 236                               I40E_FILTER_PCTYPE_L2_PAYLOAD),
 237
 238         /* IPv4 */
 239         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4,
 240                               ETH_RSS_FRAG_IPV4 | I40E_HASH_IPV4_L23_RSS_MASK,
 241                               I40E_FILTER_PCTYPE_FRAG_IPV4),
 242
 243         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4,
 244                               ETH_RSS_NONFRAG_IPV4_OTHER |
 245                               I40E_HASH_IPV4_L23_RSS_MASK,
 246                               I40E_FILTER_PCTYPE_NONF_IPV4_OTHER),
 247
 248         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4_TCP,
 249                               ETH_RSS_NONFRAG_IPV4_TCP |
 250                               I40E_HASH_IPV4_L234_RSS_MASK,
 251                               I40E_FILTER_PCTYPE_NONF_IPV4_TCP),
 252
 253         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4_UDP,
 254                               ETH_RSS_NONFRAG_IPV4_UDP |
 255                               I40E_HASH_IPV4_L234_RSS_MASK,
 256                               I40E_FILTER_PCTYPE_NONF_IPV4_UDP),
 257
 258         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4_SCTP,
 259                               ETH_RSS_NONFRAG_IPV4_SCTP |
 260                               I40E_HASH_IPV4_L234_RSS_MASK,
 261                               I40E_FILTER_PCTYPE_NONF_IPV4_SCTP),
 262
 263         /* IPv6 */
 264         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6,
 265                               ETH_RSS_FRAG_IPV6 | I40E_HASH_IPV6_L23_RSS_MASK,
 266                               I40E_FILTER_PCTYPE_FRAG_IPV6),
 267
 268         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6,
 269                               ETH_RSS_NONFRAG_IPV6_OTHER |
 270                               I40E_HASH_IPV6_L23_RSS_MASK,
 271                               I40E_FILTER_PCTYPE_NONF_IPV6_OTHER),
 272
 273         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6_TCP,
 274                               ETH_RSS_NONFRAG_IPV6_TCP |
 275                               I40E_HASH_IPV6_L234_RSS_MASK,
 276                               I40E_FILTER_PCTYPE_NONF_IPV6_TCP),
 277
 278         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6_UDP,
 279                               ETH_RSS_NONFRAG_IPV6_UDP |
 280                               I40E_HASH_IPV6_L234_RSS_MASK,
 281                               I40E_FILTER_PCTYPE_NONF_IPV6_UDP),
 282
 283         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6_SCTP,
 284                               ETH_RSS_NONFRAG_IPV6_SCTP |
 285                               I40E_HASH_IPV6_L234_RSS_MASK,
 286                               I40E_FILTER_PCTYPE_NONF_IPV6_SCTP),
 287
 288         /* ESP */
 289         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_ESP,
 290                                   ETH_RSS_ESP, I40E_CUSTOMIZED_ESP_IPV4),
 291         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_ESP,
 292                                   ETH_RSS_ESP, I40E_CUSTOMIZED_ESP_IPV6),
 293         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_UDP_ESP,
 294                                   ETH_RSS_ESP, I40E_CUSTOMIZED_ESP_IPV4_UDP),
 295         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_UDP_ESP,
 296                                   ETH_RSS_ESP, I40E_CUSTOMIZED_ESP_IPV6_UDP),
 297
 298         /* GTPC */
 299         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_GTPC,
 300                                   I40E_HASH_IPV4_L234_RSS_MASK,
 301                                   I40E_CUSTOMIZED_GTPC),
 302         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_GTPC,
 303                                   I40E_HASH_IPV6_L234_RSS_MASK,
 304                                   I40E_CUSTOMIZED_GTPC),
 305
 306         /* GTPU */
 307         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_GTPU,
 308                                   I40E_HASH_IPV4_L234_RSS_MASK,
 309                                   I40E_CUSTOMIZED_GTPU),
 310         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_GTPU_IPV4,
 311                                   ETH_RSS_GTPU, I40E_CUSTOMIZED_GTPU_IPV4),
 312         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_GTPU_IPV6,
 313                                   ETH_RSS_GTPU, I40E_CUSTOMIZED_GTPU_IPV6),
 314         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_GTPU,
 315                                   I40E_HASH_IPV6_L234_RSS_MASK,
 316                                   I40E_CUSTOMIZED_GTPU),
 317         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_GTPU_IPV4,
 318                                   ETH_RSS_GTPU, I40E_CUSTOMIZED_GTPU_IPV4),
 319         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_GTPU_IPV6,
 320                                   ETH_RSS_GTPU, I40E_CUSTOMIZED_GTPU_IPV6),
 321
 322         /* L2TPV3 */
 323         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_L2TPV3,
 324                                   ETH_RSS_L2TPV3, I40E_CUSTOMIZED_IPV4_L2TPV3),
 325         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_L2TPV3,
 326                                   ETH_RSS_L2TPV3, I40E_CUSTOMIZED_IPV6_L2TPV3),
 327
 328         /* AH */
 329         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_AH, ETH_RSS_AH,
 330                                   I40E_CUSTOMIZED_AH_IPV4),
 331         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_AH, ETH_RSS_AH,
 332                                   I40E_CUSTOMIZED_AH_IPV6),
 333 };
 334
 335 static int
 336 i40e_hash_get_pattern_type(const struct rte_flow_item pattern[],
 337                            uint64_t *pattern_types,
 338                            struct rte_flow_error *error)
 339 {
 340         const char *message = "Pattern not supported";
 341         enum rte_flow_item_type prev_item_type = RTE_FLOW_ITEM_TYPE_VOID;
 342         enum rte_flow_item_type last_item_type = prev_item_type;
 343         uint64_t item_hdr, pattern_hdrs = 0;
 344         bool inner_flag = false;
 345         int vlan_count = 0;
 346
 347         for (; pattern->type != RTE_FLOW_ITEM_TYPE_END; pattern++) {
 348                 if (pattern->type == RTE_FLOW_ITEM_TYPE_VOID)
 349                         continue;
 350
 351                 if (pattern->mask || pattern->spec || pattern->last) {
 352                         message = "Header info should not be specified";
 353                         goto not_sup;
 354                 }
 355
 356                 /* Check the previous item allows this sub-item. */
 357                 if (prev_item_type >= (enum rte_flow_item_type)
 358                                 RTE_DIM(pattern_next_allow_items) ||
 359                     !(pattern_next_allow_items[prev_item_type] &
 360                                 BIT_ULL(pattern->type)))
 361                         goto not_sup;
 362
 363                 /* For VLAN item, it does no matter about to pattern type
 364                  * recognition. So just count the number of VLAN and do not
 365                  * change the value of variable `prev_item_type`.
 366                  */
 367                 last_item_type = pattern->type;
 368                 if (last_item_type == RTE_FLOW_ITEM_TYPE_VLAN) {
 369                         if (vlan_count >= 2)
 370                                 goto not_sup;
 371                         vlan_count++;
 372                         continue;
 373                 }
 374
 375                 prev_item_type = last_item_type;
 376                 assert(last_item_type < (enum rte_flow_item_type)
 377                                 RTE_DIM(pattern_item_header));
 378                 item_hdr = pattern_item_header[last_item_type];
 379                 assert(item_hdr);
 380
 381                 if (inner_flag) {
 382                         item_hdr <<= I40E_HASH_HDR_INNER_SHIFT;
 383
 384                         /* Inner layer should not have GTPU item */
 385                         if (last_item_type == RTE_FLOW_ITEM_TYPE_GTPU)
 386                                 goto not_sup;
 387                 } else {
 388                         if (last_item_type == RTE_FLOW_ITEM_TYPE_GTPU) {
 389                                 inner_flag = true;
 390                                 vlan_count = 0;
 391                         }
 392                 }
 393
 394                 if (item_hdr & pattern_hdrs)
 395                         goto not_sup;
 396
 397                 pattern_hdrs |= item_hdr;
 398         }
 399
 400         if (pattern_hdrs && last_item_type != RTE_FLOW_ITEM_TYPE_VLAN) {
 401                 *pattern_types = pattern_hdrs;
 402                 return 0;
 403         }
 404
 405 not_sup:
 406         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 407                                   pattern, message);
 408 }
 409
 410 static uint64_t
 411 i40e_hash_get_x722_ext_pctypes(uint8_t match_pctype)
 412 {
 413         uint64_t pctypes = 0;
 414
 415         switch (match_pctype) {
 416         case I40E_FILTER_PCTYPE_NONF_IPV4_TCP:
 417                 pctypes = BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK);
 418                 break;
 419
 420         case I40E_FILTER_PCTYPE_NONF_IPV4_UDP:
 421                 pctypes = BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) |
 422                           BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP);
 423                 break;
 424
 425         case I40E_FILTER_PCTYPE_NONF_IPV6_TCP:
 426                 pctypes = BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK);
 427                 break;
 428
 429         case I40E_FILTER_PCTYPE_NONF_IPV6_UDP:
 430                 pctypes = BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) |
 431                           BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP);
 432                 break;
 433         }
 434
 435         return pctypes;
 436 }
 437
 438 static int
 439 i40e_hash_translate_gtp_inset(struct i40e_rte_flow_rss_conf *rss_conf,
 440                               struct rte_flow_error *error)
 441 {
 442         if (rss_conf->inset &
 443             (I40E_INSET_IPV4_SRC | I40E_INSET_IPV6_SRC |
 444             I40E_INSET_DST_PORT | I40E_INSET_SRC_PORT))
 445                 return rte_flow_error_set(error, ENOTSUP,
 446                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 447                                           NULL,
 448                                           "Only support external destination IP");
 449
 450         if (rss_conf->inset & I40E_INSET_IPV4_DST)
 451                 rss_conf->inset = (rss_conf->inset & ~I40E_INSET_IPV4_DST) |
 452                                   I40E_INSET_TUNNEL_IPV4_DST;
 453
 454         if (rss_conf->inset & I40E_INSET_IPV6_DST)
 455                 rss_conf->inset = (rss_conf->inset & ~I40E_INSET_IPV6_DST) |
 456                                   I40E_INSET_TUNNEL_IPV6_DST;
 457
 458         return 0;
 459 }
 460
 461 static int
 462 i40e_hash_get_pctypes(const struct rte_eth_dev *dev,
 463                       const struct i40e_hash_match_pattern *match,
 464                       struct i40e_rte_flow_rss_conf *rss_conf,
 465                       struct rte_flow_error *error)
 466 {
 467         if (match->custom_pctype_flag) {
 468                 struct i40e_pf *pf;
 469                 struct i40e_customized_pctype *custom_type;
 470
 471                 pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 472                 custom_type = i40e_find_customized_pctype(pf, match->pctype);
 473                 if (!custom_type || !custom_type->valid)
 474                         return rte_flow_error_set(error, ENOTSUP,
 475                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 476                                                   NULL, "PCTYPE not supported");
 477
 478                 rss_conf->config_pctypes |= BIT_ULL(custom_type->pctype);
 479
 480                 if (match->pctype == I40E_CUSTOMIZED_GTPU ||
 481                     match->pctype == I40E_CUSTOMIZED_GTPC)
 482                         return i40e_hash_translate_gtp_inset(rss_conf, error);
 483         } else {
 484                 struct i40e_hw *hw =
 485                                 I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 486                 uint64_t types;
 487
 488                 rss_conf->config_pctypes |= BIT_ULL(match->pctype);
 489                 if (hw->mac.type == I40E_MAC_X722) {
 490                         types = i40e_hash_get_x722_ext_pctypes(match->pctype);
 491                         rss_conf->config_pctypes |= types;
 492                 }
 493         }
 494
 495         return 0;
 496 }
 497
 498 static int
 499 i40e_hash_get_pattern_pctypes(const struct rte_eth_dev *dev,
 500                               const struct rte_flow_item pattern[],
 501                               const struct rte_flow_action_rss *rss_act,
 502                               struct i40e_rte_flow_rss_conf *rss_conf,
 503                               struct rte_flow_error *error)
 504 {
 505         uint64_t pattern_types = 0;
 506         bool match_flag = false;
 507         int i, ret;
 508
 509         ret = i40e_hash_get_pattern_type(pattern, &pattern_types, error);
 510         if (ret)
 511                 return ret;
 512
 513         for (i = 0; i < (int)RTE_DIM(match_patterns); i++) {
 514                 const struct i40e_hash_match_pattern *match =
 515                                                         &match_patterns[i];
 516
 517                 /* Check pattern types match. All items that have the same
 518                  * pattern types are together, so if the pattern types match
 519                  * previous item but they doesn't match current item, it means
 520                  * the pattern types do not match all remain items.
 521                  */
 522                 if (pattern_types != match->pattern_type) {
 523                         if (match_flag)
 524                                 break;
 525                         continue;
 526                 }
 527                 match_flag = true;
 528
 529                 /* Check RSS types match */
 530                 if (!(rss_act->types & ~match->rss_mask)) {
 531                         ret = i40e_hash_get_pctypes(dev, match,
 532                                                     rss_conf, error);
 533                         if (ret)
 534                                 return ret;
 535                 }
 536         }
 537
 538         if (rss_conf->config_pctypes)
 539                 return 0;
 540
 541         if (match_flag)
 542                 return rte_flow_error_set(error, ENOTSUP,
 543                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 544                                           NULL, "RSS types not supported");
 545
 546         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 547                                   NULL, "Pattern not supported");
 548 }
 549
 550 static uint64_t
 551 i40e_hash_get_inset(uint64_t rss_types)
 552 {
 553         uint64_t mask, inset = 0;
 554         int i;
 555
 556         for (i = 0; i < (int)RTE_DIM(i40e_hash_rss_inset); i++) {
 557                 if (rss_types & i40e_hash_rss_inset[i].rss_type)
 558                         inset |= i40e_hash_rss_inset[i].inset;
 559         }
 560
 561         if (!inset)
 562                 return 0;
 563
 564         /* If SRC_ONLY and DST_ONLY of the same level are used simultaneously,
 565          * it is the same case as none of them are added.
 566          */
 567         mask = rss_types & (ETH_RSS_L2_SRC_ONLY | ETH_RSS_L2_DST_ONLY);
 568         if (mask == ETH_RSS_L2_SRC_ONLY)
 569                 inset &= ~I40E_INSET_DMAC;
 570         else if (mask == ETH_RSS_L2_DST_ONLY)
 571                 inset &= ~I40E_INSET_SMAC;
 572
 573         mask = rss_types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY);
 574         if (mask == ETH_RSS_L3_SRC_ONLY)
 575                 inset &= ~(I40E_INSET_IPV4_DST | I40E_INSET_IPV6_DST);
 576         else if (mask == ETH_RSS_L3_DST_ONLY)
 577                 inset &= ~(I40E_INSET_IPV4_SRC | I40E_INSET_IPV6_SRC);
 578
 579         mask = rss_types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY);
 580         if (mask == ETH_RSS_L4_SRC_ONLY)
 581                 inset &= ~I40E_INSET_DST_PORT;
 582         else if (mask == ETH_RSS_L4_DST_ONLY)
 583                 inset &= ~I40E_INSET_SRC_PORT;
 584
 585         if (rss_types & I40E_HASH_L4_TYPES) {
 586                 uint64_t l3_mask = rss_types &
 587                                    (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY);
 588                 uint64_t l4_mask = rss_types &
 589                                    (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY);
 590
 591                 if (l3_mask && !l4_mask)
 592                         inset &= ~(I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT);
 593                 else if (!l3_mask && l4_mask)
 594                         inset &= ~(I40E_INSET_IPV4_DST | I40E_INSET_IPV6_DST |
 595                                  I40E_INSET_IPV4_SRC | I40E_INSET_IPV6_SRC);
 596         }
 597
 598         return inset;
 599 }
 600
 601 static int
 602 i40e_hash_config_func(struct i40e_hw *hw, enum rte_eth_hash_function func)
 603 {
 604         struct i40e_pf *pf;
 605         uint32_t reg;
 606         uint8_t symmetric = 0;
 607
 608         reg = i40e_read_rx_ctl(hw, I40E_GLQF_CTL);
 609
 610         if (func == RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) {
 611                 if (!(reg & I40E_GLQF_CTL_HTOEP_MASK))
 612                         goto set_symmetric;
 613
 614                 reg &= ~I40E_GLQF_CTL_HTOEP_MASK;
 615         } else {
 616                 if (func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ)
 617                         symmetric = 1;
 618
 619                 if (reg & I40E_GLQF_CTL_HTOEP_MASK)
 620                         goto set_symmetric;
 621
 622                 reg |= I40E_GLQF_CTL_HTOEP_MASK;
 623         }
 624
 625         pf = &((struct i40e_adapter *)hw->back)->pf;
 626         if (pf->support_multi_driver) {
 627                 PMD_DRV_LOG(ERR,
 628                             "Modify hash function is not permitted when multi-driver enabled");
 629                 return -EPERM;
 630         }
 631
 632         PMD_DRV_LOG(INFO, "NIC hash function is setting to %d", func);
 633         i40e_write_rx_ctl(hw, I40E_GLQF_CTL, reg);
 634         I40E_WRITE_FLUSH(hw);
 635
 636 set_symmetric:
 637         i40e_set_symmetric_hash_enable_per_port(hw, symmetric);
 638         return 0;
 639 }
 640
 641 static int
 642 i40e_hash_config_pctype_symmetric(struct i40e_hw *hw,
 643                                   uint32_t pctype,
 644                                   bool symmetric)
 645 {
 646         struct i40e_pf *pf = &((struct i40e_adapter *)hw->back)->pf;
 647         uint32_t reg;
 648
 649         /* For X722, get translated pctype in fd pctype register */
 650         if (hw->mac.type == I40E_MAC_X722)
 651                 pctype = i40e_read_rx_ctl(hw, I40E_GLQF_FD_PCTYPES(pctype));
 652
 653         reg = i40e_read_rx_ctl(hw, I40E_GLQF_HSYM(pctype));
 654         if (symmetric) {
 655                 if (reg & I40E_GLQF_HSYM_SYMH_ENA_MASK)
 656                         return 0;
 657                 reg |= I40E_GLQF_HSYM_SYMH_ENA_MASK;
 658         } else {
 659                 if (!(reg & I40E_GLQF_HSYM_SYMH_ENA_MASK))
 660                         return 0;
 661                 reg &= ~I40E_GLQF_HSYM_SYMH_ENA_MASK;
 662         }
 663
 664         if (pf->support_multi_driver) {
 665                 PMD_DRV_LOG(ERR,
 666                             "Enable/Disable symmetric hash is not permitted when multi-driver enabled");
 667                 return -EPERM;
 668         }
 669
 670         i40e_write_rx_ctl(hw, I40E_GLQF_HSYM(pctype), reg);
 671         I40E_WRITE_FLUSH(hw);
 672         return 0;
 673 }
 674
 675 static void
 676 i40e_hash_enable_pctype(struct i40e_hw *hw,
 677                         uint32_t pctype, bool enable)
 678 {
 679         uint32_t reg, reg_val, mask;
 680
 681         /* For X722, get translated pctype in fd pctype register */
 682         if (hw->mac.type == I40E_MAC_X722)
 683                 pctype = i40e_read_rx_ctl(hw, I40E_GLQF_FD_PCTYPES(pctype));
 684
 685         if (pctype < 32) {
 686                 mask = BIT(pctype);
 687                 reg = I40E_PFQF_HENA(0);
 688         } else {
 689                 mask = BIT(pctype - 32);
 690                 reg = I40E_PFQF_HENA(1);
 691         }
 692
 693         reg_val = i40e_read_rx_ctl(hw, reg);
 694
 695         if (enable) {
 696                 if (reg_val & mask)
 697                         return;
 698
 699                 reg_val |= mask;
 700         } else {
 701                 if (!(reg_val & mask))
 702                         return;
 703
 704                 reg_val &= ~mask;
 705         }
 706
 707         i40e_write_rx_ctl(hw, reg, reg_val);
 708         I40E_WRITE_FLUSH(hw);
 709 }
 710
 711 static int
 712 i40e_hash_config_pctype(struct i40e_hw *hw,
 713                         struct i40e_rte_flow_rss_conf *rss_conf,
 714                         uint32_t pctype)
 715 {
 716         uint64_t rss_types = rss_conf->conf.types;
 717         int ret;
 718
 719         if (rss_types == 0) {
 720                 i40e_hash_enable_pctype(hw, pctype, false);
 721                 return 0;
 722         }
 723
 724         if (rss_conf->inset) {
 725                 ret = i40e_set_hash_inset(hw, rss_conf->inset, pctype, false);
 726                 if (ret)
 727                         return ret;
 728         }
 729
 730         i40e_hash_enable_pctype(hw, pctype, true);
 731         return 0;
 732 }
 733
 734 static int
 735 i40e_hash_config_region(struct i40e_pf *pf,
 736                         const struct i40e_rte_flow_rss_conf *rss_conf)
 737 {
 738         struct i40e_hw *hw = &pf->adapter->hw;
 739         struct rte_eth_dev *dev = pf->adapter->eth_dev;
 740         struct i40e_queue_region_info *regions = pf->queue_region.region;
 741         uint32_t num = pf->queue_region.queue_region_number;
 742         uint32_t i, region_id_mask = 0;
 743
 744         /* Use a 32 bit variable to represent all regions */
 745         RTE_BUILD_BUG_ON(I40E_REGION_MAX_INDEX > 31);
 746
 747         /* Re-configure the region if it existed */
 748         for (i = 0; i < num; i++) {
 749                 if (rss_conf->region_queue_start ==
 750                     regions[i].queue_start_index &&
 751                     rss_conf->region_queue_num == regions[i].queue_num) {
 752                         uint32_t j;
 753
 754                         for (j = 0; j < regions[i].user_priority_num; j++) {
 755                                 if (regions[i].user_priority[j] ==
 756                                     rss_conf->region_priority)
 757                                         return 0;
 758                         }
 759
 760                         if (j >= I40E_MAX_USER_PRIORITY) {
 761                                 PMD_DRV_LOG(ERR,
 762                                             "Priority number exceed the maximum %d",
 763                                             I40E_MAX_USER_PRIORITY);
 764                                 return -ENOSPC;
 765                         }
 766
 767                         regions[i].user_priority[j] = rss_conf->region_priority;
 768                         regions[i].user_priority_num++;
 769                         return i40e_flush_queue_region_all_conf(dev, hw, pf, 1);
 770                 }
 771
 772                 region_id_mask |= BIT(regions[i].region_id);
 773         }
 774
 775         if (num > I40E_REGION_MAX_INDEX) {
 776                 PMD_DRV_LOG(ERR, "Queue region resource used up");
 777                 return -ENOSPC;
 778         }
 779
 780         /* Add a new region */
 781
 782         pf->queue_region.queue_region_number++;
 783         memset(&regions[num], 0, sizeof(regions[0]));
 784
 785         regions[num].region_id = rte_bsf32(~region_id_mask);
 786         regions[num].queue_num = rss_conf->region_queue_num;
 787         regions[num].queue_start_index = rss_conf->region_queue_start;
 788         regions[num].user_priority[0] = rss_conf->region_priority;
 789         regions[num].user_priority_num = 1;
 790
 791         return i40e_flush_queue_region_all_conf(dev, hw, pf, 1);
 792 }
 793
 794 static int
 795 i40e_hash_config(struct i40e_pf *pf,
 796                  struct i40e_rte_flow_rss_conf *rss_conf)
 797 {
 798         struct rte_flow_action_rss *rss_info = &rss_conf->conf;
 799         struct i40e_hw *hw = &pf->adapter->hw;
 800         uint64_t pctypes;
 801         int ret;
 802
 803         if (rss_info->func != RTE_ETH_HASH_FUNCTION_DEFAULT) {
 804                 ret = i40e_hash_config_func(hw, rss_info->func);
 805                 if (ret)
 806                         return ret;
 807
 808                 if (rss_info->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
 809                         rss_conf->misc_reset_flags |=
 810                                         I40E_HASH_FLOW_RESET_FLAG_FUNC;
 811         }
 812
 813         if (rss_conf->region_queue_num > 0) {
 814                 ret = i40e_hash_config_region(pf, rss_conf);
 815                 if (ret)
 816                         return ret;
 817
 818                 rss_conf->misc_reset_flags |= I40E_HASH_FLOW_RESET_FLAG_REGION;
 819         }
 820
 821         if (rss_info->key_len > 0) {
 822                 ret = i40e_set_rss_key(pf->main_vsi, rss_conf->key,
 823                                        rss_info->key_len);
 824                 if (ret)
 825                         return ret;
 826
 827                 rss_conf->misc_reset_flags |= I40E_HASH_FLOW_RESET_FLAG_KEY;
 828         }
 829
 830         /* Update lookup table */
 831         if (rss_info->queue_num > 0) {
 832                 uint8_t lut[ETH_RSS_RETA_SIZE_512];
 833                 uint32_t i, j = 0;
 834
 835                 for (i = 0; i < hw->func_caps.rss_table_size; i++) {
 836                         lut[i] = (uint8_t)rss_info->queue[j];
 837                         j = (j == rss_info->queue_num - 1) ? 0 : (j + 1);
 838                 }
 839
 840                 ret = i40e_set_rss_lut(pf->main_vsi, lut, (uint16_t)i);
 841                 if (ret)
 842                         return ret;
 843
 844                 pf->hash_enabled_queues = 0;
 845                 for (i = 0; i < rss_info->queue_num; i++)
 846                         pf->hash_enabled_queues |= BIT_ULL(lut[i]);
 847
 848                 pf->adapter->rss_reta_updated = 0;
 849                 rss_conf->misc_reset_flags |= I40E_HASH_FLOW_RESET_FLAG_QUEUE;
 850         }
 851
 852         /* The codes behind configure the input sets and symmetric hash
 853          * function of the packet types and enable hash on them.
 854          */
 855         pctypes = rss_conf->config_pctypes;
 856         if (!pctypes)
 857                 return 0;
 858
 859         /* For first flow that will enable hash on any packet type, we clean
 860          * the RSS sets that by legacy configuration commands and parameters.
 861          */
 862         if (!pf->hash_filter_enabled) {
 863                 i40e_pf_disable_rss(pf);
 864                 pf->hash_filter_enabled = true;
 865         }
 866
 867         do {
 868                 uint32_t idx = rte_bsf64(pctypes);
 869                 uint64_t bit = BIT_ULL(idx);
 870
 871                 if (rss_conf->symmetric_enable) {
 872                         ret = i40e_hash_config_pctype_symmetric(hw, idx, true);
 873                         if (ret)
 874                                 return ret;
 875
 876                         rss_conf->reset_symmetric_pctypes |= bit;
 877                 }
 878
 879                 ret = i40e_hash_config_pctype(hw, rss_conf, idx);
 880                 if (ret)
 881                         return ret;
 882
 883                 rss_conf->reset_config_pctypes |= bit;
 884                 pctypes &= ~bit;
 885         } while (pctypes);
 886
 887         return 0;
 888 }
 889
 890 static void
 891 i40e_hash_parse_key(const struct rte_flow_action_rss *rss_act,
 892                     struct i40e_rte_flow_rss_conf *rss_conf)
 893 {
 894         const uint8_t *key = rss_act->key;
 895
 896         if (!key || rss_act->key_len != sizeof(rss_conf->key)) {
 897                 const uint32_t rss_key_default[] = {0x6b793944,
 898                         0x23504cb5, 0x5bea75b6, 0x309f4f12, 0x3dc0a2b8,
 899                         0x024ddcdf, 0x339b8ca0, 0x4c4af64a, 0x34fac605,
 900                         0x55d85839, 0x3a58997d, 0x2ec938e1, 0x66031581};
 901
 902                 if (rss_act->key_len != sizeof(rss_conf->key))
 903                         PMD_DRV_LOG(WARNING,
 904                                     "RSS key length invalid, must be %u bytes, now set key to default",
 905                                     (uint32_t)sizeof(rss_conf->key));
 906                 key = (const uint8_t *)rss_key_default;
 907         }
 908
 909         memcpy(rss_conf->key, key, sizeof(rss_conf->key));
 910         rss_conf->conf.key = rss_conf->key;
 911         rss_conf->conf.key_len = sizeof(rss_conf->key);
 912 }
 913
 914 static int
 915 i40e_hash_parse_queues(const struct rte_eth_dev *dev,
 916                        const struct rte_flow_action_rss *rss_act,
 917                        struct i40e_rte_flow_rss_conf *rss_conf,
 918                        struct rte_flow_error *error)
 919 {
 920         struct i40e_pf *pf;
 921         struct i40e_hw *hw;
 922         uint16_t i;
 923         int max_queue;
 924
 925         hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 926         if (!rss_act->queue_num ||
 927             rss_act->queue_num > hw->func_caps.rss_table_size)
 928                 return rte_flow_error_set(error, EINVAL,
 929                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 930                                           NULL, "Invalid RSS queue number");
 931
 932         if (rss_act->key_len)
 933                 PMD_DRV_LOG(WARNING,
 934                             "RSS key is ignored when queues specified");
 935
 936         pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 937         if (pf->dev_data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 938                 max_queue = i40e_pf_calc_configured_queues_num(pf);
 939         else
 940                 max_queue = pf->dev_data->nb_rx_queues;
 941
 942         max_queue = RTE_MIN(max_queue, I40E_MAX_Q_PER_TC);
 943
 944         for (i = 0; i < rss_act->queue_num; i++) {
 945                 if ((int)rss_act->queue[i] >= max_queue)
 946                         break;
 947         }
 948
 949         if (i < rss_act->queue_num)
 950                 return rte_flow_error_set(error, EINVAL,
 951                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 952                                           NULL, "Invalid RSS queues");
 953
 954         memcpy(rss_conf->queue, rss_act->queue,
 955                rss_act->queue_num * sizeof(rss_conf->queue[0]));
 956         rss_conf->conf.queue = rss_conf->queue;
 957         rss_conf->conf.queue_num = rss_act->queue_num;
 958         return 0;
 959 }
 960
 961 static int
 962 i40e_hash_parse_queue_region(const struct rte_eth_dev *dev,
 963                              const struct rte_flow_item pattern[],
 964                              const struct rte_flow_action_rss *rss_act,
 965                              struct i40e_rte_flow_rss_conf *rss_conf,
 966                              struct rte_flow_error *error)
 967 {
 968         struct i40e_pf *pf;
 969         const struct rte_flow_item_vlan *vlan_spec, *vlan_mask;
 970         uint64_t hash_queues;
 971         uint32_t i;
 972
 973         if (pattern[1].type != RTE_FLOW_ITEM_TYPE_END)
 974                 return rte_flow_error_set(error, ENOTSUP,
 975                                           RTE_FLOW_ERROR_TYPE_ITEM_NUM,
 976                                           &pattern[1],
 977                                           "Pattern not supported.");
 978
 979         vlan_spec = pattern->spec;
 980         vlan_mask = pattern->mask;
 981         if (!vlan_spec || !vlan_mask ||
 982             (rte_be_to_cpu_16(vlan_mask->tci) >> 13) != 7)
 983                 return rte_flow_error_set(error, EINVAL,
 984                                           RTE_FLOW_ERROR_TYPE_ITEM, pattern,
 985                                           "Pattern error.");
 986
 987         if (!rss_act->queue)
 988                 return rte_flow_error_set(error, EINVAL,
 989                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 990                                           NULL, "Queues not specified");
 991
 992         if (rss_act->key_len)
 993                 PMD_DRV_LOG(WARNING,
 994                             "RSS key is ignored when configure queue region");
 995
 996         /* Use a 64 bit variable to represent all queues in a region. */
 997         RTE_BUILD_BUG_ON(I40E_MAX_Q_PER_TC > 64);
 998
 999         if (!rss_act->queue_num ||
1000             !rte_is_power_of_2(rss_act->queue_num) ||
1001             rss_act->queue_num + rss_act->queue[0] > I40E_MAX_Q_PER_TC)
1002                 return rte_flow_error_set(error, EINVAL,
1003                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1004                                           NULL, "Queue number error");
1005
1006         for (i = 1; i < rss_act->queue_num; i++) {
1007                 if (rss_act->queue[i - 1] + 1 != rss_act->queue[i])
1008                         break;
1009         }
1010
1011         if (i < rss_act->queue_num)
1012                 return rte_flow_error_set(error, EINVAL,
1013                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1014                                           NULL,
1015                                           "Queues must be incremented continuously");
1016
1017         /* Map all queues to bits of uint64_t */
1018         hash_queues = (BIT_ULL(rss_act->queue[0] + rss_act->queue_num) - 1) &
1019                       ~(BIT_ULL(rss_act->queue[0]) - 1);
1020
1021         pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1022         if (hash_queues & ~pf->hash_enabled_queues)
1023                 return rte_flow_error_set(error, EINVAL,
1024                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1025                                           NULL, "Some queues are not in LUT");
1026
1027         rss_conf->region_queue_num = (uint8_t)rss_act->queue_num;
1028         rss_conf->region_queue_start = rss_act->queue[0];
1029         rss_conf->region_priority = rte_be_to_cpu_16(vlan_spec->tci) >> 13;
1030         return 0;
1031 }
1032
1033 static int
1034 i40e_hash_parse_global_conf(const struct rte_eth_dev *dev,
1035                             const struct rte_flow_item pattern[],
1036                             const struct rte_flow_action_rss *rss_act,
1037                             struct i40e_rte_flow_rss_conf *rss_conf,
1038                             struct rte_flow_error *error)
1039 {
1040         if (rss_act->func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ)
1041                 return rte_flow_error_set(error, EINVAL,
1042                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1043                                           NULL,
1044                                           "Symmetric function should be set with pattern types");
1045
1046         rss_conf->conf.func = rss_act->func;
1047
1048         if (rss_act->types)
1049                 PMD_DRV_LOG(WARNING,
1050                             "RSS types are ignored when no pattern specified");
1051
1052         if (pattern[0].type == RTE_FLOW_ITEM_TYPE_VLAN)
1053                 return i40e_hash_parse_queue_region(dev, pattern, rss_act,
1054                                                     rss_conf, error);
1055
1056         if (rss_act->queue)
1057                 return i40e_hash_parse_queues(dev, rss_act, rss_conf, error);
1058
1059         if (rss_act->key_len) {
1060                 i40e_hash_parse_key(rss_act, rss_conf);
1061                 return 0;
1062         }
1063
1064         if (rss_act->func == RTE_ETH_HASH_FUNCTION_DEFAULT)
1065                 PMD_DRV_LOG(WARNING, "Nothing change");
1066         return 0;
1067 }
1068
1069 static bool
1070 i40e_hash_validate_rss_types(uint64_t rss_types)
1071 {
1072         uint64_t type, mask;
1073
1074         /* Validate L2 */
1075         type = ETH_RSS_ETH & rss_types;
1076         mask = (ETH_RSS_L2_SRC_ONLY | ETH_RSS_L2_DST_ONLY) & rss_types;
1077         if (!type && mask)
1078                 return false;
1079
1080         /* Validate L3 */
1081         type = (I40E_HASH_L4_TYPES | ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
1082                ETH_RSS_NONFRAG_IPV4_OTHER | ETH_RSS_IPV6 |
1083                ETH_RSS_FRAG_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER) & rss_types;
1084         mask = (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY) & rss_types;
1085         if (!type && mask)
1086                 return false;
1087
1088         /* Validate L4 */
1089         type = (I40E_HASH_L4_TYPES | ETH_RSS_PORT) & rss_types;
1090         mask = (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY) & rss_types;
1091         if (!type && mask)
1092                 return false;
1093
1094         return true;
1095 }
1096
1097 static int
1098 i40e_hash_parse_pattern_act(const struct rte_eth_dev *dev,
1099                             const struct rte_flow_item pattern[],
1100                             const struct rte_flow_action_rss *rss_act,
1101                             struct i40e_rte_flow_rss_conf *rss_conf,
1102                             struct rte_flow_error *error)
1103 {
1104         if (rss_act->queue)
1105                 return rte_flow_error_set(error, EINVAL,
1106                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1107                                           NULL,
1108                                           "RSS Queues not supported when pattern specified");
1109
1110         if (rss_act->func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ)
1111                 rss_conf->symmetric_enable = true;
1112         else if (rss_act->func != RTE_ETH_HASH_FUNCTION_DEFAULT)
1113                 return rte_flow_error_set(error, -EINVAL,
1114                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1115                                           NULL,
1116                                           "Only symmetric TOEPLITZ supported when pattern specified");
1117
1118         if (!i40e_hash_validate_rss_types(rss_act->types))
1119                 return rte_flow_error_set(error, EINVAL,
1120                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1121                                           NULL, "RSS types are invalid");
1122
1123         if (rss_act->key_len)
1124                 i40e_hash_parse_key(rss_act, rss_conf);
1125
1126         rss_conf->conf.func = rss_act->func;
1127         rss_conf->conf.types = rss_act->types;
1128         rss_conf->inset = i40e_hash_get_inset(rss_act->types);
1129
1130         return i40e_hash_get_pattern_pctypes(dev, pattern, rss_act,
1131                                              rss_conf, error);
1132 }
1133
1134 int
1135 i40e_hash_parse(const struct rte_eth_dev *dev,
1136                 const struct rte_flow_item pattern[],
1137                 const struct rte_flow_action actions[],
1138                 struct i40e_rte_flow_rss_conf *rss_conf,
1139                 struct rte_flow_error *error)
1140 {
1141         const struct rte_flow_action_rss *rss_act;
1142
1143         if (actions[1].type != RTE_FLOW_ACTION_TYPE_END)
1144                 return rte_flow_error_set(error, EINVAL,
1145                                           RTE_FLOW_ERROR_TYPE_ACTION,
1146                                           &actions[1],
1147                                           "Only support one action for RSS.");
1148
1149         rss_act = (const struct rte_flow_action_rss *)actions[0].conf;
1150         if (rss_act->level)
1151                 return rte_flow_error_set(error, ENOTSUP,
1152                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1153                                           actions,
1154                                           "RSS level is not supported");
1155
1156         while (pattern->type == RTE_FLOW_ITEM_TYPE_VOID)
1157                 pattern++;
1158
1159         if (pattern[0].type == RTE_FLOW_ITEM_TYPE_END ||
1160             pattern[0].type == RTE_FLOW_ITEM_TYPE_VLAN)
1161                 return i40e_hash_parse_global_conf(dev, pattern, rss_act,
1162                                                    rss_conf, error);
1163
1164         return i40e_hash_parse_pattern_act(dev, pattern, rss_act,
1165                                            rss_conf, error);
1166 }
1167
1168 static void
1169 i40e_invalid_rss_filter(const struct i40e_rte_flow_rss_conf *ref_conf,
1170                         struct i40e_rte_flow_rss_conf *conf)
1171 {
1172         uint32_t reset_flags = conf->misc_reset_flags;
1173
1174         conf->misc_reset_flags &= ~ref_conf->misc_reset_flags;
1175
1176         if ((reset_flags & I40E_HASH_FLOW_RESET_FLAG_REGION) &&
1177             (ref_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_REGION) &&
1178             (conf->region_queue_start != ref_conf->region_queue_start ||
1179              conf->region_queue_num != ref_conf->region_queue_num))
1180                 conf->misc_reset_flags |= I40E_HASH_FLOW_RESET_FLAG_REGION;
1181
1182         conf->reset_config_pctypes &= ~ref_conf->reset_config_pctypes;
1183         conf->reset_symmetric_pctypes &= ~ref_conf->reset_symmetric_pctypes;
1184 }
1185
1186 int
1187 i40e_hash_filter_restore(struct i40e_pf *pf)
1188 {
1189         struct i40e_rss_filter *filter;
1190         int ret;
1191
1192         TAILQ_FOREACH(filter, &pf->rss_config_list, next) {
1193                 struct i40e_rte_flow_rss_conf *rss_conf =
1194                                                 &filter->rss_filter_info;
1195                 struct i40e_rss_filter *prev;
1196
1197                 rss_conf->misc_reset_flags = 0;
1198                 rss_conf->reset_config_pctypes = 0;
1199                 rss_conf->reset_symmetric_pctypes = 0;
1200
1201                 ret = i40e_hash_config(pf, rss_conf);
1202                 if (ret) {
1203                         pf->hash_filter_enabled = 0;
1204                         i40e_pf_disable_rss(pf);
1205                         PMD_DRV_LOG(ERR,
1206                                     "Re-configure RSS failed, RSS has been disabled");
1207                         return ret;
1208                 }
1209
1210                 /* Invalid previous RSS filter */
1211                 TAILQ_FOREACH(prev, &pf->rss_config_list, next) {
1212                         if (prev == filter)
1213                                 break;
1214                         i40e_invalid_rss_filter(rss_conf,
1215                                                 &prev->rss_filter_info);
1216                 }
1217         }
1218
1219         return 0;
1220 }
1221
1222 int
1223 i40e_hash_filter_create(struct i40e_pf *pf,
1224                         struct i40e_rte_flow_rss_conf *rss_conf)
1225 {
1226         struct i40e_rss_filter *filter, *prev;
1227         struct i40e_rte_flow_rss_conf *new_conf;
1228         int ret;
1229
1230         filter = rte_zmalloc("i40e_rss_filter", sizeof(*filter), 0);
1231         if (!filter) {
1232                 PMD_DRV_LOG(ERR, "Failed to allocate memory.");
1233                 return -ENOMEM;
1234         }
1235
1236         new_conf = &filter->rss_filter_info;
1237
1238         memcpy(new_conf, rss_conf, sizeof(*new_conf));
1239         if (new_conf->conf.queue_num)
1240                 new_conf->conf.queue = new_conf->queue;
1241         if (new_conf->conf.key_len)
1242                 new_conf->conf.key = new_conf->key;
1243
1244         ret = i40e_hash_config(pf, new_conf);
1245         if (ret) {
1246                 rte_free(filter);
1247                 if (i40e_pf_config_rss(pf))
1248                         return ret;
1249
1250                 (void)i40e_hash_filter_restore(pf);
1251                 return ret;
1252         }
1253
1254         /* Invalid previous RSS filter */
1255         TAILQ_FOREACH(prev, &pf->rss_config_list, next)
1256                 i40e_invalid_rss_filter(new_conf, &prev->rss_filter_info);
1257
1258         TAILQ_INSERT_TAIL(&pf->rss_config_list, filter, next);
1259         return 0;
1260 }
1261
1262 static int
1263 i40e_hash_reset_conf(struct i40e_pf *pf,
1264                      struct i40e_rte_flow_rss_conf *rss_conf)
1265 {
1266         struct i40e_hw *hw = &pf->adapter->hw;
1267         uint64_t inset;
1268         uint32_t idx;
1269         int ret;
1270
1271         if (rss_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_FUNC) {
1272                 ret = i40e_hash_config_func(hw, RTE_ETH_HASH_FUNCTION_TOEPLITZ);
1273                 if (ret)
1274                         return ret;
1275
1276                 rss_conf->misc_reset_flags &= ~I40E_HASH_FLOW_RESET_FLAG_FUNC;
1277         }
1278
1279         if (rss_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_REGION) {
1280                 ret = i40e_flush_queue_region_all_conf(pf->adapter->eth_dev,
1281                                                        hw, pf, 0);
1282                 if (ret)
1283                         return ret;
1284
1285                 rss_conf->misc_reset_flags &= ~I40E_HASH_FLOW_RESET_FLAG_REGION;
1286         }
1287
1288         if (rss_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_KEY) {
1289                 ret = i40e_pf_reset_rss_key(pf);
1290                 if (ret)
1291                         return ret;
1292
1293                 rss_conf->misc_reset_flags &= ~I40E_HASH_FLOW_RESET_FLAG_KEY;
1294         }
1295
1296         if (rss_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_QUEUE) {
1297                 if (!pf->adapter->rss_reta_updated) {
1298                         ret = i40e_pf_reset_rss_reta(pf);
1299                         if (ret)
1300                                 return ret;
1301                 }
1302
1303                 pf->hash_enabled_queues = 0;
1304                 rss_conf->misc_reset_flags &= ~I40E_HASH_FLOW_RESET_FLAG_QUEUE;
1305         }
1306
1307         while (rss_conf->reset_config_pctypes) {
1308                 idx = rte_bsf64(rss_conf->reset_config_pctypes);
1309
1310                 i40e_hash_enable_pctype(hw, idx, false);
1311                 inset = i40e_get_default_input_set(idx);
1312                 if (inset) {
1313                         ret = i40e_set_hash_inset(hw, inset, idx, false);
1314                         if (ret)
1315                                 return ret;
1316                 }
1317
1318                 rss_conf->reset_config_pctypes &= ~BIT_ULL(idx);
1319         }
1320
1321         while (rss_conf->reset_symmetric_pctypes) {
1322                 idx = rte_bsf64(rss_conf->reset_symmetric_pctypes);
1323
1324                 ret = i40e_hash_config_pctype_symmetric(hw, idx, false);
1325                 if (ret)
1326                         return ret;
1327
1328                 rss_conf->reset_symmetric_pctypes &= ~BIT_ULL(idx);
1329         }
1330
1331         return 0;
1332 }
1333
1334 int
1335 i40e_hash_filter_destroy(struct i40e_pf *pf,
1336                          const struct i40e_rss_filter *rss_filter)
1337 {
1338         struct i40e_rss_filter *filter;
1339         int ret;
1340
1341         TAILQ_FOREACH(filter, &pf->rss_config_list, next) {
1342                 if (rss_filter == filter) {
1343                         ret = i40e_hash_reset_conf(pf,
1344                                                    &filter->rss_filter_info);
1345                         if (ret)
1346                                 return ret;
1347
1348                         TAILQ_REMOVE(&pf->rss_config_list, filter, next);
1349                         rte_free(filter);
1350                         return 0;
1351                 }
1352         }
1353
1354         return -ENOENT;
1355 }
1356
1357 int
1358 i40e_hash_filter_flush(struct i40e_pf *pf)
1359 {
1360         struct rte_flow *flow, *next;
1361
1362         TAILQ_FOREACH_SAFE(flow, &pf->flow_list, node, next) {
1363                 if (flow->filter_type != RTE_ETH_FILTER_HASH)
1364                         continue;
1365
1366                 if (flow->rule) {
1367                         struct i40e_rss_filter *filter = flow->rule;
1368                         int ret;
1369
1370                         ret = i40e_hash_reset_conf(pf,
1371                                                    &filter->rss_filter_info);
1372                         if (ret)
1373                                 return ret;
1374
1375                         TAILQ_REMOVE(&pf->rss_config_list, filter, next);
1376                         rte_free(filter);
1377                 }
1378
1379                 TAILQ_REMOVE(&pf->flow_list, flow, node);
1380                 rte_free(flow);
1381         }
1382
1383         assert(!pf->rss_config_list.tqh_first);
1384         return 0;
1385 }