drivers/net/i40e/i40e_hash.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10 #include <assert.h>
  11
  12 #include <rte_malloc.h>
  13 #include <rte_tailq.h>
  14 #include "base/i40e_prototype.h"
  15 #include "i40e_logs.h"
  16 #include "i40e_ethdev.h"
  17 #include "i40e_hash.h"
  18
  19 #ifndef BIT
  20 #define BIT(n)                          (1UL << (n))
  21 #endif
  22
  23 #ifndef BIT_ULL
  24 #define BIT_ULL(n)                      (1ULL << (n))
  25 #endif
  26
  27 /* Pattern item headers */
  28 #define I40E_HASH_HDR_ETH               0x01ULL
  29 #define I40E_HASH_HDR_IPV4              0x10ULL
  30 #define I40E_HASH_HDR_IPV6              0x20ULL
  31 #define I40E_HASH_HDR_IPV6_FRAG         0x40ULL
  32 #define I40E_HASH_HDR_TCP               0x100ULL
  33 #define I40E_HASH_HDR_UDP               0x200ULL
  34 #define I40E_HASH_HDR_SCTP              0x400ULL
  35 #define I40E_HASH_HDR_ESP               0x10000ULL
  36 #define I40E_HASH_HDR_L2TPV3            0x20000ULL
  37 #define I40E_HASH_HDR_AH                0x40000ULL
  38 #define I40E_HASH_HDR_GTPC              0x100000ULL
  39 #define I40E_HASH_HDR_GTPU              0x200000ULL
  40
  41 #define I40E_HASH_HDR_INNER_SHIFT       32
  42 #define I40E_HASH_HDR_IPV4_INNER        (I40E_HASH_HDR_IPV4 << \
  43                                         I40E_HASH_HDR_INNER_SHIFT)
  44 #define I40E_HASH_HDR_IPV6_INNER        (I40E_HASH_HDR_IPV6 << \
  45                                         I40E_HASH_HDR_INNER_SHIFT)
  46
  47 /* ETH */
  48 #define I40E_PHINT_ETH                  I40E_HASH_HDR_ETH
  49
  50 /* IPv4 */
  51 #define I40E_PHINT_IPV4                 (I40E_HASH_HDR_ETH | I40E_HASH_HDR_IPV4)
  52 #define I40E_PHINT_IPV4_TCP             (I40E_PHINT_IPV4 | I40E_HASH_HDR_TCP)
  53 #define I40E_PHINT_IPV4_UDP             (I40E_PHINT_IPV4 | I40E_HASH_HDR_UDP)
  54 #define I40E_PHINT_IPV4_SCTP            (I40E_PHINT_IPV4 | I40E_HASH_HDR_SCTP)
  55
  56 /* IPv6 */
  57 #define I40E_PHINT_IPV6                 (I40E_HASH_HDR_ETH | I40E_HASH_HDR_IPV6)
  58 #define I40E_PHINT_IPV6_FRAG            (I40E_PHINT_IPV6 | \
  59                                          I40E_HASH_HDR_IPV6_FRAG)
  60 #define I40E_PHINT_IPV6_TCP             (I40E_PHINT_IPV6 | I40E_HASH_HDR_TCP)
  61 #define I40E_PHINT_IPV6_UDP             (I40E_PHINT_IPV6 | I40E_HASH_HDR_UDP)
  62 #define I40E_PHINT_IPV6_SCTP            (I40E_PHINT_IPV6 | I40E_HASH_HDR_SCTP)
  63
  64 /* ESP */
  65 #define I40E_PHINT_IPV4_ESP             (I40E_PHINT_IPV4 | I40E_HASH_HDR_ESP)
  66 #define I40E_PHINT_IPV6_ESP             (I40E_PHINT_IPV6 | I40E_HASH_HDR_ESP)
  67 #define I40E_PHINT_IPV4_UDP_ESP         (I40E_PHINT_IPV4_UDP | \
  68                                         I40E_HASH_HDR_ESP)
  69 #define I40E_PHINT_IPV6_UDP_ESP         (I40E_PHINT_IPV6_UDP | \
  70                                         I40E_HASH_HDR_ESP)
  71
  72 /* GTPC */
  73 #define I40E_PHINT_IPV4_GTPC            (I40E_PHINT_IPV4_UDP | \
  74                                         I40E_HASH_HDR_GTPC)
  75 #define I40E_PHINT_IPV6_GTPC            (I40E_PHINT_IPV6_UDP | \
  76                                         I40E_HASH_HDR_GTPC)
  77
  78 /* GTPU */
  79 #define I40E_PHINT_IPV4_GTPU            (I40E_PHINT_IPV4_UDP | \
  80                                         I40E_HASH_HDR_GTPU)
  81 #define I40E_PHINT_IPV4_GTPU_IPV4       (I40E_PHINT_IPV4_GTPU | \
  82                                         I40E_HASH_HDR_IPV4_INNER)
  83 #define I40E_PHINT_IPV4_GTPU_IPV6       (I40E_PHINT_IPV4_GTPU | \
  84                                         I40E_HASH_HDR_IPV6_INNER)
  85 #define I40E_PHINT_IPV6_GTPU            (I40E_PHINT_IPV6_UDP | \
  86                                         I40E_HASH_HDR_GTPU)
  87 #define I40E_PHINT_IPV6_GTPU_IPV4       (I40E_PHINT_IPV6_GTPU | \
  88                                         I40E_HASH_HDR_IPV4_INNER)
  89 #define I40E_PHINT_IPV6_GTPU_IPV6       (I40E_PHINT_IPV6_GTPU | \
  90                                         I40E_HASH_HDR_IPV6_INNER)
  91
  92 /* L2TPV3 */
  93 #define I40E_PHINT_IPV4_L2TPV3          (I40E_PHINT_IPV4 | I40E_HASH_HDR_L2TPV3)
  94 #define I40E_PHINT_IPV6_L2TPV3          (I40E_PHINT_IPV6 | I40E_HASH_HDR_L2TPV3)
  95
  96 /* AH */
  97 #define I40E_PHINT_IPV4_AH              (I40E_PHINT_IPV4 | I40E_HASH_HDR_AH)
  98 #define I40E_PHINT_IPV6_AH              (I40E_PHINT_IPV6 | I40E_HASH_HDR_AH)
  99
 100 /* Structure of mapping RSS type to input set */
 101 struct i40e_hash_map_rss_inset {
 102         uint64_t rss_type;
 103         uint64_t inset;
 104 };
 105
 106 const struct i40e_hash_map_rss_inset i40e_hash_rss_inset[] = {
 107         /* IPv4 */
 108         { ETH_RSS_IPV4, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST },
 109         { ETH_RSS_FRAG_IPV4, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST },
 110
 111         { ETH_RSS_NONFRAG_IPV4_OTHER,
 112           I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST },
 113
 114         { ETH_RSS_NONFRAG_IPV4_TCP, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
 115           I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT },
 116
 117         { ETH_RSS_NONFRAG_IPV4_UDP, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
 118           I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT },
 119
 120         { ETH_RSS_NONFRAG_IPV4_SCTP, I40E_INSET_IPV4_SRC | I40E_INSET_IPV4_DST |
 121           I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | I40E_INSET_SCTP_VT },
 122
 123         /* IPv6 */
 124         { ETH_RSS_IPV6, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST },
 125         { ETH_RSS_FRAG_IPV6, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST },
 126
 127         { ETH_RSS_NONFRAG_IPV6_OTHER,
 128           I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST },
 129
 130         { ETH_RSS_NONFRAG_IPV6_TCP, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
 131           I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT },
 132
 133         { ETH_RSS_NONFRAG_IPV6_UDP, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
 134           I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT },
 135
 136         { ETH_RSS_NONFRAG_IPV6_SCTP, I40E_INSET_IPV6_SRC | I40E_INSET_IPV6_DST |
 137           I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT | I40E_INSET_SCTP_VT },
 138
 139         /* Port */
 140         { ETH_RSS_PORT, I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT },
 141
 142         /* Ether */
 143         { ETH_RSS_L2_PAYLOAD, I40E_INSET_LAST_ETHER_TYPE },
 144         { ETH_RSS_ETH, I40E_INSET_DMAC | I40E_INSET_SMAC },
 145
 146         /* VLAN */
 147         { ETH_RSS_S_VLAN, I40E_INSET_VLAN_OUTER },
 148         { ETH_RSS_C_VLAN, I40E_INSET_VLAN_INNER },
 149 };
 150
 151 #define I40E_HASH_VOID_NEXT_ALLOW       BIT_ULL(RTE_FLOW_ITEM_TYPE_ETH)
 152
 153 #define I40E_HASH_ETH_NEXT_ALLOW        (BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV4) | \
 154                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV6) | \
 155                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_VLAN))
 156
 157 #define I40E_HASH_IP_NEXT_ALLOW         (BIT_ULL(RTE_FLOW_ITEM_TYPE_TCP) | \
 158                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_UDP) | \
 159                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_SCTP) | \
 160                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_ESP) | \
 161                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_L2TPV3OIP) |\
 162                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_AH))
 163
 164 #define I40E_HASH_IPV6_NEXT_ALLOW       (I40E_HASH_IP_NEXT_ALLOW | \
 165                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT))
 166
 167 #define I40E_HASH_UDP_NEXT_ALLOW        (BIT_ULL(RTE_FLOW_ITEM_TYPE_GTPU) | \
 168                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_GTPC))
 169
 170 #define I40E_HASH_GTPU_NEXT_ALLOW       (BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV4) | \
 171                                         BIT_ULL(RTE_FLOW_ITEM_TYPE_IPV6))
 172
 173 static const uint64_t pattern_next_allow_items[] = {
 174         [RTE_FLOW_ITEM_TYPE_VOID] = I40E_HASH_VOID_NEXT_ALLOW,
 175         [RTE_FLOW_ITEM_TYPE_ETH] = I40E_HASH_ETH_NEXT_ALLOW,
 176         [RTE_FLOW_ITEM_TYPE_IPV4] = I40E_HASH_IP_NEXT_ALLOW,
 177         [RTE_FLOW_ITEM_TYPE_IPV6] = I40E_HASH_IPV6_NEXT_ALLOW,
 178         [RTE_FLOW_ITEM_TYPE_UDP] = I40E_HASH_UDP_NEXT_ALLOW,
 179         [RTE_FLOW_ITEM_TYPE_GTPU] = I40E_HASH_GTPU_NEXT_ALLOW,
 180 };
 181
 182 static const uint64_t pattern_item_header[] = {
 183         [RTE_FLOW_ITEM_TYPE_ETH] = I40E_HASH_HDR_ETH,
 184         [RTE_FLOW_ITEM_TYPE_IPV4] = I40E_HASH_HDR_IPV4,
 185         [RTE_FLOW_ITEM_TYPE_IPV6] = I40E_HASH_HDR_IPV6,
 186         [RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT] = I40E_HASH_HDR_IPV6_FRAG,
 187         [RTE_FLOW_ITEM_TYPE_TCP] = I40E_HASH_HDR_TCP,
 188         [RTE_FLOW_ITEM_TYPE_UDP] = I40E_HASH_HDR_UDP,
 189         [RTE_FLOW_ITEM_TYPE_SCTP] = I40E_HASH_HDR_SCTP,
 190         [RTE_FLOW_ITEM_TYPE_ESP] = I40E_HASH_HDR_ESP,
 191         [RTE_FLOW_ITEM_TYPE_GTPC] = I40E_HASH_HDR_GTPC,
 192         [RTE_FLOW_ITEM_TYPE_GTPU] = I40E_HASH_HDR_GTPU,
 193         [RTE_FLOW_ITEM_TYPE_L2TPV3OIP] = I40E_HASH_HDR_L2TPV3,
 194         [RTE_FLOW_ITEM_TYPE_AH] = I40E_HASH_HDR_AH,
 195 };
 196
 197 /* Structure of matched pattern */
 198 struct i40e_hash_match_pattern {
 199         uint64_t pattern_type;
 200         uint64_t rss_mask;      /* Supported RSS type for this pattern */
 201         bool custom_pctype_flag;/* true for custom packet type */
 202         uint8_t pctype;
 203 };
 204
 205 #define I40E_HASH_MAP_PATTERN(pattern, rss_mask, pctype) { \
 206         pattern, rss_mask, false, pctype  }
 207
 208 #define I40E_HASH_MAP_CUS_PATTERN(pattern, rss_mask, cus_pctype) { \
 209         pattern, rss_mask, true, cus_pctype }
 210
 211 #define I40E_HASH_L2_RSS_MASK           (ETH_RSS_VLAN | ETH_RSS_ETH | \
 212                                         ETH_RSS_L2_SRC_ONLY | \
 213                                         ETH_RSS_L2_DST_ONLY)
 214
 215 #define I40E_HASH_L23_RSS_MASK          (I40E_HASH_L2_RSS_MASK | \
 216                                         ETH_RSS_L3_SRC_ONLY | \
 217                                         ETH_RSS_L3_DST_ONLY)
 218
 219 #define I40E_HASH_IPV4_L23_RSS_MASK     (ETH_RSS_IPV4 | I40E_HASH_L23_RSS_MASK)
 220 #define I40E_HASH_IPV6_L23_RSS_MASK     (ETH_RSS_IPV6 | I40E_HASH_L23_RSS_MASK)
 221
 222 #define I40E_HASH_L234_RSS_MASK         (I40E_HASH_L23_RSS_MASK | \
 223                                         ETH_RSS_PORT | ETH_RSS_L4_SRC_ONLY | \
 224                                         ETH_RSS_L4_DST_ONLY)
 225
 226 #define I40E_HASH_IPV4_L234_RSS_MASK    (I40E_HASH_L234_RSS_MASK | ETH_RSS_IPV4)
 227 #define I40E_HASH_IPV6_L234_RSS_MASK    (I40E_HASH_L234_RSS_MASK | ETH_RSS_IPV6)
 228
 229 #define I40E_HASH_L4_TYPES              (ETH_RSS_NONFRAG_IPV4_TCP | \
 230                                         ETH_RSS_NONFRAG_IPV4_UDP | \
 231                                         ETH_RSS_NONFRAG_IPV4_SCTP | \
 232                                         ETH_RSS_NONFRAG_IPV6_TCP | \
 233                                         ETH_RSS_NONFRAG_IPV6_UDP | \
 234                                         ETH_RSS_NONFRAG_IPV6_SCTP)
 235
 236 /* Current supported patterns and RSS types.
 237  * All items that have the same pattern types are together.
 238  */
 239 static const struct i40e_hash_match_pattern match_patterns[] = {
 240         /* Ether */
 241         I40E_HASH_MAP_PATTERN(I40E_PHINT_ETH,
 242                               ETH_RSS_L2_PAYLOAD | I40E_HASH_L2_RSS_MASK,
 243                               I40E_FILTER_PCTYPE_L2_PAYLOAD),
 244
 245         /* IPv4 */
 246         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4,
 247                               ETH_RSS_FRAG_IPV4 | I40E_HASH_IPV4_L23_RSS_MASK,
 248                               I40E_FILTER_PCTYPE_FRAG_IPV4),
 249
 250         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4,
 251                               ETH_RSS_NONFRAG_IPV4_OTHER |
 252                               I40E_HASH_IPV4_L23_RSS_MASK,
 253                               I40E_FILTER_PCTYPE_NONF_IPV4_OTHER),
 254
 255         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4_TCP,
 256                               ETH_RSS_NONFRAG_IPV4_TCP |
 257                               I40E_HASH_IPV4_L234_RSS_MASK,
 258                               I40E_FILTER_PCTYPE_NONF_IPV4_TCP),
 259
 260         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4_UDP,
 261                               ETH_RSS_NONFRAG_IPV4_UDP |
 262                               I40E_HASH_IPV4_L234_RSS_MASK,
 263                               I40E_FILTER_PCTYPE_NONF_IPV4_UDP),
 264
 265         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV4_SCTP,
 266                               ETH_RSS_NONFRAG_IPV4_SCTP |
 267                               I40E_HASH_IPV4_L234_RSS_MASK,
 268                               I40E_FILTER_PCTYPE_NONF_IPV4_SCTP),
 269
 270         /* IPv6 */
 271         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6,
 272                               ETH_RSS_FRAG_IPV6 | I40E_HASH_IPV6_L23_RSS_MASK,
 273                               I40E_FILTER_PCTYPE_FRAG_IPV6),
 274
 275         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6,
 276                               ETH_RSS_NONFRAG_IPV6_OTHER |
 277                               I40E_HASH_IPV6_L23_RSS_MASK,
 278                               I40E_FILTER_PCTYPE_NONF_IPV6_OTHER),
 279
 280         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6_FRAG,
 281                               ETH_RSS_FRAG_IPV6 | I40E_HASH_L23_RSS_MASK,
 282                               I40E_FILTER_PCTYPE_FRAG_IPV6),
 283
 284         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6_TCP,
 285                               ETH_RSS_NONFRAG_IPV6_TCP |
 286                               I40E_HASH_IPV6_L234_RSS_MASK,
 287                               I40E_FILTER_PCTYPE_NONF_IPV6_TCP),
 288
 289         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6_UDP,
 290                               ETH_RSS_NONFRAG_IPV6_UDP |
 291                               I40E_HASH_IPV6_L234_RSS_MASK,
 292                               I40E_FILTER_PCTYPE_NONF_IPV6_UDP),
 293
 294         I40E_HASH_MAP_PATTERN(I40E_PHINT_IPV6_SCTP,
 295                               ETH_RSS_NONFRAG_IPV6_SCTP |
 296                               I40E_HASH_IPV6_L234_RSS_MASK,
 297                               I40E_FILTER_PCTYPE_NONF_IPV6_SCTP),
 298
 299         /* ESP */
 300         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_ESP,
 301                                   ETH_RSS_ESP, I40E_CUSTOMIZED_ESP_IPV4),
 302         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_ESP,
 303                                   ETH_RSS_ESP, I40E_CUSTOMIZED_ESP_IPV6),
 304         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_UDP_ESP,
 305                                   ETH_RSS_ESP, I40E_CUSTOMIZED_ESP_IPV4_UDP),
 306         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_UDP_ESP,
 307                                   ETH_RSS_ESP, I40E_CUSTOMIZED_ESP_IPV6_UDP),
 308
 309         /* GTPC */
 310         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_GTPC,
 311                                   I40E_HASH_IPV4_L234_RSS_MASK,
 312                                   I40E_CUSTOMIZED_GTPC),
 313         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_GTPC,
 314                                   I40E_HASH_IPV6_L234_RSS_MASK,
 315                                   I40E_CUSTOMIZED_GTPC),
 316
 317         /* GTPU */
 318         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_GTPU,
 319                                   I40E_HASH_IPV4_L234_RSS_MASK,
 320                                   I40E_CUSTOMIZED_GTPU),
 321         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_GTPU_IPV4,
 322                                   ETH_RSS_GTPU, I40E_CUSTOMIZED_GTPU_IPV4),
 323         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_GTPU_IPV6,
 324                                   ETH_RSS_GTPU, I40E_CUSTOMIZED_GTPU_IPV6),
 325         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_GTPU,
 326                                   I40E_HASH_IPV6_L234_RSS_MASK,
 327                                   I40E_CUSTOMIZED_GTPU),
 328         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_GTPU_IPV4,
 329                                   ETH_RSS_GTPU, I40E_CUSTOMIZED_GTPU_IPV4),
 330         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_GTPU_IPV6,
 331                                   ETH_RSS_GTPU, I40E_CUSTOMIZED_GTPU_IPV6),
 332
 333         /* L2TPV3 */
 334         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_L2TPV3,
 335                                   ETH_RSS_L2TPV3, I40E_CUSTOMIZED_IPV4_L2TPV3),
 336         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_L2TPV3,
 337                                   ETH_RSS_L2TPV3, I40E_CUSTOMIZED_IPV6_L2TPV3),
 338
 339         /* AH */
 340         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV4_AH, ETH_RSS_AH,
 341                                   I40E_CUSTOMIZED_AH_IPV4),
 342         I40E_HASH_MAP_CUS_PATTERN(I40E_PHINT_IPV6_AH, ETH_RSS_AH,
 343                                   I40E_CUSTOMIZED_AH_IPV6),
 344 };
 345
 346 static int
 347 i40e_hash_get_pattern_type(const struct rte_flow_item pattern[],
 348                            uint64_t *pattern_types,
 349                            struct rte_flow_error *error)
 350 {
 351         const char *message = "Pattern not supported";
 352         enum rte_flow_item_type prev_item_type = RTE_FLOW_ITEM_TYPE_VOID;
 353         enum rte_flow_item_type last_item_type = prev_item_type;
 354         uint64_t item_hdr, pattern_hdrs = 0;
 355         bool inner_flag = false;
 356         int vlan_count = 0;
 357
 358         for (; pattern->type != RTE_FLOW_ITEM_TYPE_END; pattern++) {
 359                 if (pattern->type == RTE_FLOW_ITEM_TYPE_VOID)
 360                         continue;
 361
 362                 if (pattern->mask || pattern->spec || pattern->last) {
 363                         message = "Header info should not be specified";
 364                         goto not_sup;
 365                 }
 366
 367                 /* Check the previous item allows this sub-item. */
 368                 if (prev_item_type >= (enum rte_flow_item_type)
 369                                 RTE_DIM(pattern_next_allow_items) ||
 370                     !(pattern_next_allow_items[prev_item_type] &
 371                                 BIT_ULL(pattern->type)))
 372                         goto not_sup;
 373
 374                 /* For VLAN item, it does no matter about to pattern type
 375                  * recognition. So just count the number of VLAN and do not
 376                  * change the value of variable `prev_item_type`.
 377                  */
 378                 last_item_type = pattern->type;
 379                 if (last_item_type == RTE_FLOW_ITEM_TYPE_VLAN) {
 380                         if (vlan_count >= 2)
 381                                 goto not_sup;
 382                         vlan_count++;
 383                         continue;
 384                 }
 385
 386                 prev_item_type = last_item_type;
 387                 assert(last_item_type < (enum rte_flow_item_type)
 388                                 RTE_DIM(pattern_item_header));
 389                 item_hdr = pattern_item_header[last_item_type];
 390                 assert(item_hdr);
 391
 392                 if (inner_flag) {
 393                         item_hdr <<= I40E_HASH_HDR_INNER_SHIFT;
 394
 395                         /* Inner layer should not have GTPU item */
 396                         if (last_item_type == RTE_FLOW_ITEM_TYPE_GTPU)
 397                                 goto not_sup;
 398                 } else {
 399                         if (last_item_type == RTE_FLOW_ITEM_TYPE_GTPU) {
 400                                 inner_flag = true;
 401                                 vlan_count = 0;
 402                         }
 403                 }
 404
 405                 if (item_hdr & pattern_hdrs)
 406                         goto not_sup;
 407
 408                 pattern_hdrs |= item_hdr;
 409         }
 410
 411         if (pattern_hdrs && last_item_type != RTE_FLOW_ITEM_TYPE_VLAN) {
 412                 *pattern_types = pattern_hdrs;
 413                 return 0;
 414         }
 415
 416 not_sup:
 417         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 418                                   pattern, message);
 419 }
 420
 421 static uint64_t
 422 i40e_hash_get_x722_ext_pctypes(uint8_t match_pctype)
 423 {
 424         uint64_t pctypes = 0;
 425
 426         switch (match_pctype) {
 427         case I40E_FILTER_PCTYPE_NONF_IPV4_TCP:
 428                 pctypes = BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK);
 429                 break;
 430
 431         case I40E_FILTER_PCTYPE_NONF_IPV4_UDP:
 432                 pctypes = BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) |
 433                           BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP);
 434                 break;
 435
 436         case I40E_FILTER_PCTYPE_NONF_IPV6_TCP:
 437                 pctypes = BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK);
 438                 break;
 439
 440         case I40E_FILTER_PCTYPE_NONF_IPV6_UDP:
 441                 pctypes = BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) |
 442                           BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP);
 443                 break;
 444         }
 445
 446         return pctypes;
 447 }
 448
 449 static int
 450 i40e_hash_translate_gtp_inset(struct i40e_rte_flow_rss_conf *rss_conf,
 451                               struct rte_flow_error *error)
 452 {
 453         if (rss_conf->inset &
 454             (I40E_INSET_IPV4_SRC | I40E_INSET_IPV6_SRC |
 455             I40E_INSET_DST_PORT | I40E_INSET_SRC_PORT))
 456                 return rte_flow_error_set(error, ENOTSUP,
 457                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 458                                           NULL,
 459                                           "Only support external destination IP");
 460
 461         if (rss_conf->inset & I40E_INSET_IPV4_DST)
 462                 rss_conf->inset = (rss_conf->inset & ~I40E_INSET_IPV4_DST) |
 463                                   I40E_INSET_TUNNEL_IPV4_DST;
 464
 465         if (rss_conf->inset & I40E_INSET_IPV6_DST)
 466                 rss_conf->inset = (rss_conf->inset & ~I40E_INSET_IPV6_DST) |
 467                                   I40E_INSET_TUNNEL_IPV6_DST;
 468
 469         return 0;
 470 }
 471
 472 static int
 473 i40e_hash_get_pctypes(const struct rte_eth_dev *dev,
 474                       const struct i40e_hash_match_pattern *match,
 475                       struct i40e_rte_flow_rss_conf *rss_conf,
 476                       struct rte_flow_error *error)
 477 {
 478         if (match->custom_pctype_flag) {
 479                 struct i40e_pf *pf;
 480                 struct i40e_customized_pctype *custom_type;
 481
 482                 pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 483                 custom_type = i40e_find_customized_pctype(pf, match->pctype);
 484                 if (!custom_type || !custom_type->valid)
 485                         return rte_flow_error_set(error, ENOTSUP,
 486                                                   RTE_FLOW_ERROR_TYPE_ITEM,
 487                                                   NULL, "PCTYPE not supported");
 488
 489                 rss_conf->config_pctypes |= BIT_ULL(custom_type->pctype);
 490
 491                 if (match->pctype == I40E_CUSTOMIZED_GTPU ||
 492                     match->pctype == I40E_CUSTOMIZED_GTPC)
 493                         return i40e_hash_translate_gtp_inset(rss_conf, error);
 494         } else {
 495                 struct i40e_hw *hw =
 496                                 I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 497                 uint64_t types;
 498
 499                 rss_conf->config_pctypes |= BIT_ULL(match->pctype);
 500                 if (hw->mac.type == I40E_MAC_X722) {
 501                         types = i40e_hash_get_x722_ext_pctypes(match->pctype);
 502                         rss_conf->config_pctypes |= types;
 503                 }
 504         }
 505
 506         return 0;
 507 }
 508
 509 static int
 510 i40e_hash_get_pattern_pctypes(const struct rte_eth_dev *dev,
 511                               const struct rte_flow_item pattern[],
 512                               const struct rte_flow_action_rss *rss_act,
 513                               struct i40e_rte_flow_rss_conf *rss_conf,
 514                               struct rte_flow_error *error)
 515 {
 516         uint64_t pattern_types = 0;
 517         bool match_flag = false;
 518         int i, ret;
 519
 520         ret = i40e_hash_get_pattern_type(pattern, &pattern_types, error);
 521         if (ret)
 522                 return ret;
 523
 524         for (i = 0; i < (int)RTE_DIM(match_patterns); i++) {
 525                 const struct i40e_hash_match_pattern *match =
 526                                                         &match_patterns[i];
 527
 528                 /* Check pattern types match. All items that have the same
 529                  * pattern types are together, so if the pattern types match
 530                  * previous item but they doesn't match current item, it means
 531                  * the pattern types do not match all remain items.
 532                  */
 533                 if (pattern_types != match->pattern_type) {
 534                         if (match_flag)
 535                                 break;
 536                         continue;
 537                 }
 538                 match_flag = true;
 539
 540                 /* Check RSS types match */
 541                 if (!(rss_act->types & ~match->rss_mask)) {
 542                         ret = i40e_hash_get_pctypes(dev, match,
 543                                                     rss_conf, error);
 544                         if (ret)
 545                                 return ret;
 546                 }
 547         }
 548
 549         if (rss_conf->config_pctypes)
 550                 return 0;
 551
 552         if (match_flag)
 553                 return rte_flow_error_set(error, ENOTSUP,
 554                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 555                                           NULL, "RSS types not supported");
 556
 557         return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
 558                                   NULL, "Pattern not supported");
 559 }
 560
 561 static uint64_t
 562 i40e_hash_get_inset(uint64_t rss_types)
 563 {
 564         uint64_t mask, inset = 0;
 565         int i;
 566
 567         for (i = 0; i < (int)RTE_DIM(i40e_hash_rss_inset); i++) {
 568                 if (rss_types & i40e_hash_rss_inset[i].rss_type)
 569                         inset |= i40e_hash_rss_inset[i].inset;
 570         }
 571
 572         if (!inset)
 573                 return 0;
 574
 575         /* If SRC_ONLY and DST_ONLY of the same level are used simultaneously,
 576          * it is the same case as none of them are added.
 577          */
 578         mask = rss_types & (ETH_RSS_L2_SRC_ONLY | ETH_RSS_L2_DST_ONLY);
 579         if (mask == ETH_RSS_L2_SRC_ONLY)
 580                 inset &= ~I40E_INSET_DMAC;
 581         else if (mask == ETH_RSS_L2_DST_ONLY)
 582                 inset &= ~I40E_INSET_SMAC;
 583
 584         mask = rss_types & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY);
 585         if (mask == ETH_RSS_L3_SRC_ONLY)
 586                 inset &= ~(I40E_INSET_IPV4_DST | I40E_INSET_IPV6_DST);
 587         else if (mask == ETH_RSS_L3_DST_ONLY)
 588                 inset &= ~(I40E_INSET_IPV4_SRC | I40E_INSET_IPV6_SRC);
 589
 590         mask = rss_types & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY);
 591         if (mask == ETH_RSS_L4_SRC_ONLY)
 592                 inset &= ~I40E_INSET_DST_PORT;
 593         else if (mask == ETH_RSS_L4_DST_ONLY)
 594                 inset &= ~I40E_INSET_SRC_PORT;
 595
 596         if (rss_types & I40E_HASH_L4_TYPES) {
 597                 uint64_t l3_mask = rss_types &
 598                                    (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY);
 599                 uint64_t l4_mask = rss_types &
 600                                    (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY);
 601
 602                 if (l3_mask && !l4_mask)
 603                         inset &= ~(I40E_INSET_SRC_PORT | I40E_INSET_DST_PORT);
 604                 else if (!l3_mask && l4_mask)
 605                         inset &= ~(I40E_INSET_IPV4_DST | I40E_INSET_IPV6_DST |
 606                                  I40E_INSET_IPV4_SRC | I40E_INSET_IPV6_SRC);
 607         }
 608
 609         return inset;
 610 }
 611
 612 static int
 613 i40e_hash_config_func(struct i40e_hw *hw, enum rte_eth_hash_function func)
 614 {
 615         struct i40e_pf *pf;
 616         uint32_t reg;
 617         uint8_t symmetric = 0;
 618
 619         reg = i40e_read_rx_ctl(hw, I40E_GLQF_CTL);
 620
 621         if (func == RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) {
 622                 if (!(reg & I40E_GLQF_CTL_HTOEP_MASK))
 623                         goto set_symmetric;
 624
 625                 reg &= ~I40E_GLQF_CTL_HTOEP_MASK;
 626         } else {
 627                 if (func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ)
 628                         symmetric = 1;
 629
 630                 if (reg & I40E_GLQF_CTL_HTOEP_MASK)
 631                         goto set_symmetric;
 632
 633                 reg |= I40E_GLQF_CTL_HTOEP_MASK;
 634         }
 635
 636         pf = &((struct i40e_adapter *)hw->back)->pf;
 637         if (pf->support_multi_driver) {
 638                 PMD_DRV_LOG(ERR,
 639                             "Modify hash function is not permitted when multi-driver enabled");
 640                 return -EPERM;
 641         }
 642
 643         PMD_DRV_LOG(INFO, "NIC hash function is setting to %d", func);
 644         i40e_write_rx_ctl(hw, I40E_GLQF_CTL, reg);
 645         I40E_WRITE_FLUSH(hw);
 646
 647 set_symmetric:
 648         i40e_set_symmetric_hash_enable_per_port(hw, symmetric);
 649         return 0;
 650 }
 651
 652 static int
 653 i40e_hash_config_pctype_symmetric(struct i40e_hw *hw,
 654                                   uint32_t pctype,
 655                                   bool symmetric)
 656 {
 657         struct i40e_pf *pf = &((struct i40e_adapter *)hw->back)->pf;
 658         uint32_t reg;
 659
 660         /* For X722, get translated pctype in fd pctype register */
 661         if (hw->mac.type == I40E_MAC_X722)
 662                 pctype = i40e_read_rx_ctl(hw, I40E_GLQF_FD_PCTYPES(pctype));
 663
 664         reg = i40e_read_rx_ctl(hw, I40E_GLQF_HSYM(pctype));
 665         if (symmetric) {
 666                 if (reg & I40E_GLQF_HSYM_SYMH_ENA_MASK)
 667                         return 0;
 668                 reg |= I40E_GLQF_HSYM_SYMH_ENA_MASK;
 669         } else {
 670                 if (!(reg & I40E_GLQF_HSYM_SYMH_ENA_MASK))
 671                         return 0;
 672                 reg &= ~I40E_GLQF_HSYM_SYMH_ENA_MASK;
 673         }
 674
 675         if (pf->support_multi_driver) {
 676                 PMD_DRV_LOG(ERR,
 677                             "Enable/Disable symmetric hash is not permitted when multi-driver enabled");
 678                 return -EPERM;
 679         }
 680
 681         i40e_write_rx_ctl(hw, I40E_GLQF_HSYM(pctype), reg);
 682         I40E_WRITE_FLUSH(hw);
 683         return 0;
 684 }
 685
 686 static void
 687 i40e_hash_enable_pctype(struct i40e_hw *hw,
 688                         uint32_t pctype, bool enable)
 689 {
 690         uint32_t reg, reg_val, mask;
 691
 692         if (pctype < 32) {
 693                 mask = BIT(pctype);
 694                 reg = I40E_PFQF_HENA(0);
 695         } else {
 696                 mask = BIT(pctype - 32);
 697                 reg = I40E_PFQF_HENA(1);
 698         }
 699
 700         reg_val = i40e_read_rx_ctl(hw, reg);
 701
 702         if (enable) {
 703                 if (reg_val & mask)
 704                         return;
 705
 706                 reg_val |= mask;
 707         } else {
 708                 if (!(reg_val & mask))
 709                         return;
 710
 711                 reg_val &= ~mask;
 712         }
 713
 714         i40e_write_rx_ctl(hw, reg, reg_val);
 715         I40E_WRITE_FLUSH(hw);
 716 }
 717
 718 static int
 719 i40e_hash_config_pctype(struct i40e_hw *hw,
 720                         struct i40e_rte_flow_rss_conf *rss_conf,
 721                         uint32_t pctype)
 722 {
 723         uint64_t rss_types = rss_conf->conf.types;
 724         int ret;
 725
 726         if (rss_types == 0) {
 727                 i40e_hash_enable_pctype(hw, pctype, false);
 728                 return 0;
 729         }
 730
 731         if (rss_conf->inset) {
 732                 ret = i40e_set_hash_inset(hw, rss_conf->inset, pctype, false);
 733                 if (ret)
 734                         return ret;
 735         }
 736
 737         i40e_hash_enable_pctype(hw, pctype, true);
 738         return 0;
 739 }
 740
 741 static int
 742 i40e_hash_config_region(struct i40e_pf *pf,
 743                         const struct i40e_rte_flow_rss_conf *rss_conf)
 744 {
 745         struct i40e_hw *hw = &pf->adapter->hw;
 746         struct rte_eth_dev *dev = &rte_eth_devices[pf->dev_data->port_id];
 747         struct i40e_queue_region_info *regions = pf->queue_region.region;
 748         uint32_t num = pf->queue_region.queue_region_number;
 749         uint32_t i, region_id_mask = 0;
 750
 751         /* Use a 32 bit variable to represent all regions */
 752         RTE_BUILD_BUG_ON(I40E_REGION_MAX_INDEX > 31);
 753
 754         /* Re-configure the region if it existed */
 755         for (i = 0; i < num; i++) {
 756                 if (rss_conf->region_queue_start ==
 757                     regions[i].queue_start_index &&
 758                     rss_conf->region_queue_num == regions[i].queue_num) {
 759                         uint32_t j;
 760
 761                         for (j = 0; j < regions[i].user_priority_num; j++) {
 762                                 if (regions[i].user_priority[j] ==
 763                                     rss_conf->region_priority)
 764                                         return 0;
 765                         }
 766
 767                         if (j >= I40E_MAX_USER_PRIORITY) {
 768                                 PMD_DRV_LOG(ERR,
 769                                             "Priority number exceed the maximum %d",
 770                                             I40E_MAX_USER_PRIORITY);
 771                                 return -ENOSPC;
 772                         }
 773
 774                         regions[i].user_priority[j] = rss_conf->region_priority;
 775                         regions[i].user_priority_num++;
 776                         return i40e_flush_queue_region_all_conf(dev, hw, pf, 1);
 777                 }
 778
 779                 region_id_mask |= BIT(regions[i].region_id);
 780         }
 781
 782         if (num > I40E_REGION_MAX_INDEX) {
 783                 PMD_DRV_LOG(ERR, "Queue region resource used up");
 784                 return -ENOSPC;
 785         }
 786
 787         /* Add a new region */
 788
 789         pf->queue_region.queue_region_number++;
 790         memset(&regions[num], 0, sizeof(regions[0]));
 791
 792         regions[num].region_id = rte_bsf32(~region_id_mask);
 793         regions[num].queue_num = rss_conf->region_queue_num;
 794         regions[num].queue_start_index = rss_conf->region_queue_start;
 795         regions[num].user_priority[0] = rss_conf->region_priority;
 796         regions[num].user_priority_num = 1;
 797
 798         return i40e_flush_queue_region_all_conf(dev, hw, pf, 1);
 799 }
 800
 801 static int
 802 i40e_hash_config(struct i40e_pf *pf,
 803                  struct i40e_rte_flow_rss_conf *rss_conf)
 804 {
 805         struct rte_flow_action_rss *rss_info = &rss_conf->conf;
 806         struct i40e_hw *hw = &pf->adapter->hw;
 807         uint64_t pctypes;
 808         int ret;
 809
 810         if (rss_info->func != RTE_ETH_HASH_FUNCTION_DEFAULT) {
 811                 ret = i40e_hash_config_func(hw, rss_info->func);
 812                 if (ret)
 813                         return ret;
 814
 815                 if (rss_info->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ)
 816                         rss_conf->misc_reset_flags |=
 817                                         I40E_HASH_FLOW_RESET_FLAG_FUNC;
 818         }
 819
 820         if (rss_conf->region_queue_num > 0) {
 821                 ret = i40e_hash_config_region(pf, rss_conf);
 822                 if (ret)
 823                         return ret;
 824
 825                 rss_conf->misc_reset_flags |= I40E_HASH_FLOW_RESET_FLAG_REGION;
 826         }
 827
 828         if (rss_info->key_len > 0) {
 829                 ret = i40e_set_rss_key(pf->main_vsi, rss_conf->key,
 830                                        rss_info->key_len);
 831                 if (ret)
 832                         return ret;
 833
 834                 rss_conf->misc_reset_flags |= I40E_HASH_FLOW_RESET_FLAG_KEY;
 835         }
 836
 837         /* Update lookup table */
 838         if (rss_info->queue_num > 0) {
 839                 uint8_t lut[ETH_RSS_RETA_SIZE_512];
 840                 uint32_t i, j = 0;
 841
 842                 for (i = 0; i < hw->func_caps.rss_table_size; i++) {
 843                         lut[i] = (uint8_t)rss_info->queue[j];
 844                         j = (j == rss_info->queue_num - 1) ? 0 : (j + 1);
 845                 }
 846
 847                 ret = i40e_set_rss_lut(pf->main_vsi, lut, (uint16_t)i);
 848                 if (ret)
 849                         return ret;
 850
 851                 pf->hash_enabled_queues = 0;
 852                 for (i = 0; i < rss_info->queue_num; i++)
 853                         pf->hash_enabled_queues |= BIT_ULL(lut[i]);
 854
 855                 pf->adapter->rss_reta_updated = 0;
 856                 rss_conf->misc_reset_flags |= I40E_HASH_FLOW_RESET_FLAG_QUEUE;
 857         }
 858
 859         /* The codes behind configure the input sets and symmetric hash
 860          * function of the packet types and enable hash on them.
 861          */
 862         pctypes = rss_conf->config_pctypes;
 863         if (!pctypes)
 864                 return 0;
 865
 866         /* For first flow that will enable hash on any packet type, we clean
 867          * the RSS sets that by legacy configuration commands and parameters.
 868          */
 869         if (!pf->hash_filter_enabled) {
 870                 i40e_pf_disable_rss(pf);
 871                 pf->hash_filter_enabled = true;
 872         }
 873
 874         do {
 875                 uint32_t idx = rte_bsf64(pctypes);
 876                 uint64_t bit = BIT_ULL(idx);
 877
 878                 if (rss_conf->symmetric_enable) {
 879                         ret = i40e_hash_config_pctype_symmetric(hw, idx, true);
 880                         if (ret)
 881                                 return ret;
 882
 883                         rss_conf->reset_symmetric_pctypes |= bit;
 884                 }
 885
 886                 ret = i40e_hash_config_pctype(hw, rss_conf, idx);
 887                 if (ret)
 888                         return ret;
 889
 890                 rss_conf->reset_config_pctypes |= bit;
 891                 pctypes &= ~bit;
 892         } while (pctypes);
 893
 894         return 0;
 895 }
 896
 897 static void
 898 i40e_hash_parse_key(const struct rte_flow_action_rss *rss_act,
 899                     struct i40e_rte_flow_rss_conf *rss_conf)
 900 {
 901         const uint8_t *key = rss_act->key;
 902
 903         if (!key || rss_act->key_len != sizeof(rss_conf->key)) {
 904                 const uint32_t rss_key_default[] = {0x6b793944,
 905                         0x23504cb5, 0x5bea75b6, 0x309f4f12, 0x3dc0a2b8,
 906                         0x024ddcdf, 0x339b8ca0, 0x4c4af64a, 0x34fac605,
 907                         0x55d85839, 0x3a58997d, 0x2ec938e1, 0x66031581};
 908
 909                 if (rss_act->key_len != sizeof(rss_conf->key))
 910                         PMD_DRV_LOG(WARNING,
 911                                     "RSS key length invalid, must be %u bytes, now set key to default",
 912                                     (uint32_t)sizeof(rss_conf->key));
 913
 914                 memcpy(rss_conf->key, rss_key_default, sizeof(rss_conf->key));
 915         } else {
 916                 memcpy(rss_conf->key, key, sizeof(rss_conf->key));
 917         }
 918
 919         rss_conf->conf.key = rss_conf->key;
 920         rss_conf->conf.key_len = sizeof(rss_conf->key);
 921 }
 922
 923 static int
 924 i40e_hash_parse_queues(const struct rte_eth_dev *dev,
 925                        const struct rte_flow_action_rss *rss_act,
 926                        struct i40e_rte_flow_rss_conf *rss_conf,
 927                        struct rte_flow_error *error)
 928 {
 929         struct i40e_pf *pf;
 930         struct i40e_hw *hw;
 931         uint16_t i;
 932         int max_queue;
 933
 934         hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 935         if (!rss_act->queue_num ||
 936             rss_act->queue_num > hw->func_caps.rss_table_size)
 937                 return rte_flow_error_set(error, EINVAL,
 938                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 939                                           NULL, "Invalid RSS queue number");
 940
 941         if (rss_act->key_len)
 942                 PMD_DRV_LOG(WARNING,
 943                             "RSS key is ignored when queues specified");
 944
 945         pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 946         if (pf->dev_data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 947                 max_queue = i40e_pf_calc_configured_queues_num(pf);
 948         else
 949                 max_queue = pf->dev_data->nb_rx_queues;
 950
 951         max_queue = RTE_MIN(max_queue, I40E_MAX_Q_PER_TC);
 952
 953         for (i = 0; i < rss_act->queue_num; i++) {
 954                 if ((int)rss_act->queue[i] >= max_queue)
 955                         break;
 956         }
 957
 958         if (i < rss_act->queue_num)
 959                 return rte_flow_error_set(error, EINVAL,
 960                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 961                                           NULL, "Invalid RSS queues");
 962
 963         memcpy(rss_conf->queue, rss_act->queue,
 964                rss_act->queue_num * sizeof(rss_conf->queue[0]));
 965         rss_conf->conf.queue = rss_conf->queue;
 966         rss_conf->conf.queue_num = rss_act->queue_num;
 967         return 0;
 968 }
 969
 970 static int
 971 i40e_hash_parse_queue_region(const struct rte_eth_dev *dev,
 972                              const struct rte_flow_item pattern[],
 973                              const struct rte_flow_action_rss *rss_act,
 974                              struct i40e_rte_flow_rss_conf *rss_conf,
 975                              struct rte_flow_error *error)
 976 {
 977         struct i40e_pf *pf;
 978         const struct rte_flow_item_vlan *vlan_spec, *vlan_mask;
 979         uint64_t hash_queues;
 980         uint32_t i;
 981
 982         if (pattern[1].type != RTE_FLOW_ITEM_TYPE_END)
 983                 return rte_flow_error_set(error, ENOTSUP,
 984                                           RTE_FLOW_ERROR_TYPE_ITEM_NUM,
 985                                           &pattern[1],
 986                                           "Pattern not supported.");
 987
 988         vlan_spec = pattern->spec;
 989         vlan_mask = pattern->mask;
 990         if (!vlan_spec || !vlan_mask ||
 991             (rte_be_to_cpu_16(vlan_mask->tci) >> 13) != 7)
 992                 return rte_flow_error_set(error, EINVAL,
 993                                           RTE_FLOW_ERROR_TYPE_ITEM, pattern,
 994                                           "Pattern error.");
 995
 996         if (!rss_act->queue)
 997                 return rte_flow_error_set(error, EINVAL,
 998                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
 999                                           NULL, "Queues not specified");
1000
1001         if (rss_act->key_len)
1002                 PMD_DRV_LOG(WARNING,
1003                             "RSS key is ignored when configure queue region");
1004
1005         /* Use a 64 bit variable to represent all queues in a region. */
1006         RTE_BUILD_BUG_ON(I40E_MAX_Q_PER_TC > 64);
1007
1008         if (!rss_act->queue_num ||
1009             !rte_is_power_of_2(rss_act->queue_num) ||
1010             rss_act->queue_num + rss_act->queue[0] > I40E_MAX_Q_PER_TC)
1011                 return rte_flow_error_set(error, EINVAL,
1012                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1013                                           NULL, "Queue number error");
1014
1015         for (i = 1; i < rss_act->queue_num; i++) {
1016                 if (rss_act->queue[i - 1] + 1 != rss_act->queue[i])
1017                         break;
1018         }
1019
1020         if (i < rss_act->queue_num)
1021                 return rte_flow_error_set(error, EINVAL,
1022                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1023                                           NULL,
1024                                           "Queues must be incremented continuously");
1025
1026         /* Map all queues to bits of uint64_t */
1027         hash_queues = (BIT_ULL(rss_act->queue[0] + rss_act->queue_num) - 1) &
1028                       ~(BIT_ULL(rss_act->queue[0]) - 1);
1029
1030         pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1031         if (hash_queues & ~pf->hash_enabled_queues)
1032                 return rte_flow_error_set(error, EINVAL,
1033                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1034                                           NULL, "Some queues are not in LUT");
1035
1036         rss_conf->region_queue_num = (uint8_t)rss_act->queue_num;
1037         rss_conf->region_queue_start = rss_act->queue[0];
1038         rss_conf->region_priority = rte_be_to_cpu_16(vlan_spec->tci) >> 13;
1039         return 0;
1040 }
1041
1042 static int
1043 i40e_hash_parse_global_conf(const struct rte_eth_dev *dev,
1044                             const struct rte_flow_item pattern[],
1045                             const struct rte_flow_action_rss *rss_act,
1046                             struct i40e_rte_flow_rss_conf *rss_conf,
1047                             struct rte_flow_error *error)
1048 {
1049         if (rss_act->func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ)
1050                 return rte_flow_error_set(error, EINVAL,
1051                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1052                                           NULL,
1053                                           "Symmetric function should be set with pattern types");
1054
1055         rss_conf->conf.func = rss_act->func;
1056
1057         if (rss_act->types)
1058                 PMD_DRV_LOG(WARNING,
1059                             "RSS types are ignored when no pattern specified");
1060
1061         if (pattern[0].type == RTE_FLOW_ITEM_TYPE_VLAN)
1062                 return i40e_hash_parse_queue_region(dev, pattern, rss_act,
1063                                                     rss_conf, error);
1064
1065         if (rss_act->queue)
1066                 return i40e_hash_parse_queues(dev, rss_act, rss_conf, error);
1067
1068         if (rss_act->key_len) {
1069                 i40e_hash_parse_key(rss_act, rss_conf);
1070                 return 0;
1071         }
1072
1073         if (rss_act->func == RTE_ETH_HASH_FUNCTION_DEFAULT)
1074                 PMD_DRV_LOG(WARNING, "Nothing change");
1075         return 0;
1076 }
1077
1078 static bool
1079 i40e_hash_validate_rss_types(uint64_t rss_types)
1080 {
1081         uint64_t type, mask;
1082
1083         /* Validate L2 */
1084         type = ETH_RSS_ETH & rss_types;
1085         mask = (ETH_RSS_L2_SRC_ONLY | ETH_RSS_L2_DST_ONLY) & rss_types;
1086         if (!type && mask)
1087                 return false;
1088
1089         /* Validate L3 */
1090         type = (I40E_HASH_L4_TYPES | ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
1091                ETH_RSS_NONFRAG_IPV4_OTHER | ETH_RSS_IPV6 |
1092                ETH_RSS_FRAG_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER) & rss_types;
1093         mask = (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY) & rss_types;
1094         if (!type && mask)
1095                 return false;
1096
1097         /* Validate L4 */
1098         type = (I40E_HASH_L4_TYPES | ETH_RSS_PORT) & rss_types;
1099         mask = (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY) & rss_types;
1100         if (!type && mask)
1101                 return false;
1102
1103         return true;
1104 }
1105
1106 static int
1107 i40e_hash_parse_pattern_act(const struct rte_eth_dev *dev,
1108                             const struct rte_flow_item pattern[],
1109                             const struct rte_flow_action_rss *rss_act,
1110                             struct i40e_rte_flow_rss_conf *rss_conf,
1111                             struct rte_flow_error *error)
1112 {
1113         if (rss_act->queue)
1114                 return rte_flow_error_set(error, EINVAL,
1115                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1116                                           NULL,
1117                                           "RSS Queues not supported when pattern specified");
1118
1119         switch (rss_act->func) {
1120         case RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ:
1121                 rss_conf->symmetric_enable = true;
1122                 break;
1123         case RTE_ETH_HASH_FUNCTION_DEFAULT:
1124         case RTE_ETH_HASH_FUNCTION_TOEPLITZ:
1125         case RTE_ETH_HASH_FUNCTION_SIMPLE_XOR:
1126                 break;
1127         default:
1128                 return rte_flow_error_set(error, EINVAL,
1129                                 RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1130                                 NULL,
1131                                 "RSS hash function not supported "
1132                                 "when pattern specified");
1133         }
1134
1135         if (!i40e_hash_validate_rss_types(rss_act->types))
1136                 return rte_flow_error_set(error, EINVAL,
1137                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1138                                           NULL, "RSS types are invalid");
1139
1140         if (rss_act->key_len)
1141                 i40e_hash_parse_key(rss_act, rss_conf);
1142
1143         rss_conf->conf.func = rss_act->func;
1144         rss_conf->conf.types = rss_act->types;
1145         rss_conf->inset = i40e_hash_get_inset(rss_act->types);
1146
1147         return i40e_hash_get_pattern_pctypes(dev, pattern, rss_act,
1148                                              rss_conf, error);
1149 }
1150
1151 int
1152 i40e_hash_parse(const struct rte_eth_dev *dev,
1153                 const struct rte_flow_item pattern[],
1154                 const struct rte_flow_action actions[],
1155                 struct i40e_rte_flow_rss_conf *rss_conf,
1156                 struct rte_flow_error *error)
1157 {
1158         const struct rte_flow_action_rss *rss_act;
1159
1160         if (actions[1].type != RTE_FLOW_ACTION_TYPE_END)
1161                 return rte_flow_error_set(error, EINVAL,
1162                                           RTE_FLOW_ERROR_TYPE_ACTION,
1163                                           &actions[1],
1164                                           "Only support one action for RSS.");
1165
1166         rss_act = (const struct rte_flow_action_rss *)actions[0].conf;
1167         if (rss_act->level)
1168                 return rte_flow_error_set(error, ENOTSUP,
1169                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1170                                           actions,
1171                                           "RSS level is not supported");
1172
1173         while (pattern->type == RTE_FLOW_ITEM_TYPE_VOID)
1174                 pattern++;
1175
1176         if (pattern[0].type == RTE_FLOW_ITEM_TYPE_END ||
1177             pattern[0].type == RTE_FLOW_ITEM_TYPE_VLAN)
1178                 return i40e_hash_parse_global_conf(dev, pattern, rss_act,
1179                                                    rss_conf, error);
1180
1181         return i40e_hash_parse_pattern_act(dev, pattern, rss_act,
1182                                            rss_conf, error);
1183 }
1184
1185 static void
1186 i40e_invalid_rss_filter(const struct i40e_rte_flow_rss_conf *ref_conf,
1187                         struct i40e_rte_flow_rss_conf *conf)
1188 {
1189         uint32_t reset_flags = conf->misc_reset_flags;
1190
1191         conf->misc_reset_flags &= ~ref_conf->misc_reset_flags;
1192
1193         if ((reset_flags & I40E_HASH_FLOW_RESET_FLAG_REGION) &&
1194             (ref_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_REGION) &&
1195             (conf->region_queue_start != ref_conf->region_queue_start ||
1196              conf->region_queue_num != ref_conf->region_queue_num))
1197                 conf->misc_reset_flags |= I40E_HASH_FLOW_RESET_FLAG_REGION;
1198
1199         conf->reset_config_pctypes &= ~ref_conf->reset_config_pctypes;
1200         conf->reset_symmetric_pctypes &= ~ref_conf->reset_symmetric_pctypes;
1201 }
1202
1203 int
1204 i40e_hash_filter_restore(struct i40e_pf *pf)
1205 {
1206         struct i40e_rss_filter *filter;
1207         int ret;
1208
1209         TAILQ_FOREACH(filter, &pf->rss_config_list, next) {
1210                 struct i40e_rte_flow_rss_conf *rss_conf =
1211                                                 &filter->rss_filter_info;
1212                 struct i40e_rss_filter *prev;
1213
1214                 rss_conf->misc_reset_flags = 0;
1215                 rss_conf->reset_config_pctypes = 0;
1216                 rss_conf->reset_symmetric_pctypes = 0;
1217
1218                 ret = i40e_hash_config(pf, rss_conf);
1219                 if (ret) {
1220                         pf->hash_filter_enabled = 0;
1221                         i40e_pf_disable_rss(pf);
1222                         PMD_DRV_LOG(ERR,
1223                                     "Re-configure RSS failed, RSS has been disabled");
1224                         return ret;
1225                 }
1226
1227                 /* Invalid previous RSS filter */
1228                 TAILQ_FOREACH(prev, &pf->rss_config_list, next) {
1229                         if (prev == filter)
1230                                 break;
1231                         i40e_invalid_rss_filter(rss_conf,
1232                                                 &prev->rss_filter_info);
1233                 }
1234         }
1235
1236         return 0;
1237 }
1238
1239 int
1240 i40e_hash_filter_create(struct i40e_pf *pf,
1241                         struct i40e_rte_flow_rss_conf *rss_conf)
1242 {
1243         struct i40e_rss_filter *filter, *prev;
1244         struct i40e_rte_flow_rss_conf *new_conf;
1245         int ret;
1246
1247         filter = rte_zmalloc("i40e_rss_filter", sizeof(*filter), 0);
1248         if (!filter) {
1249                 PMD_DRV_LOG(ERR, "Failed to allocate memory.");
1250                 return -ENOMEM;
1251         }
1252
1253         new_conf = &filter->rss_filter_info;
1254
1255         memcpy(new_conf, rss_conf, sizeof(*new_conf));
1256         if (new_conf->conf.queue_num)
1257                 new_conf->conf.queue = new_conf->queue;
1258         if (new_conf->conf.key_len)
1259                 new_conf->conf.key = new_conf->key;
1260
1261         ret = i40e_hash_config(pf, new_conf);
1262         if (ret) {
1263                 rte_free(filter);
1264                 if (i40e_pf_config_rss(pf))
1265                         return ret;
1266
1267                 (void)i40e_hash_filter_restore(pf);
1268                 return ret;
1269         }
1270
1271         /* Invalid previous RSS filter */
1272         TAILQ_FOREACH(prev, &pf->rss_config_list, next)
1273                 i40e_invalid_rss_filter(new_conf, &prev->rss_filter_info);
1274
1275         TAILQ_INSERT_TAIL(&pf->rss_config_list, filter, next);
1276         return 0;
1277 }
1278
1279 static int
1280 i40e_hash_reset_conf(struct i40e_pf *pf,
1281                      struct i40e_rte_flow_rss_conf *rss_conf)
1282 {
1283         struct i40e_hw *hw = &pf->adapter->hw;
1284         struct rte_eth_dev *dev;
1285         uint64_t inset;
1286         uint32_t idx;
1287         int ret;
1288
1289         if (rss_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_FUNC) {
1290                 ret = i40e_hash_config_func(hw, RTE_ETH_HASH_FUNCTION_TOEPLITZ);
1291                 if (ret)
1292                         return ret;
1293
1294                 rss_conf->misc_reset_flags &= ~I40E_HASH_FLOW_RESET_FLAG_FUNC;
1295         }
1296
1297         if (rss_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_REGION) {
1298                 dev = &rte_eth_devices[pf->dev_data->port_id];
1299                 ret = i40e_flush_queue_region_all_conf(dev, hw, pf, 0);
1300                 if (ret)
1301                         return ret;
1302
1303                 rss_conf->misc_reset_flags &= ~I40E_HASH_FLOW_RESET_FLAG_REGION;
1304         }
1305
1306         if (rss_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_KEY) {
1307                 ret = i40e_pf_reset_rss_key(pf);
1308                 if (ret)
1309                         return ret;
1310
1311                 rss_conf->misc_reset_flags &= ~I40E_HASH_FLOW_RESET_FLAG_KEY;
1312         }
1313
1314         if (rss_conf->misc_reset_flags & I40E_HASH_FLOW_RESET_FLAG_QUEUE) {
1315                 if (!pf->adapter->rss_reta_updated) {
1316                         ret = i40e_pf_reset_rss_reta(pf);
1317                         if (ret)
1318                                 return ret;
1319                 }
1320
1321                 pf->hash_enabled_queues = 0;
1322                 rss_conf->misc_reset_flags &= ~I40E_HASH_FLOW_RESET_FLAG_QUEUE;
1323         }
1324
1325         while (rss_conf->reset_config_pctypes) {
1326                 idx = rte_bsf64(rss_conf->reset_config_pctypes);
1327
1328                 i40e_hash_enable_pctype(hw, idx, false);
1329                 inset = i40e_get_default_input_set(idx);
1330                 if (inset) {
1331                         ret = i40e_set_hash_inset(hw, inset, idx, false);
1332                         if (ret)
1333                                 return ret;
1334                 }
1335
1336                 rss_conf->reset_config_pctypes &= ~BIT_ULL(idx);
1337         }
1338
1339         while (rss_conf->reset_symmetric_pctypes) {
1340                 idx = rte_bsf64(rss_conf->reset_symmetric_pctypes);
1341
1342                 ret = i40e_hash_config_pctype_symmetric(hw, idx, false);
1343                 if (ret)
1344                         return ret;
1345
1346                 rss_conf->reset_symmetric_pctypes &= ~BIT_ULL(idx);
1347         }
1348
1349         return 0;
1350 }
1351
1352 int
1353 i40e_hash_filter_destroy(struct i40e_pf *pf,
1354                          const struct i40e_rss_filter *rss_filter)
1355 {
1356         struct i40e_rss_filter *filter;
1357         int ret;
1358
1359         TAILQ_FOREACH(filter, &pf->rss_config_list, next) {
1360                 if (rss_filter == filter) {
1361                         ret = i40e_hash_reset_conf(pf,
1362                                                    &filter->rss_filter_info);
1363                         if (ret)
1364                                 return ret;
1365
1366                         TAILQ_REMOVE(&pf->rss_config_list, filter, next);
1367                         rte_free(filter);
1368                         return 0;
1369                 }
1370         }
1371
1372         return -ENOENT;
1373 }
1374
1375 int
1376 i40e_hash_filter_flush(struct i40e_pf *pf)
1377 {
1378         struct rte_flow *flow, *next;
1379
1380         RTE_TAILQ_FOREACH_SAFE(flow, &pf->flow_list, node, next) {
1381                 if (flow->filter_type != RTE_ETH_FILTER_HASH)
1382                         continue;
1383
1384                 if (flow->rule) {
1385                         struct i40e_rss_filter *filter = flow->rule;
1386                         int ret;
1387
1388                         ret = i40e_hash_reset_conf(pf,
1389                                                    &filter->rss_filter_info);
1390                         if (ret)
1391                                 return ret;
1392
1393                         TAILQ_REMOVE(&pf->rss_config_list, filter, next);
1394                         rte_free(filter);
1395                 }
1396
1397                 TAILQ_REMOVE(&pf->flow_list, flow, node);
1398                 rte_free(flow);
1399         }
1400
1401         assert(!pf->rss_config_list.tqh_first);
1402         return 0;
1403 }