drivers/net/iavf/iavf_hash.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10 #include <unistd.h>
  11 #include <stdarg.h>
  12
  13 #include <rte_debug.h>
  14 #include <rte_ether.h>
  15 #include <ethdev_driver.h>
  16 #include <rte_log.h>
  17 #include <rte_malloc.h>
  18 #include <rte_eth_ctrl.h>
  19 #include <rte_tailq.h>
  20 #include <rte_flow_driver.h>
  21
  22 #include "iavf_log.h"
  23 #include "iavf.h"
  24 #include "iavf_generic_flow.h"
  25
  26 #define IAVF_PHINT_NONE                         0
  27 #define IAVF_PHINT_GTPU                         BIT_ULL(0)
  28 #define IAVF_PHINT_GTPU_EH                      BIT_ULL(1)
  29 #define IAVF_PHINT_GTPU_EH_DWN                  BIT_ULL(2)
  30 #define IAVF_PHINT_GTPU_EH_UP                   BIT_ULL(3)
  31 #define IAVF_PHINT_OUTER_IPV4                   BIT_ULL(4)
  32 #define IAVF_PHINT_OUTER_IPV6                   BIT_ULL(5)
  33
  34 #define IAVF_PHINT_GTPU_MSK     (IAVF_PHINT_GTPU        | \
  35                                  IAVF_PHINT_GTPU_EH     | \
  36                                  IAVF_PHINT_GTPU_EH_DWN | \
  37                                  IAVF_PHINT_GTPU_EH_UP)
  38
  39 #define IAVF_PHINT_LAYERS_MSK   (IAVF_PHINT_OUTER_IPV4  | \
  40                                  IAVF_PHINT_OUTER_IPV6)
  41
  42 #define IAVF_GTPU_EH_DWNLINK    0
  43 #define IAVF_GTPU_EH_UPLINK     1
  44
  45 struct iavf_hash_match_type {
  46         uint64_t hash_type;
  47         struct virtchnl_proto_hdrs *proto_hdrs;
  48         uint64_t pattern_hint;
  49 };
  50
  51 struct iavf_rss_meta {
  52         struct virtchnl_proto_hdrs proto_hdrs;
  53         enum virtchnl_rss_algorithm rss_algorithm;
  54 };
  55
  56 struct iavf_hash_flow_cfg {
  57         struct virtchnl_rss_cfg *rss_cfg;
  58         bool simple_xor;
  59 };
  60
  61 static int
  62 iavf_hash_init(struct iavf_adapter *ad);
  63 static int
  64 iavf_hash_create(struct iavf_adapter *ad, struct rte_flow *flow, void *meta,
  65                  struct rte_flow_error *error);
  66 static int
  67 iavf_hash_destroy(struct iavf_adapter *ad, struct rte_flow *flow,
  68                   struct rte_flow_error *error);
  69 static void
  70 iavf_hash_uninit(struct iavf_adapter *ad);
  71 static void
  72 iavf_hash_free(struct rte_flow *flow);
  73 static int
  74 iavf_hash_parse_pattern_action(struct iavf_adapter *ad,
  75                                struct iavf_pattern_match_item *array,
  76                                uint32_t array_len,
  77                                const struct rte_flow_item pattern[],
  78                                const struct rte_flow_action actions[],
  79                                void **meta,
  80                                struct rte_flow_error *error);
  81
  82 #define FIELD_SELECTOR(proto_hdr_field) \
  83                 (1UL << ((proto_hdr_field) & PROTO_HDR_FIELD_MASK))
  84 #define BUFF_NOUSED                     0
  85
  86 #define proto_hdr_eth { \
  87         VIRTCHNL_PROTO_HDR_ETH, \
  88         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_SRC) | \
  89         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_DST), {BUFF_NOUSED} }
  90
  91 #define proto_hdr_svlan { \
  92         VIRTCHNL_PROTO_HDR_S_VLAN, \
  93         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_S_VLAN_ID), {BUFF_NOUSED} }
  94
  95 #define proto_hdr_cvlan { \
  96         VIRTCHNL_PROTO_HDR_C_VLAN, \
  97         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_C_VLAN_ID), {BUFF_NOUSED} }
  98
  99 #define proto_hdr_ipv4 { \
 100         VIRTCHNL_PROTO_HDR_IPV4, \
 101         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_SRC) | \
 102         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_DST), {BUFF_NOUSED} }
 103
 104 #define proto_hdr_ipv4_with_prot { \
 105         VIRTCHNL_PROTO_HDR_IPV4, \
 106         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_SRC) | \
 107         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_DST) | \
 108         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_PROT), {BUFF_NOUSED} }
 109
 110 #define proto_hdr_ipv6 { \
 111         VIRTCHNL_PROTO_HDR_IPV6, \
 112         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_SRC) | \
 113         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_DST), {BUFF_NOUSED} }
 114
 115 #define proto_hdr_ipv6_with_prot { \
 116         VIRTCHNL_PROTO_HDR_IPV6, \
 117         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_SRC) | \
 118         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_DST) | \
 119         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_PROT), {BUFF_NOUSED} }
 120
 121 #define proto_hdr_udp { \
 122         VIRTCHNL_PROTO_HDR_UDP, \
 123         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_UDP_SRC_PORT) | \
 124         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_UDP_DST_PORT), {BUFF_NOUSED} }
 125
 126 #define proto_hdr_tcp { \
 127         VIRTCHNL_PROTO_HDR_TCP, \
 128         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_TCP_SRC_PORT) | \
 129         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_TCP_DST_PORT), {BUFF_NOUSED} }
 130
 131 #define proto_hdr_sctp { \
 132         VIRTCHNL_PROTO_HDR_SCTP, \
 133         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_SCTP_SRC_PORT) | \
 134         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_SCTP_DST_PORT), {BUFF_NOUSED} }
 135
 136 #define proto_hdr_esp { \
 137         VIRTCHNL_PROTO_HDR_ESP, \
 138         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ESP_SPI), {BUFF_NOUSED} }
 139
 140 #define proto_hdr_ah { \
 141         VIRTCHNL_PROTO_HDR_AH, \
 142         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_AH_SPI), {BUFF_NOUSED} }
 143
 144 #define proto_hdr_l2tpv3 { \
 145         VIRTCHNL_PROTO_HDR_L2TPV3, \
 146         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_L2TPV3_SESS_ID), {BUFF_NOUSED} }
 147
 148 #define proto_hdr_pfcp { \
 149         VIRTCHNL_PROTO_HDR_PFCP, \
 150         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_PFCP_SEID), {BUFF_NOUSED} }
 151
 152 #define proto_hdr_gtpc { \
 153         VIRTCHNL_PROTO_HDR_GTPC, 0, {BUFF_NOUSED} }
 154
 155 #define proto_hdr_ecpri { \
 156         VIRTCHNL_PROTO_HDR_ECPRI, \
 157         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ECPRI_PC_RTC_ID), {BUFF_NOUSED} }
 158
 159 #define TUNNEL_LEVEL_OUTER              0
 160 #define TUNNEL_LEVEL_INNER              1
 161
 162 /* proto_hdrs template */
 163 struct virtchnl_proto_hdrs outer_ipv4_tmplt = {
 164         TUNNEL_LEVEL_OUTER, 4,
 165         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv4}
 166 };
 167
 168 struct virtchnl_proto_hdrs outer_ipv4_udp_tmplt = {
 169         TUNNEL_LEVEL_OUTER, 5,
 170         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan,
 171          proto_hdr_ipv4_with_prot,
 172          proto_hdr_udp}
 173 };
 174
 175 struct virtchnl_proto_hdrs outer_ipv4_tcp_tmplt = {
 176         TUNNEL_LEVEL_OUTER, 5,
 177         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan,
 178          proto_hdr_ipv4_with_prot,
 179          proto_hdr_tcp}
 180 };
 181
 182 struct virtchnl_proto_hdrs outer_ipv4_sctp_tmplt = {
 183         TUNNEL_LEVEL_OUTER, 5,
 184         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv4,
 185          proto_hdr_sctp}
 186 };
 187
 188 struct virtchnl_proto_hdrs outer_ipv6_tmplt = {
 189         TUNNEL_LEVEL_OUTER, 4,
 190         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv6}
 191 };
 192
 193 struct virtchnl_proto_hdrs outer_ipv6_udp_tmplt = {
 194         TUNNEL_LEVEL_OUTER, 5,
 195         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan,
 196          proto_hdr_ipv6_with_prot,
 197          proto_hdr_udp}
 198 };
 199
 200 struct virtchnl_proto_hdrs outer_ipv6_tcp_tmplt = {
 201         TUNNEL_LEVEL_OUTER, 5,
 202         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan,
 203          proto_hdr_ipv6_with_prot,
 204          proto_hdr_tcp}
 205 };
 206
 207 struct virtchnl_proto_hdrs outer_ipv6_sctp_tmplt = {
 208         TUNNEL_LEVEL_OUTER, 5,
 209         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv6,
 210          proto_hdr_sctp}
 211 };
 212
 213 struct virtchnl_proto_hdrs inner_ipv4_tmplt = {
 214         TUNNEL_LEVEL_INNER, 1, {proto_hdr_ipv4}
 215 };
 216
 217 struct virtchnl_proto_hdrs inner_ipv4_udp_tmplt = {
 218         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv4_with_prot, proto_hdr_udp}
 219 };
 220
 221 struct virtchnl_proto_hdrs inner_ipv4_tcp_tmplt = {
 222         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv4_with_prot, proto_hdr_tcp}
 223 };
 224
 225 struct virtchnl_proto_hdrs inner_ipv4_sctp_tmplt = {
 226         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv4, proto_hdr_sctp}
 227 };
 228
 229 struct virtchnl_proto_hdrs inner_ipv6_tmplt = {
 230         TUNNEL_LEVEL_INNER, 1, {proto_hdr_ipv6}
 231 };
 232
 233 struct virtchnl_proto_hdrs inner_ipv6_udp_tmplt = {
 234         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv6_with_prot, proto_hdr_udp}
 235 };
 236
 237 struct virtchnl_proto_hdrs inner_ipv6_tcp_tmplt = {
 238         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv6_with_prot, proto_hdr_tcp}
 239 };
 240
 241 struct virtchnl_proto_hdrs inner_ipv6_sctp_tmplt = {
 242         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv6, proto_hdr_sctp}
 243 };
 244
 245 struct virtchnl_proto_hdrs ipv4_esp_tmplt = {
 246         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_esp}
 247 };
 248
 249 struct virtchnl_proto_hdrs ipv4_udp_esp_tmplt = {
 250         TUNNEL_LEVEL_OUTER, 3,
 251         {proto_hdr_ipv4, proto_hdr_udp, proto_hdr_esp}
 252 };
 253
 254 struct virtchnl_proto_hdrs ipv4_ah_tmplt = {
 255         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_ah}
 256 };
 257
 258 struct virtchnl_proto_hdrs ipv6_esp_tmplt = {
 259         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_esp}
 260 };
 261
 262 struct virtchnl_proto_hdrs ipv6_udp_esp_tmplt = {
 263         TUNNEL_LEVEL_OUTER, 3,
 264         {proto_hdr_ipv6, proto_hdr_udp, proto_hdr_esp}
 265 };
 266
 267 struct virtchnl_proto_hdrs ipv6_ah_tmplt = {
 268         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_ah}
 269 };
 270
 271 struct virtchnl_proto_hdrs ipv4_l2tpv3_tmplt = {
 272         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_l2tpv3}
 273 };
 274
 275 struct virtchnl_proto_hdrs ipv6_l2tpv3_tmplt = {
 276         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_l2tpv3}
 277 };
 278
 279 struct virtchnl_proto_hdrs ipv4_pfcp_tmplt = {
 280         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_pfcp}
 281 };
 282
 283 struct virtchnl_proto_hdrs ipv6_pfcp_tmplt = {
 284         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_pfcp}
 285 };
 286
 287 struct virtchnl_proto_hdrs ipv4_udp_gtpc_tmplt = {
 288         TUNNEL_LEVEL_OUTER, 3, {proto_hdr_ipv4, proto_hdr_udp, proto_hdr_gtpc}
 289 };
 290
 291 struct virtchnl_proto_hdrs ipv6_udp_gtpc_tmplt = {
 292         TUNNEL_LEVEL_OUTER, 3, {proto_hdr_ipv6, proto_hdr_udp, proto_hdr_gtpc}
 293 };
 294
 295 struct virtchnl_proto_hdrs eth_ecpri_tmplt = {
 296         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_eth, proto_hdr_ecpri}
 297 };
 298
 299 struct virtchnl_proto_hdrs ipv4_ecpri_tmplt = {
 300         TUNNEL_LEVEL_OUTER, 3, {proto_hdr_ipv4, proto_hdr_udp, proto_hdr_ecpri}
 301 };
 302
 303 /* rss type super set */
 304
 305 /* IPv4 outer */
 306 #define IAVF_RSS_TYPE_OUTER_IPV4        (ETH_RSS_ETH | ETH_RSS_IPV4)
 307 #define IAVF_RSS_TYPE_OUTER_IPV4_UDP    (IAVF_RSS_TYPE_OUTER_IPV4 | \
 308                                          ETH_RSS_NONFRAG_IPV4_UDP)
 309 #define IAVF_RSS_TYPE_OUTER_IPV4_TCP    (IAVF_RSS_TYPE_OUTER_IPV4 | \
 310                                          ETH_RSS_NONFRAG_IPV4_TCP)
 311 #define IAVF_RSS_TYPE_OUTER_IPV4_SCTP   (IAVF_RSS_TYPE_OUTER_IPV4 | \
 312                                          ETH_RSS_NONFRAG_IPV4_SCTP)
 313 /* IPv6 outer */
 314 #define IAVF_RSS_TYPE_OUTER_IPV6        (ETH_RSS_ETH | ETH_RSS_IPV6)
 315 #define IAVF_RSS_TYPE_OUTER_IPV6_UDP    (IAVF_RSS_TYPE_OUTER_IPV6 | \
 316                                          ETH_RSS_NONFRAG_IPV6_UDP)
 317 #define IAVF_RSS_TYPE_OUTER_IPV6_TCP    (IAVF_RSS_TYPE_OUTER_IPV6 | \
 318                                          ETH_RSS_NONFRAG_IPV6_TCP)
 319 #define IAVF_RSS_TYPE_OUTER_IPV6_SCTP   (IAVF_RSS_TYPE_OUTER_IPV6 | \
 320                                          ETH_RSS_NONFRAG_IPV6_SCTP)
 321 /* VLAN IPV4 */
 322 #define IAVF_RSS_TYPE_VLAN_IPV4         (IAVF_RSS_TYPE_OUTER_IPV4 | \
 323                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 324 #define IAVF_RSS_TYPE_VLAN_IPV4_UDP     (IAVF_RSS_TYPE_OUTER_IPV4_UDP | \
 325                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 326 #define IAVF_RSS_TYPE_VLAN_IPV4_TCP     (IAVF_RSS_TYPE_OUTER_IPV4_TCP | \
 327                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 328 #define IAVF_RSS_TYPE_VLAN_IPV4_SCTP    (IAVF_RSS_TYPE_OUTER_IPV4_SCTP | \
 329                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 330 /* VLAN IPv6 */
 331 #define IAVF_RSS_TYPE_VLAN_IPV6         (IAVF_RSS_TYPE_OUTER_IPV6 | \
 332                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 333 #define IAVF_RSS_TYPE_VLAN_IPV6_UDP     (IAVF_RSS_TYPE_OUTER_IPV6_UDP | \
 334                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 335 #define IAVF_RSS_TYPE_VLAN_IPV6_TCP     (IAVF_RSS_TYPE_OUTER_IPV6_TCP | \
 336                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 337 #define IAVF_RSS_TYPE_VLAN_IPV6_SCTP    (IAVF_RSS_TYPE_OUTER_IPV6_SCTP | \
 338                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 339 /* IPv4 inner */
 340 #define IAVF_RSS_TYPE_INNER_IPV4        ETH_RSS_IPV4
 341 #define IAVF_RSS_TYPE_INNER_IPV4_UDP    (ETH_RSS_IPV4 | \
 342                                          ETH_RSS_NONFRAG_IPV4_UDP)
 343 #define IAVF_RSS_TYPE_INNER_IPV4_TCP    (ETH_RSS_IPV4 | \
 344                                          ETH_RSS_NONFRAG_IPV4_TCP)
 345 #define IAVF_RSS_TYPE_INNER_IPV4_SCTP   (ETH_RSS_IPV4 | \
 346                                          ETH_RSS_NONFRAG_IPV4_SCTP)
 347 /* IPv6 inner */
 348 #define IAVF_RSS_TYPE_INNER_IPV6        ETH_RSS_IPV6
 349 #define IAVF_RSS_TYPE_INNER_IPV6_UDP    (ETH_RSS_IPV6 | \
 350                                          ETH_RSS_NONFRAG_IPV6_UDP)
 351 #define IAVF_RSS_TYPE_INNER_IPV6_TCP    (ETH_RSS_IPV6 | \
 352                                          ETH_RSS_NONFRAG_IPV6_TCP)
 353 #define IAVF_RSS_TYPE_INNER_IPV6_SCTP   (ETH_RSS_IPV6 | \
 354                                          ETH_RSS_NONFRAG_IPV6_SCTP)
 355 /* GTPU IPv4 */
 356 #define IAVF_RSS_TYPE_GTPU_IPV4         (IAVF_RSS_TYPE_INNER_IPV4 | \
 357                                          ETH_RSS_GTPU)
 358 #define IAVF_RSS_TYPE_GTPU_IPV4_UDP     (IAVF_RSS_TYPE_INNER_IPV4_UDP | \
 359                                          ETH_RSS_GTPU)
 360 #define IAVF_RSS_TYPE_GTPU_IPV4_TCP     (IAVF_RSS_TYPE_INNER_IPV4_TCP | \
 361                                          ETH_RSS_GTPU)
 362 /* GTPU IPv6 */
 363 #define IAVF_RSS_TYPE_GTPU_IPV6         (IAVF_RSS_TYPE_INNER_IPV6 | \
 364                                          ETH_RSS_GTPU)
 365 #define IAVF_RSS_TYPE_GTPU_IPV6_UDP     (IAVF_RSS_TYPE_INNER_IPV6_UDP | \
 366                                          ETH_RSS_GTPU)
 367 #define IAVF_RSS_TYPE_GTPU_IPV6_TCP     (IAVF_RSS_TYPE_INNER_IPV6_TCP | \
 368                                          ETH_RSS_GTPU)
 369 /* ESP, AH, L2TPV3 and PFCP */
 370 #define IAVF_RSS_TYPE_IPV4_ESP          (ETH_RSS_ESP | ETH_RSS_IPV4)
 371 #define IAVF_RSS_TYPE_IPV4_AH           (ETH_RSS_AH | ETH_RSS_IPV4)
 372 #define IAVF_RSS_TYPE_IPV6_ESP          (ETH_RSS_ESP | ETH_RSS_IPV6)
 373 #define IAVF_RSS_TYPE_IPV6_AH           (ETH_RSS_AH | ETH_RSS_IPV6)
 374 #define IAVF_RSS_TYPE_IPV4_L2TPV3       (ETH_RSS_L2TPV3 | ETH_RSS_IPV4)
 375 #define IAVF_RSS_TYPE_IPV6_L2TPV3       (ETH_RSS_L2TPV3 | ETH_RSS_IPV6)
 376 #define IAVF_RSS_TYPE_IPV4_PFCP         (ETH_RSS_PFCP | ETH_RSS_IPV4)
 377 #define IAVF_RSS_TYPE_IPV6_PFCP         (ETH_RSS_PFCP | ETH_RSS_IPV6)
 378
 379 /**
 380  * Supported pattern for hash.
 381  * The first member is pattern item type,
 382  * the second member is input set mask,
 383  * the third member is virtchnl_proto_hdrs template
 384  */
 385 static struct iavf_pattern_match_item iavf_hash_pattern_list[] = {
 386         /* IPv4 */
 387         {iavf_pattern_eth_ipv4,                         IAVF_RSS_TYPE_OUTER_IPV4,       &outer_ipv4_tmplt},
 388         {iavf_pattern_eth_ipv4_udp,                     IAVF_RSS_TYPE_OUTER_IPV4_UDP,   &outer_ipv4_udp_tmplt},
 389         {iavf_pattern_eth_ipv4_tcp,                     IAVF_RSS_TYPE_OUTER_IPV4_TCP,   &outer_ipv4_tcp_tmplt},
 390         {iavf_pattern_eth_ipv4_sctp,                    IAVF_RSS_TYPE_OUTER_IPV4_SCTP,  &outer_ipv4_sctp_tmplt},
 391         {iavf_pattern_eth_vlan_ipv4,                    IAVF_RSS_TYPE_VLAN_IPV4,        &outer_ipv4_tmplt},
 392         {iavf_pattern_eth_vlan_ipv4_udp,                IAVF_RSS_TYPE_VLAN_IPV4_UDP,    &outer_ipv4_udp_tmplt},
 393         {iavf_pattern_eth_vlan_ipv4_tcp,                IAVF_RSS_TYPE_VLAN_IPV4_TCP,    &outer_ipv4_tcp_tmplt},
 394         {iavf_pattern_eth_vlan_ipv4_sctp,               IAVF_RSS_TYPE_VLAN_IPV4_SCTP,   &outer_ipv4_sctp_tmplt},
 395         {iavf_pattern_eth_ipv4_gtpu,                    ETH_RSS_IPV4,                   &outer_ipv4_udp_tmplt},
 396         {iavf_pattern_eth_ipv4_gtpu_ipv4,               IAVF_RSS_TYPE_GTPU_IPV4,        &inner_ipv4_tmplt},
 397         {iavf_pattern_eth_ipv4_gtpu_ipv4_udp,           IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &inner_ipv4_udp_tmplt},
 398         {iavf_pattern_eth_ipv4_gtpu_ipv4_tcp,           IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &inner_ipv4_tcp_tmplt},
 399         {iavf_pattern_eth_ipv6_gtpu_ipv4,               IAVF_RSS_TYPE_GTPU_IPV4,        &inner_ipv4_tmplt},
 400         {iavf_pattern_eth_ipv6_gtpu_ipv4_udp,           IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &inner_ipv4_udp_tmplt},
 401         {iavf_pattern_eth_ipv6_gtpu_ipv4_tcp,           IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &inner_ipv4_tcp_tmplt},
 402         {iavf_pattern_eth_ipv4_gtpu_eh_ipv4,            IAVF_RSS_TYPE_GTPU_IPV4,        &inner_ipv4_tmplt},
 403         {iavf_pattern_eth_ipv4_gtpu_eh_ipv4_udp,        IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &inner_ipv4_udp_tmplt},
 404         {iavf_pattern_eth_ipv4_gtpu_eh_ipv4_tcp,        IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &inner_ipv4_tcp_tmplt},
 405         {iavf_pattern_eth_ipv6_gtpu_eh_ipv4,            IAVF_RSS_TYPE_GTPU_IPV4,        &inner_ipv4_tmplt},
 406         {iavf_pattern_eth_ipv6_gtpu_eh_ipv4_udp,        IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &inner_ipv4_udp_tmplt},
 407         {iavf_pattern_eth_ipv6_gtpu_eh_ipv4_tcp,        IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &inner_ipv4_tcp_tmplt},
 408         {iavf_pattern_eth_ipv4_esp,                     IAVF_RSS_TYPE_IPV4_ESP,         &ipv4_esp_tmplt},
 409         {iavf_pattern_eth_ipv4_udp_esp,                 IAVF_RSS_TYPE_IPV4_ESP,         &ipv4_udp_esp_tmplt},
 410         {iavf_pattern_eth_ipv4_ah,                      IAVF_RSS_TYPE_IPV4_AH,          &ipv4_ah_tmplt},
 411         {iavf_pattern_eth_ipv4_l2tpv3,                  IAVF_RSS_TYPE_IPV4_L2TPV3,      &ipv4_l2tpv3_tmplt},
 412         {iavf_pattern_eth_ipv4_pfcp,                    IAVF_RSS_TYPE_IPV4_PFCP,        &ipv4_pfcp_tmplt},
 413         {iavf_pattern_eth_ipv4_gtpc,                    ETH_RSS_IPV4,                   &ipv4_udp_gtpc_tmplt},
 414         {iavf_pattern_eth_ecpri,                        ETH_RSS_ECPRI,                  &eth_ecpri_tmplt},
 415         {iavf_pattern_eth_ipv4_ecpri,                   ETH_RSS_ECPRI,                  &ipv4_ecpri_tmplt},
 416         /* IPv6 */
 417         {iavf_pattern_eth_ipv6,                         IAVF_RSS_TYPE_OUTER_IPV6,       &outer_ipv6_tmplt},
 418         {iavf_pattern_eth_ipv6_udp,                     IAVF_RSS_TYPE_OUTER_IPV6_UDP,   &outer_ipv6_udp_tmplt},
 419         {iavf_pattern_eth_ipv6_tcp,                     IAVF_RSS_TYPE_OUTER_IPV6_TCP,   &outer_ipv6_tcp_tmplt},
 420         {iavf_pattern_eth_ipv6_sctp,                    IAVF_RSS_TYPE_OUTER_IPV6_SCTP,  &outer_ipv6_sctp_tmplt},
 421         {iavf_pattern_eth_vlan_ipv6,                    IAVF_RSS_TYPE_VLAN_IPV6,        &outer_ipv6_tmplt},
 422         {iavf_pattern_eth_vlan_ipv6_udp,                IAVF_RSS_TYPE_VLAN_IPV6_UDP,    &outer_ipv6_udp_tmplt},
 423         {iavf_pattern_eth_vlan_ipv6_tcp,                IAVF_RSS_TYPE_VLAN_IPV6_TCP,    &outer_ipv6_tcp_tmplt},
 424         {iavf_pattern_eth_vlan_ipv6_sctp,               IAVF_RSS_TYPE_VLAN_IPV6_SCTP,   &outer_ipv6_sctp_tmplt},
 425         {iavf_pattern_eth_ipv6_gtpu,                    ETH_RSS_IPV6,                   &outer_ipv6_udp_tmplt},
 426         {iavf_pattern_eth_ipv4_gtpu_ipv6,               IAVF_RSS_TYPE_GTPU_IPV6,        &inner_ipv6_tmplt},
 427         {iavf_pattern_eth_ipv4_gtpu_ipv6_udp,           IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &inner_ipv6_udp_tmplt},
 428         {iavf_pattern_eth_ipv4_gtpu_ipv6_tcp,           IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &inner_ipv6_tcp_tmplt},
 429         {iavf_pattern_eth_ipv6_gtpu_ipv6,               IAVF_RSS_TYPE_GTPU_IPV6,        &inner_ipv6_tmplt},
 430         {iavf_pattern_eth_ipv6_gtpu_ipv6_udp,           IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &inner_ipv6_udp_tmplt},
 431         {iavf_pattern_eth_ipv6_gtpu_ipv6_tcp,           IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &inner_ipv6_tcp_tmplt},
 432         {iavf_pattern_eth_ipv4_gtpu_eh_ipv6,            IAVF_RSS_TYPE_GTPU_IPV6,        &inner_ipv6_tmplt},
 433         {iavf_pattern_eth_ipv4_gtpu_eh_ipv6_udp,        IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &inner_ipv6_udp_tmplt},
 434         {iavf_pattern_eth_ipv4_gtpu_eh_ipv6_tcp,        IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &inner_ipv6_tcp_tmplt},
 435         {iavf_pattern_eth_ipv6_gtpu_eh_ipv6,            IAVF_RSS_TYPE_GTPU_IPV6,        &inner_ipv6_tmplt},
 436         {iavf_pattern_eth_ipv6_gtpu_eh_ipv6_udp,        IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &inner_ipv6_udp_tmplt},
 437         {iavf_pattern_eth_ipv6_gtpu_eh_ipv6_tcp,        IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &inner_ipv6_tcp_tmplt},
 438         {iavf_pattern_eth_ipv6_esp,                     IAVF_RSS_TYPE_IPV6_ESP,         &ipv6_esp_tmplt},
 439         {iavf_pattern_eth_ipv6_udp_esp,                 IAVF_RSS_TYPE_IPV6_ESP,         &ipv6_udp_esp_tmplt},
 440         {iavf_pattern_eth_ipv6_ah,                      IAVF_RSS_TYPE_IPV6_AH,          &ipv6_ah_tmplt},
 441         {iavf_pattern_eth_ipv6_l2tpv3,                  IAVF_RSS_TYPE_IPV6_L2TPV3,      &ipv6_l2tpv3_tmplt},
 442         {iavf_pattern_eth_ipv6_pfcp,                    IAVF_RSS_TYPE_IPV6_PFCP,        &ipv6_pfcp_tmplt},
 443         {iavf_pattern_eth_ipv6_gtpc,                    ETH_RSS_IPV6,                   &ipv6_udp_gtpc_tmplt},
 444 };
 445
 446 static struct iavf_flow_engine iavf_hash_engine = {
 447         .init = iavf_hash_init,
 448         .create = iavf_hash_create,
 449         .destroy = iavf_hash_destroy,
 450         .uninit = iavf_hash_uninit,
 451         .free = iavf_hash_free,
 452         .type = IAVF_FLOW_ENGINE_HASH,
 453 };
 454
 455 /* Register parser for comms package. */
 456 static struct iavf_flow_parser iavf_hash_parser = {
 457         .engine = &iavf_hash_engine,
 458         .array = iavf_hash_pattern_list,
 459         .array_len = RTE_DIM(iavf_hash_pattern_list),
 460         .parse_pattern_action = iavf_hash_parse_pattern_action,
 461         .stage = IAVF_FLOW_STAGE_RSS,
 462 };
 463
 464 int
 465 iavf_rss_hash_set(struct iavf_adapter *ad, uint64_t rss_hf, bool add)
 466 {
 467         struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(ad);
 468         struct virtchnl_rss_cfg rss_cfg;
 469
 470 #define IAVF_RSS_HF_ALL ( \
 471         ETH_RSS_IPV4 | \
 472         ETH_RSS_IPV6 | \
 473         ETH_RSS_NONFRAG_IPV4_UDP | \
 474         ETH_RSS_NONFRAG_IPV6_UDP | \
 475         ETH_RSS_NONFRAG_IPV4_TCP | \
 476         ETH_RSS_NONFRAG_IPV6_TCP | \
 477         ETH_RSS_NONFRAG_IPV4_SCTP | \
 478         ETH_RSS_NONFRAG_IPV6_SCTP)
 479
 480         rss_cfg.rss_algorithm = VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC;
 481         if (rss_hf & ETH_RSS_IPV4) {
 482                 rss_cfg.proto_hdrs = inner_ipv4_tmplt;
 483                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 484         }
 485
 486         if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
 487                 rss_cfg.proto_hdrs = inner_ipv4_udp_tmplt;
 488                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 489         }
 490
 491         if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
 492                 rss_cfg.proto_hdrs = inner_ipv4_tcp_tmplt;
 493                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 494         }
 495
 496         if (rss_hf & ETH_RSS_NONFRAG_IPV4_SCTP) {
 497                 rss_cfg.proto_hdrs = inner_ipv4_sctp_tmplt;
 498                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 499         }
 500
 501         if (rss_hf & ETH_RSS_IPV6) {
 502                 rss_cfg.proto_hdrs = inner_ipv6_tmplt;
 503                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 504         }
 505
 506         if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) {
 507                 rss_cfg.proto_hdrs = inner_ipv6_udp_tmplt;
 508                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 509         }
 510
 511         if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) {
 512                 rss_cfg.proto_hdrs = inner_ipv6_tcp_tmplt;
 513                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 514         }
 515
 516         if (rss_hf & ETH_RSS_NONFRAG_IPV6_SCTP) {
 517                 rss_cfg.proto_hdrs = inner_ipv6_sctp_tmplt;
 518                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 519         }
 520
 521         vf->rss_hf = rss_hf & IAVF_RSS_HF_ALL;
 522         return 0;
 523 }
 524
 525 RTE_INIT(iavf_hash_engine_init)
 526 {
 527         struct iavf_flow_engine *engine = &iavf_hash_engine;
 528
 529         iavf_register_flow_engine(engine);
 530 }
 531
 532 static int
 533 iavf_hash_init(struct iavf_adapter *ad)
 534 {
 535         struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
 536         struct iavf_flow_parser *parser;
 537         int ret;
 538
 539         if (vf->vf_reset)
 540                 return -EIO;
 541
 542         if (!vf->vf_res)
 543                 return -EINVAL;
 544
 545         if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF))
 546                 return -ENOTSUP;
 547
 548         parser = &iavf_hash_parser;
 549
 550         ret = iavf_register_parser(parser, ad);
 551         if (ret) {
 552                 PMD_DRV_LOG(ERR, "fail to register hash parser");
 553                 return ret;
 554         }
 555
 556         return ret;
 557 }
 558
 559 static int
 560 iavf_hash_parse_pattern(const struct rte_flow_item pattern[], uint64_t *phint,
 561                         struct rte_flow_error *error)
 562 {
 563         const struct rte_flow_item *item = pattern;
 564         const struct rte_flow_item_gtp_psc *psc;
 565         const struct rte_flow_item_ecpri *ecpri;
 566         struct rte_ecpri_common_hdr ecpri_common;
 567
 568         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 569                 if (item->last) {
 570                         rte_flow_error_set(error, EINVAL,
 571                                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 572                                            "Not support range");
 573                         return -rte_errno;
 574                 }
 575
 576                 switch (item->type) {
 577                 case RTE_FLOW_ITEM_TYPE_IPV4:
 578                         if (!(*phint & IAVF_PHINT_GTPU_MSK))
 579                                 *phint |= IAVF_PHINT_OUTER_IPV4;
 580                         break;
 581                 case RTE_FLOW_ITEM_TYPE_IPV6:
 582                         if (!(*phint & IAVF_PHINT_GTPU_MSK))
 583                                 *phint |= IAVF_PHINT_OUTER_IPV6;
 584                         break;
 585                 case RTE_FLOW_ITEM_TYPE_GTPU:
 586                         *phint |= IAVF_PHINT_GTPU;
 587                         break;
 588                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
 589                         *phint |= IAVF_PHINT_GTPU_EH;
 590                         psc = item->spec;
 591                         if (!psc)
 592                                 break;
 593                         else if (psc->pdu_type == IAVF_GTPU_EH_UPLINK)
 594                                 *phint |= IAVF_PHINT_GTPU_EH_UP;
 595                         else if (psc->pdu_type == IAVF_GTPU_EH_DWNLINK)
 596                                 *phint |= IAVF_PHINT_GTPU_EH_DWN;
 597                         break;
 598                 case RTE_FLOW_ITEM_TYPE_ECPRI:
 599                         ecpri = item->spec;
 600                         if (!ecpri)
 601                                 break;
 602
 603                         ecpri_common.u32 = rte_be_to_cpu_32(ecpri->hdr.common.u32);
 604
 605                         if (ecpri_common.type !=
 606                                  RTE_ECPRI_MSG_TYPE_IQ_DATA) {
 607                                 rte_flow_error_set(error, EINVAL,
 608                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 609                                         "Unsupported common type.");
 610                                 return -rte_errno;
 611                         }
 612                         break;
 613                 default:
 614                         break;
 615                 }
 616         }
 617
 618         return 0;
 619 }
 620
 621 #define REFINE_PROTO_FLD(op, fld) \
 622         VIRTCHNL_##op##_PROTO_HDR_FIELD(hdr, VIRTCHNL_PROTO_HDR_##fld)
 623 #define REPALCE_PROTO_FLD(fld_1, fld_2) \
 624 do { \
 625         REFINE_PROTO_FLD(DEL, fld_1);   \
 626         REFINE_PROTO_FLD(ADD, fld_2);   \
 627 } while (0)
 628
 629 /* refine proto hdrs base on l2, l3, l4 rss type */
 630 static void
 631 iavf_refine_proto_hdrs_l234(struct virtchnl_proto_hdrs *proto_hdrs,
 632                             uint64_t rss_type)
 633 {
 634         struct virtchnl_proto_hdr *hdr;
 635         int i;
 636
 637         for (i = 0; i < proto_hdrs->count; i++) {
 638                 hdr = &proto_hdrs->proto_hdr[i];
 639                 switch (hdr->type) {
 640                 case VIRTCHNL_PROTO_HDR_ETH:
 641                         if (!(rss_type & ETH_RSS_ETH))
 642                                 hdr->field_selector = 0;
 643                         else if (rss_type & ETH_RSS_L2_SRC_ONLY)
 644                                 REFINE_PROTO_FLD(DEL, ETH_DST);
 645                         else if (rss_type & ETH_RSS_L2_DST_ONLY)
 646                                 REFINE_PROTO_FLD(DEL, ETH_SRC);
 647                         break;
 648                 case VIRTCHNL_PROTO_HDR_IPV4:
 649                         if (rss_type &
 650                             (ETH_RSS_IPV4 |
 651                              ETH_RSS_NONFRAG_IPV4_UDP |
 652                              ETH_RSS_NONFRAG_IPV4_TCP |
 653                              ETH_RSS_NONFRAG_IPV4_SCTP)) {
 654                                 if (rss_type & ETH_RSS_L3_SRC_ONLY) {
 655                                         REFINE_PROTO_FLD(DEL, IPV4_DST);
 656                                 } else if (rss_type & ETH_RSS_L3_DST_ONLY) {
 657                                         REFINE_PROTO_FLD(DEL, IPV4_SRC);
 658                                 } else if (rss_type &
 659                                          (ETH_RSS_L4_SRC_ONLY |
 660                                           ETH_RSS_L4_DST_ONLY)) {
 661                                         REFINE_PROTO_FLD(DEL, IPV4_DST);
 662                                         REFINE_PROTO_FLD(DEL, IPV4_SRC);
 663                                 }
 664                         } else {
 665                                 hdr->field_selector = 0;
 666                         }
 667                         break;
 668                 case VIRTCHNL_PROTO_HDR_IPV6:
 669                         if (rss_type &
 670                             (ETH_RSS_IPV6 |
 671                              ETH_RSS_NONFRAG_IPV6_UDP |
 672                              ETH_RSS_NONFRAG_IPV6_TCP |
 673                              ETH_RSS_NONFRAG_IPV6_SCTP)) {
 674                                 if (rss_type & ETH_RSS_L3_SRC_ONLY) {
 675                                         REFINE_PROTO_FLD(DEL, IPV6_DST);
 676                                 } else if (rss_type & ETH_RSS_L3_DST_ONLY) {
 677                                         REFINE_PROTO_FLD(DEL, IPV6_SRC);
 678                                 } else if (rss_type &
 679                                          (ETH_RSS_L4_SRC_ONLY |
 680                                           ETH_RSS_L4_DST_ONLY)) {
 681                                         REFINE_PROTO_FLD(DEL, IPV6_DST);
 682                                         REFINE_PROTO_FLD(DEL, IPV6_SRC);
 683                                 }
 684                         } else {
 685                                 hdr->field_selector = 0;
 686                         }
 687                         if (rss_type & RTE_ETH_RSS_L3_PRE64) {
 688                                 if (REFINE_PROTO_FLD(TEST, IPV6_SRC))
 689                                         REPALCE_PROTO_FLD(IPV6_SRC,
 690                                                           IPV6_PREFIX64_SRC);
 691                                 if (REFINE_PROTO_FLD(TEST, IPV6_DST))
 692                                         REPALCE_PROTO_FLD(IPV6_DST,
 693                                                           IPV6_PREFIX64_DST);
 694                         }
 695                         break;
 696                 case VIRTCHNL_PROTO_HDR_UDP:
 697                         if (rss_type &
 698                             (ETH_RSS_NONFRAG_IPV4_UDP |
 699                              ETH_RSS_NONFRAG_IPV6_UDP)) {
 700                                 if (rss_type & ETH_RSS_L4_SRC_ONLY)
 701                                         REFINE_PROTO_FLD(DEL, UDP_DST_PORT);
 702                                 else if (rss_type & ETH_RSS_L4_DST_ONLY)
 703                                         REFINE_PROTO_FLD(DEL, UDP_SRC_PORT);
 704                                 else if (rss_type &
 705                                          (ETH_RSS_L3_SRC_ONLY |
 706                                           ETH_RSS_L3_DST_ONLY))
 707                                         hdr->field_selector = 0;
 708                         } else {
 709                                 hdr->field_selector = 0;
 710                         }
 711                         break;
 712                 case VIRTCHNL_PROTO_HDR_TCP:
 713                         if (rss_type &
 714                             (ETH_RSS_NONFRAG_IPV4_TCP |
 715                              ETH_RSS_NONFRAG_IPV6_TCP)) {
 716                                 if (rss_type & ETH_RSS_L4_SRC_ONLY)
 717                                         REFINE_PROTO_FLD(DEL, TCP_DST_PORT);
 718                                 else if (rss_type & ETH_RSS_L4_DST_ONLY)
 719                                         REFINE_PROTO_FLD(DEL, TCP_SRC_PORT);
 720                                 else if (rss_type &
 721                                          (ETH_RSS_L3_SRC_ONLY |
 722                                           ETH_RSS_L3_DST_ONLY))
 723                                         hdr->field_selector = 0;
 724                         } else {
 725                                 hdr->field_selector = 0;
 726                         }
 727                         break;
 728                 case VIRTCHNL_PROTO_HDR_SCTP:
 729                         if (rss_type &
 730                             (ETH_RSS_NONFRAG_IPV4_SCTP |
 731                              ETH_RSS_NONFRAG_IPV6_SCTP)) {
 732                                 if (rss_type & ETH_RSS_L4_SRC_ONLY)
 733                                         REFINE_PROTO_FLD(DEL, SCTP_DST_PORT);
 734                                 else if (rss_type & ETH_RSS_L4_DST_ONLY)
 735                                         REFINE_PROTO_FLD(DEL, SCTP_SRC_PORT);
 736                                 else if (rss_type &
 737                                          (ETH_RSS_L3_SRC_ONLY |
 738                                           ETH_RSS_L3_DST_ONLY))
 739                                         hdr->field_selector = 0;
 740                         } else {
 741                                 hdr->field_selector = 0;
 742                         }
 743                         break;
 744                 case VIRTCHNL_PROTO_HDR_S_VLAN:
 745                         if (!(rss_type & ETH_RSS_S_VLAN))
 746                                 hdr->field_selector = 0;
 747                         break;
 748                 case VIRTCHNL_PROTO_HDR_C_VLAN:
 749                         if (!(rss_type & ETH_RSS_C_VLAN))
 750                                 hdr->field_selector = 0;
 751                         break;
 752                 case VIRTCHNL_PROTO_HDR_L2TPV3:
 753                         if (!(rss_type & ETH_RSS_L2TPV3))
 754                                 hdr->field_selector = 0;
 755                         break;
 756                 case VIRTCHNL_PROTO_HDR_ESP:
 757                         if (!(rss_type & ETH_RSS_ESP))
 758                                 hdr->field_selector = 0;
 759                         break;
 760                 case VIRTCHNL_PROTO_HDR_AH:
 761                         if (!(rss_type & ETH_RSS_AH))
 762                                 hdr->field_selector = 0;
 763                         break;
 764                 case VIRTCHNL_PROTO_HDR_PFCP:
 765                         if (!(rss_type & ETH_RSS_PFCP))
 766                                 hdr->field_selector = 0;
 767                         break;
 768                 case VIRTCHNL_PROTO_HDR_ECPRI:
 769                         if (!(rss_type & ETH_RSS_ECPRI))
 770                                 hdr->field_selector = 0;
 771                         break;
 772                 default:
 773                         break;
 774                 }
 775         }
 776 }
 777
 778 /* refine proto hdrs base on gtpu rss type */
 779 static void
 780 iavf_refine_proto_hdrs_gtpu(struct virtchnl_proto_hdrs *proto_hdrs,
 781                             uint64_t rss_type)
 782 {
 783         struct virtchnl_proto_hdr *hdr;
 784         int i;
 785
 786         if (!(rss_type & ETH_RSS_GTPU))
 787                 return;
 788
 789         for (i = 0; i < proto_hdrs->count; i++) {
 790                 hdr = &proto_hdrs->proto_hdr[i];
 791                 switch (hdr->type) {
 792                 case VIRTCHNL_PROTO_HDR_GTPU_IP:
 793                         REFINE_PROTO_FLD(ADD, GTPU_IP_TEID);
 794                         break;
 795                 default:
 796                         break;
 797                 }
 798         }
 799 }
 800
 801 static void
 802 iavf_refine_proto_hdrs_by_pattern(struct virtchnl_proto_hdrs *proto_hdrs,
 803                                   uint64_t phint)
 804 {
 805         struct virtchnl_proto_hdr *hdr1;
 806         struct virtchnl_proto_hdr *hdr2;
 807         int i, shift_count = 1;
 808
 809         if (!(phint & IAVF_PHINT_GTPU_MSK))
 810                 return;
 811
 812         if (phint & IAVF_PHINT_LAYERS_MSK)
 813                 shift_count++;
 814
 815         if (proto_hdrs->tunnel_level == TUNNEL_LEVEL_INNER) {
 816                 /* shift headers layer */
 817                 for (i = proto_hdrs->count - 1 + shift_count;
 818                      i > shift_count - 1; i--) {
 819                         hdr1 = &proto_hdrs->proto_hdr[i];
 820                         hdr2 = &proto_hdrs->proto_hdr[i - shift_count];
 821                         *hdr1 = *hdr2;
 822                 }
 823
 824                 if (shift_count == 1) {
 825                         /* adding gtpu header at layer 0 */
 826                         hdr1 = &proto_hdrs->proto_hdr[0];
 827                 } else {
 828                         /* adding gtpu header and outer ip header */
 829                         hdr1 = &proto_hdrs->proto_hdr[1];
 830                         hdr2 = &proto_hdrs->proto_hdr[0];
 831                         hdr2->field_selector = 0;
 832                         proto_hdrs->count++;
 833                         proto_hdrs->tunnel_level = TUNNEL_LEVEL_OUTER;
 834
 835                         if (phint & IAVF_PHINT_OUTER_IPV4)
 836                                 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr2, IPV4);
 837                         else if (phint & IAVF_PHINT_OUTER_IPV6)
 838                                 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr2, IPV6);
 839                 }
 840         } else {
 841                 hdr1 = &proto_hdrs->proto_hdr[proto_hdrs->count];
 842         }
 843
 844         hdr1->field_selector = 0;
 845         proto_hdrs->count++;
 846
 847         if (phint & IAVF_PHINT_GTPU_EH_DWN)
 848                 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_EH_PDU_DWN);
 849         else if (phint & IAVF_PHINT_GTPU_EH_UP)
 850                 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_EH_PDU_UP);
 851         else if (phint & IAVF_PHINT_GTPU_EH)
 852                 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_EH);
 853         else if (phint & IAVF_PHINT_GTPU)
 854                 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_IP);
 855 }
 856
 857 static void iavf_refine_proto_hdrs(struct virtchnl_proto_hdrs *proto_hdrs,
 858                                    uint64_t rss_type, uint64_t phint)
 859 {
 860         iavf_refine_proto_hdrs_l234(proto_hdrs, rss_type);
 861         iavf_refine_proto_hdrs_by_pattern(proto_hdrs, phint);
 862         iavf_refine_proto_hdrs_gtpu(proto_hdrs, rss_type);
 863 }
 864
 865 static uint64_t invalid_rss_comb[] = {
 866         ETH_RSS_IPV4 | ETH_RSS_NONFRAG_IPV4_UDP,
 867         ETH_RSS_IPV4 | ETH_RSS_NONFRAG_IPV4_TCP,
 868         ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_UDP,
 869         ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_TCP,
 870         RTE_ETH_RSS_L3_PRE32 | RTE_ETH_RSS_L3_PRE40 |
 871         RTE_ETH_RSS_L3_PRE48 | RTE_ETH_RSS_L3_PRE56 |
 872         RTE_ETH_RSS_L3_PRE96
 873 };
 874
 875 struct rss_attr_type {
 876         uint64_t attr;
 877         uint64_t type;
 878 };
 879
 880 #define VALID_RSS_IPV4_L4       (ETH_RSS_NONFRAG_IPV4_UDP       | \
 881                                  ETH_RSS_NONFRAG_IPV4_TCP       | \
 882                                  ETH_RSS_NONFRAG_IPV4_SCTP)
 883
 884 #define VALID_RSS_IPV6_L4       (ETH_RSS_NONFRAG_IPV6_UDP       | \
 885                                  ETH_RSS_NONFRAG_IPV6_TCP       | \
 886                                  ETH_RSS_NONFRAG_IPV6_SCTP)
 887
 888 #define VALID_RSS_IPV4          (ETH_RSS_IPV4 | VALID_RSS_IPV4_L4)
 889 #define VALID_RSS_IPV6          (ETH_RSS_IPV6 | VALID_RSS_IPV6_L4)
 890 #define VALID_RSS_L3            (VALID_RSS_IPV4 | VALID_RSS_IPV6)
 891 #define VALID_RSS_L4            (VALID_RSS_IPV4_L4 | VALID_RSS_IPV6_L4)
 892
 893 #define VALID_RSS_ATTR          (ETH_RSS_L3_SRC_ONLY    | \
 894                                  ETH_RSS_L3_DST_ONLY    | \
 895                                  ETH_RSS_L4_SRC_ONLY    | \
 896                                  ETH_RSS_L4_DST_ONLY    | \
 897                                  ETH_RSS_L2_SRC_ONLY    | \
 898                                  ETH_RSS_L2_DST_ONLY    | \
 899                                  RTE_ETH_RSS_L3_PRE64)
 900
 901 #define INVALID_RSS_ATTR        (RTE_ETH_RSS_L3_PRE32   | \
 902                                  RTE_ETH_RSS_L3_PRE40   | \
 903                                  RTE_ETH_RSS_L3_PRE48   | \
 904                                  RTE_ETH_RSS_L3_PRE56   | \
 905                                  RTE_ETH_RSS_L3_PRE96)
 906
 907 static struct rss_attr_type rss_attr_to_valid_type[] = {
 908         {ETH_RSS_L2_SRC_ONLY | ETH_RSS_L2_DST_ONLY,     ETH_RSS_ETH},
 909         {ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY,     VALID_RSS_L3},
 910         {ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY,     VALID_RSS_L4},
 911         /* current ipv6 prefix only supports prefix 64 bits*/
 912         {RTE_ETH_RSS_L3_PRE64,                          VALID_RSS_IPV6},
 913         {INVALID_RSS_ATTR,                              0}
 914 };
 915
 916 static bool
 917 iavf_any_invalid_rss_type(enum rte_eth_hash_function rss_func,
 918                           uint64_t rss_type, uint64_t allow_rss_type)
 919 {
 920         uint32_t i;
 921
 922         /**
 923          * Check if l3/l4 SRC/DST_ONLY is set for SYMMETRIC_TOEPLITZ
 924          * hash function.
 925          */
 926         if (rss_func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) {
 927                 if (rss_type & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY |
 928                     ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY))
 929                         return true;
 930         }
 931
 932         /* check invalid combination */
 933         for (i = 0; i < RTE_DIM(invalid_rss_comb); i++) {
 934                 if (__builtin_popcountll(rss_type & invalid_rss_comb[i]) > 1)
 935                         return true;
 936         }
 937
 938         /* check invalid RSS attribute */
 939         for (i = 0; i < RTE_DIM(rss_attr_to_valid_type); i++) {
 940                 struct rss_attr_type *rat = &rss_attr_to_valid_type[i];
 941
 942                 if (rat->attr & rss_type && !(rat->type & rss_type))
 943                         return true;
 944         }
 945
 946         /* check not allowed RSS type */
 947         rss_type &= ~VALID_RSS_ATTR;
 948
 949         return ((rss_type & allow_rss_type) != rss_type);
 950 }
 951
 952 static int
 953 iavf_hash_parse_action(struct iavf_pattern_match_item *match_item,
 954                        const struct rte_flow_action actions[],
 955                        uint64_t pattern_hint, void **meta,
 956                        struct rte_flow_error *error)
 957 {
 958         struct iavf_rss_meta *rss_meta = (struct iavf_rss_meta *)*meta;
 959         struct virtchnl_proto_hdrs *proto_hdrs;
 960         enum rte_flow_action_type action_type;
 961         const struct rte_flow_action_rss *rss;
 962         const struct rte_flow_action *action;
 963         uint64_t rss_type;
 964
 965         /* Supported action is RSS. */
 966         for (action = actions; action->type !=
 967                 RTE_FLOW_ACTION_TYPE_END; action++) {
 968                 action_type = action->type;
 969                 switch (action_type) {
 970                 case RTE_FLOW_ACTION_TYPE_RSS:
 971                         rss = action->conf;
 972                         rss_type = rss->types;
 973
 974                         if (rss->func ==
 975                             RTE_ETH_HASH_FUNCTION_SIMPLE_XOR){
 976                                 rss_meta->rss_algorithm =
 977                                         VIRTCHNL_RSS_ALG_XOR_ASYMMETRIC;
 978                                 return rte_flow_error_set(error, ENOTSUP,
 979                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 980                                         "function simple_xor is not supported");
 981                         } else if (rss->func ==
 982                                    RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) {
 983                                 rss_meta->rss_algorithm =
 984                                         VIRTCHNL_RSS_ALG_TOEPLITZ_SYMMETRIC;
 985                         } else {
 986                                 rss_meta->rss_algorithm =
 987                                         VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC;
 988                         }
 989
 990                         if (rss->level)
 991                                 return rte_flow_error_set(error, ENOTSUP,
 992                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 993                                         "a nonzero RSS encapsulation level is not supported");
 994
 995                         if (rss->key_len)
 996                                 return rte_flow_error_set(error, ENOTSUP,
 997                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 998                                         "a nonzero RSS key_len is not supported");
 999
1000                         if (rss->queue_num)
1001                                 return rte_flow_error_set(error, ENOTSUP,
1002                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
1003                                         "a non-NULL RSS queue is not supported");
1004
1005                         /**
1006                          * Check simultaneous use of SRC_ONLY and DST_ONLY
1007                          * of the same level.
1008                          */
1009                         rss_type = rte_eth_rss_hf_refine(rss_type);
1010
1011                         if (iavf_any_invalid_rss_type(rss->func, rss_type,
1012                                         match_item->input_set_mask))
1013                                 return rte_flow_error_set(error, ENOTSUP,
1014                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1015                                                 action, "RSS type not supported");
1016                         proto_hdrs = match_item->meta;
1017                         rss_meta->proto_hdrs = *proto_hdrs;
1018                         iavf_refine_proto_hdrs(&rss_meta->proto_hdrs,
1019                                                rss_type, pattern_hint);
1020                         break;
1021
1022                 case RTE_FLOW_ACTION_TYPE_END:
1023                         break;
1024
1025                 default:
1026                         rte_flow_error_set(error, EINVAL,
1027                                            RTE_FLOW_ERROR_TYPE_ACTION, action,
1028                                            "Invalid action.");
1029                         return -rte_errno;
1030                 }
1031         }
1032
1033         return 0;
1034 }
1035
1036 static int
1037 iavf_hash_parse_pattern_action(__rte_unused struct iavf_adapter *ad,
1038                                struct iavf_pattern_match_item *array,
1039                                uint32_t array_len,
1040                                const struct rte_flow_item pattern[],
1041                                const struct rte_flow_action actions[],
1042                                void **meta,
1043                                struct rte_flow_error *error)
1044 {
1045         struct iavf_pattern_match_item *pattern_match_item;
1046         struct iavf_rss_meta *rss_meta_ptr;
1047         uint64_t phint = IAVF_PHINT_NONE;
1048         int ret = 0;
1049
1050         rss_meta_ptr = rte_zmalloc(NULL, sizeof(*rss_meta_ptr), 0);
1051         if (!rss_meta_ptr) {
1052                 rte_flow_error_set(error, EINVAL,
1053                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1054                                    "No memory for rss_meta_ptr");
1055                 return -ENOMEM;
1056         }
1057
1058         /* Check rss supported pattern and find matched pattern. */
1059         pattern_match_item =
1060                 iavf_search_pattern_match_item(pattern, array, array_len,
1061                                                error);
1062         if (!pattern_match_item) {
1063                 ret = -rte_errno;
1064                 goto error;
1065         }
1066
1067         ret = iavf_hash_parse_pattern(pattern, &phint, error);
1068         if (ret)
1069                 goto error;
1070
1071         ret = iavf_hash_parse_action(pattern_match_item, actions, phint,
1072                                      (void **)&rss_meta_ptr, error);
1073
1074 error:
1075         if (!ret && meta)
1076                 *meta = rss_meta_ptr;
1077         else
1078                 rte_free(rss_meta_ptr);
1079
1080         rte_free(pattern_match_item);
1081
1082         return ret;
1083 }
1084
1085 static int
1086 iavf_hash_create(__rte_unused struct iavf_adapter *ad,
1087                  __rte_unused struct rte_flow *flow, void *meta,
1088                  __rte_unused struct rte_flow_error *error)
1089 {
1090         struct iavf_rss_meta *rss_meta = (struct iavf_rss_meta *)meta;
1091         struct virtchnl_rss_cfg *rss_cfg;
1092         int ret = 0;
1093
1094         rss_cfg = rte_zmalloc("iavf rss rule",
1095                               sizeof(struct virtchnl_rss_cfg), 0);
1096         if (!rss_cfg) {
1097                 rte_flow_error_set(error, EINVAL,
1098                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1099                                    "No memory for rss rule");
1100                 return -ENOMEM;
1101         }
1102
1103         rss_cfg->proto_hdrs = rss_meta->proto_hdrs;
1104         rss_cfg->rss_algorithm = rss_meta->rss_algorithm;
1105
1106         ret = iavf_add_del_rss_cfg(ad, rss_cfg, true);
1107         if (!ret) {
1108                 flow->rule = rss_cfg;
1109         } else {
1110                 PMD_DRV_LOG(ERR, "fail to add RSS configure");
1111                 rte_flow_error_set(error, -ret,
1112                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1113                                    "Failed to add rss rule.");
1114                 rte_free(rss_cfg);
1115                 return -rte_errno;
1116         }
1117
1118         rte_free(meta);
1119
1120         return ret;
1121 }
1122
1123 static int
1124 iavf_hash_destroy(__rte_unused struct iavf_adapter *ad,
1125                   struct rte_flow *flow,
1126                   __rte_unused struct rte_flow_error *error)
1127 {
1128         struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
1129         struct virtchnl_rss_cfg *rss_cfg;
1130         int ret = 0;
1131
1132         if (vf->vf_reset)
1133                 return 0;
1134
1135         rss_cfg = (struct virtchnl_rss_cfg *)flow->rule;
1136
1137         ret = iavf_add_del_rss_cfg(ad, rss_cfg, false);
1138         if (ret) {
1139                 PMD_DRV_LOG(ERR, "fail to del RSS configure");
1140                 rte_flow_error_set(error, -ret,
1141                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1142                                    "Failed to delete rss rule.");
1143                 return -rte_errno;
1144         }
1145         return ret;
1146 }
1147
1148 static void
1149 iavf_hash_uninit(struct iavf_adapter *ad)
1150 {
1151         struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
1152         struct rte_eth_rss_conf *rss_conf;
1153
1154         if (vf->vf_reset)
1155                 return;
1156
1157         if (!vf->vf_res)
1158                 return;
1159
1160         if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF))
1161                 return;
1162
1163         rss_conf = &ad->eth_dev->data->dev_conf.rx_adv_conf.rss_conf;
1164         if (iavf_rss_hash_set(ad, rss_conf->rss_hf, false))
1165                 PMD_DRV_LOG(ERR, "fail to delete default RSS");
1166
1167         iavf_unregister_parser(&iavf_hash_parser, ad);
1168 }
1169
1170 static void
1171 iavf_hash_free(struct rte_flow *flow)
1172 {
1173         rte_free(flow->rule);
1174 }