drivers/net/iavf/iavf_hash.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10 #include <unistd.h>
  11 #include <stdarg.h>
  12
  13 #include <rte_debug.h>
  14 #include <rte_ether.h>
  15 #include <rte_ethdev_driver.h>
  16 #include <rte_log.h>
  17 #include <rte_malloc.h>
  18 #include <rte_eth_ctrl.h>
  19 #include <rte_tailq.h>
  20 #include <rte_flow_driver.h>
  21
  22 #include "iavf_log.h"
  23 #include "iavf.h"
  24 #include "iavf_generic_flow.h"
  25
  26 #define IAVF_PHINT_NONE                         0
  27 #define IAVF_PHINT_GTPU                         BIT_ULL(0)
  28 #define IAVF_PHINT_GTPU_EH                      BIT_ULL(1)
  29 #define IAVF_PHINT_GTPU_EH_DWN                  BIT_ULL(2)
  30 #define IAVF_PHINT_GTPU_EH_UP                   BIT_ULL(3)
  31 #define IAVF_PHINT_OUTER_IPV4                   BIT_ULL(4)
  32 #define IAVF_PHINT_OUTER_IPV6                   BIT_ULL(5)
  33
  34 #define IAVF_PHINT_GTPU_MSK     (IAVF_PHINT_GTPU        | \
  35                                  IAVF_PHINT_GTPU_EH     | \
  36                                  IAVF_PHINT_GTPU_EH_DWN | \
  37                                  IAVF_PHINT_GTPU_EH_UP)
  38
  39 #define IAVF_PHINT_LAYERS_MSK   (IAVF_PHINT_OUTER_IPV4  | \
  40                                  IAVF_PHINT_OUTER_IPV6)
  41
  42 #define IAVF_GTPU_EH_DWNLINK    0
  43 #define IAVF_GTPU_EH_UPLINK     1
  44
  45 struct iavf_hash_match_type {
  46         uint64_t hash_type;
  47         struct virtchnl_proto_hdrs *proto_hdrs;
  48         uint64_t pattern_hint;
  49 };
  50
  51 struct iavf_rss_meta {
  52         struct virtchnl_proto_hdrs proto_hdrs;
  53         enum virtchnl_rss_algorithm rss_algorithm;
  54 };
  55
  56 struct iavf_hash_flow_cfg {
  57         struct virtchnl_rss_cfg *rss_cfg;
  58         bool simple_xor;
  59 };
  60
  61 static int
  62 iavf_hash_init(struct iavf_adapter *ad);
  63 static int
  64 iavf_hash_create(struct iavf_adapter *ad, struct rte_flow *flow, void *meta,
  65                  struct rte_flow_error *error);
  66 static int
  67 iavf_hash_destroy(struct iavf_adapter *ad, struct rte_flow *flow,
  68                   struct rte_flow_error *error);
  69 static void
  70 iavf_hash_uninit(struct iavf_adapter *ad);
  71 static void
  72 iavf_hash_free(struct rte_flow *flow);
  73 static int
  74 iavf_hash_parse_pattern_action(struct iavf_adapter *ad,
  75                                struct iavf_pattern_match_item *array,
  76                                uint32_t array_len,
  77                                const struct rte_flow_item pattern[],
  78                                const struct rte_flow_action actions[],
  79                                void **meta,
  80                                struct rte_flow_error *error);
  81
  82 #define FIELD_SELECTOR(proto_hdr_field) \
  83                 (1UL << ((proto_hdr_field) & PROTO_HDR_FIELD_MASK))
  84 #define BUFF_NOUSED                     0
  85
  86 #define proto_hdr_eth { \
  87         VIRTCHNL_PROTO_HDR_ETH, \
  88         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_SRC) | \
  89         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_DST), {BUFF_NOUSED} }
  90
  91 #define proto_hdr_svlan { \
  92         VIRTCHNL_PROTO_HDR_S_VLAN, \
  93         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_S_VLAN_ID), {BUFF_NOUSED} }
  94
  95 #define proto_hdr_cvlan { \
  96         VIRTCHNL_PROTO_HDR_C_VLAN, \
  97         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_C_VLAN_ID), {BUFF_NOUSED} }
  98
  99 #define proto_hdr_ipv4 { \
 100         VIRTCHNL_PROTO_HDR_IPV4, \
 101         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_SRC) | \
 102         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_DST), {BUFF_NOUSED} }
 103
 104 #define proto_hdr_ipv6 { \
 105         VIRTCHNL_PROTO_HDR_IPV6, \
 106         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_SRC) | \
 107         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_DST), {BUFF_NOUSED} }
 108
 109 #define proto_hdr_udp { \
 110         VIRTCHNL_PROTO_HDR_UDP, \
 111         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_UDP_SRC_PORT) | \
 112         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_UDP_DST_PORT), {BUFF_NOUSED} }
 113
 114 #define proto_hdr_tcp { \
 115         VIRTCHNL_PROTO_HDR_TCP, \
 116         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_TCP_SRC_PORT) | \
 117         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_TCP_DST_PORT), {BUFF_NOUSED} }
 118
 119 #define proto_hdr_sctp { \
 120         VIRTCHNL_PROTO_HDR_SCTP, \
 121         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_SCTP_SRC_PORT) | \
 122         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_SCTP_DST_PORT), {BUFF_NOUSED} }
 123
 124 #define proto_hdr_esp { \
 125         VIRTCHNL_PROTO_HDR_ESP, \
 126         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ESP_SPI), {BUFF_NOUSED} }
 127
 128 #define proto_hdr_ah { \
 129         VIRTCHNL_PROTO_HDR_AH, \
 130         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_AH_SPI), {BUFF_NOUSED} }
 131
 132 #define proto_hdr_l2tpv3 { \
 133         VIRTCHNL_PROTO_HDR_L2TPV3, \
 134         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_L2TPV3_SESS_ID), {BUFF_NOUSED} }
 135
 136 #define proto_hdr_pfcp { \
 137         VIRTCHNL_PROTO_HDR_PFCP, \
 138         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_PFCP_SEID), {BUFF_NOUSED} }
 139
 140 #define proto_hdr_gtpc { \
 141         VIRTCHNL_PROTO_HDR_GTPC, 0, {BUFF_NOUSED} }
 142
 143 #define TUNNEL_LEVEL_OUTER              0
 144 #define TUNNEL_LEVEL_INNER              1
 145
 146 /* proto_hdrs template */
 147 struct virtchnl_proto_hdrs outer_ipv4_tmplt = {
 148         TUNNEL_LEVEL_OUTER, 4,
 149         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv4}
 150 };
 151
 152 struct virtchnl_proto_hdrs outer_ipv4_udp_tmplt = {
 153         TUNNEL_LEVEL_OUTER, 5,
 154         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv4,
 155          proto_hdr_udp}
 156 };
 157
 158 struct virtchnl_proto_hdrs outer_ipv4_tcp_tmplt = {
 159         TUNNEL_LEVEL_OUTER, 5,
 160         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv4,
 161          proto_hdr_tcp}
 162 };
 163
 164 struct virtchnl_proto_hdrs outer_ipv4_sctp_tmplt = {
 165         TUNNEL_LEVEL_OUTER, 5,
 166         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv4,
 167          proto_hdr_sctp}
 168 };
 169
 170 struct virtchnl_proto_hdrs outer_ipv6_tmplt = {
 171         TUNNEL_LEVEL_OUTER, 4,
 172         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv6}
 173 };
 174
 175 struct virtchnl_proto_hdrs outer_ipv6_udp_tmplt = {
 176         TUNNEL_LEVEL_OUTER, 5,
 177         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv6,
 178          proto_hdr_udp}
 179 };
 180
 181 struct virtchnl_proto_hdrs outer_ipv6_tcp_tmplt = {
 182         TUNNEL_LEVEL_OUTER, 5,
 183         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv6,
 184          proto_hdr_tcp}
 185 };
 186
 187 struct virtchnl_proto_hdrs outer_ipv6_sctp_tmplt = {
 188         TUNNEL_LEVEL_OUTER, 5,
 189         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv6,
 190          proto_hdr_sctp}
 191 };
 192
 193 struct virtchnl_proto_hdrs inner_ipv4_tmplt = {
 194         TUNNEL_LEVEL_INNER, 1, {proto_hdr_ipv4}
 195 };
 196
 197 struct virtchnl_proto_hdrs inner_ipv4_udp_tmplt = {
 198         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv4, proto_hdr_udp}
 199 };
 200
 201 struct virtchnl_proto_hdrs inner_ipv4_tcp_tmplt = {
 202         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv4, proto_hdr_tcp}
 203 };
 204
 205 struct virtchnl_proto_hdrs inner_ipv4_sctp_tmplt = {
 206         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv4, proto_hdr_sctp}
 207 };
 208
 209 struct virtchnl_proto_hdrs inner_ipv6_tmplt = {
 210         TUNNEL_LEVEL_INNER, 1, {proto_hdr_ipv6}
 211 };
 212
 213 struct virtchnl_proto_hdrs inner_ipv6_udp_tmplt = {
 214         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv6, proto_hdr_udp}
 215 };
 216
 217 struct virtchnl_proto_hdrs inner_ipv6_tcp_tmplt = {
 218         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv6, proto_hdr_tcp}
 219 };
 220
 221 struct virtchnl_proto_hdrs inner_ipv6_sctp_tmplt = {
 222         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv6, proto_hdr_sctp}
 223 };
 224
 225 struct virtchnl_proto_hdrs ipv4_esp_tmplt = {
 226         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_esp}
 227 };
 228
 229 struct virtchnl_proto_hdrs ipv4_udp_esp_tmplt = {
 230         TUNNEL_LEVEL_OUTER, 3,
 231         {proto_hdr_ipv4, proto_hdr_udp, proto_hdr_esp}
 232 };
 233
 234 struct virtchnl_proto_hdrs ipv4_ah_tmplt = {
 235         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_ah}
 236 };
 237
 238 struct virtchnl_proto_hdrs ipv6_esp_tmplt = {
 239         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_esp}
 240 };
 241
 242 struct virtchnl_proto_hdrs ipv6_udp_esp_tmplt = {
 243         TUNNEL_LEVEL_OUTER, 3,
 244         {proto_hdr_ipv6, proto_hdr_udp, proto_hdr_esp}
 245 };
 246
 247 struct virtchnl_proto_hdrs ipv6_ah_tmplt = {
 248         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_ah}
 249 };
 250
 251 struct virtchnl_proto_hdrs ipv4_l2tpv3_tmplt = {
 252         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_l2tpv3}
 253 };
 254
 255 struct virtchnl_proto_hdrs ipv6_l2tpv3_tmplt = {
 256         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_l2tpv3}
 257 };
 258
 259 struct virtchnl_proto_hdrs ipv4_pfcp_tmplt = {
 260         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_pfcp}
 261 };
 262
 263 struct virtchnl_proto_hdrs ipv6_pfcp_tmplt = {
 264         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_pfcp}
 265 };
 266
 267 struct virtchnl_proto_hdrs ipv4_udp_gtpc_tmplt = {
 268         TUNNEL_LEVEL_OUTER, 3, {proto_hdr_ipv4, proto_hdr_udp, proto_hdr_gtpc}
 269 };
 270
 271 struct virtchnl_proto_hdrs ipv6_udp_gtpc_tmplt = {
 272         TUNNEL_LEVEL_OUTER, 3, {proto_hdr_ipv6, proto_hdr_udp, proto_hdr_gtpc}
 273 };
 274
 275 /* rss type super set */
 276
 277 /* IPv4 outer */
 278 #define IAVF_RSS_TYPE_OUTER_IPV4        (ETH_RSS_ETH | ETH_RSS_IPV4)
 279 #define IAVF_RSS_TYPE_OUTER_IPV4_UDP    (IAVF_RSS_TYPE_OUTER_IPV4 | \
 280                                          ETH_RSS_NONFRAG_IPV4_UDP)
 281 #define IAVF_RSS_TYPE_OUTER_IPV4_TCP    (IAVF_RSS_TYPE_OUTER_IPV4 | \
 282                                          ETH_RSS_NONFRAG_IPV4_TCP)
 283 #define IAVF_RSS_TYPE_OUTER_IPV4_SCTP   (IAVF_RSS_TYPE_OUTER_IPV4 | \
 284                                          ETH_RSS_NONFRAG_IPV4_SCTP)
 285 /* IPv6 outer */
 286 #define IAVF_RSS_TYPE_OUTER_IPV6        (ETH_RSS_ETH | ETH_RSS_IPV6)
 287 #define IAVF_RSS_TYPE_OUTER_IPV6_UDP    (IAVF_RSS_TYPE_OUTER_IPV6 | \
 288                                          ETH_RSS_NONFRAG_IPV6_UDP)
 289 #define IAVF_RSS_TYPE_OUTER_IPV6_TCP    (IAVF_RSS_TYPE_OUTER_IPV6 | \
 290                                          ETH_RSS_NONFRAG_IPV6_TCP)
 291 #define IAVF_RSS_TYPE_OUTER_IPV6_SCTP   (IAVF_RSS_TYPE_OUTER_IPV6 | \
 292                                          ETH_RSS_NONFRAG_IPV6_SCTP)
 293 /* VLAN IPV4 */
 294 #define IAVF_RSS_TYPE_VLAN_IPV4         (IAVF_RSS_TYPE_OUTER_IPV4 | \
 295                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 296 #define IAVF_RSS_TYPE_VLAN_IPV4_UDP     (IAVF_RSS_TYPE_OUTER_IPV4_UDP | \
 297                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 298 #define IAVF_RSS_TYPE_VLAN_IPV4_TCP     (IAVF_RSS_TYPE_OUTER_IPV4_TCP | \
 299                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 300 #define IAVF_RSS_TYPE_VLAN_IPV4_SCTP    (IAVF_RSS_TYPE_OUTER_IPV4_SCTP | \
 301                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 302 /* VLAN IPv6 */
 303 #define IAVF_RSS_TYPE_VLAN_IPV6         (IAVF_RSS_TYPE_OUTER_IPV6 | \
 304                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 305 #define IAVF_RSS_TYPE_VLAN_IPV6_UDP     (IAVF_RSS_TYPE_OUTER_IPV6_UDP | \
 306                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 307 #define IAVF_RSS_TYPE_VLAN_IPV6_TCP     (IAVF_RSS_TYPE_OUTER_IPV6_TCP | \
 308                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 309 #define IAVF_RSS_TYPE_VLAN_IPV6_SCTP    (IAVF_RSS_TYPE_OUTER_IPV6_SCTP | \
 310                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 311 /* IPv4 inner */
 312 #define IAVF_RSS_TYPE_INNER_IPV4        ETH_RSS_IPV4
 313 #define IAVF_RSS_TYPE_INNER_IPV4_UDP    (ETH_RSS_IPV4 | \
 314                                          ETH_RSS_NONFRAG_IPV4_UDP)
 315 #define IAVF_RSS_TYPE_INNER_IPV4_TCP    (ETH_RSS_IPV4 | \
 316                                          ETH_RSS_NONFRAG_IPV4_TCP)
 317 #define IAVF_RSS_TYPE_INNER_IPV4_SCTP   (ETH_RSS_IPV4 | \
 318                                          ETH_RSS_NONFRAG_IPV4_SCTP)
 319 /* IPv6 inner */
 320 #define IAVF_RSS_TYPE_INNER_IPV6        ETH_RSS_IPV6
 321 #define IAVF_RSS_TYPE_INNER_IPV6_UDP    (ETH_RSS_IPV6 | \
 322                                          ETH_RSS_NONFRAG_IPV6_UDP)
 323 #define IAVF_RSS_TYPE_INNER_IPV6_TCP    (ETH_RSS_IPV6 | \
 324                                          ETH_RSS_NONFRAG_IPV6_TCP)
 325 #define IAVF_RSS_TYPE_INNER_IPV6_SCTP   (ETH_RSS_IPV6 | \
 326                                          ETH_RSS_NONFRAG_IPV6_SCTP)
 327 /* GTPU IPv4 */
 328 #define IAVF_RSS_TYPE_GTPU_IPV4         (IAVF_RSS_TYPE_INNER_IPV4 | \
 329                                          ETH_RSS_GTPU)
 330 #define IAVF_RSS_TYPE_GTPU_IPV4_UDP     (IAVF_RSS_TYPE_INNER_IPV4_UDP | \
 331                                          ETH_RSS_GTPU)
 332 #define IAVF_RSS_TYPE_GTPU_IPV4_TCP     (IAVF_RSS_TYPE_INNER_IPV4_TCP | \
 333                                          ETH_RSS_GTPU)
 334 /* GTPU IPv6 */
 335 #define IAVF_RSS_TYPE_GTPU_IPV6         (IAVF_RSS_TYPE_INNER_IPV6 | \
 336                                          ETH_RSS_GTPU)
 337 #define IAVF_RSS_TYPE_GTPU_IPV6_UDP     (IAVF_RSS_TYPE_INNER_IPV6_UDP | \
 338                                          ETH_RSS_GTPU)
 339 #define IAVF_RSS_TYPE_GTPU_IPV6_TCP     (IAVF_RSS_TYPE_INNER_IPV6_TCP | \
 340                                          ETH_RSS_GTPU)
 341 /* ESP, AH, L2TPV3 and PFCP */
 342 #define IAVF_RSS_TYPE_IPV4_ESP          (ETH_RSS_ESP | ETH_RSS_IPV4)
 343 #define IAVF_RSS_TYPE_IPV4_AH           (ETH_RSS_AH | ETH_RSS_IPV4)
 344 #define IAVF_RSS_TYPE_IPV6_ESP          (ETH_RSS_ESP | ETH_RSS_IPV6)
 345 #define IAVF_RSS_TYPE_IPV6_AH           (ETH_RSS_AH | ETH_RSS_IPV6)
 346 #define IAVF_RSS_TYPE_IPV4_L2TPV3       (ETH_RSS_L2TPV3 | ETH_RSS_IPV4)
 347 #define IAVF_RSS_TYPE_IPV6_L2TPV3       (ETH_RSS_L2TPV3 | ETH_RSS_IPV6)
 348 #define IAVF_RSS_TYPE_IPV4_PFCP         (ETH_RSS_PFCP | ETH_RSS_IPV4)
 349 #define IAVF_RSS_TYPE_IPV6_PFCP         (ETH_RSS_PFCP | ETH_RSS_IPV6)
 350
 351 /**
 352  * Supported pattern for hash.
 353  * The first member is pattern item type,
 354  * the second member is input set mask,
 355  * the third member is virtchnl_proto_hdrs template
 356  */
 357 static struct iavf_pattern_match_item iavf_hash_pattern_list[] = {
 358         /* IPv4 */
 359         {iavf_pattern_eth_ipv4,                         IAVF_RSS_TYPE_OUTER_IPV4,       &outer_ipv4_tmplt},
 360         {iavf_pattern_eth_ipv4_udp,                     IAVF_RSS_TYPE_OUTER_IPV4_UDP,   &outer_ipv4_udp_tmplt},
 361         {iavf_pattern_eth_ipv4_tcp,                     IAVF_RSS_TYPE_OUTER_IPV4_TCP,   &outer_ipv4_tcp_tmplt},
 362         {iavf_pattern_eth_ipv4_sctp,                    IAVF_RSS_TYPE_OUTER_IPV4_SCTP,  &outer_ipv4_sctp_tmplt},
 363         {iavf_pattern_eth_vlan_ipv4,                    IAVF_RSS_TYPE_VLAN_IPV4,        &outer_ipv4_tmplt},
 364         {iavf_pattern_eth_vlan_ipv4_udp,                IAVF_RSS_TYPE_VLAN_IPV4_UDP,    &outer_ipv4_udp_tmplt},
 365         {iavf_pattern_eth_vlan_ipv4_tcp,                IAVF_RSS_TYPE_VLAN_IPV4_TCP,    &outer_ipv4_tcp_tmplt},
 366         {iavf_pattern_eth_vlan_ipv4_sctp,               IAVF_RSS_TYPE_VLAN_IPV4_SCTP,   &outer_ipv4_sctp_tmplt},
 367         {iavf_pattern_eth_ipv4_gtpu,                    ETH_RSS_IPV4,                   &outer_ipv4_udp_tmplt},
 368         {iavf_pattern_eth_ipv4_gtpu_ipv4,               IAVF_RSS_TYPE_GTPU_IPV4,        &inner_ipv4_tmplt},
 369         {iavf_pattern_eth_ipv4_gtpu_ipv4_udp,           IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &inner_ipv4_udp_tmplt},
 370         {iavf_pattern_eth_ipv4_gtpu_ipv4_tcp,           IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &inner_ipv4_tcp_tmplt},
 371         {iavf_pattern_eth_ipv6_gtpu_ipv4,               IAVF_RSS_TYPE_GTPU_IPV4,        &inner_ipv4_tmplt},
 372         {iavf_pattern_eth_ipv6_gtpu_ipv4_udp,           IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &inner_ipv4_udp_tmplt},
 373         {iavf_pattern_eth_ipv6_gtpu_ipv4_tcp,           IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &inner_ipv4_tcp_tmplt},
 374         {iavf_pattern_eth_ipv4_gtpu_eh_ipv4,            IAVF_RSS_TYPE_GTPU_IPV4,        &inner_ipv4_tmplt},
 375         {iavf_pattern_eth_ipv4_gtpu_eh_ipv4_udp,        IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &inner_ipv4_udp_tmplt},
 376         {iavf_pattern_eth_ipv4_gtpu_eh_ipv4_tcp,        IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &inner_ipv4_tcp_tmplt},
 377         {iavf_pattern_eth_ipv6_gtpu_eh_ipv4,            IAVF_RSS_TYPE_GTPU_IPV4,        &inner_ipv4_tmplt},
 378         {iavf_pattern_eth_ipv6_gtpu_eh_ipv4_udp,        IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &inner_ipv4_udp_tmplt},
 379         {iavf_pattern_eth_ipv6_gtpu_eh_ipv4_tcp,        IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &inner_ipv4_tcp_tmplt},
 380         {iavf_pattern_eth_ipv4_esp,                     IAVF_RSS_TYPE_IPV4_ESP,         &ipv4_esp_tmplt},
 381         {iavf_pattern_eth_ipv4_udp_esp,                 IAVF_RSS_TYPE_IPV4_ESP,         &ipv4_udp_esp_tmplt},
 382         {iavf_pattern_eth_ipv4_ah,                      IAVF_RSS_TYPE_IPV4_AH,          &ipv4_ah_tmplt},
 383         {iavf_pattern_eth_ipv4_l2tpv3,                  IAVF_RSS_TYPE_IPV4_L2TPV3,      &ipv4_l2tpv3_tmplt},
 384         {iavf_pattern_eth_ipv4_pfcp,                    IAVF_RSS_TYPE_IPV4_PFCP,        &ipv4_pfcp_tmplt},
 385         {iavf_pattern_eth_ipv4_gtpc,                    ETH_RSS_IPV4,                   &ipv4_udp_gtpc_tmplt},
 386         /* IPv6 */
 387         {iavf_pattern_eth_ipv6,                         IAVF_RSS_TYPE_OUTER_IPV6,       &outer_ipv6_tmplt},
 388         {iavf_pattern_eth_ipv6_udp,                     IAVF_RSS_TYPE_OUTER_IPV6_UDP,   &outer_ipv6_udp_tmplt},
 389         {iavf_pattern_eth_ipv6_tcp,                     IAVF_RSS_TYPE_OUTER_IPV6_TCP,   &outer_ipv6_tcp_tmplt},
 390         {iavf_pattern_eth_ipv6_sctp,                    IAVF_RSS_TYPE_OUTER_IPV6_SCTP,  &outer_ipv6_sctp_tmplt},
 391         {iavf_pattern_eth_vlan_ipv6,                    IAVF_RSS_TYPE_VLAN_IPV6,        &outer_ipv6_tmplt},
 392         {iavf_pattern_eth_vlan_ipv6_udp,                IAVF_RSS_TYPE_VLAN_IPV6_UDP,    &outer_ipv6_udp_tmplt},
 393         {iavf_pattern_eth_vlan_ipv6_tcp,                IAVF_RSS_TYPE_VLAN_IPV6_TCP,    &outer_ipv6_tcp_tmplt},
 394         {iavf_pattern_eth_vlan_ipv6_sctp,               IAVF_RSS_TYPE_VLAN_IPV6_SCTP,   &outer_ipv6_sctp_tmplt},
 395         {iavf_pattern_eth_ipv6_gtpu,                    ETH_RSS_IPV6,                   &outer_ipv6_udp_tmplt},
 396         {iavf_pattern_eth_ipv4_gtpu_ipv6,               IAVF_RSS_TYPE_GTPU_IPV6,        &inner_ipv6_tmplt},
 397         {iavf_pattern_eth_ipv4_gtpu_ipv6_udp,           IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &inner_ipv6_udp_tmplt},
 398         {iavf_pattern_eth_ipv4_gtpu_ipv6_tcp,           IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &inner_ipv6_tcp_tmplt},
 399         {iavf_pattern_eth_ipv6_gtpu_ipv6,               IAVF_RSS_TYPE_GTPU_IPV6,        &inner_ipv6_tmplt},
 400         {iavf_pattern_eth_ipv6_gtpu_ipv6_udp,           IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &inner_ipv6_udp_tmplt},
 401         {iavf_pattern_eth_ipv6_gtpu_ipv6_tcp,           IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &inner_ipv6_tcp_tmplt},
 402         {iavf_pattern_eth_ipv4_gtpu_eh_ipv6,            IAVF_RSS_TYPE_GTPU_IPV6,        &inner_ipv6_tmplt},
 403         {iavf_pattern_eth_ipv4_gtpu_eh_ipv6_udp,        IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &inner_ipv6_udp_tmplt},
 404         {iavf_pattern_eth_ipv4_gtpu_eh_ipv6_tcp,        IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &inner_ipv6_tcp_tmplt},
 405         {iavf_pattern_eth_ipv6_gtpu_eh_ipv6,            IAVF_RSS_TYPE_GTPU_IPV6,        &inner_ipv6_tmplt},
 406         {iavf_pattern_eth_ipv6_gtpu_eh_ipv6_udp,        IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &inner_ipv6_udp_tmplt},
 407         {iavf_pattern_eth_ipv6_gtpu_eh_ipv6_tcp,        IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &inner_ipv6_tcp_tmplt},
 408         {iavf_pattern_eth_ipv6_esp,                     IAVF_RSS_TYPE_IPV6_ESP,         &ipv6_esp_tmplt},
 409         {iavf_pattern_eth_ipv6_udp_esp,                 IAVF_RSS_TYPE_IPV6_ESP,         &ipv6_udp_esp_tmplt},
 410         {iavf_pattern_eth_ipv6_ah,                      IAVF_RSS_TYPE_IPV6_AH,          &ipv6_ah_tmplt},
 411         {iavf_pattern_eth_ipv6_l2tpv3,                  IAVF_RSS_TYPE_IPV6_L2TPV3,      &ipv6_l2tpv3_tmplt},
 412         {iavf_pattern_eth_ipv6_pfcp,                    IAVF_RSS_TYPE_IPV6_PFCP,        &ipv6_pfcp_tmplt},
 413         {iavf_pattern_eth_ipv6_gtpc,                    ETH_RSS_IPV6,                   &ipv6_udp_gtpc_tmplt},
 414 };
 415
 416 struct virtchnl_proto_hdrs *iavf_hash_default_hdrs[] = {
 417         &inner_ipv4_tmplt,
 418         &inner_ipv4_udp_tmplt,
 419         &inner_ipv4_tcp_tmplt,
 420         &inner_ipv4_sctp_tmplt,
 421         &inner_ipv6_tmplt,
 422         &inner_ipv6_udp_tmplt,
 423         &inner_ipv6_tcp_tmplt,
 424         &inner_ipv6_sctp_tmplt,
 425 };
 426
 427 static struct iavf_flow_engine iavf_hash_engine = {
 428         .init = iavf_hash_init,
 429         .create = iavf_hash_create,
 430         .destroy = iavf_hash_destroy,
 431         .uninit = iavf_hash_uninit,
 432         .free = iavf_hash_free,
 433         .type = IAVF_FLOW_ENGINE_HASH,
 434 };
 435
 436 /* Register parser for comms package. */
 437 static struct iavf_flow_parser iavf_hash_parser = {
 438         .engine = &iavf_hash_engine,
 439         .array = iavf_hash_pattern_list,
 440         .array_len = RTE_DIM(iavf_hash_pattern_list),
 441         .parse_pattern_action = iavf_hash_parse_pattern_action,
 442         .stage = IAVF_FLOW_STAGE_RSS,
 443 };
 444
 445 static int
 446 iavf_hash_default_set(struct iavf_adapter *ad, bool add)
 447 {
 448         struct virtchnl_rss_cfg *rss_cfg;
 449         uint16_t i;
 450
 451         rss_cfg = rte_zmalloc("iavf rss rule",
 452                               sizeof(struct virtchnl_rss_cfg), 0);
 453         if (!rss_cfg)
 454                 return -ENOMEM;
 455
 456         for (i = 0; i < RTE_DIM(iavf_hash_default_hdrs); i++) {
 457                 rss_cfg->proto_hdrs = *iavf_hash_default_hdrs[i];
 458                 rss_cfg->rss_algorithm = VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC;
 459
 460                 iavf_add_del_rss_cfg(ad, rss_cfg, add);
 461         }
 462
 463         return 0;
 464 }
 465
 466 RTE_INIT(iavf_hash_engine_init)
 467 {
 468         struct iavf_flow_engine *engine = &iavf_hash_engine;
 469
 470         iavf_register_flow_engine(engine);
 471 }
 472
 473 static int
 474 iavf_hash_init(struct iavf_adapter *ad)
 475 {
 476         struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
 477         struct iavf_flow_parser *parser;
 478         int ret;
 479
 480         if (vf->vf_reset)
 481                 return -EIO;
 482
 483         if (!vf->vf_res)
 484                 return -EINVAL;
 485
 486         if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF))
 487                 return -ENOTSUP;
 488
 489         parser = &iavf_hash_parser;
 490
 491         ret = iavf_register_parser(parser, ad);
 492         if (ret) {
 493                 PMD_DRV_LOG(ERR, "fail to register hash parser");
 494                 return ret;
 495         }
 496
 497         ret = iavf_hash_default_set(ad, true);
 498         if (ret) {
 499                 PMD_DRV_LOG(ERR, "fail to set default RSS");
 500                 iavf_unregister_parser(parser, ad);
 501         }
 502
 503         return ret;
 504 }
 505
 506 static int
 507 iavf_hash_parse_pattern(const struct rte_flow_item pattern[], uint64_t *phint,
 508                         struct rte_flow_error *error)
 509 {
 510         const struct rte_flow_item *item = pattern;
 511         const struct rte_flow_item_gtp_psc *psc;
 512
 513         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 514                 if (item->last) {
 515                         rte_flow_error_set(error, EINVAL,
 516                                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 517                                            "Not support range");
 518                         return -rte_errno;
 519                 }
 520
 521                 switch (item->type) {
 522                 case RTE_FLOW_ITEM_TYPE_IPV4:
 523                         if (!(*phint & IAVF_PHINT_GTPU_MSK))
 524                                 *phint |= IAVF_PHINT_OUTER_IPV4;
 525                         break;
 526                 case RTE_FLOW_ITEM_TYPE_IPV6:
 527                         if (!(*phint & IAVF_PHINT_GTPU_MSK))
 528                                 *phint |= IAVF_PHINT_OUTER_IPV6;
 529                         break;
 530                 case RTE_FLOW_ITEM_TYPE_GTPU:
 531                         *phint |= IAVF_PHINT_GTPU;
 532                         break;
 533                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
 534                         *phint |= IAVF_PHINT_GTPU_EH;
 535                         psc = item->spec;
 536                         if (!psc)
 537                                 break;
 538                         else if (psc->pdu_type == IAVF_GTPU_EH_UPLINK)
 539                                 *phint |= IAVF_PHINT_GTPU_EH_UP;
 540                         else if (psc->pdu_type == IAVF_GTPU_EH_DWNLINK)
 541                                 *phint |= IAVF_PHINT_GTPU_EH_DWN;
 542                         break;
 543                 default:
 544                         break;
 545                 }
 546         }
 547
 548         return 0;
 549 }
 550
 551 #define REFINE_PROTO_FLD(op, fld) \
 552         VIRTCHNL_##op##_PROTO_HDR_FIELD(hdr, VIRTCHNL_PROTO_HDR_##fld)
 553 #define REPALCE_PROTO_FLD(fld_1, fld_2) \
 554 do { \
 555         REFINE_PROTO_FLD(DEL, fld_1);   \
 556         REFINE_PROTO_FLD(ADD, fld_2);   \
 557 } while (0)
 558
 559 /* refine proto hdrs base on l2, l3, l4 rss type */
 560 static void
 561 iavf_refine_proto_hdrs_l234(struct virtchnl_proto_hdrs *proto_hdrs,
 562                             uint64_t rss_type)
 563 {
 564         struct virtchnl_proto_hdr *hdr;
 565         int i;
 566
 567         for (i = 0; i < proto_hdrs->count; i++) {
 568                 hdr = &proto_hdrs->proto_hdr[i];
 569                 switch (hdr->type) {
 570                 case VIRTCHNL_PROTO_HDR_ETH:
 571                         if (!(rss_type & ETH_RSS_ETH))
 572                                 hdr->field_selector = 0;
 573                         else if (rss_type & ETH_RSS_L2_SRC_ONLY)
 574                                 REFINE_PROTO_FLD(DEL, ETH_DST);
 575                         else if (rss_type & ETH_RSS_L2_DST_ONLY)
 576                                 REFINE_PROTO_FLD(DEL, ETH_SRC);
 577                         break;
 578                 case VIRTCHNL_PROTO_HDR_IPV4:
 579                         if (rss_type &
 580                             (ETH_RSS_IPV4 |
 581                              ETH_RSS_NONFRAG_IPV4_UDP |
 582                              ETH_RSS_NONFRAG_IPV4_TCP |
 583                              ETH_RSS_NONFRAG_IPV4_SCTP)) {
 584                                 if (rss_type & ETH_RSS_L3_SRC_ONLY)
 585                                         REFINE_PROTO_FLD(DEL, IPV4_DST);
 586                                 else if (rss_type & ETH_RSS_L3_DST_ONLY)
 587                                         REFINE_PROTO_FLD(DEL, IPV4_SRC);
 588                                 else if (rss_type &
 589                                          (ETH_RSS_L4_SRC_ONLY |
 590                                           ETH_RSS_L4_DST_ONLY))
 591                                         hdr->field_selector = 0;
 592                         } else {
 593                                 hdr->field_selector = 0;
 594                         }
 595                         break;
 596                 case VIRTCHNL_PROTO_HDR_IPV6:
 597                         if (rss_type &
 598                             (ETH_RSS_IPV6 |
 599                              ETH_RSS_NONFRAG_IPV6_UDP |
 600                              ETH_RSS_NONFRAG_IPV6_TCP |
 601                              ETH_RSS_NONFRAG_IPV6_SCTP)) {
 602                                 if (rss_type & ETH_RSS_L3_SRC_ONLY)
 603                                         REFINE_PROTO_FLD(DEL, IPV6_DST);
 604                                 else if (rss_type & ETH_RSS_L3_DST_ONLY)
 605                                         REFINE_PROTO_FLD(DEL, IPV6_SRC);
 606                                 else if (rss_type &
 607                                          (ETH_RSS_L4_SRC_ONLY |
 608                                           ETH_RSS_L4_DST_ONLY))
 609                                         hdr->field_selector = 0;
 610                         } else {
 611                                 hdr->field_selector = 0;
 612                         }
 613                         if (rss_type & RTE_ETH_RSS_L3_PRE64) {
 614                                 if (REFINE_PROTO_FLD(TEST, IPV6_SRC))
 615                                         REPALCE_PROTO_FLD(IPV6_SRC,
 616                                                           IPV6_PREFIX64_SRC);
 617                                 if (REFINE_PROTO_FLD(TEST, IPV6_DST))
 618                                         REPALCE_PROTO_FLD(IPV6_DST,
 619                                                           IPV6_PREFIX64_DST);
 620                         }
 621                         break;
 622                 case VIRTCHNL_PROTO_HDR_UDP:
 623                         if (rss_type &
 624                             (ETH_RSS_NONFRAG_IPV4_UDP |
 625                              ETH_RSS_NONFRAG_IPV6_UDP)) {
 626                                 if (rss_type & ETH_RSS_L4_SRC_ONLY)
 627                                         REFINE_PROTO_FLD(DEL, UDP_DST_PORT);
 628                                 else if (rss_type & ETH_RSS_L4_DST_ONLY)
 629                                         REFINE_PROTO_FLD(DEL, UDP_SRC_PORT);
 630                                 else if (rss_type &
 631                                          (ETH_RSS_L3_SRC_ONLY |
 632                                           ETH_RSS_L3_DST_ONLY))
 633                                         hdr->field_selector = 0;
 634                         } else {
 635                                 hdr->field_selector = 0;
 636                         }
 637                         break;
 638                 case VIRTCHNL_PROTO_HDR_TCP:
 639                         if (rss_type &
 640                             (ETH_RSS_NONFRAG_IPV4_TCP |
 641                              ETH_RSS_NONFRAG_IPV6_TCP)) {
 642                                 if (rss_type & ETH_RSS_L4_SRC_ONLY)
 643                                         REFINE_PROTO_FLD(DEL, TCP_DST_PORT);
 644                                 else if (rss_type & ETH_RSS_L4_DST_ONLY)
 645                                         REFINE_PROTO_FLD(DEL, TCP_SRC_PORT);
 646                                 else if (rss_type &
 647                                          (ETH_RSS_L3_SRC_ONLY |
 648                                           ETH_RSS_L3_DST_ONLY))
 649                                         hdr->field_selector = 0;
 650                         } else {
 651                                 hdr->field_selector = 0;
 652                         }
 653                         break;
 654                 case VIRTCHNL_PROTO_HDR_SCTP:
 655                         if (rss_type &
 656                             (ETH_RSS_NONFRAG_IPV4_SCTP |
 657                              ETH_RSS_NONFRAG_IPV6_SCTP)) {
 658                                 if (rss_type & ETH_RSS_L4_SRC_ONLY)
 659                                         REFINE_PROTO_FLD(DEL, SCTP_DST_PORT);
 660                                 else if (rss_type & ETH_RSS_L4_DST_ONLY)
 661                                         REFINE_PROTO_FLD(DEL, SCTP_SRC_PORT);
 662                                 else if (rss_type &
 663                                          (ETH_RSS_L3_SRC_ONLY |
 664                                           ETH_RSS_L3_DST_ONLY))
 665                                         hdr->field_selector = 0;
 666                         } else {
 667                                 hdr->field_selector = 0;
 668                         }
 669                         break;
 670                 case VIRTCHNL_PROTO_HDR_S_VLAN:
 671                         if (!(rss_type & ETH_RSS_S_VLAN))
 672                                 hdr->field_selector = 0;
 673                         break;
 674                 case VIRTCHNL_PROTO_HDR_C_VLAN:
 675                         if (!(rss_type & ETH_RSS_C_VLAN))
 676                                 hdr->field_selector = 0;
 677                         break;
 678                 case VIRTCHNL_PROTO_HDR_L2TPV3:
 679                         if (!(rss_type & ETH_RSS_L2TPV3))
 680                                 hdr->field_selector = 0;
 681                         break;
 682                 case VIRTCHNL_PROTO_HDR_ESP:
 683                         if (!(rss_type & ETH_RSS_ESP))
 684                                 hdr->field_selector = 0;
 685                         break;
 686                 case VIRTCHNL_PROTO_HDR_AH:
 687                         if (!(rss_type & ETH_RSS_AH))
 688                                 hdr->field_selector = 0;
 689                         break;
 690                 case VIRTCHNL_PROTO_HDR_PFCP:
 691                         if (!(rss_type & ETH_RSS_PFCP))
 692                                 hdr->field_selector = 0;
 693                         break;
 694                 default:
 695                         break;
 696                 }
 697         }
 698 }
 699
 700 /* refine proto hdrs base on gtpu rss type */
 701 static void
 702 iavf_refine_proto_hdrs_gtpu(struct virtchnl_proto_hdrs *proto_hdrs,
 703                             uint64_t rss_type)
 704 {
 705         struct virtchnl_proto_hdr *hdr;
 706         int i;
 707
 708         if (!(rss_type & ETH_RSS_GTPU))
 709                 return;
 710
 711         for (i = 0; i < proto_hdrs->count; i++) {
 712                 hdr = &proto_hdrs->proto_hdr[i];
 713                 switch (hdr->type) {
 714                 case VIRTCHNL_PROTO_HDR_GTPU_IP:
 715                         REFINE_PROTO_FLD(ADD, GTPU_IP_TEID);
 716                         break;
 717                 default:
 718                         break;
 719                 }
 720         }
 721 }
 722
 723 static void
 724 iavf_refine_proto_hdrs_by_pattern(struct virtchnl_proto_hdrs *proto_hdrs,
 725                                   uint64_t phint)
 726 {
 727         struct virtchnl_proto_hdr *hdr1;
 728         struct virtchnl_proto_hdr *hdr2;
 729         int i, shift_count = 1;
 730
 731         if (!(phint & IAVF_PHINT_GTPU_MSK))
 732                 return;
 733
 734         if (phint & IAVF_PHINT_LAYERS_MSK)
 735                 shift_count++;
 736
 737         if (proto_hdrs->tunnel_level == TUNNEL_LEVEL_INNER) {
 738                 /* shift headers layer */
 739                 for (i = proto_hdrs->count - 1 + shift_count;
 740                      i > shift_count - 1; i--) {
 741                         hdr1 = &proto_hdrs->proto_hdr[i];
 742                         hdr2 = &proto_hdrs->proto_hdr[i - shift_count];
 743                         *hdr1 = *hdr2;
 744                 }
 745
 746                 if (shift_count == 1) {
 747                         /* adding gtpu header at layer 0 */
 748                         hdr1 = &proto_hdrs->proto_hdr[0];
 749                 } else {
 750                         /* adding gtpu header and outer ip header */
 751                         hdr1 = &proto_hdrs->proto_hdr[1];
 752                         hdr2 = &proto_hdrs->proto_hdr[0];
 753                         hdr2->field_selector = 0;
 754                         proto_hdrs->count++;
 755                         proto_hdrs->tunnel_level = TUNNEL_LEVEL_OUTER;
 756
 757                         if (phint & IAVF_PHINT_OUTER_IPV4)
 758                                 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr2, IPV4);
 759                         else if (phint & IAVF_PHINT_OUTER_IPV6)
 760                                 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr2, IPV6);
 761                 }
 762         } else {
 763                 hdr1 = &proto_hdrs->proto_hdr[proto_hdrs->count];
 764         }
 765
 766         hdr1->field_selector = 0;
 767         proto_hdrs->count++;
 768
 769         if (phint & IAVF_PHINT_GTPU_EH_DWN)
 770                 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_EH_PDU_DWN);
 771         else if (phint & IAVF_PHINT_GTPU_EH_UP)
 772                 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_EH_PDU_UP);
 773         else if (phint & IAVF_PHINT_GTPU_EH)
 774                 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_EH);
 775         else if (phint & IAVF_PHINT_GTPU)
 776                 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_IP);
 777 }
 778
 779 static void iavf_refine_proto_hdrs(struct virtchnl_proto_hdrs *proto_hdrs,
 780                                    uint64_t rss_type, uint64_t phint)
 781 {
 782         iavf_refine_proto_hdrs_l234(proto_hdrs, rss_type);
 783         iavf_refine_proto_hdrs_by_pattern(proto_hdrs, phint);
 784         iavf_refine_proto_hdrs_gtpu(proto_hdrs, rss_type);
 785 }
 786
 787 static uint64_t invalid_rss_comb[] = {
 788         ETH_RSS_IPV4 | ETH_RSS_NONFRAG_IPV4_UDP,
 789         ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_UDP,
 790         RTE_ETH_RSS_L3_PRE32 | RTE_ETH_RSS_L3_PRE40 |
 791         RTE_ETH_RSS_L3_PRE48 | RTE_ETH_RSS_L3_PRE56 |
 792         RTE_ETH_RSS_L3_PRE96
 793 };
 794
 795 struct rss_attr_type {
 796         uint64_t attr;
 797         uint64_t type;
 798 };
 799
 800 #define VALID_RSS_IPV4_L4       (ETH_RSS_NONFRAG_IPV4_UDP       | \
 801                                  ETH_RSS_NONFRAG_IPV4_TCP       | \
 802                                  ETH_RSS_NONFRAG_IPV4_SCTP)
 803
 804 #define VALID_RSS_IPV6_L4       (ETH_RSS_NONFRAG_IPV6_UDP       | \
 805                                  ETH_RSS_NONFRAG_IPV6_TCP       | \
 806                                  ETH_RSS_NONFRAG_IPV6_SCTP)
 807
 808 #define VALID_RSS_IPV4          (ETH_RSS_IPV4 | VALID_RSS_IPV4_L4)
 809 #define VALID_RSS_IPV6          (ETH_RSS_IPV6 | VALID_RSS_IPV6_L4)
 810 #define VALID_RSS_L3            (VALID_RSS_IPV4 | VALID_RSS_IPV6)
 811 #define VALID_RSS_L4            (VALID_RSS_IPV4_L4 | VALID_RSS_IPV6_L4)
 812
 813 #define VALID_RSS_ATTR          (ETH_RSS_L3_SRC_ONLY    | \
 814                                  ETH_RSS_L3_DST_ONLY    | \
 815                                  ETH_RSS_L4_SRC_ONLY    | \
 816                                  ETH_RSS_L4_DST_ONLY    | \
 817                                  ETH_RSS_L2_SRC_ONLY    | \
 818                                  ETH_RSS_L2_DST_ONLY    | \
 819                                  RTE_ETH_RSS_L3_PRE64)
 820
 821 #define INVALID_RSS_ATTR        (RTE_ETH_RSS_L3_PRE32   | \
 822                                  RTE_ETH_RSS_L3_PRE40   | \
 823                                  RTE_ETH_RSS_L3_PRE48   | \
 824                                  RTE_ETH_RSS_L3_PRE56   | \
 825                                  RTE_ETH_RSS_L3_PRE96)
 826
 827 static struct rss_attr_type rss_attr_to_valid_type[] = {
 828         {ETH_RSS_L2_SRC_ONLY | ETH_RSS_L2_DST_ONLY,     ETH_RSS_ETH},
 829         {ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY,     VALID_RSS_L3},
 830         {ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY,     VALID_RSS_L4},
 831         /* current ipv6 prefix only supports prefix 64 bits*/
 832         {RTE_ETH_RSS_L3_PRE64,                          VALID_RSS_IPV6},
 833         {INVALID_RSS_ATTR,                              0}
 834 };
 835
 836 static bool
 837 iavf_any_invalid_rss_type(uint64_t rss_type, uint64_t allow_rss_type)
 838 {
 839         uint32_t i;
 840
 841         /* check invalid combination */
 842         for (i = 0; i < RTE_DIM(invalid_rss_comb); i++) {
 843                 if (__builtin_popcountll(rss_type & invalid_rss_comb[i]) > 1)
 844                         return true;
 845         }
 846
 847         /* check invalid RSS attribute */
 848         for (i = 0; i < RTE_DIM(rss_attr_to_valid_type); i++) {
 849                 struct rss_attr_type *rat = &rss_attr_to_valid_type[i];
 850
 851                 if (rat->attr & rss_type && !(rat->type & rss_type))
 852                         return true;
 853         }
 854
 855         /* check not allowed RSS type */
 856         rss_type &= ~VALID_RSS_ATTR;
 857
 858         return ((rss_type & allow_rss_type) != rss_type);
 859 }
 860
 861 static int
 862 iavf_hash_parse_action(struct iavf_pattern_match_item *match_item,
 863                        const struct rte_flow_action actions[],
 864                        uint64_t pattern_hint, void **meta,
 865                        struct rte_flow_error *error)
 866 {
 867         struct iavf_rss_meta *rss_meta = (struct iavf_rss_meta *)*meta;
 868         struct virtchnl_proto_hdrs *proto_hdrs;
 869         enum rte_flow_action_type action_type;
 870         const struct rte_flow_action_rss *rss;
 871         const struct rte_flow_action *action;
 872         uint64_t rss_type;
 873
 874         /* Supported action is RSS. */
 875         for (action = actions; action->type !=
 876                 RTE_FLOW_ACTION_TYPE_END; action++) {
 877                 action_type = action->type;
 878                 switch (action_type) {
 879                 case RTE_FLOW_ACTION_TYPE_RSS:
 880                         rss = action->conf;
 881                         rss_type = rss->types;
 882
 883                         if (rss->func ==
 884                             RTE_ETH_HASH_FUNCTION_SIMPLE_XOR){
 885                                 rss_meta->rss_algorithm =
 886                                         VIRTCHNL_RSS_ALG_XOR_ASYMMETRIC;
 887                                 return rte_flow_error_set(error, ENOTSUP,
 888                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 889                                         "function simple_xor is not supported");
 890                         } else if (rss->func ==
 891                                    RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) {
 892                                 rss_meta->rss_algorithm =
 893                                         VIRTCHNL_RSS_ALG_TOEPLITZ_SYMMETRIC;
 894                         } else {
 895                                 rss_meta->rss_algorithm =
 896                                         VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC;
 897                         }
 898
 899                         if (rss->level)
 900                                 return rte_flow_error_set(error, ENOTSUP,
 901                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 902                                         "a nonzero RSS encapsulation level is not supported");
 903
 904                         if (rss->key_len)
 905                                 return rte_flow_error_set(error, ENOTSUP,
 906                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 907                                         "a nonzero RSS key_len is not supported");
 908
 909                         if (rss->queue_num)
 910                                 return rte_flow_error_set(error, ENOTSUP,
 911                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 912                                         "a non-NULL RSS queue is not supported");
 913
 914                         /**
 915                          * Check simultaneous use of SRC_ONLY and DST_ONLY
 916                          * of the same level.
 917                          */
 918                         rss_type = rte_eth_rss_hf_refine(rss_type);
 919
 920                         if (iavf_any_invalid_rss_type(rss_type,
 921                                         match_item->input_set_mask))
 922                                 return rte_flow_error_set(error, ENOTSUP,
 923                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 924                                                 action, "RSS type not supported");
 925                         proto_hdrs = match_item->meta;
 926                         rss_meta->proto_hdrs = *proto_hdrs;
 927                         iavf_refine_proto_hdrs(&rss_meta->proto_hdrs,
 928                                                rss_type, pattern_hint);
 929                         break;
 930
 931                 case RTE_FLOW_ACTION_TYPE_END:
 932                         break;
 933
 934                 default:
 935                         rte_flow_error_set(error, EINVAL,
 936                                            RTE_FLOW_ERROR_TYPE_ACTION, action,
 937                                            "Invalid action.");
 938                         return -rte_errno;
 939                 }
 940         }
 941
 942         return 0;
 943 }
 944
 945 static int
 946 iavf_hash_parse_pattern_action(__rte_unused struct iavf_adapter *ad,
 947                                struct iavf_pattern_match_item *array,
 948                                uint32_t array_len,
 949                                const struct rte_flow_item pattern[],
 950                                const struct rte_flow_action actions[],
 951                                void **meta,
 952                                struct rte_flow_error *error)
 953 {
 954         struct iavf_pattern_match_item *pattern_match_item;
 955         struct iavf_rss_meta *rss_meta_ptr;
 956         uint64_t phint = IAVF_PHINT_NONE;
 957         int ret = 0;
 958
 959         rss_meta_ptr = rte_zmalloc(NULL, sizeof(*rss_meta_ptr), 0);
 960         if (!rss_meta_ptr) {
 961                 rte_flow_error_set(error, EINVAL,
 962                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 963                                    "No memory for rss_meta_ptr");
 964                 return -ENOMEM;
 965         }
 966
 967         /* Check rss supported pattern and find matched pattern. */
 968         pattern_match_item =
 969                 iavf_search_pattern_match_item(pattern, array, array_len,
 970                                                error);
 971         if (!pattern_match_item) {
 972                 ret = -rte_errno;
 973                 goto error;
 974         }
 975
 976         ret = iavf_hash_parse_pattern(pattern, &phint, error);
 977         if (ret)
 978                 goto error;
 979
 980         ret = iavf_hash_parse_action(pattern_match_item, actions, phint,
 981                                      (void **)&rss_meta_ptr, error);
 982
 983 error:
 984         if (!ret && meta)
 985                 *meta = rss_meta_ptr;
 986         else
 987                 rte_free(rss_meta_ptr);
 988
 989         rte_free(pattern_match_item);
 990
 991         return ret;
 992 }
 993
 994 static int
 995 iavf_hash_create(__rte_unused struct iavf_adapter *ad,
 996                  __rte_unused struct rte_flow *flow, void *meta,
 997                  __rte_unused struct rte_flow_error *error)
 998 {
 999         struct iavf_rss_meta *rss_meta = (struct iavf_rss_meta *)meta;
1000         struct virtchnl_rss_cfg *rss_cfg;
1001         int ret = 0;
1002
1003         rss_cfg = rte_zmalloc("iavf rss rule",
1004                               sizeof(struct virtchnl_rss_cfg), 0);
1005         if (!rss_cfg) {
1006                 rte_flow_error_set(error, EINVAL,
1007                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1008                                    "No memory for rss rule");
1009                 return -ENOMEM;
1010         }
1011
1012         rss_cfg->proto_hdrs = rss_meta->proto_hdrs;
1013         rss_cfg->rss_algorithm = rss_meta->rss_algorithm;
1014
1015         ret = iavf_add_del_rss_cfg(ad, rss_cfg, true);
1016         if (!ret) {
1017                 flow->rule = rss_cfg;
1018         } else {
1019                 PMD_DRV_LOG(ERR, "fail to add RSS configure");
1020                 rte_flow_error_set(error, -ret,
1021                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1022                                    "Failed to add rss rule.");
1023                 rte_free(rss_cfg);
1024                 return -rte_errno;
1025         }
1026
1027         rte_free(meta);
1028
1029         return ret;
1030 }
1031
1032 static int
1033 iavf_hash_destroy(__rte_unused struct iavf_adapter *ad,
1034                   struct rte_flow *flow,
1035                   __rte_unused struct rte_flow_error *error)
1036 {
1037         struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
1038         struct virtchnl_rss_cfg *rss_cfg;
1039         int ret = 0;
1040
1041         if (vf->vf_reset)
1042                 return 0;
1043
1044         rss_cfg = (struct virtchnl_rss_cfg *)flow->rule;
1045
1046         ret = iavf_add_del_rss_cfg(ad, rss_cfg, false);
1047         if (ret) {
1048                 PMD_DRV_LOG(ERR, "fail to del RSS configure");
1049                 rte_flow_error_set(error, -ret,
1050                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1051                                    "Failed to delete rss rule.");
1052                 return -rte_errno;
1053         }
1054         return ret;
1055 }
1056
1057 static void
1058 iavf_hash_uninit(struct iavf_adapter *ad)
1059 {
1060         struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
1061
1062         if (vf->vf_reset)
1063                 return;
1064
1065         if (iavf_hash_default_set(ad, false))
1066                 PMD_DRV_LOG(ERR, "fail to delete default RSS");
1067
1068         iavf_unregister_parser(&iavf_hash_parser, ad);
1069 }
1070
1071 static void
1072 iavf_hash_free(struct rte_flow *flow)
1073 {
1074         rte_free(flow->rule);
1075 }