drivers/net/iavf/iavf_hash.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10 #include <unistd.h>
  11 #include <stdarg.h>
  12
  13 #include <rte_debug.h>
  14 #include <rte_ether.h>
  15 #include <rte_ethdev_driver.h>
  16 #include <rte_log.h>
  17 #include <rte_malloc.h>
  18 #include <rte_eth_ctrl.h>
  19 #include <rte_tailq.h>
  20 #include <rte_flow_driver.h>
  21
  22 #include "iavf_log.h"
  23 #include "iavf.h"
  24 #include "iavf_generic_flow.h"
  25
  26 #define IAVF_PHINT_NONE                         0
  27 #define IAVF_PHINT_GTPU                         BIT_ULL(0)
  28 #define IAVF_PHINT_GTPU_EH                      BIT_ULL(1)
  29 #define IAVF_PHINT_GTPU_EH_DWN                  BIT_ULL(2)
  30 #define IAVF_PHINT_GTPU_EH_UP                   BIT_ULL(3)
  31
  32 #define IAVF_PHINT_GTPU_MSK     (IAVF_PHINT_GTPU        | \
  33                                  IAVF_PHINT_GTPU_EH     | \
  34                                  IAVF_PHINT_GTPU_EH_DWN | \
  35                                  IAVF_PHINT_GTPU_EH_UP)
  36
  37 #define IAVF_GTPU_EH_DWNLINK    0
  38 #define IAVF_GTPU_EH_UPLINK     1
  39
  40 struct iavf_hash_match_type {
  41         uint64_t hash_type;
  42         struct virtchnl_proto_hdrs *proto_hdrs;
  43         uint64_t pattern_hint;
  44 };
  45
  46 struct iavf_rss_meta {
  47         struct virtchnl_proto_hdrs proto_hdrs;
  48         enum virtchnl_rss_algorithm rss_algorithm;
  49 };
  50
  51 struct iavf_hash_flow_cfg {
  52         struct virtchnl_rss_cfg *rss_cfg;
  53         bool simple_xor;
  54 };
  55
  56 static int
  57 iavf_hash_init(struct iavf_adapter *ad);
  58 static int
  59 iavf_hash_create(struct iavf_adapter *ad, struct rte_flow *flow, void *meta,
  60                  struct rte_flow_error *error);
  61 static int
  62 iavf_hash_destroy(struct iavf_adapter *ad, struct rte_flow *flow,
  63                   struct rte_flow_error *error);
  64 static void
  65 iavf_hash_uninit(struct iavf_adapter *ad);
  66 static void
  67 iavf_hash_free(struct rte_flow *flow);
  68 static int
  69 iavf_hash_parse_pattern_action(struct iavf_adapter *ad,
  70                                struct iavf_pattern_match_item *array,
  71                                uint32_t array_len,
  72                                const struct rte_flow_item pattern[],
  73                                const struct rte_flow_action actions[],
  74                                void **meta,
  75                                struct rte_flow_error *error);
  76
  77 #define FIELD_SELECTOR(proto_hdr_field) \
  78                 (1UL << ((proto_hdr_field) & PROTO_HDR_FIELD_MASK))
  79 #define BUFF_NOUSED                     0
  80
  81 #define proto_hdr_eth { \
  82         VIRTCHNL_PROTO_HDR_ETH, \
  83         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_SRC) | \
  84         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_DST), {BUFF_NOUSED} }
  85
  86 #define proto_hdr_svlan { \
  87         VIRTCHNL_PROTO_HDR_S_VLAN, \
  88         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_S_VLAN_ID), {BUFF_NOUSED} }
  89
  90 #define proto_hdr_cvlan { \
  91         VIRTCHNL_PROTO_HDR_C_VLAN, \
  92         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_C_VLAN_ID), {BUFF_NOUSED} }
  93
  94 #define proto_hdr_ipv4 { \
  95         VIRTCHNL_PROTO_HDR_IPV4, \
  96         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_SRC) | \
  97         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_DST), {BUFF_NOUSED} }
  98
  99 #define proto_hdr_ipv6 { \
 100         VIRTCHNL_PROTO_HDR_IPV6, \
 101         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_SRC) | \
 102         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_DST), {BUFF_NOUSED} }
 103
 104 #define proto_hdr_udp { \
 105         VIRTCHNL_PROTO_HDR_UDP, \
 106         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_UDP_SRC_PORT) | \
 107         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_UDP_DST_PORT), {BUFF_NOUSED} }
 108
 109 #define proto_hdr_tcp { \
 110         VIRTCHNL_PROTO_HDR_TCP, \
 111         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_TCP_SRC_PORT) | \
 112         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_TCP_DST_PORT), {BUFF_NOUSED} }
 113
 114 #define proto_hdr_sctp { \
 115         VIRTCHNL_PROTO_HDR_SCTP, \
 116         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_SCTP_SRC_PORT) | \
 117         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_SCTP_DST_PORT), {BUFF_NOUSED} }
 118
 119 #define proto_hdr_esp { \
 120         VIRTCHNL_PROTO_HDR_ESP, \
 121         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ESP_SPI), {BUFF_NOUSED} }
 122
 123 #define proto_hdr_ah { \
 124         VIRTCHNL_PROTO_HDR_AH, \
 125         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_AH_SPI), {BUFF_NOUSED} }
 126
 127 #define proto_hdr_l2tpv3 { \
 128         VIRTCHNL_PROTO_HDR_L2TPV3, \
 129         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_L2TPV3_SESS_ID), {BUFF_NOUSED} }
 130
 131 #define proto_hdr_pfcp { \
 132         VIRTCHNL_PROTO_HDR_PFCP, \
 133         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_PFCP_SEID), {BUFF_NOUSED} }
 134
 135 #define proto_hdr_gtpc { \
 136         VIRTCHNL_PROTO_HDR_GTPC, 0, {BUFF_NOUSED} }
 137
 138 #define TUNNEL_LEVEL_OUTER              0
 139 #define TUNNEL_LEVEL_INNER              1
 140
 141 /* proto_hdrs template */
 142 struct virtchnl_proto_hdrs outer_ipv4_tmplt = {
 143         TUNNEL_LEVEL_OUTER, 4,
 144         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv4}
 145 };
 146
 147 struct virtchnl_proto_hdrs outer_ipv4_udp_tmplt = {
 148         TUNNEL_LEVEL_OUTER, 5,
 149         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv4,
 150          proto_hdr_udp}
 151 };
 152
 153 struct virtchnl_proto_hdrs outer_ipv4_tcp_tmplt = {
 154         TUNNEL_LEVEL_OUTER, 5,
 155         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv4,
 156          proto_hdr_tcp}
 157 };
 158
 159 struct virtchnl_proto_hdrs outer_ipv4_sctp_tmplt = {
 160         TUNNEL_LEVEL_OUTER, 5,
 161         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv4,
 162          proto_hdr_sctp}
 163 };
 164
 165 struct virtchnl_proto_hdrs outer_ipv6_tmplt = {
 166         TUNNEL_LEVEL_OUTER, 4,
 167         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv6}
 168 };
 169
 170 struct virtchnl_proto_hdrs outer_ipv6_udp_tmplt = {
 171         TUNNEL_LEVEL_OUTER, 5,
 172         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv6,
 173          proto_hdr_udp}
 174 };
 175
 176 struct virtchnl_proto_hdrs outer_ipv6_tcp_tmplt = {
 177         TUNNEL_LEVEL_OUTER, 5,
 178         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv6,
 179          proto_hdr_tcp}
 180 };
 181
 182 struct virtchnl_proto_hdrs outer_ipv6_sctp_tmplt = {
 183         TUNNEL_LEVEL_OUTER, 5,
 184         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv6,
 185          proto_hdr_sctp}
 186 };
 187
 188 struct virtchnl_proto_hdrs inner_ipv4_tmplt = {
 189         TUNNEL_LEVEL_INNER, 1, {proto_hdr_ipv4}
 190 };
 191
 192 struct virtchnl_proto_hdrs inner_ipv4_udp_tmplt = {
 193         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv4, proto_hdr_udp}
 194 };
 195
 196 struct virtchnl_proto_hdrs inner_ipv4_tcp_tmplt = {
 197         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv4, proto_hdr_tcp}
 198 };
 199
 200 struct virtchnl_proto_hdrs inner_ipv4_sctp_tmplt = {
 201         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv4, proto_hdr_sctp}
 202 };
 203
 204 struct virtchnl_proto_hdrs inner_ipv6_tmplt = {
 205         TUNNEL_LEVEL_INNER, 1, {proto_hdr_ipv6}
 206 };
 207
 208 struct virtchnl_proto_hdrs inner_ipv6_udp_tmplt = {
 209         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv6, proto_hdr_udp}
 210 };
 211
 212 struct virtchnl_proto_hdrs inner_ipv6_tcp_tmplt = {
 213         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv6, proto_hdr_tcp}
 214 };
 215
 216 struct virtchnl_proto_hdrs inner_ipv6_sctp_tmplt = {
 217         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv6, proto_hdr_sctp}
 218 };
 219
 220 struct virtchnl_proto_hdrs ipv4_esp_tmplt = {
 221         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_esp}
 222 };
 223
 224 struct virtchnl_proto_hdrs ipv4_udp_esp_tmplt = {
 225         TUNNEL_LEVEL_OUTER, 3,
 226         {proto_hdr_ipv4, proto_hdr_udp, proto_hdr_esp}
 227 };
 228
 229 struct virtchnl_proto_hdrs ipv4_ah_tmplt = {
 230         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_ah}
 231 };
 232
 233 struct virtchnl_proto_hdrs ipv6_esp_tmplt = {
 234         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_esp}
 235 };
 236
 237 struct virtchnl_proto_hdrs ipv6_udp_esp_tmplt = {
 238         TUNNEL_LEVEL_OUTER, 3,
 239         {proto_hdr_ipv6, proto_hdr_udp, proto_hdr_esp}
 240 };
 241
 242 struct virtchnl_proto_hdrs ipv6_ah_tmplt = {
 243         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_ah}
 244 };
 245
 246 struct virtchnl_proto_hdrs ipv4_l2tpv3_tmplt = {
 247         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_l2tpv3}
 248 };
 249
 250 struct virtchnl_proto_hdrs ipv6_l2tpv3_tmplt = {
 251         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_l2tpv3}
 252 };
 253
 254 struct virtchnl_proto_hdrs ipv4_pfcp_tmplt = {
 255         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_pfcp}
 256 };
 257
 258 struct virtchnl_proto_hdrs ipv6_pfcp_tmplt = {
 259         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_pfcp}
 260 };
 261
 262 struct virtchnl_proto_hdrs ipv4_udp_gtpc_tmplt = {
 263         TUNNEL_LEVEL_OUTER, 3, {proto_hdr_ipv4, proto_hdr_udp, proto_hdr_gtpc}
 264 };
 265
 266 struct virtchnl_proto_hdrs ipv6_udp_gtpc_tmplt = {
 267         TUNNEL_LEVEL_OUTER, 3, {proto_hdr_ipv6, proto_hdr_udp, proto_hdr_gtpc}
 268 };
 269
 270 /* rss type super set */
 271
 272 /* IPv4 outer */
 273 #define IAVF_RSS_TYPE_OUTER_IPV4        (ETH_RSS_ETH | ETH_RSS_IPV4)
 274 #define IAVF_RSS_TYPE_OUTER_IPV4_UDP    (IAVF_RSS_TYPE_OUTER_IPV4 | \
 275                                          ETH_RSS_NONFRAG_IPV4_UDP)
 276 #define IAVF_RSS_TYPE_OUTER_IPV4_TCP    (IAVF_RSS_TYPE_OUTER_IPV4 | \
 277                                          ETH_RSS_NONFRAG_IPV4_TCP)
 278 #define IAVF_RSS_TYPE_OUTER_IPV4_SCTP   (IAVF_RSS_TYPE_OUTER_IPV4 | \
 279                                          ETH_RSS_NONFRAG_IPV4_SCTP)
 280 /* IPv6 outer */
 281 #define IAVF_RSS_TYPE_OUTER_IPV6        (ETH_RSS_ETH | ETH_RSS_IPV6)
 282 #define IAVF_RSS_TYPE_OUTER_IPV6_UDP    (IAVF_RSS_TYPE_OUTER_IPV6 | \
 283                                          ETH_RSS_NONFRAG_IPV6_UDP)
 284 #define IAVF_RSS_TYPE_OUTER_IPV6_TCP    (IAVF_RSS_TYPE_OUTER_IPV6 | \
 285                                          ETH_RSS_NONFRAG_IPV6_TCP)
 286 #define IAVF_RSS_TYPE_OUTER_IPV6_SCTP   (IAVF_RSS_TYPE_OUTER_IPV6 | \
 287                                          ETH_RSS_NONFRAG_IPV6_SCTP)
 288 /* VLAN IPV4 */
 289 #define IAVF_RSS_TYPE_VLAN_IPV4         (IAVF_RSS_TYPE_OUTER_IPV4 | \
 290                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 291 #define IAVF_RSS_TYPE_VLAN_IPV4_UDP     (IAVF_RSS_TYPE_OUTER_IPV4_UDP | \
 292                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 293 #define IAVF_RSS_TYPE_VLAN_IPV4_TCP     (IAVF_RSS_TYPE_OUTER_IPV4_TCP | \
 294                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 295 #define IAVF_RSS_TYPE_VLAN_IPV4_SCTP    (IAVF_RSS_TYPE_OUTER_IPV4_SCTP | \
 296                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 297 /* VLAN IPv6 */
 298 #define IAVF_RSS_TYPE_VLAN_IPV6         (IAVF_RSS_TYPE_OUTER_IPV6 | \
 299                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 300 #define IAVF_RSS_TYPE_VLAN_IPV6_UDP     (IAVF_RSS_TYPE_OUTER_IPV6_UDP | \
 301                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 302 #define IAVF_RSS_TYPE_VLAN_IPV6_TCP     (IAVF_RSS_TYPE_OUTER_IPV6_TCP | \
 303                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 304 #define IAVF_RSS_TYPE_VLAN_IPV6_SCTP    (IAVF_RSS_TYPE_OUTER_IPV6_SCTP | \
 305                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 306 /* IPv4 inner */
 307 #define IAVF_RSS_TYPE_INNER_IPV4        ETH_RSS_IPV4
 308 #define IAVF_RSS_TYPE_INNER_IPV4_UDP    (ETH_RSS_IPV4 | \
 309                                          ETH_RSS_NONFRAG_IPV4_UDP)
 310 #define IAVF_RSS_TYPE_INNER_IPV4_TCP    (ETH_RSS_IPV4 | \
 311                                          ETH_RSS_NONFRAG_IPV4_TCP)
 312 #define IAVF_RSS_TYPE_INNER_IPV4_SCTP   (ETH_RSS_IPV4 | \
 313                                          ETH_RSS_NONFRAG_IPV4_SCTP)
 314 /* IPv6 inner */
 315 #define IAVF_RSS_TYPE_INNER_IPV6        ETH_RSS_IPV6
 316 #define IAVF_RSS_TYPE_INNER_IPV6_UDP    (ETH_RSS_IPV6 | \
 317                                          ETH_RSS_NONFRAG_IPV6_UDP)
 318 #define IAVF_RSS_TYPE_INNER_IPV6_TCP    (ETH_RSS_IPV6 | \
 319                                          ETH_RSS_NONFRAG_IPV6_TCP)
 320 #define IAVF_RSS_TYPE_INNER_IPV6_SCTP   (ETH_RSS_IPV6 | \
 321                                          ETH_RSS_NONFRAG_IPV6_SCTP)
 322 /* GTPU IPv4 */
 323 #define IAVF_RSS_TYPE_GTPU_IPV4         (IAVF_RSS_TYPE_INNER_IPV4 | \
 324                                          ETH_RSS_GTPU)
 325 #define IAVF_RSS_TYPE_GTPU_IPV4_UDP     (IAVF_RSS_TYPE_INNER_IPV4_UDP | \
 326                                          ETH_RSS_GTPU)
 327 #define IAVF_RSS_TYPE_GTPU_IPV4_TCP     (IAVF_RSS_TYPE_INNER_IPV4_TCP | \
 328                                          ETH_RSS_GTPU)
 329 /* GTPU IPv6 */
 330 #define IAVF_RSS_TYPE_GTPU_IPV6         (IAVF_RSS_TYPE_INNER_IPV6 | \
 331                                          ETH_RSS_GTPU)
 332 #define IAVF_RSS_TYPE_GTPU_IPV6_UDP     (IAVF_RSS_TYPE_INNER_IPV6_UDP | \
 333                                          ETH_RSS_GTPU)
 334 #define IAVF_RSS_TYPE_GTPU_IPV6_TCP     (IAVF_RSS_TYPE_INNER_IPV6_TCP | \
 335                                          ETH_RSS_GTPU)
 336 /* ESP, AH, L2TPV3 and PFCP */
 337 #define IAVF_RSS_TYPE_IPV4_ESP          (ETH_RSS_ESP | ETH_RSS_IPV4)
 338 #define IAVF_RSS_TYPE_IPV4_AH           (ETH_RSS_AH | ETH_RSS_IPV4)
 339 #define IAVF_RSS_TYPE_IPV6_ESP          (ETH_RSS_ESP | ETH_RSS_IPV6)
 340 #define IAVF_RSS_TYPE_IPV6_AH           (ETH_RSS_AH | ETH_RSS_IPV6)
 341 #define IAVF_RSS_TYPE_IPV4_L2TPV3       (ETH_RSS_L2TPV3 | ETH_RSS_IPV4)
 342 #define IAVF_RSS_TYPE_IPV6_L2TPV3       (ETH_RSS_L2TPV3 | ETH_RSS_IPV6)
 343 #define IAVF_RSS_TYPE_IPV4_PFCP         (ETH_RSS_PFCP | ETH_RSS_IPV4)
 344 #define IAVF_RSS_TYPE_IPV6_PFCP         (ETH_RSS_PFCP | ETH_RSS_IPV6)
 345
 346 /**
 347  * Supported pattern for hash.
 348  * The first member is pattern item type,
 349  * the second member is input set mask,
 350  * the third member is virtchnl_proto_hdrs template
 351  */
 352 static struct iavf_pattern_match_item iavf_hash_pattern_list[] = {
 353         /* IPv4 */
 354         {iavf_pattern_eth_ipv4,                         IAVF_RSS_TYPE_OUTER_IPV4,       &outer_ipv4_tmplt},
 355         {iavf_pattern_eth_ipv4_udp,                     IAVF_RSS_TYPE_OUTER_IPV4_UDP,   &outer_ipv4_udp_tmplt},
 356         {iavf_pattern_eth_ipv4_tcp,                     IAVF_RSS_TYPE_OUTER_IPV4_TCP,   &outer_ipv4_tcp_tmplt},
 357         {iavf_pattern_eth_ipv4_sctp,                    IAVF_RSS_TYPE_OUTER_IPV4_SCTP,  &outer_ipv4_sctp_tmplt},
 358         {iavf_pattern_eth_vlan_ipv4,                    IAVF_RSS_TYPE_VLAN_IPV4,        &outer_ipv4_tmplt},
 359         {iavf_pattern_eth_vlan_ipv4_udp,                IAVF_RSS_TYPE_VLAN_IPV4_UDP,    &outer_ipv4_udp_tmplt},
 360         {iavf_pattern_eth_vlan_ipv4_tcp,                IAVF_RSS_TYPE_VLAN_IPV4_TCP,    &outer_ipv4_tcp_tmplt},
 361         {iavf_pattern_eth_vlan_ipv4_sctp,               IAVF_RSS_TYPE_VLAN_IPV4_SCTP,   &outer_ipv4_sctp_tmplt},
 362         {iavf_pattern_eth_ipv4_gtpu,                    ETH_RSS_IPV4,                   &outer_ipv4_udp_tmplt},
 363         {iavf_pattern_eth_ipv4_gtpu_ipv4,               IAVF_RSS_TYPE_GTPU_IPV4,        &inner_ipv4_tmplt},
 364         {iavf_pattern_eth_ipv4_gtpu_ipv4_udp,           IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &inner_ipv4_udp_tmplt},
 365         {iavf_pattern_eth_ipv4_gtpu_ipv4_tcp,           IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &inner_ipv4_tcp_tmplt},
 366         {iavf_pattern_eth_ipv6_gtpu_ipv4,               IAVF_RSS_TYPE_GTPU_IPV4,        &inner_ipv4_tmplt},
 367         {iavf_pattern_eth_ipv6_gtpu_ipv4_udp,           IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &inner_ipv4_udp_tmplt},
 368         {iavf_pattern_eth_ipv6_gtpu_ipv4_tcp,           IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &inner_ipv4_tcp_tmplt},
 369         {iavf_pattern_eth_ipv4_gtpu_eh_ipv4,            IAVF_RSS_TYPE_GTPU_IPV4,        &inner_ipv4_tmplt},
 370         {iavf_pattern_eth_ipv4_gtpu_eh_ipv4_udp,        IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &inner_ipv4_udp_tmplt},
 371         {iavf_pattern_eth_ipv4_gtpu_eh_ipv4_tcp,        IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &inner_ipv4_tcp_tmplt},
 372         {iavf_pattern_eth_ipv6_gtpu_eh_ipv4,            IAVF_RSS_TYPE_GTPU_IPV4,        &inner_ipv4_tmplt},
 373         {iavf_pattern_eth_ipv6_gtpu_eh_ipv4_udp,        IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &inner_ipv4_udp_tmplt},
 374         {iavf_pattern_eth_ipv6_gtpu_eh_ipv4_tcp,        IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &inner_ipv4_tcp_tmplt},
 375         {iavf_pattern_eth_ipv4_esp,                     IAVF_RSS_TYPE_IPV4_ESP,         &ipv4_esp_tmplt},
 376         {iavf_pattern_eth_ipv4_udp_esp,                 IAVF_RSS_TYPE_IPV4_ESP,         &ipv4_udp_esp_tmplt},
 377         {iavf_pattern_eth_ipv4_ah,                      IAVF_RSS_TYPE_IPV4_AH,          &ipv4_ah_tmplt},
 378         {iavf_pattern_eth_ipv4_l2tpv3,                  IAVF_RSS_TYPE_IPV4_L2TPV3,      &ipv4_l2tpv3_tmplt},
 379         {iavf_pattern_eth_ipv4_pfcp,                    IAVF_RSS_TYPE_IPV4_PFCP,        &ipv4_pfcp_tmplt},
 380         {iavf_pattern_eth_ipv4_gtpc,                    ETH_RSS_IPV4,                   &ipv4_udp_gtpc_tmplt},
 381         /* IPv6 */
 382         {iavf_pattern_eth_ipv6,                         IAVF_RSS_TYPE_OUTER_IPV6,       &outer_ipv6_tmplt},
 383         {iavf_pattern_eth_ipv6_udp,                     IAVF_RSS_TYPE_OUTER_IPV6_UDP,   &outer_ipv6_udp_tmplt},
 384         {iavf_pattern_eth_ipv6_tcp,                     IAVF_RSS_TYPE_OUTER_IPV6_TCP,   &outer_ipv6_tcp_tmplt},
 385         {iavf_pattern_eth_ipv6_sctp,                    IAVF_RSS_TYPE_OUTER_IPV6_SCTP,  &outer_ipv6_sctp_tmplt},
 386         {iavf_pattern_eth_vlan_ipv6,                    IAVF_RSS_TYPE_VLAN_IPV6,        &outer_ipv6_tmplt},
 387         {iavf_pattern_eth_vlan_ipv6_udp,                IAVF_RSS_TYPE_VLAN_IPV6_UDP,    &outer_ipv6_udp_tmplt},
 388         {iavf_pattern_eth_vlan_ipv6_tcp,                IAVF_RSS_TYPE_VLAN_IPV6_TCP,    &outer_ipv6_tcp_tmplt},
 389         {iavf_pattern_eth_vlan_ipv6_sctp,               IAVF_RSS_TYPE_VLAN_IPV6_SCTP,   &outer_ipv6_sctp_tmplt},
 390         {iavf_pattern_eth_ipv6_gtpu,                    ETH_RSS_IPV6,                   &outer_ipv6_udp_tmplt},
 391         {iavf_pattern_eth_ipv4_gtpu_ipv6,               IAVF_RSS_TYPE_GTPU_IPV6,        &inner_ipv6_tmplt},
 392         {iavf_pattern_eth_ipv4_gtpu_ipv6_udp,           IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &inner_ipv6_udp_tmplt},
 393         {iavf_pattern_eth_ipv4_gtpu_ipv6_tcp,           IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &inner_ipv6_tcp_tmplt},
 394         {iavf_pattern_eth_ipv6_gtpu_ipv6,               IAVF_RSS_TYPE_GTPU_IPV6,        &inner_ipv6_tmplt},
 395         {iavf_pattern_eth_ipv6_gtpu_ipv6_udp,           IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &inner_ipv6_udp_tmplt},
 396         {iavf_pattern_eth_ipv6_gtpu_ipv6_tcp,           IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &inner_ipv6_tcp_tmplt},
 397         {iavf_pattern_eth_ipv4_gtpu_eh_ipv6,            IAVF_RSS_TYPE_GTPU_IPV6,        &inner_ipv6_tmplt},
 398         {iavf_pattern_eth_ipv4_gtpu_eh_ipv6_udp,        IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &inner_ipv6_udp_tmplt},
 399         {iavf_pattern_eth_ipv4_gtpu_eh_ipv6_tcp,        IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &inner_ipv6_tcp_tmplt},
 400         {iavf_pattern_eth_ipv6_gtpu_eh_ipv6,            IAVF_RSS_TYPE_GTPU_IPV6,        &inner_ipv6_tmplt},
 401         {iavf_pattern_eth_ipv6_gtpu_eh_ipv6_udp,        IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &inner_ipv6_udp_tmplt},
 402         {iavf_pattern_eth_ipv6_gtpu_eh_ipv6_tcp,        IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &inner_ipv6_tcp_tmplt},
 403         {iavf_pattern_eth_ipv6_esp,                     IAVF_RSS_TYPE_IPV6_ESP,         &ipv6_esp_tmplt},
 404         {iavf_pattern_eth_ipv6_udp_esp,                 IAVF_RSS_TYPE_IPV6_ESP,         &ipv6_udp_esp_tmplt},
 405         {iavf_pattern_eth_ipv6_ah,                      IAVF_RSS_TYPE_IPV6_AH,          &ipv6_ah_tmplt},
 406         {iavf_pattern_eth_ipv6_l2tpv3,                  IAVF_RSS_TYPE_IPV6_L2TPV3,      &ipv6_l2tpv3_tmplt},
 407         {iavf_pattern_eth_ipv6_pfcp,                    IAVF_RSS_TYPE_IPV6_PFCP,        &ipv6_pfcp_tmplt},
 408         {iavf_pattern_eth_ipv6_gtpc,                    ETH_RSS_IPV6,                   &ipv6_udp_gtpc_tmplt},
 409 };
 410
 411 struct virtchnl_proto_hdrs *iavf_hash_default_hdrs[] = {
 412         &inner_ipv4_tmplt,
 413         &inner_ipv4_udp_tmplt,
 414         &inner_ipv4_tcp_tmplt,
 415         &inner_ipv4_sctp_tmplt,
 416         &inner_ipv6_tmplt,
 417         &inner_ipv6_udp_tmplt,
 418         &inner_ipv6_tcp_tmplt,
 419         &inner_ipv6_sctp_tmplt,
 420 };
 421
 422 static struct iavf_flow_engine iavf_hash_engine = {
 423         .init = iavf_hash_init,
 424         .create = iavf_hash_create,
 425         .destroy = iavf_hash_destroy,
 426         .uninit = iavf_hash_uninit,
 427         .free = iavf_hash_free,
 428         .type = IAVF_FLOW_ENGINE_HASH,
 429 };
 430
 431 /* Register parser for comms package. */
 432 static struct iavf_flow_parser iavf_hash_parser = {
 433         .engine = &iavf_hash_engine,
 434         .array = iavf_hash_pattern_list,
 435         .array_len = RTE_DIM(iavf_hash_pattern_list),
 436         .parse_pattern_action = iavf_hash_parse_pattern_action,
 437         .stage = IAVF_FLOW_STAGE_RSS,
 438 };
 439
 440 static int
 441 iavf_hash_default_set(struct iavf_adapter *ad, bool add)
 442 {
 443         struct virtchnl_rss_cfg *rss_cfg;
 444         uint16_t i;
 445
 446         rss_cfg = rte_zmalloc("iavf rss rule",
 447                               sizeof(struct virtchnl_rss_cfg), 0);
 448         if (!rss_cfg)
 449                 return -ENOMEM;
 450
 451         for (i = 0; i < RTE_DIM(iavf_hash_default_hdrs); i++) {
 452                 rss_cfg->proto_hdrs = *iavf_hash_default_hdrs[i];
 453                 rss_cfg->rss_algorithm = VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC;
 454
 455                 iavf_add_del_rss_cfg(ad, rss_cfg, add);
 456         }
 457
 458         return 0;
 459 }
 460
 461 RTE_INIT(iavf_hash_engine_init)
 462 {
 463         struct iavf_flow_engine *engine = &iavf_hash_engine;
 464
 465         iavf_register_flow_engine(engine);
 466 }
 467
 468 static int
 469 iavf_hash_init(struct iavf_adapter *ad)
 470 {
 471         struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
 472         struct iavf_flow_parser *parser;
 473         int ret;
 474
 475         if (vf->vf_reset)
 476                 return -EIO;
 477
 478         if (!vf->vf_res)
 479                 return -EINVAL;
 480
 481         if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF))
 482                 return -ENOTSUP;
 483
 484         parser = &iavf_hash_parser;
 485
 486         ret = iavf_register_parser(parser, ad);
 487         if (ret) {
 488                 PMD_DRV_LOG(ERR, "fail to register hash parser");
 489                 return ret;
 490         }
 491
 492         ret = iavf_hash_default_set(ad, true);
 493         if (ret) {
 494                 PMD_DRV_LOG(ERR, "fail to set default RSS");
 495                 iavf_unregister_parser(parser, ad);
 496         }
 497
 498         return ret;
 499 }
 500
 501 static int
 502 iavf_hash_parse_pattern(struct iavf_pattern_match_item *pattern_match_item,
 503                         const struct rte_flow_item pattern[], uint64_t *phint,
 504                         struct rte_flow_error *error)
 505 {
 506         const struct rte_flow_item *item = pattern;
 507         const struct rte_flow_item_gtp_psc *psc;
 508
 509         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 510                 if (item->last) {
 511                         rte_flow_error_set(error, EINVAL,
 512                                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 513                                            "Not support range");
 514                         return -rte_errno;
 515                 }
 516
 517                 switch (item->type) {
 518                 case RTE_FLOW_ITEM_TYPE_GTPU:
 519                         *phint |= IAVF_PHINT_GTPU;
 520                         break;
 521                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
 522                         *phint |= IAVF_PHINT_GTPU_EH;
 523                         psc = item->spec;
 524                         if (!psc)
 525                                 break;
 526                         else if (psc->pdu_type == IAVF_GTPU_EH_UPLINK)
 527                                 *phint |= IAVF_PHINT_GTPU_EH_UP;
 528                         else if (psc->pdu_type == IAVF_GTPU_EH_DWNLINK)
 529                                 *phint |= IAVF_PHINT_GTPU_EH_DWN;
 530                         break;
 531                 default:
 532                         break;
 533                 }
 534         }
 535
 536         /* update and restore pattern hint */
 537         *phint |= *(uint64_t *)(pattern_match_item->meta);
 538
 539         return 0;
 540 }
 541
 542 #define REFINE_PROTO_FLD(op, fld) \
 543         VIRTCHNL_##op##_PROTO_HDR_FIELD(hdr, VIRTCHNL_PROTO_HDR_##fld)
 544 #define REPALCE_PROTO_FLD(fld_1, fld_2) \
 545 do { \
 546         REFINE_PROTO_FLD(DEL, fld_1);   \
 547         REFINE_PROTO_FLD(ADD, fld_2);   \
 548 } while (0)
 549
 550 /* refine proto hdrs base on l2, l3, l4 rss type */
 551 static void
 552 iavf_refine_proto_hdrs_l234(struct virtchnl_proto_hdrs *proto_hdrs,
 553                             uint64_t rss_type)
 554 {
 555         struct virtchnl_proto_hdr *hdr;
 556         int i;
 557
 558         for (i = 0; i < proto_hdrs->count; i++) {
 559                 hdr = &proto_hdrs->proto_hdr[i];
 560                 switch (hdr->type) {
 561                 case VIRTCHNL_PROTO_HDR_ETH:
 562                         if (!(rss_type & ETH_RSS_ETH))
 563                                 hdr->field_selector = 0;
 564                         else if (rss_type & ETH_RSS_L2_SRC_ONLY)
 565                                 REFINE_PROTO_FLD(DEL, ETH_DST);
 566                         else if (rss_type & ETH_RSS_L2_DST_ONLY)
 567                                 REFINE_PROTO_FLD(DEL, ETH_SRC);
 568                         break;
 569                 case VIRTCHNL_PROTO_HDR_IPV4:
 570                         if (rss_type &
 571                             (ETH_RSS_IPV4 |
 572                              ETH_RSS_NONFRAG_IPV4_UDP |
 573                              ETH_RSS_NONFRAG_IPV4_TCP |
 574                              ETH_RSS_NONFRAG_IPV4_SCTP)) {
 575                                 if (rss_type & ETH_RSS_L3_SRC_ONLY)
 576                                         REFINE_PROTO_FLD(DEL, IPV4_DST);
 577                                 else if (rss_type & ETH_RSS_L3_DST_ONLY)
 578                                         REFINE_PROTO_FLD(DEL, IPV4_SRC);
 579                                 else if (rss_type &
 580                                          (ETH_RSS_L4_SRC_ONLY |
 581                                           ETH_RSS_L4_DST_ONLY))
 582                                         hdr->field_selector = 0;
 583                         } else {
 584                                 hdr->field_selector = 0;
 585                         }
 586                         break;
 587                 case VIRTCHNL_PROTO_HDR_IPV6:
 588                         if (rss_type &
 589                             (ETH_RSS_IPV6 |
 590                              ETH_RSS_NONFRAG_IPV6_UDP |
 591                              ETH_RSS_NONFRAG_IPV6_TCP |
 592                              ETH_RSS_NONFRAG_IPV6_SCTP)) {
 593                                 if (rss_type & ETH_RSS_L3_SRC_ONLY)
 594                                         REFINE_PROTO_FLD(DEL, IPV6_DST);
 595                                 else if (rss_type & ETH_RSS_L3_DST_ONLY)
 596                                         REFINE_PROTO_FLD(DEL, IPV6_SRC);
 597                                 else if (rss_type &
 598                                          (ETH_RSS_L4_SRC_ONLY |
 599                                           ETH_RSS_L4_DST_ONLY))
 600                                         hdr->field_selector = 0;
 601                         } else {
 602                                 hdr->field_selector = 0;
 603                         }
 604                         if (rss_type & RTE_ETH_RSS_L3_PRE64) {
 605                                 if (REFINE_PROTO_FLD(TEST, IPV6_SRC))
 606                                         REPALCE_PROTO_FLD(IPV6_SRC,
 607                                                           IPV6_PREFIX64_SRC);
 608                                 if (REFINE_PROTO_FLD(TEST, IPV6_DST))
 609                                         REPALCE_PROTO_FLD(IPV6_DST,
 610                                                           IPV6_PREFIX64_DST);
 611                         }
 612                         break;
 613                 case VIRTCHNL_PROTO_HDR_UDP:
 614                         if (rss_type &
 615                             (ETH_RSS_NONFRAG_IPV4_UDP |
 616                              ETH_RSS_NONFRAG_IPV6_UDP)) {
 617                                 if (rss_type & ETH_RSS_L4_SRC_ONLY)
 618                                         REFINE_PROTO_FLD(DEL, UDP_DST_PORT);
 619                                 else if (rss_type & ETH_RSS_L4_DST_ONLY)
 620                                         REFINE_PROTO_FLD(DEL, UDP_SRC_PORT);
 621                                 else if (rss_type &
 622                                          (ETH_RSS_L3_SRC_ONLY |
 623                                           ETH_RSS_L3_DST_ONLY))
 624                                         hdr->field_selector = 0;
 625                         } else {
 626                                 hdr->field_selector = 0;
 627                         }
 628                         break;
 629                 case VIRTCHNL_PROTO_HDR_TCP:
 630                         if (rss_type &
 631                             (ETH_RSS_NONFRAG_IPV4_TCP |
 632                              ETH_RSS_NONFRAG_IPV6_TCP)) {
 633                                 if (rss_type & ETH_RSS_L4_SRC_ONLY)
 634                                         REFINE_PROTO_FLD(DEL, TCP_DST_PORT);
 635                                 else if (rss_type & ETH_RSS_L4_DST_ONLY)
 636                                         REFINE_PROTO_FLD(DEL, TCP_SRC_PORT);
 637                                 else if (rss_type &
 638                                          (ETH_RSS_L3_SRC_ONLY |
 639                                           ETH_RSS_L3_DST_ONLY))
 640                                         hdr->field_selector = 0;
 641                         } else {
 642                                 hdr->field_selector = 0;
 643                         }
 644                         break;
 645                 case VIRTCHNL_PROTO_HDR_SCTP:
 646                         if (rss_type &
 647                             (ETH_RSS_NONFRAG_IPV4_SCTP |
 648                              ETH_RSS_NONFRAG_IPV6_SCTP)) {
 649                                 if (rss_type & ETH_RSS_L4_SRC_ONLY)
 650                                         REFINE_PROTO_FLD(DEL, SCTP_DST_PORT);
 651                                 else if (rss_type & ETH_RSS_L4_DST_ONLY)
 652                                         REFINE_PROTO_FLD(DEL, SCTP_SRC_PORT);
 653                                 else if (rss_type &
 654                                          (ETH_RSS_L3_SRC_ONLY |
 655                                           ETH_RSS_L3_DST_ONLY))
 656                                         hdr->field_selector = 0;
 657                         } else {
 658                                 hdr->field_selector = 0;
 659                         }
 660                         break;
 661                 case VIRTCHNL_PROTO_HDR_S_VLAN:
 662                         if (!(rss_type & ETH_RSS_S_VLAN))
 663                                 hdr->field_selector = 0;
 664                         break;
 665                 case VIRTCHNL_PROTO_HDR_C_VLAN:
 666                         if (!(rss_type & ETH_RSS_C_VLAN))
 667                                 hdr->field_selector = 0;
 668                         break;
 669                 case VIRTCHNL_PROTO_HDR_L2TPV3:
 670                         if (!(rss_type & ETH_RSS_L2TPV3))
 671                                 hdr->field_selector = 0;
 672                         break;
 673                 case VIRTCHNL_PROTO_HDR_ESP:
 674                         if (!(rss_type & ETH_RSS_ESP))
 675                                 hdr->field_selector = 0;
 676                         break;
 677                 case VIRTCHNL_PROTO_HDR_AH:
 678                         if (!(rss_type & ETH_RSS_AH))
 679                                 hdr->field_selector = 0;
 680                         break;
 681                 case VIRTCHNL_PROTO_HDR_PFCP:
 682                         if (!(rss_type & ETH_RSS_PFCP))
 683                                 hdr->field_selector = 0;
 684                         break;
 685                 default:
 686                         break;
 687                 }
 688         }
 689 }
 690
 691 /* refine proto hdrs base on gtpu rss type */
 692 static void
 693 iavf_refine_proto_hdrs_gtpu(struct virtchnl_proto_hdrs *proto_hdrs,
 694                             uint64_t rss_type)
 695 {
 696         struct virtchnl_proto_hdr *hdr;
 697         int i;
 698
 699         if (!(rss_type & ETH_RSS_GTPU))
 700                 return;
 701
 702         for (i = 0; i < proto_hdrs->count; i++) {
 703                 hdr = &proto_hdrs->proto_hdr[i];
 704                 switch (hdr->type) {
 705                 case VIRTCHNL_PROTO_HDR_GTPU_IP:
 706                         REFINE_PROTO_FLD(ADD, GTPU_IP_TEID);
 707                         break;
 708                 default:
 709                         break;
 710                 }
 711         }
 712 }
 713
 714 static void
 715 iavf_refine_proto_hdrs_by_pattern(struct virtchnl_proto_hdrs *proto_hdrs,
 716                                   uint64_t phint)
 717 {
 718         struct virtchnl_proto_hdr *hdr1;
 719         struct virtchnl_proto_hdr *hdr2;
 720         int i;
 721
 722         if (!(phint & IAVF_PHINT_GTPU_MSK))
 723                 return;
 724
 725         if (proto_hdrs->tunnel_level == TUNNEL_LEVEL_INNER) {
 726                 /* shift headers 1 layer */
 727                 for (i = proto_hdrs->count; i > 0; i--) {
 728                         hdr1 = &proto_hdrs->proto_hdr[i];
 729                         hdr2 = &proto_hdrs->proto_hdr[i - 1];
 730
 731                         *hdr1 = *hdr2;
 732                 }
 733
 734                 /* adding gtpu header at layer 0 */
 735                 hdr1 = &proto_hdrs->proto_hdr[0];
 736         } else {
 737                 hdr1 = &proto_hdrs->proto_hdr[proto_hdrs->count];
 738         }
 739
 740         hdr1->field_selector = 0;
 741         proto_hdrs->count++;
 742
 743         if (phint & IAVF_PHINT_GTPU_EH_DWN)
 744                 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_EH_PDU_DWN);
 745         else if (phint & IAVF_PHINT_GTPU_EH_UP)
 746                 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_EH_PDU_UP);
 747         else if (phint & IAVF_PHINT_GTPU_EH)
 748                 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_EH);
 749         else if (phint & IAVF_PHINT_GTPU)
 750                 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_IP);
 751 }
 752
 753 static void iavf_refine_proto_hdrs(struct virtchnl_proto_hdrs *proto_hdrs,
 754                                    uint64_t rss_type, uint64_t phint)
 755 {
 756         iavf_refine_proto_hdrs_l234(proto_hdrs, rss_type);
 757         iavf_refine_proto_hdrs_by_pattern(proto_hdrs, phint);
 758         iavf_refine_proto_hdrs_gtpu(proto_hdrs, rss_type);
 759 }
 760
 761 static uint64_t invalid_rss_comb[] = {
 762         ETH_RSS_IPV4 | ETH_RSS_NONFRAG_IPV4_UDP,
 763         ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_UDP,
 764         RTE_ETH_RSS_L3_PRE32 | RTE_ETH_RSS_L3_PRE40 |
 765         RTE_ETH_RSS_L3_PRE48 | RTE_ETH_RSS_L3_PRE56 |
 766         RTE_ETH_RSS_L3_PRE96
 767 };
 768
 769 struct rss_attr_type {
 770         uint64_t attr;
 771         uint64_t type;
 772 };
 773
 774 #define VALID_RSS_IPV4_L4       (ETH_RSS_NONFRAG_IPV4_UDP       | \
 775                                  ETH_RSS_NONFRAG_IPV4_TCP       | \
 776                                  ETH_RSS_NONFRAG_IPV4_SCTP)
 777
 778 #define VALID_RSS_IPV6_L4       (ETH_RSS_NONFRAG_IPV6_UDP       | \
 779                                  ETH_RSS_NONFRAG_IPV6_TCP       | \
 780                                  ETH_RSS_NONFRAG_IPV6_SCTP)
 781
 782 #define VALID_RSS_IPV4          (ETH_RSS_IPV4 | VALID_RSS_IPV4_L4)
 783 #define VALID_RSS_IPV6          (ETH_RSS_IPV6 | VALID_RSS_IPV6_L4)
 784 #define VALID_RSS_L3            (VALID_RSS_IPV4 | VALID_RSS_IPV6)
 785 #define VALID_RSS_L4            (VALID_RSS_IPV4_L4 | VALID_RSS_IPV6_L4)
 786
 787 #define VALID_RSS_ATTR          (ETH_RSS_L3_SRC_ONLY    | \
 788                                  ETH_RSS_L3_DST_ONLY    | \
 789                                  ETH_RSS_L4_SRC_ONLY    | \
 790                                  ETH_RSS_L4_DST_ONLY    | \
 791                                  ETH_RSS_L2_SRC_ONLY    | \
 792                                  ETH_RSS_L2_DST_ONLY    | \
 793                                  RTE_ETH_RSS_L3_PRE64)
 794
 795 #define INVALID_RSS_ATTR        (RTE_ETH_RSS_L3_PRE32   | \
 796                                  RTE_ETH_RSS_L3_PRE40   | \
 797                                  RTE_ETH_RSS_L3_PRE48   | \
 798                                  RTE_ETH_RSS_L3_PRE56   | \
 799                                  RTE_ETH_RSS_L3_PRE96)
 800
 801 static struct rss_attr_type rss_attr_to_valid_type[] = {
 802         {ETH_RSS_L2_SRC_ONLY | ETH_RSS_L2_DST_ONLY,     ETH_RSS_ETH},
 803         {ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY,     VALID_RSS_L3},
 804         {ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY,     VALID_RSS_L4},
 805         /* current ipv6 prefix only supports prefix 64 bits*/
 806         {RTE_ETH_RSS_L3_PRE64,                          VALID_RSS_IPV6},
 807         {INVALID_RSS_ATTR,                              0}
 808 };
 809
 810 static bool
 811 iavf_any_invalid_rss_type(uint64_t rss_type, uint64_t allow_rss_type)
 812 {
 813         uint32_t i;
 814
 815         /* check invalid combination */
 816         for (i = 0; i < RTE_DIM(invalid_rss_comb); i++) {
 817                 if (__builtin_popcountll(rss_type & invalid_rss_comb[i]) > 1)
 818                         return true;
 819         }
 820
 821         /* check invalid RSS attribute */
 822         for (i = 0; i < RTE_DIM(rss_attr_to_valid_type); i++) {
 823                 struct rss_attr_type *rat = &rss_attr_to_valid_type[i];
 824
 825                 if (rat->attr & rss_type && !(rat->type & rss_type))
 826                         return true;
 827         }
 828
 829         /* check not allowed RSS type */
 830         rss_type &= ~VALID_RSS_ATTR;
 831
 832         return ((rss_type & allow_rss_type) != rss_type);
 833 }
 834
 835 static int
 836 iavf_hash_parse_action(struct iavf_pattern_match_item *match_item,
 837                        const struct rte_flow_action actions[],
 838                        uint64_t pattern_hint, void **meta,
 839                        struct rte_flow_error *error)
 840 {
 841         struct iavf_rss_meta *rss_meta = (struct iavf_rss_meta *)*meta;
 842         struct virtchnl_proto_hdrs *proto_hdrs;
 843         enum rte_flow_action_type action_type;
 844         const struct rte_flow_action_rss *rss;
 845         const struct rte_flow_action *action;
 846         uint64_t rss_type;
 847
 848         /* Supported action is RSS. */
 849         for (action = actions; action->type !=
 850                 RTE_FLOW_ACTION_TYPE_END; action++) {
 851                 action_type = action->type;
 852                 switch (action_type) {
 853                 case RTE_FLOW_ACTION_TYPE_RSS:
 854                         rss = action->conf;
 855                         rss_type = rss->types;
 856
 857                         if (rss->func ==
 858                             RTE_ETH_HASH_FUNCTION_SIMPLE_XOR){
 859                                 rss_meta->rss_algorithm =
 860                                         VIRTCHNL_RSS_ALG_XOR_ASYMMETRIC;
 861                                 return rte_flow_error_set(error, ENOTSUP,
 862                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 863                                         "function simple_xor is not supported");
 864                         } else if (rss->func ==
 865                                    RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) {
 866                                 rss_meta->rss_algorithm =
 867                                         VIRTCHNL_RSS_ALG_TOEPLITZ_SYMMETRIC;
 868                         } else {
 869                                 rss_meta->rss_algorithm =
 870                                         VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC;
 871                         }
 872
 873                         if (rss->level)
 874                                 return rte_flow_error_set(error, ENOTSUP,
 875                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 876                                         "a nonzero RSS encapsulation level is not supported");
 877
 878                         if (rss->key_len)
 879                                 return rte_flow_error_set(error, ENOTSUP,
 880                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 881                                         "a nonzero RSS key_len is not supported");
 882
 883                         if (rss->queue_num)
 884                                 return rte_flow_error_set(error, ENOTSUP,
 885                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 886                                         "a non-NULL RSS queue is not supported");
 887
 888                         /**
 889                          * Check simultaneous use of SRC_ONLY and DST_ONLY
 890                          * of the same level.
 891                          */
 892                         rss_type = rte_eth_rss_hf_refine(rss_type);
 893
 894                         if (iavf_any_invalid_rss_type(rss_type,
 895                                         match_item->input_set_mask))
 896                                 return rte_flow_error_set(error, ENOTSUP,
 897                                                 RTE_FLOW_ERROR_TYPE_ACTION,
 898                                                 action, "RSS type not supported");
 899                         proto_hdrs = match_item->meta;
 900                         rss_meta->proto_hdrs = *proto_hdrs;
 901                         iavf_refine_proto_hdrs(&rss_meta->proto_hdrs,
 902                                                rss_type, pattern_hint);
 903                         break;
 904
 905                 case RTE_FLOW_ACTION_TYPE_END:
 906                         break;
 907
 908                 default:
 909                         rte_flow_error_set(error, EINVAL,
 910                                            RTE_FLOW_ERROR_TYPE_ACTION, action,
 911                                            "Invalid action.");
 912                         return -rte_errno;
 913                 }
 914         }
 915
 916         return 0;
 917 }
 918
 919 static int
 920 iavf_hash_parse_pattern_action(__rte_unused struct iavf_adapter *ad,
 921                                struct iavf_pattern_match_item *array,
 922                                uint32_t array_len,
 923                                const struct rte_flow_item pattern[],
 924                                const struct rte_flow_action actions[],
 925                                void **meta,
 926                                struct rte_flow_error *error)
 927 {
 928         struct iavf_pattern_match_item *pattern_match_item;
 929         struct iavf_rss_meta *rss_meta_ptr;
 930         uint64_t phint = IAVF_PHINT_NONE;
 931         int ret = 0;
 932
 933         rss_meta_ptr = rte_zmalloc(NULL, sizeof(*rss_meta_ptr), 0);
 934         if (!rss_meta_ptr) {
 935                 rte_flow_error_set(error, EINVAL,
 936                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 937                                    "No memory for rss_meta_ptr");
 938                 return -ENOMEM;
 939         }
 940
 941         /* Check rss supported pattern and find matched pattern. */
 942         pattern_match_item =
 943                 iavf_search_pattern_match_item(pattern, array, array_len,
 944                                                error);
 945         if (!pattern_match_item) {
 946                 ret = -rte_errno;
 947                 goto error;
 948         }
 949
 950         ret = iavf_hash_parse_pattern(pattern_match_item, pattern, &phint,
 951                                       error);
 952         if (ret)
 953                 goto error;
 954
 955         ret = iavf_hash_parse_action(pattern_match_item, actions, phint,
 956                                      (void **)&rss_meta_ptr, error);
 957
 958 error:
 959         if (!ret && meta)
 960                 *meta = rss_meta_ptr;
 961         else
 962                 rte_free(rss_meta_ptr);
 963
 964         rte_free(pattern_match_item);
 965
 966         return ret;
 967 }
 968
 969 static int
 970 iavf_hash_create(__rte_unused struct iavf_adapter *ad,
 971                  __rte_unused struct rte_flow *flow, void *meta,
 972                  __rte_unused struct rte_flow_error *error)
 973 {
 974         struct iavf_rss_meta *rss_meta = (struct iavf_rss_meta *)meta;
 975         struct virtchnl_rss_cfg *rss_cfg;
 976         int ret = 0;
 977
 978         rss_cfg = rte_zmalloc("iavf rss rule",
 979                               sizeof(struct virtchnl_rss_cfg), 0);
 980         if (!rss_cfg) {
 981                 rte_flow_error_set(error, EINVAL,
 982                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 983                                    "No memory for rss rule");
 984                 return -ENOMEM;
 985         }
 986
 987         rss_cfg->proto_hdrs = rss_meta->proto_hdrs;
 988         rss_cfg->rss_algorithm = rss_meta->rss_algorithm;
 989
 990         ret = iavf_add_del_rss_cfg(ad, rss_cfg, true);
 991         if (!ret) {
 992                 flow->rule = rss_cfg;
 993         } else {
 994                 PMD_DRV_LOG(ERR, "fail to add RSS configure");
 995                 rte_flow_error_set(error, -ret,
 996                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 997                                    "Failed to add rss rule.");
 998                 rte_free(rss_cfg);
 999                 return -rte_errno;
1000         }
1001
1002         rte_free(meta);
1003
1004         return ret;
1005 }
1006
1007 static int
1008 iavf_hash_destroy(__rte_unused struct iavf_adapter *ad,
1009                   struct rte_flow *flow,
1010                   __rte_unused struct rte_flow_error *error)
1011 {
1012         struct virtchnl_rss_cfg *rss_cfg;
1013         int ret = 0;
1014
1015         rss_cfg = (struct virtchnl_rss_cfg *)flow->rule;
1016
1017         ret = iavf_add_del_rss_cfg(ad, rss_cfg, false);
1018         if (ret) {
1019                 PMD_DRV_LOG(ERR, "fail to del RSS configure");
1020                 rte_flow_error_set(error, -ret,
1021                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1022                                    "Failed to delete rss rule.");
1023                 return -rte_errno;
1024         }
1025         return ret;
1026 }
1027
1028 static void
1029 iavf_hash_uninit(struct iavf_adapter *ad)
1030 {
1031         struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
1032
1033         if (vf->vf_reset)
1034                 return;
1035
1036         if (iavf_hash_default_set(ad, false))
1037                 PMD_DRV_LOG(ERR, "fail to delete default RSS");
1038
1039         iavf_unregister_parser(&iavf_hash_parser, ad);
1040 }
1041
1042 static void
1043 iavf_hash_free(struct rte_flow *flow)
1044 {
1045         rte_free(flow->rule);
1046 }