drivers/net/iavf/iavf_hash.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10 #include <unistd.h>
  11 #include <stdarg.h>
  12
  13 #include <rte_debug.h>
  14 #include <rte_ether.h>
  15 #include <ethdev_driver.h>
  16 #include <rte_log.h>
  17 #include <rte_malloc.h>
  18 #include <rte_eth_ctrl.h>
  19 #include <rte_tailq.h>
  20 #include <rte_flow_driver.h>
  21
  22 #include "iavf_log.h"
  23 #include "iavf.h"
  24 #include "iavf_generic_flow.h"
  25
  26 #define IAVF_PHINT_NONE                         0
  27 #define IAVF_PHINT_GTPU                         BIT_ULL(0)
  28 #define IAVF_PHINT_GTPU_EH                      BIT_ULL(1)
  29 #define IAVF_PHINT_GTPU_EH_DWN                  BIT_ULL(2)
  30 #define IAVF_PHINT_GTPU_EH_UP                   BIT_ULL(3)
  31 #define IAVF_PHINT_OUTER_IPV4                   BIT_ULL(4)
  32 #define IAVF_PHINT_OUTER_IPV6                   BIT_ULL(5)
  33
  34 #define IAVF_PHINT_GTPU_MSK     (IAVF_PHINT_GTPU        | \
  35                                  IAVF_PHINT_GTPU_EH     | \
  36                                  IAVF_PHINT_GTPU_EH_DWN | \
  37                                  IAVF_PHINT_GTPU_EH_UP)
  38
  39 #define IAVF_PHINT_LAYERS_MSK   (IAVF_PHINT_OUTER_IPV4  | \
  40                                  IAVF_PHINT_OUTER_IPV6)
  41
  42 #define IAVF_GTPU_EH_DWNLINK    0
  43 #define IAVF_GTPU_EH_UPLINK     1
  44
  45 struct iavf_hash_match_type {
  46         uint64_t hash_type;
  47         struct virtchnl_proto_hdrs *proto_hdrs;
  48         uint64_t pattern_hint;
  49 };
  50
  51 struct iavf_rss_meta {
  52         struct virtchnl_proto_hdrs proto_hdrs;
  53         enum virtchnl_rss_algorithm rss_algorithm;
  54 };
  55
  56 struct iavf_hash_flow_cfg {
  57         struct virtchnl_rss_cfg *rss_cfg;
  58         bool simple_xor;
  59 };
  60
  61 static int
  62 iavf_hash_init(struct iavf_adapter *ad);
  63 static int
  64 iavf_hash_create(struct iavf_adapter *ad, struct rte_flow *flow, void *meta,
  65                  struct rte_flow_error *error);
  66 static int
  67 iavf_hash_destroy(struct iavf_adapter *ad, struct rte_flow *flow,
  68                   struct rte_flow_error *error);
  69 static void
  70 iavf_hash_uninit(struct iavf_adapter *ad);
  71 static void
  72 iavf_hash_free(struct rte_flow *flow);
  73 static int
  74 iavf_hash_parse_pattern_action(struct iavf_adapter *ad,
  75                                struct iavf_pattern_match_item *array,
  76                                uint32_t array_len,
  77                                const struct rte_flow_item pattern[],
  78                                const struct rte_flow_action actions[],
  79                                void **meta,
  80                                struct rte_flow_error *error);
  81
  82 #define FIELD_SELECTOR(proto_hdr_field) \
  83                 (1UL << ((proto_hdr_field) & PROTO_HDR_FIELD_MASK))
  84 #define BUFF_NOUSED                     0
  85
  86 #define proto_hdr_eth { \
  87         VIRTCHNL_PROTO_HDR_ETH, \
  88         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_SRC) | \
  89         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_DST), {BUFF_NOUSED} }
  90
  91 #define proto_hdr_svlan { \
  92         VIRTCHNL_PROTO_HDR_S_VLAN, \
  93         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_S_VLAN_ID), {BUFF_NOUSED} }
  94
  95 #define proto_hdr_cvlan { \
  96         VIRTCHNL_PROTO_HDR_C_VLAN, \
  97         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_C_VLAN_ID), {BUFF_NOUSED} }
  98
  99 #define proto_hdr_ipv4 { \
 100         VIRTCHNL_PROTO_HDR_IPV4, \
 101         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_SRC) | \
 102         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_DST), {BUFF_NOUSED} }
 103
 104 #define proto_hdr_ipv4_with_prot { \
 105         VIRTCHNL_PROTO_HDR_IPV4, \
 106         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_SRC) | \
 107         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_DST) | \
 108         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_PROT), {BUFF_NOUSED} }
 109
 110 #define proto_hdr_ipv6 { \
 111         VIRTCHNL_PROTO_HDR_IPV6, \
 112         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_SRC) | \
 113         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_DST), {BUFF_NOUSED} }
 114
 115 #define proto_hdr_ipv6_frag { \
 116         VIRTCHNL_PROTO_HDR_IPV6_EH_FRAG, \
 117         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_EH_FRAG_PKID), {BUFF_NOUSED} }
 118
 119 #define proto_hdr_ipv6_with_prot { \
 120         VIRTCHNL_PROTO_HDR_IPV6, \
 121         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_SRC) | \
 122         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_DST) | \
 123         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_PROT), {BUFF_NOUSED} }
 124
 125 #define proto_hdr_udp { \
 126         VIRTCHNL_PROTO_HDR_UDP, \
 127         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_UDP_SRC_PORT) | \
 128         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_UDP_DST_PORT), {BUFF_NOUSED} }
 129
 130 #define proto_hdr_tcp { \
 131         VIRTCHNL_PROTO_HDR_TCP, \
 132         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_TCP_SRC_PORT) | \
 133         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_TCP_DST_PORT), {BUFF_NOUSED} }
 134
 135 #define proto_hdr_sctp { \
 136         VIRTCHNL_PROTO_HDR_SCTP, \
 137         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_SCTP_SRC_PORT) | \
 138         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_SCTP_DST_PORT), {BUFF_NOUSED} }
 139
 140 #define proto_hdr_esp { \
 141         VIRTCHNL_PROTO_HDR_ESP, \
 142         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ESP_SPI), {BUFF_NOUSED} }
 143
 144 #define proto_hdr_ah { \
 145         VIRTCHNL_PROTO_HDR_AH, \
 146         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_AH_SPI), {BUFF_NOUSED} }
 147
 148 #define proto_hdr_l2tpv3 { \
 149         VIRTCHNL_PROTO_HDR_L2TPV3, \
 150         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_L2TPV3_SESS_ID), {BUFF_NOUSED} }
 151
 152 #define proto_hdr_pfcp { \
 153         VIRTCHNL_PROTO_HDR_PFCP, \
 154         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_PFCP_SEID), {BUFF_NOUSED} }
 155
 156 #define proto_hdr_gtpc { \
 157         VIRTCHNL_PROTO_HDR_GTPC, 0, {BUFF_NOUSED} }
 158
 159 #define proto_hdr_ecpri { \
 160         VIRTCHNL_PROTO_HDR_ECPRI, \
 161         FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ECPRI_PC_RTC_ID), {BUFF_NOUSED} }
 162
 163 #define TUNNEL_LEVEL_OUTER              0
 164 #define TUNNEL_LEVEL_INNER              1
 165
 166 /* proto_hdrs template */
 167 struct virtchnl_proto_hdrs outer_ipv4_tmplt = {
 168         TUNNEL_LEVEL_OUTER, 4,
 169         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv4}
 170 };
 171
 172 struct virtchnl_proto_hdrs outer_ipv4_udp_tmplt = {
 173         TUNNEL_LEVEL_OUTER, 5,
 174         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan,
 175          proto_hdr_ipv4_with_prot,
 176          proto_hdr_udp}
 177 };
 178
 179 struct virtchnl_proto_hdrs outer_ipv4_tcp_tmplt = {
 180         TUNNEL_LEVEL_OUTER, 5,
 181         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan,
 182          proto_hdr_ipv4_with_prot,
 183          proto_hdr_tcp}
 184 };
 185
 186 struct virtchnl_proto_hdrs outer_ipv4_sctp_tmplt = {
 187         TUNNEL_LEVEL_OUTER, 5,
 188         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv4,
 189          proto_hdr_sctp}
 190 };
 191
 192 struct virtchnl_proto_hdrs outer_ipv6_tmplt = {
 193         TUNNEL_LEVEL_OUTER, 4,
 194         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv6}
 195 };
 196
 197 struct virtchnl_proto_hdrs outer_ipv6_frag_tmplt = {
 198         TUNNEL_LEVEL_OUTER, 5,
 199         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan,
 200          proto_hdr_ipv6, proto_hdr_ipv6_frag}
 201 };
 202
 203 struct virtchnl_proto_hdrs outer_ipv6_udp_tmplt = {
 204         TUNNEL_LEVEL_OUTER, 5,
 205         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan,
 206          proto_hdr_ipv6_with_prot,
 207          proto_hdr_udp}
 208 };
 209
 210 struct virtchnl_proto_hdrs outer_ipv6_tcp_tmplt = {
 211         TUNNEL_LEVEL_OUTER, 5,
 212         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan,
 213          proto_hdr_ipv6_with_prot,
 214          proto_hdr_tcp}
 215 };
 216
 217 struct virtchnl_proto_hdrs outer_ipv6_sctp_tmplt = {
 218         TUNNEL_LEVEL_OUTER, 5,
 219         {proto_hdr_eth, proto_hdr_svlan, proto_hdr_cvlan, proto_hdr_ipv6,
 220          proto_hdr_sctp}
 221 };
 222
 223 struct virtchnl_proto_hdrs inner_ipv4_tmplt = {
 224         TUNNEL_LEVEL_INNER, 1, {proto_hdr_ipv4}
 225 };
 226
 227 struct virtchnl_proto_hdrs inner_ipv4_udp_tmplt = {
 228         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv4_with_prot, proto_hdr_udp}
 229 };
 230
 231 struct virtchnl_proto_hdrs inner_ipv4_tcp_tmplt = {
 232         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv4_with_prot, proto_hdr_tcp}
 233 };
 234
 235 struct virtchnl_proto_hdrs inner_ipv4_sctp_tmplt = {
 236         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv4, proto_hdr_sctp}
 237 };
 238
 239 struct virtchnl_proto_hdrs inner_ipv6_tmplt = {
 240         TUNNEL_LEVEL_INNER, 1, {proto_hdr_ipv6}
 241 };
 242
 243 struct virtchnl_proto_hdrs inner_ipv6_udp_tmplt = {
 244         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv6_with_prot, proto_hdr_udp}
 245 };
 246
 247 struct virtchnl_proto_hdrs inner_ipv6_tcp_tmplt = {
 248         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv6_with_prot, proto_hdr_tcp}
 249 };
 250
 251 struct virtchnl_proto_hdrs inner_ipv6_sctp_tmplt = {
 252         TUNNEL_LEVEL_INNER, 2, {proto_hdr_ipv6, proto_hdr_sctp}
 253 };
 254
 255 struct virtchnl_proto_hdrs ipv4_esp_tmplt = {
 256         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_esp}
 257 };
 258
 259 struct virtchnl_proto_hdrs ipv4_udp_esp_tmplt = {
 260         TUNNEL_LEVEL_OUTER, 3,
 261         {proto_hdr_ipv4, proto_hdr_udp, proto_hdr_esp}
 262 };
 263
 264 struct virtchnl_proto_hdrs ipv4_ah_tmplt = {
 265         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_ah}
 266 };
 267
 268 struct virtchnl_proto_hdrs ipv6_esp_tmplt = {
 269         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_esp}
 270 };
 271
 272 struct virtchnl_proto_hdrs ipv6_udp_esp_tmplt = {
 273         TUNNEL_LEVEL_OUTER, 3,
 274         {proto_hdr_ipv6, proto_hdr_udp, proto_hdr_esp}
 275 };
 276
 277 struct virtchnl_proto_hdrs ipv6_ah_tmplt = {
 278         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_ah}
 279 };
 280
 281 struct virtchnl_proto_hdrs ipv4_l2tpv3_tmplt = {
 282         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_l2tpv3}
 283 };
 284
 285 struct virtchnl_proto_hdrs ipv6_l2tpv3_tmplt = {
 286         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_l2tpv3}
 287 };
 288
 289 struct virtchnl_proto_hdrs ipv4_pfcp_tmplt = {
 290         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv4, proto_hdr_pfcp}
 291 };
 292
 293 struct virtchnl_proto_hdrs ipv6_pfcp_tmplt = {
 294         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_ipv6, proto_hdr_pfcp}
 295 };
 296
 297 struct virtchnl_proto_hdrs ipv4_udp_gtpc_tmplt = {
 298         TUNNEL_LEVEL_OUTER, 3, {proto_hdr_ipv4, proto_hdr_udp, proto_hdr_gtpc}
 299 };
 300
 301 struct virtchnl_proto_hdrs ipv6_udp_gtpc_tmplt = {
 302         TUNNEL_LEVEL_OUTER, 3, {proto_hdr_ipv6, proto_hdr_udp, proto_hdr_gtpc}
 303 };
 304
 305 struct virtchnl_proto_hdrs eth_ecpri_tmplt = {
 306         TUNNEL_LEVEL_OUTER, 2, {proto_hdr_eth, proto_hdr_ecpri}
 307 };
 308
 309 struct virtchnl_proto_hdrs ipv4_ecpri_tmplt = {
 310         TUNNEL_LEVEL_OUTER, 3, {proto_hdr_ipv4, proto_hdr_udp, proto_hdr_ecpri}
 311 };
 312
 313 /* rss type super set */
 314
 315 /* IPv4 outer */
 316 #define IAVF_RSS_TYPE_OUTER_IPV4        (ETH_RSS_ETH | ETH_RSS_IPV4 | \
 317                                          ETH_RSS_FRAG_IPV4)
 318 #define IAVF_RSS_TYPE_OUTER_IPV4_UDP    (IAVF_RSS_TYPE_OUTER_IPV4 | \
 319                                          ETH_RSS_NONFRAG_IPV4_UDP)
 320 #define IAVF_RSS_TYPE_OUTER_IPV4_TCP    (IAVF_RSS_TYPE_OUTER_IPV4 | \
 321                                          ETH_RSS_NONFRAG_IPV4_TCP)
 322 #define IAVF_RSS_TYPE_OUTER_IPV4_SCTP   (IAVF_RSS_TYPE_OUTER_IPV4 | \
 323                                          ETH_RSS_NONFRAG_IPV4_SCTP)
 324 /* IPv6 outer */
 325 #define IAVF_RSS_TYPE_OUTER_IPV6        (ETH_RSS_ETH | ETH_RSS_IPV6)
 326 #define IAVF_RSS_TYPE_OUTER_IPV6_FRAG   (IAVF_RSS_TYPE_OUTER_IPV6 | \
 327                                          ETH_RSS_FRAG_IPV6)
 328 #define IAVF_RSS_TYPE_OUTER_IPV6_UDP    (IAVF_RSS_TYPE_OUTER_IPV6 | \
 329                                          ETH_RSS_NONFRAG_IPV6_UDP)
 330 #define IAVF_RSS_TYPE_OUTER_IPV6_TCP    (IAVF_RSS_TYPE_OUTER_IPV6 | \
 331                                          ETH_RSS_NONFRAG_IPV6_TCP)
 332 #define IAVF_RSS_TYPE_OUTER_IPV6_SCTP   (IAVF_RSS_TYPE_OUTER_IPV6 | \
 333                                          ETH_RSS_NONFRAG_IPV6_SCTP)
 334 /* VLAN IPV4 */
 335 #define IAVF_RSS_TYPE_VLAN_IPV4         (IAVF_RSS_TYPE_OUTER_IPV4 | \
 336                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 337 #define IAVF_RSS_TYPE_VLAN_IPV4_UDP     (IAVF_RSS_TYPE_OUTER_IPV4_UDP | \
 338                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 339 #define IAVF_RSS_TYPE_VLAN_IPV4_TCP     (IAVF_RSS_TYPE_OUTER_IPV4_TCP | \
 340                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 341 #define IAVF_RSS_TYPE_VLAN_IPV4_SCTP    (IAVF_RSS_TYPE_OUTER_IPV4_SCTP | \
 342                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 343 /* VLAN IPv6 */
 344 #define IAVF_RSS_TYPE_VLAN_IPV6         (IAVF_RSS_TYPE_OUTER_IPV6 | \
 345                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 346 #define IAVF_RSS_TYPE_VLAN_IPV6_FRAG    (IAVF_RSS_TYPE_OUTER_IPV6_FRAG | \
 347                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 348 #define IAVF_RSS_TYPE_VLAN_IPV6_UDP     (IAVF_RSS_TYPE_OUTER_IPV6_UDP | \
 349                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 350 #define IAVF_RSS_TYPE_VLAN_IPV6_TCP     (IAVF_RSS_TYPE_OUTER_IPV6_TCP | \
 351                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 352 #define IAVF_RSS_TYPE_VLAN_IPV6_SCTP    (IAVF_RSS_TYPE_OUTER_IPV6_SCTP | \
 353                                          ETH_RSS_S_VLAN | ETH_RSS_C_VLAN)
 354 /* IPv4 inner */
 355 #define IAVF_RSS_TYPE_INNER_IPV4        ETH_RSS_IPV4
 356 #define IAVF_RSS_TYPE_INNER_IPV4_UDP    (ETH_RSS_IPV4 | \
 357                                          ETH_RSS_NONFRAG_IPV4_UDP)
 358 #define IAVF_RSS_TYPE_INNER_IPV4_TCP    (ETH_RSS_IPV4 | \
 359                                          ETH_RSS_NONFRAG_IPV4_TCP)
 360 #define IAVF_RSS_TYPE_INNER_IPV4_SCTP   (ETH_RSS_IPV4 | \
 361                                          ETH_RSS_NONFRAG_IPV4_SCTP)
 362 /* IPv6 inner */
 363 #define IAVF_RSS_TYPE_INNER_IPV6        ETH_RSS_IPV6
 364 #define IAVF_RSS_TYPE_INNER_IPV6_UDP    (ETH_RSS_IPV6 | \
 365                                          ETH_RSS_NONFRAG_IPV6_UDP)
 366 #define IAVF_RSS_TYPE_INNER_IPV6_TCP    (ETH_RSS_IPV6 | \
 367                                          ETH_RSS_NONFRAG_IPV6_TCP)
 368 #define IAVF_RSS_TYPE_INNER_IPV6_SCTP   (ETH_RSS_IPV6 | \
 369                                          ETH_RSS_NONFRAG_IPV6_SCTP)
 370 /* GTPU IPv4 */
 371 #define IAVF_RSS_TYPE_GTPU_IPV4         (IAVF_RSS_TYPE_INNER_IPV4 | \
 372                                          ETH_RSS_GTPU)
 373 #define IAVF_RSS_TYPE_GTPU_IPV4_UDP     (IAVF_RSS_TYPE_INNER_IPV4_UDP | \
 374                                          ETH_RSS_GTPU)
 375 #define IAVF_RSS_TYPE_GTPU_IPV4_TCP     (IAVF_RSS_TYPE_INNER_IPV4_TCP | \
 376                                          ETH_RSS_GTPU)
 377 /* GTPU IPv6 */
 378 #define IAVF_RSS_TYPE_GTPU_IPV6         (IAVF_RSS_TYPE_INNER_IPV6 | \
 379                                          ETH_RSS_GTPU)
 380 #define IAVF_RSS_TYPE_GTPU_IPV6_UDP     (IAVF_RSS_TYPE_INNER_IPV6_UDP | \
 381                                          ETH_RSS_GTPU)
 382 #define IAVF_RSS_TYPE_GTPU_IPV6_TCP     (IAVF_RSS_TYPE_INNER_IPV6_TCP | \
 383                                          ETH_RSS_GTPU)
 384 /* ESP, AH, L2TPV3 and PFCP */
 385 #define IAVF_RSS_TYPE_IPV4_ESP          (ETH_RSS_ESP | ETH_RSS_IPV4)
 386 #define IAVF_RSS_TYPE_IPV4_AH           (ETH_RSS_AH | ETH_RSS_IPV4)
 387 #define IAVF_RSS_TYPE_IPV6_ESP          (ETH_RSS_ESP | ETH_RSS_IPV6)
 388 #define IAVF_RSS_TYPE_IPV6_AH           (ETH_RSS_AH | ETH_RSS_IPV6)
 389 #define IAVF_RSS_TYPE_IPV4_L2TPV3       (ETH_RSS_L2TPV3 | ETH_RSS_IPV4)
 390 #define IAVF_RSS_TYPE_IPV6_L2TPV3       (ETH_RSS_L2TPV3 | ETH_RSS_IPV6)
 391 #define IAVF_RSS_TYPE_IPV4_PFCP         (ETH_RSS_PFCP | ETH_RSS_IPV4)
 392 #define IAVF_RSS_TYPE_IPV6_PFCP         (ETH_RSS_PFCP | ETH_RSS_IPV6)
 393
 394 /**
 395  * Supported pattern for hash.
 396  * The first member is pattern item type,
 397  * the second member is input set mask,
 398  * the third member is virtchnl_proto_hdrs template
 399  */
 400 static struct iavf_pattern_match_item iavf_hash_pattern_list[] = {
 401         /* IPv4 */
 402         {iavf_pattern_eth_ipv4,                         IAVF_RSS_TYPE_OUTER_IPV4,       &outer_ipv4_tmplt},
 403         {iavf_pattern_eth_ipv4_udp,                     IAVF_RSS_TYPE_OUTER_IPV4_UDP,   &outer_ipv4_udp_tmplt},
 404         {iavf_pattern_eth_ipv4_tcp,                     IAVF_RSS_TYPE_OUTER_IPV4_TCP,   &outer_ipv4_tcp_tmplt},
 405         {iavf_pattern_eth_ipv4_sctp,                    IAVF_RSS_TYPE_OUTER_IPV4_SCTP,  &outer_ipv4_sctp_tmplt},
 406         {iavf_pattern_eth_vlan_ipv4,                    IAVF_RSS_TYPE_VLAN_IPV4,        &outer_ipv4_tmplt},
 407         {iavf_pattern_eth_vlan_ipv4_udp,                IAVF_RSS_TYPE_VLAN_IPV4_UDP,    &outer_ipv4_udp_tmplt},
 408         {iavf_pattern_eth_vlan_ipv4_tcp,                IAVF_RSS_TYPE_VLAN_IPV4_TCP,    &outer_ipv4_tcp_tmplt},
 409         {iavf_pattern_eth_vlan_ipv4_sctp,               IAVF_RSS_TYPE_VLAN_IPV4_SCTP,   &outer_ipv4_sctp_tmplt},
 410         {iavf_pattern_eth_ipv4_gtpu,                    ETH_RSS_IPV4,                   &outer_ipv4_udp_tmplt},
 411         {iavf_pattern_eth_ipv4_gtpu_ipv4,               IAVF_RSS_TYPE_GTPU_IPV4,        &inner_ipv4_tmplt},
 412         {iavf_pattern_eth_ipv4_gtpu_ipv4_udp,           IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &inner_ipv4_udp_tmplt},
 413         {iavf_pattern_eth_ipv4_gtpu_ipv4_tcp,           IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &inner_ipv4_tcp_tmplt},
 414         {iavf_pattern_eth_ipv6_gtpu_ipv4,               IAVF_RSS_TYPE_GTPU_IPV4,        &inner_ipv4_tmplt},
 415         {iavf_pattern_eth_ipv6_gtpu_ipv4_udp,           IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &inner_ipv4_udp_tmplt},
 416         {iavf_pattern_eth_ipv6_gtpu_ipv4_tcp,           IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &inner_ipv4_tcp_tmplt},
 417         {iavf_pattern_eth_ipv4_gtpu_eh_ipv4,            IAVF_RSS_TYPE_GTPU_IPV4,        &inner_ipv4_tmplt},
 418         {iavf_pattern_eth_ipv4_gtpu_eh_ipv4_udp,        IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &inner_ipv4_udp_tmplt},
 419         {iavf_pattern_eth_ipv4_gtpu_eh_ipv4_tcp,        IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &inner_ipv4_tcp_tmplt},
 420         {iavf_pattern_eth_ipv6_gtpu_eh_ipv4,            IAVF_RSS_TYPE_GTPU_IPV4,        &inner_ipv4_tmplt},
 421         {iavf_pattern_eth_ipv6_gtpu_eh_ipv4_udp,        IAVF_RSS_TYPE_GTPU_IPV4_UDP,    &inner_ipv4_udp_tmplt},
 422         {iavf_pattern_eth_ipv6_gtpu_eh_ipv4_tcp,        IAVF_RSS_TYPE_GTPU_IPV4_TCP,    &inner_ipv4_tcp_tmplt},
 423         {iavf_pattern_eth_ipv4_esp,                     IAVF_RSS_TYPE_IPV4_ESP,         &ipv4_esp_tmplt},
 424         {iavf_pattern_eth_ipv4_udp_esp,                 IAVF_RSS_TYPE_IPV4_ESP,         &ipv4_udp_esp_tmplt},
 425         {iavf_pattern_eth_ipv4_ah,                      IAVF_RSS_TYPE_IPV4_AH,          &ipv4_ah_tmplt},
 426         {iavf_pattern_eth_ipv4_l2tpv3,                  IAVF_RSS_TYPE_IPV4_L2TPV3,      &ipv4_l2tpv3_tmplt},
 427         {iavf_pattern_eth_ipv4_pfcp,                    IAVF_RSS_TYPE_IPV4_PFCP,        &ipv4_pfcp_tmplt},
 428         {iavf_pattern_eth_ipv4_gtpc,                    ETH_RSS_IPV4,                   &ipv4_udp_gtpc_tmplt},
 429         {iavf_pattern_eth_ecpri,                        ETH_RSS_ECPRI,                  &eth_ecpri_tmplt},
 430         {iavf_pattern_eth_ipv4_ecpri,                   ETH_RSS_ECPRI,                  &ipv4_ecpri_tmplt},
 431         /* IPv6 */
 432         {iavf_pattern_eth_ipv6,                         IAVF_RSS_TYPE_OUTER_IPV6,       &outer_ipv6_tmplt},
 433         {iavf_pattern_eth_ipv6_frag_ext,                IAVF_RSS_TYPE_OUTER_IPV6_FRAG,  &outer_ipv6_frag_tmplt},
 434         {iavf_pattern_eth_ipv6_udp,                     IAVF_RSS_TYPE_OUTER_IPV6_UDP,   &outer_ipv6_udp_tmplt},
 435         {iavf_pattern_eth_ipv6_tcp,                     IAVF_RSS_TYPE_OUTER_IPV6_TCP,   &outer_ipv6_tcp_tmplt},
 436         {iavf_pattern_eth_ipv6_sctp,                    IAVF_RSS_TYPE_OUTER_IPV6_SCTP,  &outer_ipv6_sctp_tmplt},
 437         {iavf_pattern_eth_vlan_ipv6,                    IAVF_RSS_TYPE_VLAN_IPV6,        &outer_ipv6_tmplt},
 438         {iavf_pattern_eth_vlan_ipv6_frag_ext,           IAVF_RSS_TYPE_OUTER_IPV6_FRAG,  &outer_ipv6_frag_tmplt},
 439         {iavf_pattern_eth_vlan_ipv6_udp,                IAVF_RSS_TYPE_VLAN_IPV6_UDP,    &outer_ipv6_udp_tmplt},
 440         {iavf_pattern_eth_vlan_ipv6_tcp,                IAVF_RSS_TYPE_VLAN_IPV6_TCP,    &outer_ipv6_tcp_tmplt},
 441         {iavf_pattern_eth_vlan_ipv6_sctp,               IAVF_RSS_TYPE_VLAN_IPV6_SCTP,   &outer_ipv6_sctp_tmplt},
 442         {iavf_pattern_eth_ipv6_gtpu,                    ETH_RSS_IPV6,                   &outer_ipv6_udp_tmplt},
 443         {iavf_pattern_eth_ipv4_gtpu_ipv6,               IAVF_RSS_TYPE_GTPU_IPV6,        &inner_ipv6_tmplt},
 444         {iavf_pattern_eth_ipv4_gtpu_ipv6_udp,           IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &inner_ipv6_udp_tmplt},
 445         {iavf_pattern_eth_ipv4_gtpu_ipv6_tcp,           IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &inner_ipv6_tcp_tmplt},
 446         {iavf_pattern_eth_ipv6_gtpu_ipv6,               IAVF_RSS_TYPE_GTPU_IPV6,        &inner_ipv6_tmplt},
 447         {iavf_pattern_eth_ipv6_gtpu_ipv6_udp,           IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &inner_ipv6_udp_tmplt},
 448         {iavf_pattern_eth_ipv6_gtpu_ipv6_tcp,           IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &inner_ipv6_tcp_tmplt},
 449         {iavf_pattern_eth_ipv4_gtpu_eh_ipv6,            IAVF_RSS_TYPE_GTPU_IPV6,        &inner_ipv6_tmplt},
 450         {iavf_pattern_eth_ipv4_gtpu_eh_ipv6_udp,        IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &inner_ipv6_udp_tmplt},
 451         {iavf_pattern_eth_ipv4_gtpu_eh_ipv6_tcp,        IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &inner_ipv6_tcp_tmplt},
 452         {iavf_pattern_eth_ipv6_gtpu_eh_ipv6,            IAVF_RSS_TYPE_GTPU_IPV6,        &inner_ipv6_tmplt},
 453         {iavf_pattern_eth_ipv6_gtpu_eh_ipv6_udp,        IAVF_RSS_TYPE_GTPU_IPV6_UDP,    &inner_ipv6_udp_tmplt},
 454         {iavf_pattern_eth_ipv6_gtpu_eh_ipv6_tcp,        IAVF_RSS_TYPE_GTPU_IPV6_TCP,    &inner_ipv6_tcp_tmplt},
 455         {iavf_pattern_eth_ipv6_esp,                     IAVF_RSS_TYPE_IPV6_ESP,         &ipv6_esp_tmplt},
 456         {iavf_pattern_eth_ipv6_udp_esp,                 IAVF_RSS_TYPE_IPV6_ESP,         &ipv6_udp_esp_tmplt},
 457         {iavf_pattern_eth_ipv6_ah,                      IAVF_RSS_TYPE_IPV6_AH,          &ipv6_ah_tmplt},
 458         {iavf_pattern_eth_ipv6_l2tpv3,                  IAVF_RSS_TYPE_IPV6_L2TPV3,      &ipv6_l2tpv3_tmplt},
 459         {iavf_pattern_eth_ipv6_pfcp,                    IAVF_RSS_TYPE_IPV6_PFCP,        &ipv6_pfcp_tmplt},
 460         {iavf_pattern_eth_ipv6_gtpc,                    ETH_RSS_IPV6,                   &ipv6_udp_gtpc_tmplt},
 461 };
 462
 463 static struct iavf_flow_engine iavf_hash_engine = {
 464         .init = iavf_hash_init,
 465         .create = iavf_hash_create,
 466         .destroy = iavf_hash_destroy,
 467         .uninit = iavf_hash_uninit,
 468         .free = iavf_hash_free,
 469         .type = IAVF_FLOW_ENGINE_HASH,
 470 };
 471
 472 /* Register parser for comms package. */
 473 static struct iavf_flow_parser iavf_hash_parser = {
 474         .engine = &iavf_hash_engine,
 475         .array = iavf_hash_pattern_list,
 476         .array_len = RTE_DIM(iavf_hash_pattern_list),
 477         .parse_pattern_action = iavf_hash_parse_pattern_action,
 478         .stage = IAVF_FLOW_STAGE_RSS,
 479 };
 480
 481 int
 482 iavf_rss_hash_set(struct iavf_adapter *ad, uint64_t rss_hf, bool add)
 483 {
 484         struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(ad);
 485         struct virtchnl_rss_cfg rss_cfg;
 486
 487 #define IAVF_RSS_HF_ALL ( \
 488         ETH_RSS_IPV4 | \
 489         ETH_RSS_IPV6 | \
 490         ETH_RSS_NONFRAG_IPV4_UDP | \
 491         ETH_RSS_NONFRAG_IPV6_UDP | \
 492         ETH_RSS_NONFRAG_IPV4_TCP | \
 493         ETH_RSS_NONFRAG_IPV6_TCP | \
 494         ETH_RSS_NONFRAG_IPV4_SCTP | \
 495         ETH_RSS_NONFRAG_IPV6_SCTP)
 496
 497         rss_cfg.rss_algorithm = VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC;
 498         if (rss_hf & ETH_RSS_IPV4) {
 499                 rss_cfg.proto_hdrs = inner_ipv4_tmplt;
 500                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 501         }
 502
 503         if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
 504                 rss_cfg.proto_hdrs = inner_ipv4_udp_tmplt;
 505                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 506         }
 507
 508         if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
 509                 rss_cfg.proto_hdrs = inner_ipv4_tcp_tmplt;
 510                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 511         }
 512
 513         if (rss_hf & ETH_RSS_NONFRAG_IPV4_SCTP) {
 514                 rss_cfg.proto_hdrs = inner_ipv4_sctp_tmplt;
 515                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 516         }
 517
 518         if (rss_hf & ETH_RSS_IPV6) {
 519                 rss_cfg.proto_hdrs = inner_ipv6_tmplt;
 520                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 521         }
 522
 523         if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) {
 524                 rss_cfg.proto_hdrs = inner_ipv6_udp_tmplt;
 525                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 526         }
 527
 528         if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) {
 529                 rss_cfg.proto_hdrs = inner_ipv6_tcp_tmplt;
 530                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 531         }
 532
 533         if (rss_hf & ETH_RSS_NONFRAG_IPV6_SCTP) {
 534                 rss_cfg.proto_hdrs = inner_ipv6_sctp_tmplt;
 535                 iavf_add_del_rss_cfg(ad, &rss_cfg, add);
 536         }
 537
 538         vf->rss_hf = rss_hf & IAVF_RSS_HF_ALL;
 539         return 0;
 540 }
 541
 542 RTE_INIT(iavf_hash_engine_init)
 543 {
 544         struct iavf_flow_engine *engine = &iavf_hash_engine;
 545
 546         iavf_register_flow_engine(engine);
 547 }
 548
 549 static int
 550 iavf_hash_init(struct iavf_adapter *ad)
 551 {
 552         struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
 553         struct iavf_flow_parser *parser;
 554         int ret;
 555
 556         if (vf->vf_reset)
 557                 return -EIO;
 558
 559         if (!vf->vf_res)
 560                 return -EINVAL;
 561
 562         if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF))
 563                 return -ENOTSUP;
 564
 565         parser = &iavf_hash_parser;
 566
 567         ret = iavf_register_parser(parser, ad);
 568         if (ret) {
 569                 PMD_DRV_LOG(ERR, "fail to register hash parser");
 570                 return ret;
 571         }
 572
 573         return ret;
 574 }
 575
 576 static int
 577 iavf_hash_parse_pattern(const struct rte_flow_item pattern[], uint64_t *phint,
 578                         struct rte_flow_error *error)
 579 {
 580         const struct rte_flow_item *item = pattern;
 581         const struct rte_flow_item_gtp_psc *psc;
 582         const struct rte_flow_item_ecpri *ecpri;
 583         struct rte_ecpri_common_hdr ecpri_common;
 584
 585         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 586                 if (item->last) {
 587                         rte_flow_error_set(error, EINVAL,
 588                                            RTE_FLOW_ERROR_TYPE_ITEM, item,
 589                                            "Not support range");
 590                         return -rte_errno;
 591                 }
 592
 593                 switch (item->type) {
 594                 case RTE_FLOW_ITEM_TYPE_IPV4:
 595                         if (!(*phint & IAVF_PHINT_GTPU_MSK))
 596                                 *phint |= IAVF_PHINT_OUTER_IPV4;
 597                         break;
 598                 case RTE_FLOW_ITEM_TYPE_IPV6:
 599                         if (!(*phint & IAVF_PHINT_GTPU_MSK))
 600                                 *phint |= IAVF_PHINT_OUTER_IPV6;
 601                         break;
 602                 case RTE_FLOW_ITEM_TYPE_GTPU:
 603                         *phint |= IAVF_PHINT_GTPU;
 604                         break;
 605                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
 606                         *phint |= IAVF_PHINT_GTPU_EH;
 607                         psc = item->spec;
 608                         if (!psc)
 609                                 break;
 610                         else if (psc->pdu_type == IAVF_GTPU_EH_UPLINK)
 611                                 *phint |= IAVF_PHINT_GTPU_EH_UP;
 612                         else if (psc->pdu_type == IAVF_GTPU_EH_DWNLINK)
 613                                 *phint |= IAVF_PHINT_GTPU_EH_DWN;
 614                         break;
 615                 case RTE_FLOW_ITEM_TYPE_ECPRI:
 616                         ecpri = item->spec;
 617                         if (!ecpri)
 618                                 break;
 619
 620                         ecpri_common.u32 = rte_be_to_cpu_32(ecpri->hdr.common.u32);
 621
 622                         if (ecpri_common.type !=
 623                                  RTE_ECPRI_MSG_TYPE_IQ_DATA) {
 624                                 rte_flow_error_set(error, EINVAL,
 625                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 626                                         "Unsupported common type.");
 627                                 return -rte_errno;
 628                         }
 629                         break;
 630                 default:
 631                         break;
 632                 }
 633         }
 634
 635         return 0;
 636 }
 637
 638 #define REFINE_PROTO_FLD(op, fld) \
 639         VIRTCHNL_##op##_PROTO_HDR_FIELD(hdr, VIRTCHNL_PROTO_HDR_##fld)
 640 #define REPALCE_PROTO_FLD(fld_1, fld_2) \
 641 do { \
 642         REFINE_PROTO_FLD(DEL, fld_1);   \
 643         REFINE_PROTO_FLD(ADD, fld_2);   \
 644 } while (0)
 645
 646 static void
 647 iavf_hash_add_fragment_hdr(struct virtchnl_proto_hdrs *hdrs, int layer)
 648 {
 649         struct virtchnl_proto_hdr *hdr1;
 650         struct virtchnl_proto_hdr *hdr2;
 651         int i;
 652
 653         if (layer < 0 || layer > hdrs->count)
 654                 return;
 655
 656         /* shift headers layer */
 657         for (i = hdrs->count; i >= layer; i--) {
 658                 hdr1 = &hdrs->proto_hdr[i];
 659                 hdr2 = &hdrs->proto_hdr[i - 1];
 660                 *hdr1 = *hdr2;
 661         }
 662
 663         /* adding dummy fragment header */
 664         hdr1 = &hdrs->proto_hdr[layer];
 665         VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, IPV4_FRAG);
 666         hdrs->count = ++layer;
 667 }
 668
 669 /* refine proto hdrs base on l2, l3, l4 rss type */
 670 static void
 671 iavf_refine_proto_hdrs_l234(struct virtchnl_proto_hdrs *proto_hdrs,
 672                             uint64_t rss_type)
 673 {
 674         struct virtchnl_proto_hdr *hdr;
 675         int i;
 676
 677         for (i = 0; i < proto_hdrs->count; i++) {
 678                 hdr = &proto_hdrs->proto_hdr[i];
 679                 switch (hdr->type) {
 680                 case VIRTCHNL_PROTO_HDR_ETH:
 681                         if (!(rss_type & ETH_RSS_ETH))
 682                                 hdr->field_selector = 0;
 683                         else if (rss_type & ETH_RSS_L2_SRC_ONLY)
 684                                 REFINE_PROTO_FLD(DEL, ETH_DST);
 685                         else if (rss_type & ETH_RSS_L2_DST_ONLY)
 686                                 REFINE_PROTO_FLD(DEL, ETH_SRC);
 687                         break;
 688                 case VIRTCHNL_PROTO_HDR_IPV4:
 689                         if (rss_type &
 690                             (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 691                              ETH_RSS_NONFRAG_IPV4_UDP |
 692                              ETH_RSS_NONFRAG_IPV4_TCP |
 693                              ETH_RSS_NONFRAG_IPV4_SCTP)) {
 694                                 if (rss_type & ETH_RSS_FRAG_IPV4) {
 695                                         iavf_hash_add_fragment_hdr(proto_hdrs, i + 1);
 696                                 } else if (rss_type & ETH_RSS_L3_SRC_ONLY) {
 697                                         REFINE_PROTO_FLD(DEL, IPV4_DST);
 698                                 } else if (rss_type & ETH_RSS_L3_DST_ONLY) {
 699                                         REFINE_PROTO_FLD(DEL, IPV4_SRC);
 700                                 } else if (rss_type &
 701                                            (ETH_RSS_L4_SRC_ONLY |
 702                                             ETH_RSS_L4_DST_ONLY)) {
 703                                         REFINE_PROTO_FLD(DEL, IPV4_DST);
 704                                         REFINE_PROTO_FLD(DEL, IPV4_SRC);
 705                                 }
 706                         } else {
 707                                 hdr->field_selector = 0;
 708                         }
 709                         break;
 710                 case VIRTCHNL_PROTO_HDR_IPV4_FRAG:
 711                         if (rss_type &
 712                             (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
 713                              ETH_RSS_NONFRAG_IPV4_UDP |
 714                              ETH_RSS_NONFRAG_IPV4_TCP |
 715                              ETH_RSS_NONFRAG_IPV4_SCTP)) {
 716                                 if (rss_type & ETH_RSS_FRAG_IPV4)
 717                                         REFINE_PROTO_FLD(ADD, IPV4_FRAG_PKID);
 718                         } else {
 719                                 hdr->field_selector = 0;
 720                         }
 721                         break;
 722                 case VIRTCHNL_PROTO_HDR_IPV6:
 723                         if (rss_type &
 724                             (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
 725                              ETH_RSS_NONFRAG_IPV6_UDP |
 726                              ETH_RSS_NONFRAG_IPV6_TCP |
 727                              ETH_RSS_NONFRAG_IPV6_SCTP)) {
 728                                 if (rss_type & ETH_RSS_L3_SRC_ONLY) {
 729                                         REFINE_PROTO_FLD(DEL, IPV6_DST);
 730                                 } else if (rss_type & ETH_RSS_L3_DST_ONLY) {
 731                                         REFINE_PROTO_FLD(DEL, IPV6_SRC);
 732                                 } else if (rss_type &
 733                                            (ETH_RSS_L4_SRC_ONLY |
 734                                             ETH_RSS_L4_DST_ONLY)) {
 735                                         REFINE_PROTO_FLD(DEL, IPV6_DST);
 736                                         REFINE_PROTO_FLD(DEL, IPV6_SRC);
 737                                 }
 738                         } else {
 739                                 hdr->field_selector = 0;
 740                         }
 741                         if (rss_type & RTE_ETH_RSS_L3_PRE64) {
 742                                 if (REFINE_PROTO_FLD(TEST, IPV6_SRC))
 743                                         REPALCE_PROTO_FLD(IPV6_SRC,
 744                                                           IPV6_PREFIX64_SRC);
 745                                 if (REFINE_PROTO_FLD(TEST, IPV6_DST))
 746                                         REPALCE_PROTO_FLD(IPV6_DST,
 747                                                           IPV6_PREFIX64_DST);
 748                         }
 749                         break;
 750                 case VIRTCHNL_PROTO_HDR_IPV6_EH_FRAG:
 751                         if (rss_type & ETH_RSS_FRAG_IPV6)
 752                                 REFINE_PROTO_FLD(ADD, IPV6_EH_FRAG_PKID);
 753                         else
 754                                 hdr->field_selector = 0;
 755
 756                         break;
 757                 case VIRTCHNL_PROTO_HDR_UDP:
 758                         if (rss_type &
 759                             (ETH_RSS_NONFRAG_IPV4_UDP |
 760                              ETH_RSS_NONFRAG_IPV6_UDP)) {
 761                                 if (rss_type & ETH_RSS_L4_SRC_ONLY)
 762                                         REFINE_PROTO_FLD(DEL, UDP_DST_PORT);
 763                                 else if (rss_type & ETH_RSS_L4_DST_ONLY)
 764                                         REFINE_PROTO_FLD(DEL, UDP_SRC_PORT);
 765                                 else if (rss_type &
 766                                          (ETH_RSS_L3_SRC_ONLY |
 767                                           ETH_RSS_L3_DST_ONLY))
 768                                         hdr->field_selector = 0;
 769                         } else {
 770                                 hdr->field_selector = 0;
 771                         }
 772                         break;
 773                 case VIRTCHNL_PROTO_HDR_TCP:
 774                         if (rss_type &
 775                             (ETH_RSS_NONFRAG_IPV4_TCP |
 776                              ETH_RSS_NONFRAG_IPV6_TCP)) {
 777                                 if (rss_type & ETH_RSS_L4_SRC_ONLY)
 778                                         REFINE_PROTO_FLD(DEL, TCP_DST_PORT);
 779                                 else if (rss_type & ETH_RSS_L4_DST_ONLY)
 780                                         REFINE_PROTO_FLD(DEL, TCP_SRC_PORT);
 781                                 else if (rss_type &
 782                                          (ETH_RSS_L3_SRC_ONLY |
 783                                           ETH_RSS_L3_DST_ONLY))
 784                                         hdr->field_selector = 0;
 785                         } else {
 786                                 hdr->field_selector = 0;
 787                         }
 788                         break;
 789                 case VIRTCHNL_PROTO_HDR_SCTP:
 790                         if (rss_type &
 791                             (ETH_RSS_NONFRAG_IPV4_SCTP |
 792                              ETH_RSS_NONFRAG_IPV6_SCTP)) {
 793                                 if (rss_type & ETH_RSS_L4_SRC_ONLY)
 794                                         REFINE_PROTO_FLD(DEL, SCTP_DST_PORT);
 795                                 else if (rss_type & ETH_RSS_L4_DST_ONLY)
 796                                         REFINE_PROTO_FLD(DEL, SCTP_SRC_PORT);
 797                                 else if (rss_type &
 798                                          (ETH_RSS_L3_SRC_ONLY |
 799                                           ETH_RSS_L3_DST_ONLY))
 800                                         hdr->field_selector = 0;
 801                         } else {
 802                                 hdr->field_selector = 0;
 803                         }
 804                         break;
 805                 case VIRTCHNL_PROTO_HDR_S_VLAN:
 806                         if (!(rss_type & ETH_RSS_S_VLAN))
 807                                 hdr->field_selector = 0;
 808                         break;
 809                 case VIRTCHNL_PROTO_HDR_C_VLAN:
 810                         if (!(rss_type & ETH_RSS_C_VLAN))
 811                                 hdr->field_selector = 0;
 812                         break;
 813                 case VIRTCHNL_PROTO_HDR_L2TPV3:
 814                         if (!(rss_type & ETH_RSS_L2TPV3))
 815                                 hdr->field_selector = 0;
 816                         break;
 817                 case VIRTCHNL_PROTO_HDR_ESP:
 818                         if (!(rss_type & ETH_RSS_ESP))
 819                                 hdr->field_selector = 0;
 820                         break;
 821                 case VIRTCHNL_PROTO_HDR_AH:
 822                         if (!(rss_type & ETH_RSS_AH))
 823                                 hdr->field_selector = 0;
 824                         break;
 825                 case VIRTCHNL_PROTO_HDR_PFCP:
 826                         if (!(rss_type & ETH_RSS_PFCP))
 827                                 hdr->field_selector = 0;
 828                         break;
 829                 case VIRTCHNL_PROTO_HDR_ECPRI:
 830                         if (!(rss_type & ETH_RSS_ECPRI))
 831                                 hdr->field_selector = 0;
 832                         break;
 833                 default:
 834                         break;
 835                 }
 836         }
 837 }
 838
 839 /* refine proto hdrs base on gtpu rss type */
 840 static void
 841 iavf_refine_proto_hdrs_gtpu(struct virtchnl_proto_hdrs *proto_hdrs,
 842                             uint64_t rss_type)
 843 {
 844         struct virtchnl_proto_hdr *hdr;
 845         int i;
 846
 847         if (!(rss_type & ETH_RSS_GTPU))
 848                 return;
 849
 850         for (i = 0; i < proto_hdrs->count; i++) {
 851                 hdr = &proto_hdrs->proto_hdr[i];
 852                 switch (hdr->type) {
 853                 case VIRTCHNL_PROTO_HDR_GTPU_IP:
 854                         REFINE_PROTO_FLD(ADD, GTPU_IP_TEID);
 855                         break;
 856                 default:
 857                         break;
 858                 }
 859         }
 860 }
 861
 862 static void
 863 iavf_refine_proto_hdrs_by_pattern(struct virtchnl_proto_hdrs *proto_hdrs,
 864                                   uint64_t phint)
 865 {
 866         struct virtchnl_proto_hdr *hdr1;
 867         struct virtchnl_proto_hdr *hdr2;
 868         int i, shift_count = 1;
 869
 870         if (!(phint & IAVF_PHINT_GTPU_MSK))
 871                 return;
 872
 873         if (phint & IAVF_PHINT_LAYERS_MSK)
 874                 shift_count++;
 875
 876         if (proto_hdrs->tunnel_level == TUNNEL_LEVEL_INNER) {
 877                 /* shift headers layer */
 878                 for (i = proto_hdrs->count - 1 + shift_count;
 879                      i > shift_count - 1; i--) {
 880                         hdr1 = &proto_hdrs->proto_hdr[i];
 881                         hdr2 = &proto_hdrs->proto_hdr[i - shift_count];
 882                         *hdr1 = *hdr2;
 883                 }
 884
 885                 if (shift_count == 1) {
 886                         /* adding gtpu header at layer 0 */
 887                         hdr1 = &proto_hdrs->proto_hdr[0];
 888                 } else {
 889                         /* adding gtpu header and outer ip header */
 890                         hdr1 = &proto_hdrs->proto_hdr[1];
 891                         hdr2 = &proto_hdrs->proto_hdr[0];
 892                         hdr2->field_selector = 0;
 893                         proto_hdrs->count++;
 894                         proto_hdrs->tunnel_level = TUNNEL_LEVEL_OUTER;
 895
 896                         if (phint & IAVF_PHINT_OUTER_IPV4)
 897                                 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr2, IPV4);
 898                         else if (phint & IAVF_PHINT_OUTER_IPV6)
 899                                 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr2, IPV6);
 900                 }
 901         } else {
 902                 hdr1 = &proto_hdrs->proto_hdr[proto_hdrs->count];
 903         }
 904
 905         hdr1->field_selector = 0;
 906         proto_hdrs->count++;
 907
 908         if (phint & IAVF_PHINT_GTPU_EH_DWN)
 909                 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_EH_PDU_DWN);
 910         else if (phint & IAVF_PHINT_GTPU_EH_UP)
 911                 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_EH_PDU_UP);
 912         else if (phint & IAVF_PHINT_GTPU_EH)
 913                 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_EH);
 914         else if (phint & IAVF_PHINT_GTPU)
 915                 VIRTCHNL_SET_PROTO_HDR_TYPE(hdr1, GTPU_IP);
 916 }
 917
 918 static void iavf_refine_proto_hdrs(struct virtchnl_proto_hdrs *proto_hdrs,
 919                                    uint64_t rss_type, uint64_t phint)
 920 {
 921         iavf_refine_proto_hdrs_l234(proto_hdrs, rss_type);
 922         iavf_refine_proto_hdrs_by_pattern(proto_hdrs, phint);
 923         iavf_refine_proto_hdrs_gtpu(proto_hdrs, rss_type);
 924 }
 925
 926 static uint64_t invalid_rss_comb[] = {
 927         ETH_RSS_IPV4 | ETH_RSS_NONFRAG_IPV4_UDP,
 928         ETH_RSS_IPV4 | ETH_RSS_NONFRAG_IPV4_TCP,
 929         ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_UDP,
 930         ETH_RSS_IPV6 | ETH_RSS_NONFRAG_IPV6_TCP,
 931         RTE_ETH_RSS_L3_PRE32 | RTE_ETH_RSS_L3_PRE40 |
 932         RTE_ETH_RSS_L3_PRE48 | RTE_ETH_RSS_L3_PRE56 |
 933         RTE_ETH_RSS_L3_PRE96
 934 };
 935
 936 struct rss_attr_type {
 937         uint64_t attr;
 938         uint64_t type;
 939 };
 940
 941 #define VALID_RSS_IPV4_L4       (ETH_RSS_NONFRAG_IPV4_UDP       | \
 942                                  ETH_RSS_NONFRAG_IPV4_TCP       | \
 943                                  ETH_RSS_NONFRAG_IPV4_SCTP)
 944
 945 #define VALID_RSS_IPV6_L4       (ETH_RSS_NONFRAG_IPV6_UDP       | \
 946                                  ETH_RSS_NONFRAG_IPV6_TCP       | \
 947                                  ETH_RSS_NONFRAG_IPV6_SCTP)
 948
 949 #define VALID_RSS_IPV4          (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | \
 950                                  VALID_RSS_IPV4_L4)
 951 #define VALID_RSS_IPV6          (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 | \
 952                                  VALID_RSS_IPV6_L4)
 953 #define VALID_RSS_L3            (VALID_RSS_IPV4 | VALID_RSS_IPV6)
 954 #define VALID_RSS_L4            (VALID_RSS_IPV4_L4 | VALID_RSS_IPV6_L4)
 955
 956 #define VALID_RSS_ATTR          (ETH_RSS_L3_SRC_ONLY    | \
 957                                  ETH_RSS_L3_DST_ONLY    | \
 958                                  ETH_RSS_L4_SRC_ONLY    | \
 959                                  ETH_RSS_L4_DST_ONLY    | \
 960                                  ETH_RSS_L2_SRC_ONLY    | \
 961                                  ETH_RSS_L2_DST_ONLY    | \
 962                                  RTE_ETH_RSS_L3_PRE64)
 963
 964 #define INVALID_RSS_ATTR        (RTE_ETH_RSS_L3_PRE32   | \
 965                                  RTE_ETH_RSS_L3_PRE40   | \
 966                                  RTE_ETH_RSS_L3_PRE48   | \
 967                                  RTE_ETH_RSS_L3_PRE56   | \
 968                                  RTE_ETH_RSS_L3_PRE96)
 969
 970 static struct rss_attr_type rss_attr_to_valid_type[] = {
 971         {ETH_RSS_L2_SRC_ONLY | ETH_RSS_L2_DST_ONLY,     ETH_RSS_ETH},
 972         {ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY,     VALID_RSS_L3},
 973         {ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY,     VALID_RSS_L4},
 974         /* current ipv6 prefix only supports prefix 64 bits*/
 975         {RTE_ETH_RSS_L3_PRE64,                          VALID_RSS_IPV6},
 976         {INVALID_RSS_ATTR,                              0}
 977 };
 978
 979 static bool
 980 iavf_any_invalid_rss_type(enum rte_eth_hash_function rss_func,
 981                           uint64_t rss_type, uint64_t allow_rss_type)
 982 {
 983         uint32_t i;
 984
 985         /**
 986          * Check if l3/l4 SRC/DST_ONLY is set for SYMMETRIC_TOEPLITZ
 987          * hash function.
 988          */
 989         if (rss_func == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) {
 990                 if (rss_type & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY |
 991                     ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY))
 992                         return true;
 993
 994                 if (!(rss_type &
 995                    (ETH_RSS_IPV4 | ETH_RSS_IPV6 |
 996                     ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_NONFRAG_IPV6_UDP |
 997                     ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_NONFRAG_IPV6_TCP |
 998                     ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_NONFRAG_IPV6_SCTP)))
 999                         return true;
1000         }
1001
1002         /* check invalid combination */
1003         for (i = 0; i < RTE_DIM(invalid_rss_comb); i++) {
1004                 if (__builtin_popcountll(rss_type & invalid_rss_comb[i]) > 1)
1005                         return true;
1006         }
1007
1008         /* check invalid RSS attribute */
1009         for (i = 0; i < RTE_DIM(rss_attr_to_valid_type); i++) {
1010                 struct rss_attr_type *rat = &rss_attr_to_valid_type[i];
1011
1012                 if (rat->attr & rss_type && !(rat->type & rss_type))
1013                         return true;
1014         }
1015
1016         /* check not allowed RSS type */
1017         rss_type &= ~VALID_RSS_ATTR;
1018
1019         return ((rss_type & allow_rss_type) != rss_type);
1020 }
1021
1022 static int
1023 iavf_hash_parse_action(struct iavf_pattern_match_item *match_item,
1024                        const struct rte_flow_action actions[],
1025                        uint64_t pattern_hint, void **meta,
1026                        struct rte_flow_error *error)
1027 {
1028         struct iavf_rss_meta *rss_meta = (struct iavf_rss_meta *)*meta;
1029         struct virtchnl_proto_hdrs *proto_hdrs;
1030         enum rte_flow_action_type action_type;
1031         const struct rte_flow_action_rss *rss;
1032         const struct rte_flow_action *action;
1033         uint64_t rss_type;
1034
1035         /* Supported action is RSS. */
1036         for (action = actions; action->type !=
1037                 RTE_FLOW_ACTION_TYPE_END; action++) {
1038                 action_type = action->type;
1039                 switch (action_type) {
1040                 case RTE_FLOW_ACTION_TYPE_RSS:
1041                         rss = action->conf;
1042                         rss_type = rss->types;
1043
1044                         if (rss->func ==
1045                             RTE_ETH_HASH_FUNCTION_SIMPLE_XOR){
1046                                 rss_meta->rss_algorithm =
1047                                         VIRTCHNL_RSS_ALG_XOR_ASYMMETRIC;
1048                                 return rte_flow_error_set(error, ENOTSUP,
1049                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
1050                                         "function simple_xor is not supported");
1051                         } else if (rss->func ==
1052                                    RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) {
1053                                 rss_meta->rss_algorithm =
1054                                         VIRTCHNL_RSS_ALG_TOEPLITZ_SYMMETRIC;
1055                         } else {
1056                                 rss_meta->rss_algorithm =
1057                                         VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC;
1058                         }
1059
1060                         if (rss->level)
1061                                 return rte_flow_error_set(error, ENOTSUP,
1062                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
1063                                         "a nonzero RSS encapsulation level is not supported");
1064
1065                         if (rss->key_len)
1066                                 return rte_flow_error_set(error, ENOTSUP,
1067                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
1068                                         "a nonzero RSS key_len is not supported");
1069
1070                         if (rss->queue_num)
1071                                 return rte_flow_error_set(error, ENOTSUP,
1072                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
1073                                         "a non-NULL RSS queue is not supported");
1074
1075                         /**
1076                          * Check simultaneous use of SRC_ONLY and DST_ONLY
1077                          * of the same level.
1078                          */
1079                         rss_type = rte_eth_rss_hf_refine(rss_type);
1080
1081                         if (iavf_any_invalid_rss_type(rss->func, rss_type,
1082                                         match_item->input_set_mask))
1083                                 return rte_flow_error_set(error, ENOTSUP,
1084                                                 RTE_FLOW_ERROR_TYPE_ACTION,
1085                                                 action, "RSS type not supported");
1086                         proto_hdrs = match_item->meta;
1087                         rss_meta->proto_hdrs = *proto_hdrs;
1088                         iavf_refine_proto_hdrs(&rss_meta->proto_hdrs,
1089                                                rss_type, pattern_hint);
1090                         break;
1091
1092                 case RTE_FLOW_ACTION_TYPE_END:
1093                         break;
1094
1095                 default:
1096                         rte_flow_error_set(error, EINVAL,
1097                                            RTE_FLOW_ERROR_TYPE_ACTION, action,
1098                                            "Invalid action.");
1099                         return -rte_errno;
1100                 }
1101         }
1102
1103         return 0;
1104 }
1105
1106 static int
1107 iavf_hash_parse_pattern_action(__rte_unused struct iavf_adapter *ad,
1108                                struct iavf_pattern_match_item *array,
1109                                uint32_t array_len,
1110                                const struct rte_flow_item pattern[],
1111                                const struct rte_flow_action actions[],
1112                                void **meta,
1113                                struct rte_flow_error *error)
1114 {
1115         struct iavf_pattern_match_item *pattern_match_item;
1116         struct iavf_rss_meta *rss_meta_ptr;
1117         uint64_t phint = IAVF_PHINT_NONE;
1118         int ret = 0;
1119
1120         rss_meta_ptr = rte_zmalloc(NULL, sizeof(*rss_meta_ptr), 0);
1121         if (!rss_meta_ptr) {
1122                 rte_flow_error_set(error, EINVAL,
1123                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1124                                    "No memory for rss_meta_ptr");
1125                 return -ENOMEM;
1126         }
1127
1128         /* Check rss supported pattern and find matched pattern. */
1129         pattern_match_item =
1130                 iavf_search_pattern_match_item(pattern, array, array_len,
1131                                                error);
1132         if (!pattern_match_item) {
1133                 ret = -rte_errno;
1134                 goto error;
1135         }
1136
1137         ret = iavf_hash_parse_pattern(pattern, &phint, error);
1138         if (ret)
1139                 goto error;
1140
1141         ret = iavf_hash_parse_action(pattern_match_item, actions, phint,
1142                                      (void **)&rss_meta_ptr, error);
1143
1144 error:
1145         if (!ret && meta)
1146                 *meta = rss_meta_ptr;
1147         else
1148                 rte_free(rss_meta_ptr);
1149
1150         rte_free(pattern_match_item);
1151
1152         return ret;
1153 }
1154
1155 static int
1156 iavf_hash_create(__rte_unused struct iavf_adapter *ad,
1157                  __rte_unused struct rte_flow *flow, void *meta,
1158                  __rte_unused struct rte_flow_error *error)
1159 {
1160         struct iavf_rss_meta *rss_meta = (struct iavf_rss_meta *)meta;
1161         struct virtchnl_rss_cfg *rss_cfg;
1162         int ret = 0;
1163
1164         rss_cfg = rte_zmalloc("iavf rss rule",
1165                               sizeof(struct virtchnl_rss_cfg), 0);
1166         if (!rss_cfg) {
1167                 rte_flow_error_set(error, EINVAL,
1168                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1169                                    "No memory for rss rule");
1170                 return -ENOMEM;
1171         }
1172
1173         rss_cfg->proto_hdrs = rss_meta->proto_hdrs;
1174         rss_cfg->rss_algorithm = rss_meta->rss_algorithm;
1175
1176         ret = iavf_add_del_rss_cfg(ad, rss_cfg, true);
1177         if (!ret) {
1178                 flow->rule = rss_cfg;
1179         } else {
1180                 PMD_DRV_LOG(ERR, "fail to add RSS configure");
1181                 rte_flow_error_set(error, -ret,
1182                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1183                                    "Failed to add rss rule.");
1184                 rte_free(rss_cfg);
1185                 return -rte_errno;
1186         }
1187
1188         rte_free(meta);
1189
1190         return ret;
1191 }
1192
1193 static int
1194 iavf_hash_destroy(__rte_unused struct iavf_adapter *ad,
1195                   struct rte_flow *flow,
1196                   __rte_unused struct rte_flow_error *error)
1197 {
1198         struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
1199         struct virtchnl_rss_cfg *rss_cfg;
1200         int ret = 0;
1201
1202         if (vf->vf_reset)
1203                 return 0;
1204
1205         rss_cfg = (struct virtchnl_rss_cfg *)flow->rule;
1206
1207         ret = iavf_add_del_rss_cfg(ad, rss_cfg, false);
1208         if (ret) {
1209                 PMD_DRV_LOG(ERR, "fail to del RSS configure");
1210                 rte_flow_error_set(error, -ret,
1211                                    RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1212                                    "Failed to delete rss rule.");
1213                 return -rte_errno;
1214         }
1215         return ret;
1216 }
1217
1218 static void
1219 iavf_hash_uninit(struct iavf_adapter *ad)
1220 {
1221         struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
1222         struct rte_eth_rss_conf *rss_conf;
1223
1224         if (vf->vf_reset)
1225                 return;
1226
1227         if (!vf->vf_res)
1228                 return;
1229
1230         if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF))
1231                 return;
1232
1233         rss_conf = &ad->eth_dev->data->dev_conf.rx_adv_conf.rss_conf;
1234         if (iavf_rss_hash_set(ad, rss_conf->rss_hf, false))
1235                 PMD_DRV_LOG(ERR, "fail to delete default RSS");
1236
1237         iavf_unregister_parser(&iavf_hash_parser, ad);
1238 }
1239
1240 static void
1241 iavf_hash_free(struct rte_flow *flow)
1242 {
1243         rte_free(flow->rule);
1244 }