drivers/net/ice/ice_hash.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10 #include <unistd.h>
  11 #include <stdarg.h>
  12
  13 #include <rte_debug.h>
  14 #include <rte_ether.h>
  15 #include <rte_ethdev_driver.h>
  16 #include <rte_log.h>
  17 #include <rte_malloc.h>
  18 #include <rte_eth_ctrl.h>
  19 #include <rte_tailq.h>
  20 #include <rte_flow_driver.h>
  21
  22 #include "ice_logs.h"
  23 #include "base/ice_type.h"
  24 #include "base/ice_flow.h"
  25 #include "ice_ethdev.h"
  26 #include "ice_generic_flow.h"
  27
  28 struct rss_type_match_hdr {
  29         uint32_t hdr_mask;
  30         uint64_t eth_rss_hint;
  31 };
  32
  33 struct ice_hash_match_type {
  34         uint64_t hash_type;
  35         uint64_t hash_flds;
  36 };
  37
  38 struct rss_meta {
  39         uint32_t pkt_hdr;
  40         uint64_t hash_flds;
  41         uint8_t hash_function;
  42 };
  43
  44 struct ice_hash_flow_cfg {
  45         bool simple_xor;
  46         struct ice_rss_cfg rss_cfg;
  47 };
  48
  49 static int
  50 ice_hash_init(struct ice_adapter *ad);
  51
  52 static int
  53 ice_hash_create(struct ice_adapter *ad,
  54                 struct rte_flow *flow,
  55                 void *meta,
  56                 struct rte_flow_error *error);
  57
  58 static int
  59 ice_hash_destroy(struct ice_adapter *ad,
  60                 struct rte_flow *flow,
  61                 struct rte_flow_error *error);
  62
  63 static void
  64 ice_hash_uninit(struct ice_adapter *ad);
  65
  66 static void
  67 ice_hash_free(struct rte_flow *flow);
  68
  69 static int
  70 ice_hash_parse_pattern_action(struct ice_adapter *ad,
  71                         struct ice_pattern_match_item *array,
  72                         uint32_t array_len,
  73                         const struct rte_flow_item pattern[],
  74                         const struct rte_flow_action actions[],
  75                         void **meta,
  76                         struct rte_flow_error *error);
  77
  78 /* The first member is protocol header, the second member is ETH_RSS_*. */
  79 struct rss_type_match_hdr hint_empty = {
  80         ICE_FLOW_SEG_HDR_NONE,  0};
  81 struct rss_type_match_hdr hint_eth_ipv4 = {
  82         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER,
  83         ETH_RSS_ETH | ETH_RSS_IPV4};
  84 struct rss_type_match_hdr hint_eth_ipv4_udp = {
  85         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
  86         ICE_FLOW_SEG_HDR_UDP,
  87         ETH_RSS_ETH | ETH_RSS_NONFRAG_IPV4_UDP};
  88 struct rss_type_match_hdr hint_eth_ipv4_tcp = {
  89         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
  90         ICE_FLOW_SEG_HDR_TCP,
  91         ETH_RSS_ETH | ETH_RSS_NONFRAG_IPV4_TCP};
  92 struct rss_type_match_hdr hint_eth_ipv4_sctp = {
  93         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER |
  94         ICE_FLOW_SEG_HDR_SCTP,
  95         ETH_RSS_ETH | ETH_RSS_NONFRAG_IPV4_SCTP};
  96 struct rss_type_match_hdr hint_eth_ipv4_gtpu_eh_ipv4 = {
  97         ICE_FLOW_SEG_HDR_GTPU_EH | ICE_FLOW_SEG_HDR_IPV4 |
  98         ICE_FLOW_SEG_HDR_IPV_OTHER,
  99         ETH_RSS_GTPU | ETH_RSS_IPV4};
 100 struct rss_type_match_hdr hint_eth_ipv4_gtpu_eh_ipv4_udp = {
 101         ICE_FLOW_SEG_HDR_GTPU_EH | ICE_FLOW_SEG_HDR_IPV4 |
 102         ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_UDP,
 103         ETH_RSS_GTPU | ETH_RSS_NONFRAG_IPV4_UDP};
 104 struct rss_type_match_hdr hint_eth_ipv4_gtpu_eh_ipv4_tcp = {
 105         ICE_FLOW_SEG_HDR_GTPU_EH | ICE_FLOW_SEG_HDR_IPV4 |
 106         ICE_FLOW_SEG_HDR_IPV_OTHER | ICE_FLOW_SEG_HDR_TCP,
 107         ETH_RSS_GTPU | ETH_RSS_NONFRAG_IPV4_TCP};
 108 struct rss_type_match_hdr hint_eth_pppoes_ipv4 = {
 109         ICE_FLOW_SEG_HDR_PPPOE,
 110         ETH_RSS_ETH | ETH_RSS_PPPOE | ETH_RSS_IPV4};
 111 struct rss_type_match_hdr hint_eth_pppoes_ipv4_udp = {
 112         ICE_FLOW_SEG_HDR_PPPOE,
 113         ETH_RSS_ETH | ETH_RSS_PPPOE | ETH_RSS_NONFRAG_IPV4_UDP};
 114 struct rss_type_match_hdr hint_eth_pppoes_ipv4_tcp = {
 115         ICE_FLOW_SEG_HDR_PPPOE,
 116         ETH_RSS_ETH | ETH_RSS_PPPOE | ETH_RSS_NONFRAG_IPV4_TCP};
 117 struct rss_type_match_hdr hint_eth_pppoes_ipv4_sctp = {
 118         ICE_FLOW_SEG_HDR_PPPOE,
 119         ETH_RSS_ETH | ETH_RSS_PPPOE | ETH_RSS_NONFRAG_IPV4_SCTP};
 120 struct rss_type_match_hdr hint_eth_ipv6 = {
 121         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER,
 122         ETH_RSS_ETH | ETH_RSS_IPV6};
 123 struct rss_type_match_hdr hint_eth_ipv6_udp = {
 124         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 125         ICE_FLOW_SEG_HDR_UDP,
 126         ETH_RSS_ETH | ETH_RSS_NONFRAG_IPV6_UDP};
 127 struct rss_type_match_hdr hint_eth_ipv6_tcp = {
 128         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 129         ICE_FLOW_SEG_HDR_TCP,
 130         ETH_RSS_ETH | ETH_RSS_NONFRAG_IPV6_TCP};
 131 struct rss_type_match_hdr hint_eth_ipv6_sctp = {
 132         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER |
 133         ICE_FLOW_SEG_HDR_SCTP,
 134         ETH_RSS_ETH | ETH_RSS_NONFRAG_IPV6_SCTP};
 135 struct rss_type_match_hdr hint_eth_pppoes_ipv6 = {
 136         ICE_FLOW_SEG_HDR_PPPOE,
 137         ETH_RSS_ETH | ETH_RSS_PPPOE | ETH_RSS_IPV6};
 138 struct rss_type_match_hdr hint_eth_pppoes_ipv6_udp = {
 139         ICE_FLOW_SEG_HDR_PPPOE,
 140         ETH_RSS_ETH | ETH_RSS_PPPOE | ETH_RSS_NONFRAG_IPV6_UDP};
 141 struct rss_type_match_hdr hint_eth_pppoes_ipv6_tcp = {
 142         ICE_FLOW_SEG_HDR_PPPOE,
 143         ETH_RSS_ETH | ETH_RSS_PPPOE | ETH_RSS_NONFRAG_IPV6_TCP};
 144 struct rss_type_match_hdr hint_eth_pppoes_ipv6_sctp = {
 145         ICE_FLOW_SEG_HDR_PPPOE,
 146         ETH_RSS_ETH | ETH_RSS_PPPOE | ETH_RSS_NONFRAG_IPV6_SCTP};
 147 struct rss_type_match_hdr hint_eth_pppoes = {
 148         ICE_FLOW_SEG_HDR_PPPOE,
 149         ETH_RSS_ETH | ETH_RSS_PPPOE};
 150
 151 /* Supported pattern for os default package. */
 152 static struct ice_pattern_match_item ice_hash_pattern_list_os[] = {
 153         {pattern_eth_ipv4,      ICE_INSET_NONE, &hint_eth_ipv4},
 154         {pattern_eth_ipv4_udp,  ICE_INSET_NONE, &hint_eth_ipv4_udp},
 155         {pattern_eth_ipv4_tcp,  ICE_INSET_NONE, &hint_eth_ipv4_tcp},
 156         {pattern_eth_ipv4_sctp, ICE_INSET_NONE, &hint_eth_ipv4_sctp},
 157         {pattern_eth_ipv6,      ICE_INSET_NONE, &hint_eth_ipv6},
 158         {pattern_eth_ipv6_udp,  ICE_INSET_NONE, &hint_eth_ipv6_udp},
 159         {pattern_eth_ipv6_tcp,  ICE_INSET_NONE, &hint_eth_ipv6_tcp},
 160         {pattern_eth_ipv6_sctp, ICE_INSET_NONE, &hint_eth_ipv6_sctp},
 161         {pattern_empty,         ICE_INSET_NONE, &hint_empty},
 162 };
 163
 164 /* Supported pattern for comms package. */
 165 static struct ice_pattern_match_item ice_hash_pattern_list_comms[] = {
 166         {pattern_empty,                     ICE_INSET_NONE,
 167                 &hint_empty},
 168         {pattern_eth_ipv4,                  ICE_INSET_NONE,
 169                 &hint_eth_ipv4},
 170         {pattern_eth_ipv4_udp,              ICE_INSET_NONE,
 171                 &hint_eth_ipv4_udp},
 172         {pattern_eth_ipv4_tcp,              ICE_INSET_NONE,
 173                 &hint_eth_ipv4_tcp},
 174         {pattern_eth_ipv4_sctp,             ICE_INSET_NONE,
 175                 &hint_eth_ipv4_sctp},
 176         {pattern_eth_ipv4_gtpu_eh_ipv4,     ICE_INSET_NONE,
 177                 &hint_eth_ipv4_gtpu_eh_ipv4},
 178         {pattern_eth_ipv4_gtpu_eh_ipv4_udp, ICE_INSET_NONE,
 179                 &hint_eth_ipv4_gtpu_eh_ipv4_udp},
 180         {pattern_eth_ipv4_gtpu_eh_ipv4_tcp, ICE_INSET_NONE,
 181                 &hint_eth_ipv4_gtpu_eh_ipv4_tcp},
 182         {pattern_eth_pppoes_ipv4,           ICE_INSET_NONE,
 183                 &hint_eth_pppoes_ipv4},
 184         {pattern_eth_pppoes_ipv4_udp,       ICE_INSET_NONE,
 185                 &hint_eth_pppoes_ipv4_udp},
 186         {pattern_eth_pppoes_ipv4_tcp,       ICE_INSET_NONE,
 187                 &hint_eth_pppoes_ipv4_tcp},
 188         {pattern_eth_pppoes_ipv4_sctp,      ICE_INSET_NONE,
 189                 &hint_eth_pppoes_ipv4_sctp},
 190         {pattern_eth_ipv6,                  ICE_INSET_NONE,
 191                 &hint_eth_ipv6},
 192         {pattern_eth_ipv6_udp,              ICE_INSET_NONE,
 193                 &hint_eth_ipv6_udp},
 194         {pattern_eth_ipv6_tcp,              ICE_INSET_NONE,
 195                 &hint_eth_ipv6_tcp},
 196         {pattern_eth_ipv6_sctp,             ICE_INSET_NONE,
 197                 &hint_eth_ipv6_sctp},
 198         {pattern_eth_pppoes_ipv6,           ICE_INSET_NONE,
 199                 &hint_eth_pppoes_ipv6},
 200         {pattern_eth_pppoes_ipv6_udp,       ICE_INSET_NONE,
 201                 &hint_eth_pppoes_ipv6_udp},
 202         {pattern_eth_pppoes_ipv6_tcp,       ICE_INSET_NONE,
 203                 &hint_eth_pppoes_ipv6_tcp},
 204         {pattern_eth_pppoes_ipv6_sctp,      ICE_INSET_NONE,
 205                 &hint_eth_pppoes_ipv6_sctp},
 206         {pattern_eth_pppoes,                ICE_INSET_NONE,
 207                 &hint_eth_pppoes},
 208 };
 209
 210 /**
 211  * The first member is input set combination,
 212  * the second member is hash fields.
 213  */
 214 struct ice_hash_match_type ice_hash_type_list[] = {
 215         {ETH_RSS_PPPOE,
 216                 ICE_FLOW_HASH_PPPOE_SESS_ID},
 217         {ETH_RSS_ETH | ETH_RSS_PPPOE | ETH_RSS_L2_SRC_ONLY,
 218                 ICE_FLOW_HASH_PPPOE_SESS_ID |
 219                 BIT_ULL(ICE_FLOW_FIELD_IDX_ETH_SA)},
 220         /* IPV4 */
 221         {ETH_RSS_IPV4 | ETH_RSS_L3_SRC_ONLY,
 222                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA)},
 223         {ETH_RSS_IPV4 | ETH_RSS_L3_DST_ONLY,
 224                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA)},
 225         {ETH_RSS_IPV4, ICE_FLOW_HASH_IPV4},
 226         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY,
 227                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) |
 228                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT) |
 229                 BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)},
 230         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY,
 231                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) |
 232                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT) |
 233                 BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)},
 234         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L3_SRC_ONLY,
 235                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA)},
 236         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY,
 237                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) |
 238                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT) |
 239                 BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)},
 240         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY,
 241                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) |
 242                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT) |
 243                 BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)},
 244         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L3_DST_ONLY,
 245                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA)},
 246         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L4_SRC_ONLY,
 247                 BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT) |
 248                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT)},
 249         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L4_DST_ONLY,
 250                 BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT) |
 251                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT)},
 252         {ETH_RSS_NONFRAG_IPV4_UDP,
 253                 ICE_HASH_UDP_IPV4 |
 254                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT)},
 255         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY,
 256                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) |
 257                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT) |
 258                 BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)},
 259         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY,
 260                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) |
 261                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT) |
 262                 BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)},
 263         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L3_SRC_ONLY,
 264                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA)},
 265         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY,
 266                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) |
 267                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT) |
 268                 BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)},
 269         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY,
 270                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) |
 271                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT) |
 272                 BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)},
 273         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L3_DST_ONLY,
 274                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA)},
 275         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L4_SRC_ONLY,
 276                 BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT) |
 277                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT)},
 278         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L4_DST_ONLY,
 279                 BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT) |
 280                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT)},
 281         {ETH_RSS_NONFRAG_IPV4_TCP,
 282                 ICE_HASH_TCP_IPV4 |
 283                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT)},
 284         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY,
 285                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) |
 286                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT) |
 287                 BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)},
 288         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY,
 289                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) |
 290                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT) |
 291                 BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)},
 292         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L3_SRC_ONLY,
 293                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA)},
 294         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY,
 295                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) |
 296                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT) |
 297                 BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)},
 298         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY,
 299                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) |
 300                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT) |
 301                 BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)},
 302         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L3_DST_ONLY,
 303                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA)},
 304         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L4_SRC_ONLY,
 305                 BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT) |
 306                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT)},
 307         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L4_DST_ONLY,
 308                 BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT) |
 309                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT)},
 310         {ETH_RSS_NONFRAG_IPV4_SCTP,
 311                 ICE_HASH_SCTP_IPV4 |
 312                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT)},
 313         /* IPV6 */
 314         {ETH_RSS_IPV6 | ETH_RSS_L3_SRC_ONLY,
 315                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)},
 316         {ETH_RSS_IPV6 | ETH_RSS_L3_DST_ONLY,
 317                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)},
 318         {ETH_RSS_IPV6, ICE_FLOW_HASH_IPV6},
 319         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY,
 320                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) |
 321                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT) |
 322                 BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)},
 323         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY,
 324                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) |
 325                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT) |
 326                 BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)},
 327         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L3_SRC_ONLY,
 328                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)},
 329         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY,
 330                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) |
 331                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT) |
 332                 BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)},
 333         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY,
 334                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) |
 335                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT) |
 336                 BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)},
 337         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L3_DST_ONLY,
 338                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)},
 339         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L4_SRC_ONLY,
 340                 BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT) |
 341                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT)},
 342         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L4_DST_ONLY,
 343                 BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT) |
 344                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT)},
 345         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_PFCP,
 346                 BIT_ULL(ICE_FLOW_FIELD_IDX_PFCP_SEID)},
 347         {ETH_RSS_NONFRAG_IPV6_UDP,
 348                 ICE_HASH_UDP_IPV6 |
 349                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT)},
 350         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY,
 351                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) |
 352                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT) |
 353                 BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)},
 354         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY,
 355                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) |
 356                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT) |
 357                 BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)},
 358         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L3_SRC_ONLY,
 359                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)},
 360         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY,
 361                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) |
 362                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT) |
 363                 BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)},
 364         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY,
 365                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) |
 366                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT) |
 367                 BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)},
 368         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L3_DST_ONLY,
 369                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)},
 370         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L4_SRC_ONLY,
 371                 BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT) |
 372                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT)},
 373         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L4_DST_ONLY,
 374                 BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT) |
 375                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT)},
 376         {ETH_RSS_NONFRAG_IPV6_TCP,
 377                 ICE_HASH_TCP_IPV6 |
 378                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT)},
 379         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY,
 380                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) |
 381                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT) |
 382                 BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)},
 383         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY,
 384                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) |
 385                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT) |
 386                 BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)},
 387         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L3_SRC_ONLY,
 388                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)},
 389         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY,
 390                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) |
 391                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT) |
 392                 BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)},
 393         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY,
 394                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) |
 395                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT) |
 396                 BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)},
 397         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L3_DST_ONLY,
 398                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)},
 399         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L4_SRC_ONLY,
 400                 BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT) |
 401                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT)},
 402         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L4_DST_ONLY,
 403                 BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT) |
 404                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT)},
 405         {ETH_RSS_NONFRAG_IPV6_SCTP,
 406                 ICE_HASH_SCTP_IPV6 |
 407                 BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT)},
 408 };
 409
 410 static struct ice_flow_engine ice_hash_engine = {
 411         .init = ice_hash_init,
 412         .create = ice_hash_create,
 413         .destroy = ice_hash_destroy,
 414         .uninit = ice_hash_uninit,
 415         .free = ice_hash_free,
 416         .type = ICE_FLOW_ENGINE_HASH,
 417 };
 418
 419 /* Register parser for os package. */
 420 static struct ice_flow_parser ice_hash_parser_os = {
 421         .engine = &ice_hash_engine,
 422         .array = ice_hash_pattern_list_os,
 423         .array_len = RTE_DIM(ice_hash_pattern_list_os),
 424         .parse_pattern_action = ice_hash_parse_pattern_action,
 425         .stage = ICE_FLOW_STAGE_RSS,
 426 };
 427
 428 /* Register parser for comms package. */
 429 static struct ice_flow_parser ice_hash_parser_comms = {
 430         .engine = &ice_hash_engine,
 431         .array = ice_hash_pattern_list_comms,
 432         .array_len = RTE_DIM(ice_hash_pattern_list_comms),
 433         .parse_pattern_action = ice_hash_parse_pattern_action,
 434         .stage = ICE_FLOW_STAGE_RSS,
 435 };
 436
 437 RTE_INIT(ice_hash_engine_init)
 438 {
 439         struct ice_flow_engine *engine = &ice_hash_engine;
 440         ice_register_flow_engine(engine);
 441 }
 442
 443 static int
 444 ice_hash_init(struct ice_adapter *ad)
 445 {
 446         struct ice_flow_parser *parser = NULL;
 447
 448         if (ad->hw.dcf_enabled)
 449                 return 0;
 450
 451         if (ad->active_pkg_type == ICE_PKG_TYPE_OS_DEFAULT)
 452                 parser = &ice_hash_parser_os;
 453         else if (ad->active_pkg_type == ICE_PKG_TYPE_COMMS)
 454                 parser = &ice_hash_parser_comms;
 455         else
 456                 return -EINVAL;
 457
 458         return ice_register_parser(parser, ad);
 459 }
 460
 461 static int
 462 ice_hash_check_inset(const struct rte_flow_item pattern[],
 463                 struct rte_flow_error *error)
 464 {
 465         const struct rte_flow_item *item = pattern;
 466
 467         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 468                 if (item->last) {
 469                         rte_flow_error_set(error, EINVAL,
 470                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 471                                         "Not support range");
 472                         return -rte_errno;
 473                 }
 474
 475                 /* Ignore spec and mask. */
 476                 if (item->spec || item->mask) {
 477                         rte_flow_error_set(error, EINVAL,
 478                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 479                                         "Invalid spec/mask.");
 480                         return -rte_errno;
 481                 }
 482         }
 483
 484         return 0;
 485 }
 486
 487 static int
 488 ice_hash_parse_action(struct ice_pattern_match_item *pattern_match_item,
 489                 const struct rte_flow_action actions[],
 490                 void **meta,
 491                 struct rte_flow_error *error)
 492 {
 493         struct rss_type_match_hdr *m = (struct rss_type_match_hdr *)
 494                                 (pattern_match_item->meta);
 495         struct rss_meta *hash_meta = (struct rss_meta *)*meta;
 496         uint32_t type_list_len = RTE_DIM(ice_hash_type_list);
 497         enum rte_flow_action_type action_type;
 498         const struct rte_flow_action_rss *rss;
 499         const struct rte_flow_action *action;
 500         uint64_t combine_type;
 501         uint64_t rss_type;
 502         uint16_t i;
 503
 504         /* Supported action is RSS. */
 505         for (action = actions; action->type !=
 506                 RTE_FLOW_ACTION_TYPE_END; action++) {
 507                 action_type = action->type;
 508                 switch (action_type) {
 509                 case RTE_FLOW_ACTION_TYPE_RSS:
 510                         rss = action->conf;
 511                         rss_type = rss->types;
 512
 513                         /* Check hash function and save it to rss_meta. */
 514                         if (pattern_match_item->pattern_list !=
 515                             pattern_empty && rss->func ==
 516                             RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) {
 517                                 return rte_flow_error_set(error, ENOTSUP,
 518                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 519                                         "Not supported flow");
 520                         } else if (rss->func ==
 521                                    RTE_ETH_HASH_FUNCTION_SIMPLE_XOR){
 522                                 ((struct rss_meta *)*meta)->hash_function =
 523                                 RTE_ETH_HASH_FUNCTION_SIMPLE_XOR;
 524                                 return 0;
 525                         } else if (rss->func ==
 526                                    RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) {
 527                                 ((struct rss_meta *)*meta)->hash_function =
 528                                 RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ;
 529                         }
 530
 531                         if (rss->level)
 532                                 return rte_flow_error_set(error, ENOTSUP,
 533                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 534                                         "a nonzero RSS encapsulation level is not supported");
 535
 536                         if (rss->key_len)
 537                                 return rte_flow_error_set(error, ENOTSUP,
 538                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 539                                         "a nonzero RSS key_len is not supported");
 540
 541                         if (rss->queue)
 542                                 return rte_flow_error_set(error, ENOTSUP,
 543                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 544                                         "a non-NULL RSS queue is not supported");
 545
 546                         /**
 547                          * Check simultaneous use of SRC_ONLY and DST_ONLY
 548                          * of the same level.
 549                          */
 550                         rss_type = rte_eth_rss_hf_refine(rss_type);
 551
 552                         combine_type = ETH_RSS_L2_SRC_ONLY |
 553                                         ETH_RSS_L2_DST_ONLY |
 554                                         ETH_RSS_L3_SRC_ONLY |
 555                                         ETH_RSS_L3_DST_ONLY |
 556                                         ETH_RSS_L4_SRC_ONLY |
 557                                         ETH_RSS_L4_DST_ONLY;
 558
 559                         /* Check if rss types match pattern. */
 560                         if (rss_type & ~combine_type & ~m->eth_rss_hint) {
 561                                 return rte_flow_error_set(error,
 562                                 ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
 563                                 action, "Not supported RSS types");
 564                         }
 565
 566                         /* Find matched hash fields according to hash type. */
 567                         for (i = 0; i < type_list_len; i++) {
 568                                 struct ice_hash_match_type *ht_map =
 569                                         &ice_hash_type_list[i];
 570
 571                                 if (rss_type == ht_map->hash_type) {
 572                                         hash_meta->hash_flds =
 573                                                         ht_map->hash_flds;
 574                                         break;
 575                                 }
 576                         }
 577
 578                         break;
 579
 580                 case RTE_FLOW_ACTION_TYPE_END:
 581                         break;
 582
 583                 default:
 584                         rte_flow_error_set(error, EINVAL,
 585                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 586                                         "Invalid action.");
 587                         return -rte_errno;
 588                 }
 589         }
 590
 591         return 0;
 592 }
 593
 594 static int
 595 ice_hash_parse_pattern_action(__rte_unused struct ice_adapter *ad,
 596                         struct ice_pattern_match_item *array,
 597                         uint32_t array_len,
 598                         const struct rte_flow_item pattern[],
 599                         const struct rte_flow_action actions[],
 600                         void **meta,
 601                         struct rte_flow_error *error)
 602 {
 603         int ret = 0;
 604         struct ice_pattern_match_item *pattern_match_item;
 605         struct rss_meta *rss_meta_ptr;
 606
 607         rss_meta_ptr = rte_zmalloc(NULL, sizeof(*rss_meta_ptr), 0);
 608         if (!rss_meta_ptr) {
 609                 rte_flow_error_set(error, EINVAL,
 610                                 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 611                                 "No memory for rss_meta_ptr");
 612                 return -ENOMEM;
 613         }
 614
 615         /* Check rss supported pattern and find matched pattern. */
 616         pattern_match_item = ice_search_pattern_match_item(pattern,
 617                                         array, array_len, error);
 618         if (!pattern_match_item) {
 619                 ret = -rte_errno;
 620                 goto error;
 621         }
 622
 623         ret = ice_hash_check_inset(pattern, error);
 624         if (ret)
 625                 goto error;
 626
 627         /* Save protocol header to rss_meta. */
 628         rss_meta_ptr->pkt_hdr = ((struct rss_type_match_hdr *)
 629                 (pattern_match_item->meta))->hdr_mask;
 630
 631         /* Check rss action. */
 632         ret = ice_hash_parse_action(pattern_match_item, actions,
 633                                     (void **)&rss_meta_ptr, error);
 634
 635 error:
 636         if (!ret && meta)
 637                 *meta = rss_meta_ptr;
 638         else
 639                 rte_free(rss_meta_ptr);
 640         rte_free(pattern_match_item);
 641
 642         return ret;
 643 }
 644
 645 static int
 646 ice_hash_create(struct ice_adapter *ad,
 647                 struct rte_flow *flow,
 648                 void *meta,
 649                 struct rte_flow_error *error)
 650 {
 651         struct ice_pf *pf = &ad->pf;
 652         struct ice_hw *hw = ICE_PF_TO_HW(pf);
 653         struct ice_vsi *vsi = pf->main_vsi;
 654         int ret;
 655         uint32_t reg;
 656         struct ice_hash_flow_cfg *filter_ptr;
 657
 658         uint32_t headermask = ((struct rss_meta *)meta)->pkt_hdr;
 659         uint64_t hash_field = ((struct rss_meta *)meta)->hash_flds;
 660         uint8_t hash_function = ((struct rss_meta *)meta)->hash_function;
 661
 662         filter_ptr = rte_zmalloc("ice_rss_filter",
 663                                 sizeof(struct ice_hash_flow_cfg), 0);
 664         if (!filter_ptr) {
 665                 rte_flow_error_set(error, EINVAL,
 666                                 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 667                                 "No memory for filter_ptr");
 668                 return -ENOMEM;
 669         }
 670
 671         if (hash_function == RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) {
 672                 /* Enable registers for simple_xor hash function. */
 673                 reg = ICE_READ_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id));
 674                 reg = (reg & (~VSIQF_HASH_CTL_HASH_SCHEME_M)) |
 675                         (2 << VSIQF_HASH_CTL_HASH_SCHEME_S);
 676                 ICE_WRITE_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id), reg);
 677
 678                 filter_ptr->simple_xor = 1;
 679
 680                 goto out;
 681         } else {
 682                 filter_ptr->rss_cfg.packet_hdr = headermask;
 683                 filter_ptr->rss_cfg.hashed_flds = hash_field;
 684                 filter_ptr->rss_cfg.symm =
 685                         (hash_function ==
 686                                 RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ);
 687
 688                 ret = ice_add_rss_cfg(hw, vsi->idx,
 689                                 filter_ptr->rss_cfg.hashed_flds,
 690                                 filter_ptr->rss_cfg.packet_hdr,
 691                                 filter_ptr->rss_cfg.symm);
 692                 if (ret) {
 693                         rte_flow_error_set(error, EINVAL,
 694                                         RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 695                                         "rss flow create fail");
 696                         goto error;
 697                 }
 698         }
 699
 700 out:
 701         flow->rule = filter_ptr;
 702         rte_free(meta);
 703         return 0;
 704
 705 error:
 706         rte_free(filter_ptr);
 707         rte_free(meta);
 708         return -rte_errno;
 709 }
 710
 711 static int
 712 ice_hash_destroy(struct ice_adapter *ad,
 713                 struct rte_flow *flow,
 714                 struct rte_flow_error *error)
 715 {
 716         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(ad);
 717         struct ice_hw *hw = ICE_PF_TO_HW(pf);
 718         struct ice_vsi *vsi = pf->main_vsi;
 719         int ret;
 720         uint32_t reg;
 721         struct ice_hash_flow_cfg *filter_ptr;
 722
 723         filter_ptr = (struct ice_hash_flow_cfg *)flow->rule;
 724
 725         if (filter_ptr->simple_xor == 1) {
 726                 /* Return to symmetric_toeplitz state. */
 727                 reg = ICE_READ_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id));
 728                 reg = (reg & (~VSIQF_HASH_CTL_HASH_SCHEME_M)) |
 729                         (1 << VSIQF_HASH_CTL_HASH_SCHEME_S);
 730                 ICE_WRITE_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id), reg);
 731         } else {
 732                 ret = ice_rem_rss_cfg(hw, vsi->idx,
 733                                 filter_ptr->rss_cfg.hashed_flds,
 734                                 filter_ptr->rss_cfg.packet_hdr);
 735                 /* Fixme: Ignore the error if a rule does not exist.
 736                  * Currently a rule for inputset change or symm turn on/off
 737                  * will overwrite an exist rule, while application still
 738                  * have 2 rte_flow handles.
 739                  **/
 740                 if (ret && ret != ICE_ERR_DOES_NOT_EXIST) {
 741                         rte_flow_error_set(error, EINVAL,
 742                                         RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 743                                         "rss flow destroy fail");
 744                         goto error;
 745                 }
 746         }
 747
 748         rte_free(filter_ptr);
 749         return 0;
 750
 751 error:
 752         rte_free(filter_ptr);
 753         return -rte_errno;
 754 }
 755
 756 static void
 757 ice_hash_uninit(struct ice_adapter *ad)
 758 {
 759         if (ad->hw.dcf_enabled)
 760                 return;
 761
 762         if (ad->active_pkg_type == ICE_PKG_TYPE_OS_DEFAULT)
 763                 ice_unregister_parser(&ice_hash_parser_os, ad);
 764         else if (ad->active_pkg_type == ICE_PKG_TYPE_COMMS)
 765                 ice_unregister_parser(&ice_hash_parser_comms, ad);
 766 }
 767
 768 static void
 769 ice_hash_free(struct rte_flow *flow)
 770 {
 771         rte_free(flow->rule);
 772 }