drivers/net/ice/ice_hash.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10 #include <unistd.h>
  11 #include <stdarg.h>
  12
  13 #include <rte_debug.h>
  14 #include <rte_ether.h>
  15 #include <rte_ethdev_driver.h>
  16 #include <rte_log.h>
  17 #include <rte_malloc.h>
  18 #include <rte_eth_ctrl.h>
  19 #include <rte_tailq.h>
  20 #include <rte_flow_driver.h>
  21
  22 #include "ice_logs.h"
  23 #include "base/ice_type.h"
  24 #include "base/ice_flow.h"
  25 #include "ice_ethdev.h"
  26 #include "ice_generic_flow.h"
  27
  28 #define ICE_ACTION_RSS_MAX_QUEUE_NUM 32
  29
  30 struct rss_type_match_hdr {
  31         uint32_t hdr_mask;
  32         uint64_t eth_rss_hint;
  33 };
  34
  35 struct ice_hash_match_type {
  36         uint64_t hash_type;
  37         uint64_t hash_flds;
  38 };
  39
  40 struct rss_meta {
  41         uint32_t pkt_hdr;
  42         uint64_t hash_flds;
  43         uint8_t hash_function;
  44 };
  45
  46 static int
  47 ice_hash_init(struct ice_adapter *ad);
  48
  49 static int
  50 ice_hash_create(struct ice_adapter *ad,
  51                 struct rte_flow *flow,
  52                 void *meta,
  53                 struct rte_flow_error *error);
  54
  55 static int
  56 ice_hash_destroy(struct ice_adapter *ad,
  57                 struct rte_flow *flow,
  58                 struct rte_flow_error *error);
  59
  60 static void
  61 ice_hash_uninit(struct ice_adapter *ad);
  62
  63 static void
  64 ice_hash_free(struct rte_flow *flow);
  65
  66 static int
  67 ice_hash_parse_pattern_action(struct ice_adapter *ad,
  68                         struct ice_pattern_match_item *array,
  69                         uint32_t array_len,
  70                         const struct rte_flow_item pattern[],
  71                         const struct rte_flow_action actions[],
  72                         void **meta,
  73                         struct rte_flow_error *error);
  74
  75 /* The first member is protocol header, the second member is ETH_RSS_*. */
  76 struct rss_type_match_hdr hint_0 = {
  77         ICE_FLOW_SEG_HDR_NONE,  0};
  78 struct rss_type_match_hdr hint_1 = {
  79         ICE_FLOW_SEG_HDR_IPV4,  ETH_RSS_IPV4};
  80 struct rss_type_match_hdr hint_2 = {
  81         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_UDP, ETH_RSS_NONFRAG_IPV4_UDP};
  82 struct rss_type_match_hdr hint_3 = {
  83         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_TCP, ETH_RSS_NONFRAG_IPV4_TCP};
  84 struct rss_type_match_hdr hint_4 = {
  85         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_SCTP, ETH_RSS_NONFRAG_IPV4_SCTP};
  86 struct rss_type_match_hdr hint_5 = {
  87         ICE_FLOW_SEG_HDR_IPV6,  ETH_RSS_IPV6};
  88 struct rss_type_match_hdr hint_6 = {
  89         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_UDP, ETH_RSS_NONFRAG_IPV6_UDP};
  90 struct rss_type_match_hdr hint_7 = {
  91         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_TCP, ETH_RSS_NONFRAG_IPV6_TCP};
  92 struct rss_type_match_hdr hint_8 = {
  93         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_SCTP, ETH_RSS_NONFRAG_IPV6_SCTP};
  94 struct rss_type_match_hdr hint_9 = {
  95         ICE_FLOW_SEG_HDR_GTPU_IP,       ETH_RSS_IPV4};
  96 struct rss_type_match_hdr hint_10 = {
  97         ICE_FLOW_SEG_HDR_PPPOE, ETH_RSS_IPV4};
  98 struct rss_type_match_hdr hint_11 = {
  99         ICE_FLOW_SEG_HDR_PPPOE, ETH_RSS_NONFRAG_IPV4_UDP};
 100 struct rss_type_match_hdr hint_12 = {
 101         ICE_FLOW_SEG_HDR_PPPOE, ETH_RSS_NONFRAG_IPV4_TCP};
 102 struct rss_type_match_hdr hint_13 = {
 103         ICE_FLOW_SEG_HDR_PPPOE, ETH_RSS_NONFRAG_IPV4_SCTP};
 104
 105 /* Supported pattern for os default package. */
 106 static struct ice_pattern_match_item ice_hash_pattern_list_os[] = {
 107         {pattern_eth_ipv4,      ICE_INSET_NONE, &hint_1},
 108         {pattern_eth_ipv4_udp,  ICE_INSET_NONE, &hint_2},
 109         {pattern_eth_ipv4_tcp,  ICE_INSET_NONE, &hint_3},
 110         {pattern_eth_ipv4_sctp, ICE_INSET_NONE, &hint_4},
 111         {pattern_eth_ipv6,      ICE_INSET_NONE, &hint_5},
 112         {pattern_eth_ipv6_udp,  ICE_INSET_NONE, &hint_6},
 113         {pattern_eth_ipv6_tcp,  ICE_INSET_NONE, &hint_7},
 114         {pattern_eth_ipv6_sctp, ICE_INSET_NONE, &hint_8},
 115         {pattern_empty,         ICE_INSET_NONE, &hint_0},
 116 };
 117
 118 /* Supported pattern for comms package. */
 119 static struct ice_pattern_match_item ice_hash_pattern_list_comms[] = {
 120         {pattern_eth_ipv4,               ICE_INSET_NONE,  &hint_1},
 121         {pattern_eth_ipv4_udp,           ICE_INSET_NONE,  &hint_2},
 122         {pattern_eth_ipv4_tcp,           ICE_INSET_NONE,  &hint_3},
 123         {pattern_eth_ipv4_sctp,          ICE_INSET_NONE,  &hint_4},
 124         {pattern_eth_ipv6,               ICE_INSET_NONE,  &hint_5},
 125         {pattern_eth_ipv6_udp,           ICE_INSET_NONE,  &hint_6},
 126         {pattern_eth_ipv6_tcp,           ICE_INSET_NONE,  &hint_7},
 127         {pattern_eth_ipv6_sctp,          ICE_INSET_NONE,  &hint_8},
 128         {pattern_empty,                  ICE_INSET_NONE,  &hint_0},
 129         {pattern_eth_ipv4_gtpu_ipv4,     ICE_INSET_NONE,  &hint_9},
 130         {pattern_eth_ipv4_gtpu_ipv4_udp, ICE_INSET_NONE,  &hint_9},
 131         {pattern_eth_ipv4_gtpu_ipv4_tcp, ICE_INSET_NONE,  &hint_9},
 132         {pattern_eth_pppoes_ipv4,        ICE_INSET_NONE,  &hint_10},
 133         {pattern_eth_pppoes_ipv4_udp,    ICE_INSET_NONE,  &hint_11},
 134         {pattern_eth_pppoes_ipv4_tcp,    ICE_INSET_NONE,  &hint_12},
 135         {pattern_eth_pppoes_ipv4_sctp,   ICE_INSET_NONE,  &hint_13},
 136 };
 137
 138 /**
 139  * The first member is input set combination,
 140  * the second member is hash fields.
 141  */
 142 struct ice_hash_match_type ice_hash_type_list[] = {
 143         {ETH_RSS_IPV4 | ETH_RSS_L3_SRC_ONLY,                                    BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA)},
 144         {ETH_RSS_IPV4 | ETH_RSS_L3_DST_ONLY,                                    BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA)},
 145         {ETH_RSS_IPV4,                                                          ICE_FLOW_HASH_IPV4},
 146         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)},
 147         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)},
 148         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L3_SRC_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA)},
 149         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)},
 150         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)},
 151         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L3_DST_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA)},
 152         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L4_SRC_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)},
 153         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L4_DST_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)},
 154         {ETH_RSS_NONFRAG_IPV4_UDP,                                              ICE_HASH_UDP_IPV4},
 155         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)},
 156         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)},
 157         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L3_SRC_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA)},
 158         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)},
 159         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)},
 160         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L3_DST_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA)},
 161         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L4_SRC_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)},
 162         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L4_DST_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)},
 163         {ETH_RSS_NONFRAG_IPV4_TCP,                                              ICE_HASH_TCP_IPV4},
 164         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY, BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)},
 165         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY, BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)},
 166         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L3_SRC_ONLY,                       BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA)},
 167         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY, BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)},
 168         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY, BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)},
 169         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L3_DST_ONLY,                       BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA)},
 170         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L4_SRC_ONLY,                       BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)},
 171         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L4_DST_ONLY,                       BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)},
 172         {ETH_RSS_NONFRAG_IPV4_SCTP,                                             ICE_HASH_SCTP_IPV4},
 173         {ETH_RSS_IPV6 | ETH_RSS_L3_SRC_ONLY,                                    BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)},
 174         {ETH_RSS_IPV6 | ETH_RSS_L3_DST_ONLY,                                    BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)},
 175         {ETH_RSS_IPV6,                                                          ICE_FLOW_HASH_IPV6},
 176         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)},
 177         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)},
 178         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L3_SRC_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)},
 179         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)},
 180         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)},
 181         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L3_DST_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)},
 182         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L4_SRC_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)},
 183         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L4_DST_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)},
 184         {ETH_RSS_NONFRAG_IPV6_UDP,                                              ICE_HASH_UDP_IPV6},
 185         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)},
 186         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)},
 187         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L3_SRC_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)},
 188         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)},
 189         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)},
 190         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L3_DST_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)},
 191         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L4_SRC_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)},
 192         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L4_DST_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)},
 193         {ETH_RSS_NONFRAG_IPV6_TCP,                                              ICE_HASH_TCP_IPV6},
 194         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY, BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)},
 195         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY, BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)},
 196         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L3_SRC_ONLY,                       BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)},
 197         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY, BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)},
 198         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY, BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)},
 199         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L3_DST_ONLY,                       BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)},
 200         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L4_SRC_ONLY,                       BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)},
 201         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L4_DST_ONLY,                       BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)},
 202         {ETH_RSS_NONFRAG_IPV6_SCTP,                                             ICE_HASH_SCTP_IPV6},
 203 };
 204
 205 static struct ice_flow_engine ice_hash_engine = {
 206         .init = ice_hash_init,
 207         .create = ice_hash_create,
 208         .destroy = ice_hash_destroy,
 209         .uninit = ice_hash_uninit,
 210         .free = ice_hash_free,
 211         .type = ICE_FLOW_ENGINE_HASH,
 212 };
 213
 214 /* Register parser for os package. */
 215 static struct ice_flow_parser ice_hash_parser_os = {
 216         .engine = &ice_hash_engine,
 217         .array = ice_hash_pattern_list_os,
 218         .array_len = RTE_DIM(ice_hash_pattern_list_os),
 219         .parse_pattern_action = ice_hash_parse_pattern_action,
 220         .stage = ICE_FLOW_STAGE_RSS,
 221 };
 222
 223 /* Register parser for comms package. */
 224 static struct ice_flow_parser ice_hash_parser_comms = {
 225         .engine = &ice_hash_engine,
 226         .array = ice_hash_pattern_list_comms,
 227         .array_len = RTE_DIM(ice_hash_pattern_list_comms),
 228         .parse_pattern_action = ice_hash_parse_pattern_action,
 229         .stage = ICE_FLOW_STAGE_RSS,
 230 };
 231
 232 RTE_INIT(ice_hash_engine_init)
 233 {
 234         struct ice_flow_engine *engine = &ice_hash_engine;
 235         ice_register_flow_engine(engine);
 236 }
 237
 238 static int
 239 ice_hash_init(struct ice_adapter *ad)
 240 {
 241         struct ice_flow_parser *parser = NULL;
 242
 243         if (ad->active_pkg_type == ICE_PKG_TYPE_OS_DEFAULT)
 244                 parser = &ice_hash_parser_os;
 245         else if (ad->active_pkg_type == ICE_PKG_TYPE_COMMS)
 246                 parser = &ice_hash_parser_comms;
 247
 248         return ice_register_parser(parser, ad);
 249 }
 250
 251 static int
 252 ice_hash_check_inset(const struct rte_flow_item pattern[],
 253                 struct rte_flow_error *error)
 254 {
 255         const struct rte_flow_item *item = pattern;
 256
 257         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 258                 if (item->last) {
 259                         rte_flow_error_set(error, EINVAL,
 260                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 261                                         "Not support range");
 262                         return -rte_errno;
 263                 }
 264
 265                 /* Ignore spec and mask. */
 266                 if (item->spec || item->mask) {
 267                         rte_flow_error_set(error, EINVAL,
 268                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 269                                         "Invalid spec/mask.");
 270                         return -rte_errno;
 271                 }
 272         }
 273
 274         return 0;
 275 }
 276
 277 static int
 278 ice_hash_parse_action(struct ice_pattern_match_item *pattern_match_item,
 279                 const struct rte_flow_action actions[],
 280                 void **meta,
 281                 struct rte_flow_error *error)
 282 {
 283         const struct rte_flow_action *action;
 284         enum rte_flow_action_type action_type;
 285         const struct rte_flow_action_rss *rss;
 286         struct rss_type_match_hdr *m = (struct rss_type_match_hdr *)
 287                                 (pattern_match_item->meta);
 288         uint32_t type_list_len = RTE_DIM(ice_hash_type_list);
 289         struct ice_hash_match_type *type_match_item;
 290         uint64_t rss_hf;
 291         uint16_t i;
 292
 293         /* Supported action is RSS. */
 294         for (action = actions; action->type !=
 295                 RTE_FLOW_ACTION_TYPE_END; action++) {
 296                 action_type = action->type;
 297                 switch (action_type) {
 298                 case RTE_FLOW_ACTION_TYPE_RSS:
 299                         rss = action->conf;
 300                         rss_hf = rss->types;
 301
 302                         /**
 303                          * Check simultaneous use of SRC_ONLY and DST_ONLY
 304                          * of the same level.
 305                          */
 306                         rss_hf = rte_eth_rss_hf_refine(rss_hf);
 307
 308                         /* Check if pattern is empty. */
 309                         if (pattern_match_item->pattern_list !=
 310                                 pattern_empty && rss->func ==
 311                                 RTE_ETH_HASH_FUNCTION_SIMPLE_XOR)
 312                                 return rte_flow_error_set(error, ENOTSUP,
 313                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 314                                         "Not supported flow");
 315
 316                         /* Check if rss types match pattern. */
 317                         if (rss->func != RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) {
 318                                 if (((rss_hf & ETH_RSS_IPV4) != m->eth_rss_hint) &&
 319                                 ((rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) != m->eth_rss_hint) &&
 320                                 ((rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) != m->eth_rss_hint) &&
 321                                 ((rss_hf & ETH_RSS_NONFRAG_IPV4_SCTP) != m->eth_rss_hint) &&
 322                                 ((rss_hf & ETH_RSS_IPV6) != m->eth_rss_hint) &&
 323                                 ((rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) != m->eth_rss_hint) &&
 324                                 ((rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) != m->eth_rss_hint) &&
 325                                 ((rss_hf & ETH_RSS_NONFRAG_IPV6_SCTP) != m->eth_rss_hint))
 326                                         return rte_flow_error_set(error,
 327                                         ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
 328                                         action, "Not supported RSS types");
 329                         }
 330
 331                         if (rss->level)
 332                                 return rte_flow_error_set(error, ENOTSUP,
 333                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 334                                         "a nonzero RSS encapsulation level is not supported");
 335
 336                         if (rss->key_len == 0)
 337                                 return rte_flow_error_set(error, ENOTSUP,
 338                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 339                                         "RSS hash key_len mustn't be 0");
 340
 341                         if (rss->queue_num > ICE_ACTION_RSS_MAX_QUEUE_NUM)
 342                                 return rte_flow_error_set(error, ENOTSUP,
 343                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 344                                         "too many queues for RSS context");
 345
 346                         /* Check hash function and save it to rss_meta. */
 347                         if (rss->func ==
 348                                 RTE_ETH_HASH_FUNCTION_SIMPLE_XOR)
 349                                 ((struct rss_meta *)*meta)->hash_function =
 350                                 RTE_ETH_HASH_FUNCTION_SIMPLE_XOR;
 351
 352                         if (rss->func ==
 353                                 RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ)
 354                                 ((struct rss_meta *)*meta)->hash_function =
 355                                 RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ;
 356
 357                         type_match_item = rte_zmalloc("ice_type_match_item",
 358                                         sizeof(struct ice_hash_match_type), 0);
 359                         if (!type_match_item) {
 360                                 rte_flow_error_set(error, EINVAL,
 361                                         RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 362                                         "No memory for type_match_item");
 363                                 return -ENOMEM;
 364                         }
 365
 366                         /* Find matched hash fields according to hash type. */
 367                         for (i = 0; i < type_list_len; i++) {
 368                                 if (rss_hf ==
 369                                         ice_hash_type_list[i].hash_type) {
 370                                         type_match_item->hash_type =
 371                                                 ice_hash_type_list[i].hash_type;
 372                                         type_match_item->hash_flds =
 373                                                 ice_hash_type_list[i].hash_flds;
 374                                 }
 375                         }
 376
 377                         /* Save hash fileds to rss_meta. */
 378                         ((struct rss_meta *)*meta)->hash_flds =
 379                                         type_match_item->hash_flds;
 380
 381                         rte_free(type_match_item);
 382                         break;
 383
 384                 case RTE_FLOW_ACTION_TYPE_END:
 385                         break;
 386
 387                 default:
 388                         rte_flow_error_set(error, EINVAL,
 389                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 390                                         "Invalid action.");
 391                         return -rte_errno;
 392                 }
 393         }
 394
 395         return 0;
 396 }
 397
 398 static int
 399 ice_hash_parse_pattern_action(__rte_unused struct ice_adapter *ad,
 400                         struct ice_pattern_match_item *array,
 401                         uint32_t array_len,
 402                         const struct rte_flow_item pattern[],
 403                         const struct rte_flow_action actions[],
 404                         void **meta,
 405                         struct rte_flow_error *error)
 406 {
 407         int ret = 0;
 408         struct ice_pattern_match_item *pattern_match_item;
 409         struct rss_meta *rss_meta_ptr;
 410
 411         rss_meta_ptr = rte_zmalloc(NULL, sizeof(*rss_meta_ptr), 0);
 412         if (!rss_meta_ptr) {
 413                 rte_flow_error_set(error, EINVAL,
 414                                 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 415                                 "No memory for rss_meta_ptr");
 416                 return -ENOMEM;
 417         }
 418
 419         /* Check rss supported pattern and find matched pattern. */
 420         pattern_match_item = ice_search_pattern_match_item(pattern,
 421                                         array, array_len, error);
 422         if (!pattern_match_item)
 423                 return -rte_errno;
 424
 425         ret = ice_hash_check_inset(pattern, error);
 426         if (ret)
 427                 return -rte_errno;
 428
 429         /* Save protocol header to rss_meta. */
 430         *meta = rss_meta_ptr;
 431         ((struct rss_meta *)*meta)->pkt_hdr = ((struct rss_type_match_hdr *)
 432                 (pattern_match_item->meta))->hdr_mask;
 433
 434         /* Check rss action. */
 435         ret = ice_hash_parse_action(pattern_match_item, actions, meta, error);
 436         if (ret)
 437                 return -rte_errno;
 438
 439         rte_free(pattern_match_item);
 440
 441         return 0;
 442 }
 443
 444 static int
 445 ice_hash_create(struct ice_adapter *ad,
 446                 struct rte_flow *flow,
 447                 void *meta,
 448                 struct rte_flow_error *error)
 449 {
 450         struct ice_pf *pf = &ad->pf;
 451         struct ice_hw *hw = ICE_PF_TO_HW(pf);
 452         struct ice_vsi *vsi = pf->main_vsi;
 453         int ret;
 454         uint32_t reg;
 455         struct ice_rss_cfg *filter_ptr;
 456
 457         uint32_t headermask = ((struct rss_meta *)meta)->pkt_hdr;
 458         uint64_t hash_field = ((struct rss_meta *)meta)->hash_flds;
 459         uint8_t hash_function = ((struct rss_meta *)meta)->hash_function;
 460
 461         filter_ptr = rte_zmalloc("ice_rss_filter",
 462                                 sizeof(struct ice_rss_cfg), 0);
 463         if (!filter_ptr) {
 464                 rte_flow_error_set(error, EINVAL,
 465                                 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 466                                 "No memory for filter_ptr");
 467                 return -ENOMEM;
 468         }
 469
 470         if (hash_function == RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) {
 471                 /* Enable registers for simple_xor hash function. */
 472                 reg = ICE_READ_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id));
 473                 reg = (reg & (~VSIQF_HASH_CTL_HASH_SCHEME_M)) |
 474                         (2 << VSIQF_HASH_CTL_HASH_SCHEME_S);
 475                 ICE_WRITE_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id), reg);
 476
 477                 filter_ptr->symm = 0;
 478
 479                 goto out;
 480         } else if (hash_function == RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ) {
 481                 ret = ice_add_rss_cfg(hw, vsi->idx, hash_field, headermask, 1);
 482                 if (ret) {
 483                         rte_flow_error_set(error, EINVAL,
 484                                         RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 485                                         "rss flow create fail");
 486                         goto error;
 487                 }
 488         } else {
 489                 ret = ice_add_rss_cfg(hw, vsi->idx, hash_field, headermask, 0);
 490                 if (ret) {
 491                         rte_flow_error_set(error, EINVAL,
 492                                         RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 493                                         "rss flow create fail");
 494                         goto error;
 495                 }
 496         }
 497
 498         filter_ptr->packet_hdr = headermask;
 499         filter_ptr->hashed_flds = hash_field;
 500
 501 out:
 502         flow->rule = filter_ptr;
 503         rte_free(meta);
 504         return 0;
 505
 506 error:
 507         rte_free(filter_ptr);
 508         rte_free(meta);
 509         return -rte_errno;
 510 }
 511
 512 static int
 513 ice_hash_destroy(struct ice_adapter *ad,
 514                 struct rte_flow *flow,
 515                 struct rte_flow_error *error)
 516 {
 517         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(ad);
 518         struct ice_hw *hw = ICE_PF_TO_HW(pf);
 519         struct ice_vsi *vsi = pf->main_vsi;
 520         int ret;
 521         uint32_t reg;
 522         struct ice_rss_cfg *filter_ptr;
 523
 524         filter_ptr = (struct ice_rss_cfg *)flow->rule;
 525
 526         if (filter_ptr->symm == 0) {
 527                 /* Return to symmetric_toeplitz state. */
 528                 reg = ICE_READ_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id));
 529                 reg = (reg & (~VSIQF_HASH_CTL_HASH_SCHEME_M)) |
 530                         (1 << VSIQF_HASH_CTL_HASH_SCHEME_S);
 531                 ICE_WRITE_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id), reg);
 532         } else {
 533                 ret = ice_rem_vsi_rss_cfg(hw, vsi->idx);
 534                 if (ret) {
 535                         rte_flow_error_set(error, EINVAL,
 536                                         RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 537                                         "rss flow destroy fail");
 538                         goto error;
 539                 }
 540         }
 541
 542         rte_free(filter_ptr);
 543         return 0;
 544
 545 error:
 546         rte_free(filter_ptr);
 547         return -rte_errno;
 548 }
 549
 550 static void
 551 ice_hash_uninit(struct ice_adapter *ad)
 552 {
 553         if (ad->active_pkg_type == ICE_PKG_TYPE_OS_DEFAULT)
 554                 ice_unregister_parser(&ice_hash_parser_os, ad);
 555         else if (ad->active_pkg_type == ICE_PKG_TYPE_COMMS)
 556                 ice_unregister_parser(&ice_hash_parser_comms, ad);
 557 }
 558
 559 static void
 560 ice_hash_free(struct rte_flow *flow)
 561 {
 562         rte_free(flow->rule);
 563 }