drivers/net/ice/ice_hash.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10 #include <unistd.h>
  11 #include <stdarg.h>
  12
  13 #include <rte_debug.h>
  14 #include <rte_ether.h>
  15 #include <rte_ethdev_driver.h>
  16 #include <rte_log.h>
  17 #include <rte_malloc.h>
  18 #include <rte_eth_ctrl.h>
  19 #include <rte_tailq.h>
  20 #include <rte_flow_driver.h>
  21
  22 #include "ice_logs.h"
  23 #include "base/ice_type.h"
  24 #include "base/ice_flow.h"
  25 #include "ice_ethdev.h"
  26 #include "ice_generic_flow.h"
  27
  28 struct rss_type_match_hdr {
  29         uint32_t hdr_mask;
  30         uint64_t eth_rss_hint;
  31 };
  32
  33 struct ice_hash_match_type {
  34         uint64_t hash_type;
  35         uint64_t hash_flds;
  36 };
  37
  38 struct rss_meta {
  39         uint32_t pkt_hdr;
  40         uint64_t hash_flds;
  41         uint8_t hash_function;
  42 };
  43
  44 struct ice_hash_flow_cfg {
  45         bool simple_xor;
  46         struct ice_rss_cfg rss_cfg;
  47 };
  48
  49 static int
  50 ice_hash_init(struct ice_adapter *ad);
  51
  52 static int
  53 ice_hash_create(struct ice_adapter *ad,
  54                 struct rte_flow *flow,
  55                 void *meta,
  56                 struct rte_flow_error *error);
  57
  58 static int
  59 ice_hash_destroy(struct ice_adapter *ad,
  60                 struct rte_flow *flow,
  61                 struct rte_flow_error *error);
  62
  63 static void
  64 ice_hash_uninit(struct ice_adapter *ad);
  65
  66 static void
  67 ice_hash_free(struct rte_flow *flow);
  68
  69 static int
  70 ice_hash_parse_pattern_action(struct ice_adapter *ad,
  71                         struct ice_pattern_match_item *array,
  72                         uint32_t array_len,
  73                         const struct rte_flow_item pattern[],
  74                         const struct rte_flow_action actions[],
  75                         void **meta,
  76                         struct rte_flow_error *error);
  77
  78 /* The first member is protocol header, the second member is ETH_RSS_*. */
  79 struct rss_type_match_hdr hint_0 = {
  80         ICE_FLOW_SEG_HDR_NONE,  0};
  81 struct rss_type_match_hdr hint_1 = {
  82         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER, ETH_RSS_IPV4};
  83 struct rss_type_match_hdr hint_2 = {
  84         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_UDP |
  85         ICE_FLOW_SEG_HDR_IPV_OTHER, ETH_RSS_NONFRAG_IPV4_UDP};
  86 struct rss_type_match_hdr hint_3 = {
  87         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_TCP |
  88         ICE_FLOW_SEG_HDR_IPV_OTHER, ETH_RSS_NONFRAG_IPV4_TCP};
  89 struct rss_type_match_hdr hint_4 = {
  90         ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_SCTP |
  91         ICE_FLOW_SEG_HDR_IPV_OTHER, ETH_RSS_NONFRAG_IPV4_SCTP};
  92 struct rss_type_match_hdr hint_5 = {
  93         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER, ETH_RSS_IPV6};
  94 struct rss_type_match_hdr hint_6 = {
  95         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_UDP |
  96         ICE_FLOW_SEG_HDR_IPV_OTHER, ETH_RSS_NONFRAG_IPV6_UDP};
  97 struct rss_type_match_hdr hint_7 = {
  98         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_TCP |
  99         ICE_FLOW_SEG_HDR_IPV_OTHER, ETH_RSS_NONFRAG_IPV6_TCP};
 100 struct rss_type_match_hdr hint_8 = {
 101         ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_SCTP |
 102         ICE_FLOW_SEG_HDR_IPV_OTHER, ETH_RSS_NONFRAG_IPV6_SCTP};
 103 struct rss_type_match_hdr hint_9 = {
 104         ICE_FLOW_SEG_HDR_GTPU_EH, ETH_RSS_IPV4};
 105 struct rss_type_match_hdr hint_10 = {
 106         ICE_FLOW_SEG_HDR_PPPOE, ETH_RSS_IPV4};
 107 struct rss_type_match_hdr hint_11 = {
 108         ICE_FLOW_SEG_HDR_PPPOE, ETH_RSS_NONFRAG_IPV4_UDP};
 109 struct rss_type_match_hdr hint_12 = {
 110         ICE_FLOW_SEG_HDR_PPPOE, ETH_RSS_NONFRAG_IPV4_TCP};
 111 struct rss_type_match_hdr hint_13 = {
 112         ICE_FLOW_SEG_HDR_PPPOE, ETH_RSS_NONFRAG_IPV4_SCTP};
 113 struct rss_type_match_hdr hint_14 = {
 114         ICE_FLOW_SEG_HDR_GTPU_EH, ETH_RSS_NONFRAG_IPV4_UDP};
 115 struct rss_type_match_hdr hint_15 = {
 116         ICE_FLOW_SEG_HDR_GTPU_EH, ETH_RSS_NONFRAG_IPV4_TCP};
 117 struct rss_type_match_hdr hint_16 = {
 118         ICE_FLOW_SEG_HDR_PPPOE, ETH_RSS_IPV6};
 119 struct rss_type_match_hdr hint_17 = {
 120         ICE_FLOW_SEG_HDR_PPPOE, ETH_RSS_NONFRAG_IPV6_UDP};
 121 struct rss_type_match_hdr hint_18 = {
 122         ICE_FLOW_SEG_HDR_PPPOE, ETH_RSS_NONFRAG_IPV6_TCP};
 123 struct rss_type_match_hdr hint_19 = {
 124         ICE_FLOW_SEG_HDR_PPPOE, ETH_RSS_NONFRAG_IPV6_SCTP};
 125 struct rss_type_match_hdr hint_20 = {
 126         ICE_FLOW_SEG_HDR_PPPOE, ETH_RSS_ETH | ETH_RSS_PPPOE};
 127
 128 /* Supported pattern for os default package. */
 129 static struct ice_pattern_match_item ice_hash_pattern_list_os[] = {
 130         {pattern_eth_ipv4,      ICE_INSET_NONE, &hint_1},
 131         {pattern_eth_ipv4_udp,  ICE_INSET_NONE, &hint_2},
 132         {pattern_eth_ipv4_tcp,  ICE_INSET_NONE, &hint_3},
 133         {pattern_eth_ipv4_sctp, ICE_INSET_NONE, &hint_4},
 134         {pattern_eth_ipv6,      ICE_INSET_NONE, &hint_5},
 135         {pattern_eth_ipv6_udp,  ICE_INSET_NONE, &hint_6},
 136         {pattern_eth_ipv6_tcp,  ICE_INSET_NONE, &hint_7},
 137         {pattern_eth_ipv6_sctp, ICE_INSET_NONE, &hint_8},
 138         {pattern_empty,         ICE_INSET_NONE, &hint_0},
 139 };
 140
 141 /* Supported pattern for comms package. */
 142 static struct ice_pattern_match_item ice_hash_pattern_list_comms[] = {
 143         {pattern_eth_ipv4,                  ICE_INSET_NONE,  &hint_1},
 144         {pattern_eth_ipv4_udp,              ICE_INSET_NONE,  &hint_2},
 145         {pattern_eth_ipv4_tcp,              ICE_INSET_NONE,  &hint_3},
 146         {pattern_eth_ipv4_sctp,             ICE_INSET_NONE,  &hint_4},
 147         {pattern_eth_ipv6,                  ICE_INSET_NONE,  &hint_5},
 148         {pattern_eth_ipv6_udp,              ICE_INSET_NONE,  &hint_6},
 149         {pattern_eth_ipv6_tcp,              ICE_INSET_NONE,  &hint_7},
 150         {pattern_eth_ipv6_sctp,             ICE_INSET_NONE,  &hint_8},
 151         {pattern_empty,                     ICE_INSET_NONE,  &hint_0},
 152         {pattern_eth_ipv4_gtpu_eh_ipv4,     ICE_INSET_NONE,  &hint_9},
 153         {pattern_eth_ipv4_gtpu_eh_ipv4_udp, ICE_INSET_NONE,  &hint_14},
 154         {pattern_eth_ipv4_gtpu_eh_ipv4_tcp, ICE_INSET_NONE,  &hint_15},
 155         {pattern_eth_pppoes_ipv4,           ICE_INSET_NONE,  &hint_10},
 156         {pattern_eth_pppoes_ipv4_udp,       ICE_INSET_NONE,  &hint_11},
 157         {pattern_eth_pppoes_ipv4_tcp,       ICE_INSET_NONE,  &hint_12},
 158         {pattern_eth_pppoes_ipv4_sctp,      ICE_INSET_NONE,  &hint_13},
 159         {pattern_eth_pppoes_ipv6,           ICE_INSET_NONE,  &hint_16},
 160         {pattern_eth_pppoes_ipv6_udp,       ICE_INSET_NONE,  &hint_17},
 161         {pattern_eth_pppoes_ipv6_tcp,       ICE_INSET_NONE,  &hint_18},
 162         {pattern_eth_pppoes_ipv6_sctp,      ICE_INSET_NONE,  &hint_19},
 163         {pattern_eth_pppoes,                ICE_INSET_NONE,  &hint_20},
 164 };
 165
 166 /**
 167  * The first member is input set combination,
 168  * the second member is hash fields.
 169  */
 170 struct ice_hash_match_type ice_hash_type_list[] = {
 171         {ETH_RSS_IPV4 | ETH_RSS_L3_SRC_ONLY,                                    BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA)},
 172         {ETH_RSS_IPV4 | ETH_RSS_L3_DST_ONLY,                                    BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA)},
 173         {ETH_RSS_IPV4,                                                          ICE_FLOW_HASH_IPV4},
 174         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)},
 175         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)},
 176         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L3_SRC_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA)},
 177         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)},
 178         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)},
 179         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L3_DST_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA)},
 180         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L4_SRC_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)},
 181         {ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_L4_DST_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)},
 182         {ETH_RSS_NONFRAG_IPV4_UDP,                                              ICE_HASH_UDP_IPV4},
 183         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)},
 184         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)},
 185         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L3_SRC_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA)},
 186         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)},
 187         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)},
 188         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L3_DST_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA)},
 189         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L4_SRC_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)},
 190         {ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_L4_DST_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)},
 191         {ETH_RSS_NONFRAG_IPV4_TCP,                                              ICE_HASH_TCP_IPV4},
 192         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY, BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)},
 193         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY, BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)},
 194         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L3_SRC_ONLY,                       BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA)},
 195         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY, BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)},
 196         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY, BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)},
 197         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L3_DST_ONLY,                       BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA)},
 198         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L4_SRC_ONLY,                       BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)},
 199         {ETH_RSS_NONFRAG_IPV4_SCTP | ETH_RSS_L4_DST_ONLY,                       BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)},
 200         {ETH_RSS_NONFRAG_IPV4_SCTP,                                             ICE_HASH_SCTP_IPV4},
 201         {ETH_RSS_IPV6 | ETH_RSS_L3_SRC_ONLY,                                    BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)},
 202         {ETH_RSS_IPV6 | ETH_RSS_L3_DST_ONLY,                                    BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)},
 203         {ETH_RSS_IPV6,                                                          ICE_FLOW_HASH_IPV6},
 204         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)},
 205         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)},
 206         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L3_SRC_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)},
 207         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)},
 208         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)},
 209         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L3_DST_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)},
 210         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L4_SRC_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)},
 211         {ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_L4_DST_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)},
 212         {ETH_RSS_NONFRAG_IPV6_UDP,                                              ICE_HASH_UDP_IPV6},
 213         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)},
 214         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)},
 215         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L3_SRC_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)},
 216         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)},
 217         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY,  BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)},
 218         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L3_DST_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)},
 219         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L4_SRC_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)},
 220         {ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_L4_DST_ONLY,                        BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)},
 221         {ETH_RSS_NONFRAG_IPV6_TCP,                                              ICE_HASH_TCP_IPV6},
 222         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_SRC_ONLY, BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)},
 223         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L3_SRC_ONLY | ETH_RSS_L4_DST_ONLY, BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) | BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)},
 224         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L3_SRC_ONLY,                       BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)},
 225         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_SRC_ONLY, BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)},
 226         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L3_DST_ONLY | ETH_RSS_L4_DST_ONLY, BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) | BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)},
 227         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L3_DST_ONLY,                       BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)},
 228         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L4_SRC_ONLY,                       BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)},
 229         {ETH_RSS_NONFRAG_IPV6_SCTP | ETH_RSS_L4_DST_ONLY,                       BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)},
 230         {ETH_RSS_NONFRAG_IPV6_SCTP,                                             ICE_HASH_SCTP_IPV6},
 231         {ETH_RSS_ETH | ETH_RSS_L2_SRC_ONLY,                                     BIT_ULL(ICE_FLOW_FIELD_IDX_ETH_SA)},
 232         {ETH_RSS_PPPOE,                                                         ICE_FLOW_HASH_PPPOE_SESS_ID},
 233         {ETH_RSS_ETH | ETH_RSS_PPPOE | ETH_RSS_L2_SRC_ONLY,                     ICE_FLOW_HASH_PPPOE_SESS_ID | BIT_ULL(ICE_FLOW_FIELD_IDX_ETH_SA)},
 234 };
 235
 236 static struct ice_flow_engine ice_hash_engine = {
 237         .init = ice_hash_init,
 238         .create = ice_hash_create,
 239         .destroy = ice_hash_destroy,
 240         .uninit = ice_hash_uninit,
 241         .free = ice_hash_free,
 242         .type = ICE_FLOW_ENGINE_HASH,
 243 };
 244
 245 /* Register parser for os package. */
 246 static struct ice_flow_parser ice_hash_parser_os = {
 247         .engine = &ice_hash_engine,
 248         .array = ice_hash_pattern_list_os,
 249         .array_len = RTE_DIM(ice_hash_pattern_list_os),
 250         .parse_pattern_action = ice_hash_parse_pattern_action,
 251         .stage = ICE_FLOW_STAGE_RSS,
 252 };
 253
 254 /* Register parser for comms package. */
 255 static struct ice_flow_parser ice_hash_parser_comms = {
 256         .engine = &ice_hash_engine,
 257         .array = ice_hash_pattern_list_comms,
 258         .array_len = RTE_DIM(ice_hash_pattern_list_comms),
 259         .parse_pattern_action = ice_hash_parse_pattern_action,
 260         .stage = ICE_FLOW_STAGE_RSS,
 261 };
 262
 263 RTE_INIT(ice_hash_engine_init)
 264 {
 265         struct ice_flow_engine *engine = &ice_hash_engine;
 266         ice_register_flow_engine(engine);
 267 }
 268
 269 static int
 270 ice_hash_init(struct ice_adapter *ad)
 271 {
 272         struct ice_flow_parser *parser = NULL;
 273
 274         if (ad->hw.dcf_enabled)
 275                 return 0;
 276
 277         if (ad->active_pkg_type == ICE_PKG_TYPE_OS_DEFAULT)
 278                 parser = &ice_hash_parser_os;
 279         else if (ad->active_pkg_type == ICE_PKG_TYPE_COMMS)
 280                 parser = &ice_hash_parser_comms;
 281         else
 282                 return -EINVAL;
 283
 284         return ice_register_parser(parser, ad);
 285 }
 286
 287 static int
 288 ice_hash_check_inset(const struct rte_flow_item pattern[],
 289                 struct rte_flow_error *error)
 290 {
 291         const struct rte_flow_item *item = pattern;
 292
 293         for (item = pattern; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 294                 if (item->last) {
 295                         rte_flow_error_set(error, EINVAL,
 296                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 297                                         "Not support range");
 298                         return -rte_errno;
 299                 }
 300
 301                 /* Ignore spec and mask. */
 302                 if (item->spec || item->mask) {
 303                         rte_flow_error_set(error, EINVAL,
 304                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
 305                                         "Invalid spec/mask.");
 306                         return -rte_errno;
 307                 }
 308         }
 309
 310         return 0;
 311 }
 312
 313 static int
 314 ice_hash_parse_action(struct ice_pattern_match_item *pattern_match_item,
 315                 const struct rte_flow_action actions[],
 316                 void **meta,
 317                 struct rte_flow_error *error)
 318 {
 319         const struct rte_flow_action *action;
 320         enum rte_flow_action_type action_type;
 321         const struct rte_flow_action_rss *rss;
 322         struct rss_type_match_hdr *m = (struct rss_type_match_hdr *)
 323                                 (pattern_match_item->meta);
 324         uint32_t type_list_len = RTE_DIM(ice_hash_type_list);
 325         struct ice_hash_match_type *type_match_item;
 326         uint64_t rss_hf;
 327         uint16_t i;
 328
 329         /* Supported action is RSS. */
 330         for (action = actions; action->type !=
 331                 RTE_FLOW_ACTION_TYPE_END; action++) {
 332                 action_type = action->type;
 333                 switch (action_type) {
 334                 case RTE_FLOW_ACTION_TYPE_RSS:
 335                         rss = action->conf;
 336                         rss_hf = rss->types;
 337
 338                         /**
 339                          * Check simultaneous use of SRC_ONLY and DST_ONLY
 340                          * of the same level.
 341                          */
 342                         rss_hf = rte_eth_rss_hf_refine(rss_hf);
 343
 344                         /* Check if pattern is empty. */
 345                         if (pattern_match_item->pattern_list !=
 346                                 pattern_empty && rss->func ==
 347                                 RTE_ETH_HASH_FUNCTION_SIMPLE_XOR)
 348                                 return rte_flow_error_set(error, ENOTSUP,
 349                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 350                                         "Not supported flow");
 351
 352                         if ((rss_hf & ETH_RSS_ETH) && (rss_hf & ~ETH_RSS_PPPOE))
 353                                 m->eth_rss_hint = ETH_RSS_ETH;
 354                         else if ((rss_hf & ETH_RSS_PPPOE) && (rss_hf & ~ETH_RSS_ETH))
 355                                 m->eth_rss_hint = ETH_RSS_PPPOE;
 356                         else if ((rss_hf & ETH_RSS_ETH) && (rss_hf & ETH_RSS_PPPOE))
 357                                 m->eth_rss_hint = ETH_RSS_ETH | ETH_RSS_PPPOE;
 358
 359                         /* Check if rss types match pattern. */
 360                         if (rss->func != RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) {
 361                                 if (((rss_hf & ETH_RSS_IPV4) != m->eth_rss_hint) &&
 362                                 ((rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) != m->eth_rss_hint) &&
 363                                 ((rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) != m->eth_rss_hint) &&
 364                                 ((rss_hf & ETH_RSS_NONFRAG_IPV4_SCTP) != m->eth_rss_hint) &&
 365                                 ((rss_hf & ETH_RSS_IPV6) != m->eth_rss_hint) &&
 366                                 ((rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) != m->eth_rss_hint) &&
 367                                 ((rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) != m->eth_rss_hint) &&
 368                                 ((rss_hf & ETH_RSS_NONFRAG_IPV6_SCTP) != m->eth_rss_hint) &&
 369                                 ((rss_hf & ETH_RSS_ETH) != m->eth_rss_hint) &&
 370                                 ((rss_hf & ETH_RSS_PPPOE) != m->eth_rss_hint) &&
 371                                 (((rss_hf & (ETH_RSS_ETH | ETH_RSS_PPPOE)) !=
 372                                                                         m->eth_rss_hint)))
 373                                         return rte_flow_error_set(error,
 374                                         ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
 375                                         action, "Not supported RSS types");
 376                         }
 377
 378                         if (rss->level)
 379                                 return rte_flow_error_set(error, ENOTSUP,
 380                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 381                                         "a nonzero RSS encapsulation level is not supported");
 382
 383                         if (rss->key_len)
 384                                 return rte_flow_error_set(error, ENOTSUP,
 385                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 386                                         "a nonzero RSS key_len is not supported");
 387
 388                         if (rss->queue)
 389                                 return rte_flow_error_set(error, ENOTSUP,
 390                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 391                                         "a non-NULL RSS queue is not supported");
 392
 393                         /* Check hash function and save it to rss_meta. */
 394                         if (rss->func ==
 395                                 RTE_ETH_HASH_FUNCTION_SIMPLE_XOR)
 396                                 ((struct rss_meta *)*meta)->hash_function =
 397                                 RTE_ETH_HASH_FUNCTION_SIMPLE_XOR;
 398
 399                         if (rss->func ==
 400                                 RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ)
 401                                 ((struct rss_meta *)*meta)->hash_function =
 402                                 RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ;
 403
 404                         type_match_item = rte_zmalloc("ice_type_match_item",
 405                                         sizeof(struct ice_hash_match_type), 0);
 406                         if (!type_match_item) {
 407                                 rte_flow_error_set(error, EINVAL,
 408                                         RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 409                                         "No memory for type_match_item");
 410                                 return -ENOMEM;
 411                         }
 412
 413                         /* Find matched hash fields according to hash type. */
 414                         for (i = 0; i < type_list_len; i++) {
 415                                 if (rss_hf ==
 416                                         ice_hash_type_list[i].hash_type) {
 417                                         type_match_item->hash_type =
 418                                                 ice_hash_type_list[i].hash_type;
 419                                         type_match_item->hash_flds =
 420                                                 ice_hash_type_list[i].hash_flds;
 421                                 }
 422                         }
 423
 424                         /* Save hash fileds to rss_meta. */
 425                         ((struct rss_meta *)*meta)->hash_flds =
 426                                         type_match_item->hash_flds;
 427
 428                         rte_free(type_match_item);
 429                         break;
 430
 431                 case RTE_FLOW_ACTION_TYPE_END:
 432                         break;
 433
 434                 default:
 435                         rte_flow_error_set(error, EINVAL,
 436                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
 437                                         "Invalid action.");
 438                         return -rte_errno;
 439                 }
 440         }
 441
 442         return 0;
 443 }
 444
 445 static int
 446 ice_hash_parse_pattern_action(__rte_unused struct ice_adapter *ad,
 447                         struct ice_pattern_match_item *array,
 448                         uint32_t array_len,
 449                         const struct rte_flow_item pattern[],
 450                         const struct rte_flow_action actions[],
 451                         void **meta,
 452                         struct rte_flow_error *error)
 453 {
 454         int ret = 0;
 455         struct ice_pattern_match_item *pattern_match_item;
 456         struct rss_meta *rss_meta_ptr;
 457
 458         rss_meta_ptr = rte_zmalloc(NULL, sizeof(*rss_meta_ptr), 0);
 459         if (!rss_meta_ptr) {
 460                 rte_flow_error_set(error, EINVAL,
 461                                 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 462                                 "No memory for rss_meta_ptr");
 463                 return -ENOMEM;
 464         }
 465
 466         /* Check rss supported pattern and find matched pattern. */
 467         pattern_match_item = ice_search_pattern_match_item(pattern,
 468                                         array, array_len, error);
 469         if (!pattern_match_item) {
 470                 ret = -rte_errno;
 471                 goto error;
 472         }
 473
 474         ret = ice_hash_check_inset(pattern, error);
 475         if (ret)
 476                 goto error;
 477
 478         /* Save protocol header to rss_meta. */
 479         rss_meta_ptr->pkt_hdr = ((struct rss_type_match_hdr *)
 480                 (pattern_match_item->meta))->hdr_mask;
 481
 482         /* Check rss action. */
 483         ret = ice_hash_parse_action(pattern_match_item, actions,
 484                                     (void **)&rss_meta_ptr, error);
 485
 486 error:
 487         if (!ret && meta)
 488                 *meta = rss_meta_ptr;
 489         else
 490                 rte_free(rss_meta_ptr);
 491         rte_free(pattern_match_item);
 492
 493         return ret;
 494 }
 495
 496 static int
 497 ice_hash_create(struct ice_adapter *ad,
 498                 struct rte_flow *flow,
 499                 void *meta,
 500                 struct rte_flow_error *error)
 501 {
 502         struct ice_pf *pf = &ad->pf;
 503         struct ice_hw *hw = ICE_PF_TO_HW(pf);
 504         struct ice_vsi *vsi = pf->main_vsi;
 505         int ret;
 506         uint32_t reg;
 507         struct ice_hash_flow_cfg *filter_ptr;
 508
 509         uint32_t headermask = ((struct rss_meta *)meta)->pkt_hdr;
 510         uint64_t hash_field = ((struct rss_meta *)meta)->hash_flds;
 511         uint8_t hash_function = ((struct rss_meta *)meta)->hash_function;
 512
 513         filter_ptr = rte_zmalloc("ice_rss_filter",
 514                                 sizeof(struct ice_hash_flow_cfg), 0);
 515         if (!filter_ptr) {
 516                 rte_flow_error_set(error, EINVAL,
 517                                 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 518                                 "No memory for filter_ptr");
 519                 return -ENOMEM;
 520         }
 521
 522         if (hash_function == RTE_ETH_HASH_FUNCTION_SIMPLE_XOR) {
 523                 /* Enable registers for simple_xor hash function. */
 524                 reg = ICE_READ_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id));
 525                 reg = (reg & (~VSIQF_HASH_CTL_HASH_SCHEME_M)) |
 526                         (2 << VSIQF_HASH_CTL_HASH_SCHEME_S);
 527                 ICE_WRITE_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id), reg);
 528
 529                 filter_ptr->simple_xor = 1;
 530
 531                 goto out;
 532         } else {
 533                 filter_ptr->rss_cfg.packet_hdr = headermask;
 534                 filter_ptr->rss_cfg.hashed_flds = hash_field;
 535                 filter_ptr->rss_cfg.symm =
 536                         (hash_function ==
 537                                 RTE_ETH_HASH_FUNCTION_SYMMETRIC_TOEPLITZ);
 538
 539                 ret = ice_add_rss_cfg(hw, vsi->idx,
 540                                 filter_ptr->rss_cfg.hashed_flds,
 541                                 filter_ptr->rss_cfg.packet_hdr,
 542                                 filter_ptr->rss_cfg.symm);
 543                 if (ret) {
 544                         rte_flow_error_set(error, EINVAL,
 545                                         RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 546                                         "rss flow create fail");
 547                         goto error;
 548                 }
 549         }
 550
 551 out:
 552         flow->rule = filter_ptr;
 553         rte_free(meta);
 554         return 0;
 555
 556 error:
 557         rte_free(filter_ptr);
 558         rte_free(meta);
 559         return -rte_errno;
 560 }
 561
 562 static int
 563 ice_hash_destroy(struct ice_adapter *ad,
 564                 struct rte_flow *flow,
 565                 struct rte_flow_error *error)
 566 {
 567         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(ad);
 568         struct ice_hw *hw = ICE_PF_TO_HW(pf);
 569         struct ice_vsi *vsi = pf->main_vsi;
 570         int ret;
 571         uint32_t reg;
 572         struct ice_hash_flow_cfg *filter_ptr;
 573
 574         filter_ptr = (struct ice_hash_flow_cfg *)flow->rule;
 575
 576         if (filter_ptr->simple_xor == 1) {
 577                 /* Return to symmetric_toeplitz state. */
 578                 reg = ICE_READ_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id));
 579                 reg = (reg & (~VSIQF_HASH_CTL_HASH_SCHEME_M)) |
 580                         (1 << VSIQF_HASH_CTL_HASH_SCHEME_S);
 581                 ICE_WRITE_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id), reg);
 582         } else {
 583                 ret = ice_rem_rss_cfg(hw, vsi->idx,
 584                                 filter_ptr->rss_cfg.hashed_flds,
 585                                 filter_ptr->rss_cfg.packet_hdr);
 586                 /* Fixme: Ignore the error if a rule does not exist.
 587                  * Currently a rule for inputset change or symm turn on/off
 588                  * will overwrite an exist rule, while application still
 589                  * have 2 rte_flow handles.
 590                  **/
 591                 if (ret && ret != ICE_ERR_DOES_NOT_EXIST) {
 592                         rte_flow_error_set(error, EINVAL,
 593                                         RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
 594                                         "rss flow destroy fail");
 595                         goto error;
 596                 }
 597         }
 598
 599         rte_free(filter_ptr);
 600         return 0;
 601
 602 error:
 603         rte_free(filter_ptr);
 604         return -rte_errno;
 605 }
 606
 607 static void
 608 ice_hash_uninit(struct ice_adapter *ad)
 609 {
 610         if (ad->hw.dcf_enabled)
 611                 return;
 612
 613         if (ad->active_pkg_type == ICE_PKG_TYPE_OS_DEFAULT)
 614                 ice_unregister_parser(&ice_hash_parser_os, ad);
 615         else if (ad->active_pkg_type == ICE_PKG_TYPE_COMMS)
 616                 ice_unregister_parser(&ice_hash_parser_comms, ad);
 617 }
 618
 619 static void
 620 ice_hash_free(struct rte_flow *flow)
 621 {
 622         rte_free(flow->rule);
 623 }