drivers/net/ice/ice_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 #include <rte_string_fns.h>
   6 #include <ethdev_pci.h>
   7
   8 #include <stdio.h>
   9 #include <sys/types.h>
  10 #include <sys/stat.h>
  11 #include <unistd.h>
  12
  13 #include <rte_tailq.h>
  14
  15 #include "eal_firmware.h"
  16
  17 #include "base/ice_sched.h"
  18 #include "base/ice_flow.h"
  19 #include "base/ice_dcb.h"
  20 #include "base/ice_common.h"
  21 #include "base/ice_ptp_hw.h"
  22
  23 #include "rte_pmd_ice.h"
  24 #include "ice_ethdev.h"
  25 #include "ice_rxtx.h"
  26 #include "ice_generic_flow.h"
  27
  28 /* devargs */
  29 #define ICE_SAFE_MODE_SUPPORT_ARG "safe-mode-support"
  30 #define ICE_PIPELINE_MODE_SUPPORT_ARG  "pipeline-mode-support"
  31 #define ICE_PROTO_XTR_ARG         "proto_xtr"
  32 #define ICE_HW_DEBUG_MASK_ARG     "hw_debug_mask"
  33 #define ICE_ONE_PPS_OUT_ARG       "pps_out"
  34 #define ICE_RX_LOW_LATENCY_ARG    "rx_low_latency"
  35
  36 #define ICE_CYCLECOUNTER_MASK  0xffffffffffffffffULL
  37
  38 uint64_t ice_timestamp_dynflag;
  39 int ice_timestamp_dynfield_offset = -1;
  40
  41 static const char * const ice_valid_args[] = {
  42         ICE_SAFE_MODE_SUPPORT_ARG,
  43         ICE_PIPELINE_MODE_SUPPORT_ARG,
  44         ICE_PROTO_XTR_ARG,
  45         ICE_HW_DEBUG_MASK_ARG,
  46         ICE_ONE_PPS_OUT_ARG,
  47         ICE_RX_LOW_LATENCY_ARG,
  48         NULL
  49 };
  50
  51 #define PPS_OUT_DELAY_NS  1
  52
  53 static const struct rte_mbuf_dynfield ice_proto_xtr_metadata_param = {
  54         .name = "intel_pmd_dynfield_proto_xtr_metadata",
  55         .size = sizeof(uint32_t),
  56         .align = __alignof__(uint32_t),
  57         .flags = 0,
  58 };
  59
  60 struct proto_xtr_ol_flag {
  61         const struct rte_mbuf_dynflag param;
  62         uint64_t *ol_flag;
  63         bool required;
  64 };
  65
  66 static bool ice_proto_xtr_hw_support[PROTO_XTR_MAX];
  67
  68 static struct proto_xtr_ol_flag ice_proto_xtr_ol_flag_params[] = {
  69         [PROTO_XTR_VLAN] = {
  70                 .param = { .name = "intel_pmd_dynflag_proto_xtr_vlan" },
  71                 .ol_flag = &rte_net_ice_dynflag_proto_xtr_vlan_mask },
  72         [PROTO_XTR_IPV4] = {
  73                 .param = { .name = "intel_pmd_dynflag_proto_xtr_ipv4" },
  74                 .ol_flag = &rte_net_ice_dynflag_proto_xtr_ipv4_mask },
  75         [PROTO_XTR_IPV6] = {
  76                 .param = { .name = "intel_pmd_dynflag_proto_xtr_ipv6" },
  77                 .ol_flag = &rte_net_ice_dynflag_proto_xtr_ipv6_mask },
  78         [PROTO_XTR_IPV6_FLOW] = {
  79                 .param = { .name = "intel_pmd_dynflag_proto_xtr_ipv6_flow" },
  80                 .ol_flag = &rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask },
  81         [PROTO_XTR_TCP] = {
  82                 .param = { .name = "intel_pmd_dynflag_proto_xtr_tcp" },
  83                 .ol_flag = &rte_net_ice_dynflag_proto_xtr_tcp_mask },
  84         [PROTO_XTR_IP_OFFSET] = {
  85                 .param = { .name = "intel_pmd_dynflag_proto_xtr_ip_offset" },
  86                 .ol_flag = &rte_net_ice_dynflag_proto_xtr_ip_offset_mask },
  87 };
  88
  89 #define ICE_OS_DEFAULT_PKG_NAME         "ICE OS Default Package"
  90 #define ICE_COMMS_PKG_NAME                      "ICE COMMS Package"
  91 #define ICE_MAX_RES_DESC_NUM        1024
  92
  93 static int ice_dev_configure(struct rte_eth_dev *dev);
  94 static int ice_dev_start(struct rte_eth_dev *dev);
  95 static int ice_dev_stop(struct rte_eth_dev *dev);
  96 static int ice_dev_close(struct rte_eth_dev *dev);
  97 static int ice_dev_reset(struct rte_eth_dev *dev);
  98 static int ice_dev_info_get(struct rte_eth_dev *dev,
  99                             struct rte_eth_dev_info *dev_info);
 100 static int ice_link_update(struct rte_eth_dev *dev,
 101                            int wait_to_complete);
 102 static int ice_dev_set_link_up(struct rte_eth_dev *dev);
 103 static int ice_dev_set_link_down(struct rte_eth_dev *dev);
 104
 105 static int ice_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 106 static int ice_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 107 static int ice_rss_reta_update(struct rte_eth_dev *dev,
 108                                struct rte_eth_rss_reta_entry64 *reta_conf,
 109                                uint16_t reta_size);
 110 static int ice_rss_reta_query(struct rte_eth_dev *dev,
 111                               struct rte_eth_rss_reta_entry64 *reta_conf,
 112                               uint16_t reta_size);
 113 static int ice_rss_hash_update(struct rte_eth_dev *dev,
 114                                struct rte_eth_rss_conf *rss_conf);
 115 static int ice_rss_hash_conf_get(struct rte_eth_dev *dev,
 116                                  struct rte_eth_rss_conf *rss_conf);
 117 static int ice_promisc_enable(struct rte_eth_dev *dev);
 118 static int ice_promisc_disable(struct rte_eth_dev *dev);
 119 static int ice_allmulti_enable(struct rte_eth_dev *dev);
 120 static int ice_allmulti_disable(struct rte_eth_dev *dev);
 121 static int ice_vlan_filter_set(struct rte_eth_dev *dev,
 122                                uint16_t vlan_id,
 123                                int on);
 124 static int ice_macaddr_set(struct rte_eth_dev *dev,
 125                            struct rte_ether_addr *mac_addr);
 126 static int ice_macaddr_add(struct rte_eth_dev *dev,
 127                            struct rte_ether_addr *mac_addr,
 128                            __rte_unused uint32_t index,
 129                            uint32_t pool);
 130 static void ice_macaddr_remove(struct rte_eth_dev *dev, uint32_t index);
 131 static int ice_rx_queue_intr_enable(struct rte_eth_dev *dev,
 132                                     uint16_t queue_id);
 133 static int ice_rx_queue_intr_disable(struct rte_eth_dev *dev,
 134                                      uint16_t queue_id);
 135 static int ice_fw_version_get(struct rte_eth_dev *dev, char *fw_version,
 136                               size_t fw_size);
 137 static int ice_vlan_pvid_set(struct rte_eth_dev *dev,
 138                              uint16_t pvid, int on);
 139 static int ice_get_eeprom_length(struct rte_eth_dev *dev);
 140 static int ice_get_eeprom(struct rte_eth_dev *dev,
 141                           struct rte_dev_eeprom_info *eeprom);
 142 static int ice_stats_get(struct rte_eth_dev *dev,
 143                          struct rte_eth_stats *stats);
 144 static int ice_stats_reset(struct rte_eth_dev *dev);
 145 static int ice_xstats_get(struct rte_eth_dev *dev,
 146                           struct rte_eth_xstat *xstats, unsigned int n);
 147 static int ice_xstats_get_names(struct rte_eth_dev *dev,
 148                                 struct rte_eth_xstat_name *xstats_names,
 149                                 unsigned int limit);
 150 static int ice_dev_flow_ops_get(struct rte_eth_dev *dev,
 151                                 const struct rte_flow_ops **ops);
 152 static int ice_dev_udp_tunnel_port_add(struct rte_eth_dev *dev,
 153                         struct rte_eth_udp_tunnel *udp_tunnel);
 154 static int ice_dev_udp_tunnel_port_del(struct rte_eth_dev *dev,
 155                         struct rte_eth_udp_tunnel *udp_tunnel);
 156 static int ice_timesync_enable(struct rte_eth_dev *dev);
 157 static int ice_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
 158                                           struct timespec *timestamp,
 159                                           uint32_t flags);
 160 static int ice_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
 161                                           struct timespec *timestamp);
 162 static int ice_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta);
 163 static int ice_timesync_read_time(struct rte_eth_dev *dev,
 164                                   struct timespec *timestamp);
 165 static int ice_timesync_write_time(struct rte_eth_dev *dev,
 166                                    const struct timespec *timestamp);
 167 static int ice_timesync_disable(struct rte_eth_dev *dev);
 168
 169 static const struct rte_pci_id pci_id_ice_map[] = {
 170         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E823L_BACKPLANE) },
 171         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E823L_SFP) },
 172         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E823L_10G_BASE_T) },
 173         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E823L_1GBE) },
 174         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E823L_QSFP) },
 175         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E810C_BACKPLANE) },
 176         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E810C_QSFP) },
 177         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E810C_SFP) },
 178         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E810_XXV_BACKPLANE) },
 179         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E810_XXV_QSFP) },
 180         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E810_XXV_SFP) },
 181         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E823C_BACKPLANE) },
 182         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E823C_QSFP) },
 183         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E823C_SFP) },
 184         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E823C_10G_BASE_T) },
 185         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E823C_SGMII) },
 186         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E822C_BACKPLANE) },
 187         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E822C_QSFP) },
 188         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E822C_SFP) },
 189         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E822C_10G_BASE_T) },
 190         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E822C_SGMII) },
 191         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E822L_BACKPLANE) },
 192         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E822L_SFP) },
 193         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E822L_10G_BASE_T) },
 194         { RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E822L_SGMII) },
 195         { .vendor_id = 0, /* sentinel */ },
 196 };
 197
 198 static const struct eth_dev_ops ice_eth_dev_ops = {
 199         .dev_configure                = ice_dev_configure,
 200         .dev_start                    = ice_dev_start,
 201         .dev_stop                     = ice_dev_stop,
 202         .dev_close                    = ice_dev_close,
 203         .dev_reset                    = ice_dev_reset,
 204         .dev_set_link_up              = ice_dev_set_link_up,
 205         .dev_set_link_down            = ice_dev_set_link_down,
 206         .rx_queue_start               = ice_rx_queue_start,
 207         .rx_queue_stop                = ice_rx_queue_stop,
 208         .tx_queue_start               = ice_tx_queue_start,
 209         .tx_queue_stop                = ice_tx_queue_stop,
 210         .rx_queue_setup               = ice_rx_queue_setup,
 211         .rx_queue_release             = ice_dev_rx_queue_release,
 212         .tx_queue_setup               = ice_tx_queue_setup,
 213         .tx_queue_release             = ice_dev_tx_queue_release,
 214         .dev_infos_get                = ice_dev_info_get,
 215         .dev_supported_ptypes_get     = ice_dev_supported_ptypes_get,
 216         .link_update                  = ice_link_update,
 217         .mtu_set                      = ice_mtu_set,
 218         .mac_addr_set                 = ice_macaddr_set,
 219         .mac_addr_add                 = ice_macaddr_add,
 220         .mac_addr_remove              = ice_macaddr_remove,
 221         .vlan_filter_set              = ice_vlan_filter_set,
 222         .vlan_offload_set             = ice_vlan_offload_set,
 223         .reta_update                  = ice_rss_reta_update,
 224         .reta_query                   = ice_rss_reta_query,
 225         .rss_hash_update              = ice_rss_hash_update,
 226         .rss_hash_conf_get            = ice_rss_hash_conf_get,
 227         .promiscuous_enable           = ice_promisc_enable,
 228         .promiscuous_disable          = ice_promisc_disable,
 229         .allmulticast_enable          = ice_allmulti_enable,
 230         .allmulticast_disable         = ice_allmulti_disable,
 231         .rx_queue_intr_enable         = ice_rx_queue_intr_enable,
 232         .rx_queue_intr_disable        = ice_rx_queue_intr_disable,
 233         .fw_version_get               = ice_fw_version_get,
 234         .vlan_pvid_set                = ice_vlan_pvid_set,
 235         .rxq_info_get                 = ice_rxq_info_get,
 236         .txq_info_get                 = ice_txq_info_get,
 237         .rx_burst_mode_get            = ice_rx_burst_mode_get,
 238         .tx_burst_mode_get            = ice_tx_burst_mode_get,
 239         .get_eeprom_length            = ice_get_eeprom_length,
 240         .get_eeprom                   = ice_get_eeprom,
 241         .stats_get                    = ice_stats_get,
 242         .stats_reset                  = ice_stats_reset,
 243         .xstats_get                   = ice_xstats_get,
 244         .xstats_get_names             = ice_xstats_get_names,
 245         .xstats_reset                 = ice_stats_reset,
 246         .flow_ops_get                 = ice_dev_flow_ops_get,
 247         .udp_tunnel_port_add          = ice_dev_udp_tunnel_port_add,
 248         .udp_tunnel_port_del          = ice_dev_udp_tunnel_port_del,
 249         .tx_done_cleanup              = ice_tx_done_cleanup,
 250         .get_monitor_addr             = ice_get_monitor_addr,
 251         .timesync_enable              = ice_timesync_enable,
 252         .timesync_read_rx_timestamp   = ice_timesync_read_rx_timestamp,
 253         .timesync_read_tx_timestamp   = ice_timesync_read_tx_timestamp,
 254         .timesync_adjust_time         = ice_timesync_adjust_time,
 255         .timesync_read_time           = ice_timesync_read_time,
 256         .timesync_write_time          = ice_timesync_write_time,
 257         .timesync_disable             = ice_timesync_disable,
 258 };
 259
 260 /* store statistics names and its offset in stats structure */
 261 struct ice_xstats_name_off {
 262         char name[RTE_ETH_XSTATS_NAME_SIZE];
 263         unsigned int offset;
 264 };
 265
 266 static const struct ice_xstats_name_off ice_stats_strings[] = {
 267         {"rx_unicast_packets", offsetof(struct ice_eth_stats, rx_unicast)},
 268         {"rx_multicast_packets", offsetof(struct ice_eth_stats, rx_multicast)},
 269         {"rx_broadcast_packets", offsetof(struct ice_eth_stats, rx_broadcast)},
 270         {"rx_dropped_packets", offsetof(struct ice_eth_stats, rx_discards)},
 271         {"rx_unknown_protocol_packets", offsetof(struct ice_eth_stats,
 272                 rx_unknown_protocol)},
 273         {"tx_unicast_packets", offsetof(struct ice_eth_stats, tx_unicast)},
 274         {"tx_multicast_packets", offsetof(struct ice_eth_stats, tx_multicast)},
 275         {"tx_broadcast_packets", offsetof(struct ice_eth_stats, tx_broadcast)},
 276         {"tx_dropped_packets", offsetof(struct ice_eth_stats, tx_discards)},
 277 };
 278
 279 #define ICE_NB_ETH_XSTATS (sizeof(ice_stats_strings) / \
 280                 sizeof(ice_stats_strings[0]))
 281
 282 static const struct ice_xstats_name_off ice_hw_port_strings[] = {
 283         {"tx_link_down_dropped", offsetof(struct ice_hw_port_stats,
 284                 tx_dropped_link_down)},
 285         {"rx_crc_errors", offsetof(struct ice_hw_port_stats, crc_errors)},
 286         {"rx_illegal_byte_errors", offsetof(struct ice_hw_port_stats,
 287                 illegal_bytes)},
 288         {"rx_error_bytes", offsetof(struct ice_hw_port_stats, error_bytes)},
 289         {"mac_local_errors", offsetof(struct ice_hw_port_stats,
 290                 mac_local_faults)},
 291         {"mac_remote_errors", offsetof(struct ice_hw_port_stats,
 292                 mac_remote_faults)},
 293         {"rx_len_errors", offsetof(struct ice_hw_port_stats,
 294                 rx_len_errors)},
 295         {"tx_xon_packets", offsetof(struct ice_hw_port_stats, link_xon_tx)},
 296         {"rx_xon_packets", offsetof(struct ice_hw_port_stats, link_xon_rx)},
 297         {"tx_xoff_packets", offsetof(struct ice_hw_port_stats, link_xoff_tx)},
 298         {"rx_xoff_packets", offsetof(struct ice_hw_port_stats, link_xoff_rx)},
 299         {"rx_size_64_packets", offsetof(struct ice_hw_port_stats, rx_size_64)},
 300         {"rx_size_65_to_127_packets", offsetof(struct ice_hw_port_stats,
 301                 rx_size_127)},
 302         {"rx_size_128_to_255_packets", offsetof(struct ice_hw_port_stats,
 303                 rx_size_255)},
 304         {"rx_size_256_to_511_packets", offsetof(struct ice_hw_port_stats,
 305                 rx_size_511)},
 306         {"rx_size_512_to_1023_packets", offsetof(struct ice_hw_port_stats,
 307                 rx_size_1023)},
 308         {"rx_size_1024_to_1522_packets", offsetof(struct ice_hw_port_stats,
 309                 rx_size_1522)},
 310         {"rx_size_1523_to_max_packets", offsetof(struct ice_hw_port_stats,
 311                 rx_size_big)},
 312         {"rx_undersized_errors", offsetof(struct ice_hw_port_stats,
 313                 rx_undersize)},
 314         {"rx_oversize_errors", offsetof(struct ice_hw_port_stats,
 315                 rx_oversize)},
 316         {"rx_mac_short_pkt_dropped", offsetof(struct ice_hw_port_stats,
 317                 mac_short_pkt_dropped)},
 318         {"rx_fragmented_errors", offsetof(struct ice_hw_port_stats,
 319                 rx_fragments)},
 320         {"rx_jabber_errors", offsetof(struct ice_hw_port_stats, rx_jabber)},
 321         {"tx_size_64_packets", offsetof(struct ice_hw_port_stats, tx_size_64)},
 322         {"tx_size_65_to_127_packets", offsetof(struct ice_hw_port_stats,
 323                 tx_size_127)},
 324         {"tx_size_128_to_255_packets", offsetof(struct ice_hw_port_stats,
 325                 tx_size_255)},
 326         {"tx_size_256_to_511_packets", offsetof(struct ice_hw_port_stats,
 327                 tx_size_511)},
 328         {"tx_size_512_to_1023_packets", offsetof(struct ice_hw_port_stats,
 329                 tx_size_1023)},
 330         {"tx_size_1024_to_1522_packets", offsetof(struct ice_hw_port_stats,
 331                 tx_size_1522)},
 332         {"tx_size_1523_to_max_packets", offsetof(struct ice_hw_port_stats,
 333                 tx_size_big)},
 334 };
 335
 336 #define ICE_NB_HW_PORT_XSTATS (sizeof(ice_hw_port_strings) / \
 337                 sizeof(ice_hw_port_strings[0]))
 338
 339 static void
 340 ice_init_controlq_parameter(struct ice_hw *hw)
 341 {
 342         /* fields for adminq */
 343         hw->adminq.num_rq_entries = ICE_ADMINQ_LEN;
 344         hw->adminq.num_sq_entries = ICE_ADMINQ_LEN;
 345         hw->adminq.rq_buf_size = ICE_ADMINQ_BUF_SZ;
 346         hw->adminq.sq_buf_size = ICE_ADMINQ_BUF_SZ;
 347
 348         /* fields for mailboxq, DPDK used as PF host */
 349         hw->mailboxq.num_rq_entries = ICE_MAILBOXQ_LEN;
 350         hw->mailboxq.num_sq_entries = ICE_MAILBOXQ_LEN;
 351         hw->mailboxq.rq_buf_size = ICE_MAILBOXQ_BUF_SZ;
 352         hw->mailboxq.sq_buf_size = ICE_MAILBOXQ_BUF_SZ;
 353 }
 354
 355 static int
 356 lookup_proto_xtr_type(const char *xtr_name)
 357 {
 358         static struct {
 359                 const char *name;
 360                 enum proto_xtr_type type;
 361         } xtr_type_map[] = {
 362                 { "vlan",      PROTO_XTR_VLAN      },
 363                 { "ipv4",      PROTO_XTR_IPV4      },
 364                 { "ipv6",      PROTO_XTR_IPV6      },
 365                 { "ipv6_flow", PROTO_XTR_IPV6_FLOW },
 366                 { "tcp",       PROTO_XTR_TCP       },
 367                 { "ip_offset", PROTO_XTR_IP_OFFSET },
 368         };
 369         uint32_t i;
 370
 371         for (i = 0; i < RTE_DIM(xtr_type_map); i++) {
 372                 if (strcmp(xtr_name, xtr_type_map[i].name) == 0)
 373                         return xtr_type_map[i].type;
 374         }
 375
 376         return -1;
 377 }
 378
 379 /*
 380  * Parse elem, the elem could be single number/range or '(' ')' group
 381  * 1) A single number elem, it's just a simple digit. e.g. 9
 382  * 2) A single range elem, two digits with a '-' between. e.g. 2-6
 383  * 3) A group elem, combines multiple 1) or 2) with '( )'. e.g (0,2-4,6)
 384  *    Within group elem, '-' used for a range separator;
 385  *                       ',' used for a single number.
 386  */
 387 static int
 388 parse_queue_set(const char *input, int xtr_type, struct ice_devargs *devargs)
 389 {
 390         const char *str = input;
 391         char *end = NULL;
 392         uint32_t min, max;
 393         uint32_t idx;
 394
 395         while (isblank(*str))
 396                 str++;
 397
 398         if (!isdigit(*str) && *str != '(')
 399                 return -1;
 400
 401         /* process single number or single range of number */
 402         if (*str != '(') {
 403                 errno = 0;
 404                 idx = strtoul(str, &end, 10);
 405                 if (errno || end == NULL || idx >= ICE_MAX_QUEUE_NUM)
 406                         return -1;
 407
 408                 while (isblank(*end))
 409                         end++;
 410
 411                 min = idx;
 412                 max = idx;
 413
 414                 /* process single <number>-<number> */
 415                 if (*end == '-') {
 416                         end++;
 417                         while (isblank(*end))
 418                                 end++;
 419                         if (!isdigit(*end))
 420                                 return -1;
 421
 422                         errno = 0;
 423                         idx = strtoul(end, &end, 10);
 424                         if (errno || end == NULL || idx >= ICE_MAX_QUEUE_NUM)
 425                                 return -1;
 426
 427                         max = idx;
 428                         while (isblank(*end))
 429                                 end++;
 430                 }
 431
 432                 if (*end != ':')
 433                         return -1;
 434
 435                 for (idx = RTE_MIN(min, max);
 436                      idx <= RTE_MAX(min, max); idx++)
 437                         devargs->proto_xtr[idx] = xtr_type;
 438
 439                 return 0;
 440         }
 441
 442         /* process set within bracket */
 443         str++;
 444         while (isblank(*str))
 445                 str++;
 446         if (*str == '\0')
 447                 return -1;
 448
 449         min = ICE_MAX_QUEUE_NUM;
 450         do {
 451                 /* go ahead to the first digit */
 452                 while (isblank(*str))
 453                         str++;
 454                 if (!isdigit(*str))
 455                         return -1;
 456
 457                 /* get the digit value */
 458                 errno = 0;
 459                 idx = strtoul(str, &end, 10);
 460                 if (errno || end == NULL || idx >= ICE_MAX_QUEUE_NUM)
 461                         return -1;
 462
 463                 /* go ahead to separator '-',',' and ')' */
 464                 while (isblank(*end))
 465                         end++;
 466                 if (*end == '-') {
 467                         if (min == ICE_MAX_QUEUE_NUM)
 468                                 min = idx;
 469                         else /* avoid continuous '-' */
 470                                 return -1;
 471                 } else if (*end == ',' || *end == ')') {
 472                         max = idx;
 473                         if (min == ICE_MAX_QUEUE_NUM)
 474                                 min = idx;
 475
 476                         for (idx = RTE_MIN(min, max);
 477                              idx <= RTE_MAX(min, max); idx++)
 478                                 devargs->proto_xtr[idx] = xtr_type;
 479
 480                         min = ICE_MAX_QUEUE_NUM;
 481                 } else {
 482                         return -1;
 483                 }
 484
 485                 str = end + 1;
 486         } while (*end != ')' && *end != '\0');
 487
 488         return 0;
 489 }
 490
 491 static int
 492 parse_queue_proto_xtr(const char *queues, struct ice_devargs *devargs)
 493 {
 494         const char *queue_start;
 495         uint32_t idx;
 496         int xtr_type;
 497         char xtr_name[32];
 498
 499         while (isblank(*queues))
 500                 queues++;
 501
 502         if (*queues != '[') {
 503                 xtr_type = lookup_proto_xtr_type(queues);
 504                 if (xtr_type < 0)
 505                         return -1;
 506
 507                 devargs->proto_xtr_dflt = xtr_type;
 508
 509                 return 0;
 510         }
 511
 512         queues++;
 513         do {
 514                 while (isblank(*queues))
 515                         queues++;
 516                 if (*queues == '\0')
 517                         return -1;
 518
 519                 queue_start = queues;
 520
 521                 /* go across a complete bracket */
 522                 if (*queue_start == '(') {
 523                         queues += strcspn(queues, ")");
 524                         if (*queues != ')')
 525                                 return -1;
 526                 }
 527
 528                 /* scan the separator ':' */
 529                 queues += strcspn(queues, ":");
 530                 if (*queues++ != ':')
 531                         return -1;
 532                 while (isblank(*queues))
 533                         queues++;
 534
 535                 for (idx = 0; ; idx++) {
 536                         if (isblank(queues[idx]) ||
 537                             queues[idx] == ',' ||
 538                             queues[idx] == ']' ||
 539                             queues[idx] == '\0')
 540                                 break;
 541
 542                         if (idx > sizeof(xtr_name) - 2)
 543                                 return -1;
 544
 545                         xtr_name[idx] = queues[idx];
 546                 }
 547                 xtr_name[idx] = '\0';
 548                 xtr_type = lookup_proto_xtr_type(xtr_name);
 549                 if (xtr_type < 0)
 550                         return -1;
 551
 552                 queues += idx;
 553
 554                 while (isblank(*queues) || *queues == ',' || *queues == ']')
 555                         queues++;
 556
 557                 if (parse_queue_set(queue_start, xtr_type, devargs) < 0)
 558                         return -1;
 559         } while (*queues != '\0');
 560
 561         return 0;
 562 }
 563
 564 static int
 565 handle_proto_xtr_arg(__rte_unused const char *key, const char *value,
 566                      void *extra_args)
 567 {
 568         struct ice_devargs *devargs = extra_args;
 569
 570         if (value == NULL || extra_args == NULL)
 571                 return -EINVAL;
 572
 573         if (parse_queue_proto_xtr(value, devargs) < 0) {
 574                 PMD_DRV_LOG(ERR,
 575                             "The protocol extraction parameter is wrong : '%s'",
 576                             value);
 577                 return -1;
 578         }
 579
 580         return 0;
 581 }
 582
 583 static void
 584 ice_check_proto_xtr_support(struct ice_hw *hw)
 585 {
 586 #define FLX_REG(val, fld, idx) \
 587         (((val) & GLFLXP_RXDID_FLX_WRD_##idx##_##fld##_M) >> \
 588          GLFLXP_RXDID_FLX_WRD_##idx##_##fld##_S)
 589         static struct {
 590                 uint32_t rxdid;
 591                 uint8_t opcode;
 592                 uint8_t protid_0;
 593                 uint8_t protid_1;
 594         } xtr_sets[] = {
 595                 [PROTO_XTR_VLAN] = { ICE_RXDID_COMMS_AUX_VLAN,
 596                                      ICE_RX_OPC_EXTRACT,
 597                                      ICE_PROT_EVLAN_O, ICE_PROT_VLAN_O},
 598                 [PROTO_XTR_IPV4] = { ICE_RXDID_COMMS_AUX_IPV4,
 599                                      ICE_RX_OPC_EXTRACT,
 600                                      ICE_PROT_IPV4_OF_OR_S,
 601                                      ICE_PROT_IPV4_OF_OR_S },
 602                 [PROTO_XTR_IPV6] = { ICE_RXDID_COMMS_AUX_IPV6,
 603                                      ICE_RX_OPC_EXTRACT,
 604                                      ICE_PROT_IPV6_OF_OR_S,
 605                                      ICE_PROT_IPV6_OF_OR_S },
 606                 [PROTO_XTR_IPV6_FLOW] = { ICE_RXDID_COMMS_AUX_IPV6_FLOW,
 607                                           ICE_RX_OPC_EXTRACT,
 608                                           ICE_PROT_IPV6_OF_OR_S,
 609                                           ICE_PROT_IPV6_OF_OR_S },
 610                 [PROTO_XTR_TCP] = { ICE_RXDID_COMMS_AUX_TCP,
 611                                     ICE_RX_OPC_EXTRACT,
 612                                     ICE_PROT_TCP_IL, ICE_PROT_ID_INVAL },
 613                 [PROTO_XTR_IP_OFFSET] = { ICE_RXDID_COMMS_AUX_IP_OFFSET,
 614                                           ICE_RX_OPC_PROTID,
 615                                           ICE_PROT_IPV4_OF_OR_S,
 616                                           ICE_PROT_IPV6_OF_OR_S },
 617         };
 618         uint32_t i;
 619
 620         for (i = 0; i < RTE_DIM(xtr_sets); i++) {
 621                 uint32_t rxdid = xtr_sets[i].rxdid;
 622                 uint32_t v;
 623
 624                 if (xtr_sets[i].protid_0 != ICE_PROT_ID_INVAL) {
 625                         v = ICE_READ_REG(hw, GLFLXP_RXDID_FLX_WRD_4(rxdid));
 626
 627                         if (FLX_REG(v, PROT_MDID, 4) == xtr_sets[i].protid_0 &&
 628                             FLX_REG(v, RXDID_OPCODE, 4) == xtr_sets[i].opcode)
 629                                 ice_proto_xtr_hw_support[i] = true;
 630                 }
 631
 632                 if (xtr_sets[i].protid_1 != ICE_PROT_ID_INVAL) {
 633                         v = ICE_READ_REG(hw, GLFLXP_RXDID_FLX_WRD_5(rxdid));
 634
 635                         if (FLX_REG(v, PROT_MDID, 5) == xtr_sets[i].protid_1 &&
 636                             FLX_REG(v, RXDID_OPCODE, 5) == xtr_sets[i].opcode)
 637                                 ice_proto_xtr_hw_support[i] = true;
 638                 }
 639         }
 640 }
 641
 642 static int
 643 ice_res_pool_init(struct ice_res_pool_info *pool, uint32_t base,
 644                   uint32_t num)
 645 {
 646         struct pool_entry *entry;
 647
 648         if (!pool || !num)
 649                 return -EINVAL;
 650
 651         entry = rte_zmalloc(NULL, sizeof(*entry), 0);
 652         if (!entry) {
 653                 PMD_INIT_LOG(ERR,
 654                              "Failed to allocate memory for resource pool");
 655                 return -ENOMEM;
 656         }
 657
 658         /* queue heap initialize */
 659         pool->num_free = num;
 660         pool->num_alloc = 0;
 661         pool->base = base;
 662         LIST_INIT(&pool->alloc_list);
 663         LIST_INIT(&pool->free_list);
 664
 665         /* Initialize element  */
 666         entry->base = 0;
 667         entry->len = num;
 668
 669         LIST_INSERT_HEAD(&pool->free_list, entry, next);
 670         return 0;
 671 }
 672
 673 static int
 674 ice_res_pool_alloc(struct ice_res_pool_info *pool,
 675                    uint16_t num)
 676 {
 677         struct pool_entry *entry, *valid_entry;
 678
 679         if (!pool || !num) {
 680                 PMD_INIT_LOG(ERR, "Invalid parameter");
 681                 return -EINVAL;
 682         }
 683
 684         if (pool->num_free < num) {
 685                 PMD_INIT_LOG(ERR, "No resource. ask:%u, available:%u",
 686                              num, pool->num_free);
 687                 return -ENOMEM;
 688         }
 689
 690         valid_entry = NULL;
 691         /* Lookup  in free list and find most fit one */
 692         LIST_FOREACH(entry, &pool->free_list, next) {
 693                 if (entry->len >= num) {
 694                         /* Find best one */
 695                         if (entry->len == num) {
 696                                 valid_entry = entry;
 697                                 break;
 698                         }
 699                         if (!valid_entry ||
 700                             valid_entry->len > entry->len)
 701                                 valid_entry = entry;
 702                 }
 703         }
 704
 705         /* Not find one to satisfy the request, return */
 706         if (!valid_entry) {
 707                 PMD_INIT_LOG(ERR, "No valid entry found");
 708                 return -ENOMEM;
 709         }
 710         /**
 711          * The entry have equal queue number as requested,
 712          * remove it from alloc_list.
 713          */
 714         if (valid_entry->len == num) {
 715                 LIST_REMOVE(valid_entry, next);
 716         } else {
 717                 /**
 718                  * The entry have more numbers than requested,
 719                  * create a new entry for alloc_list and minus its
 720                  * queue base and number in free_list.
 721                  */
 722                 entry = rte_zmalloc(NULL, sizeof(*entry), 0);
 723                 if (!entry) {
 724                         PMD_INIT_LOG(ERR,
 725                                      "Failed to allocate memory for "
 726                                      "resource pool");
 727                         return -ENOMEM;
 728                 }
 729                 entry->base = valid_entry->base;
 730                 entry->len = num;
 731                 valid_entry->base += num;
 732                 valid_entry->len -= num;
 733                 valid_entry = entry;
 734         }
 735
 736         /* Insert it into alloc list, not sorted */
 737         LIST_INSERT_HEAD(&pool->alloc_list, valid_entry, next);
 738
 739         pool->num_free -= valid_entry->len;
 740         pool->num_alloc += valid_entry->len;
 741
 742         return valid_entry->base + pool->base;
 743 }
 744
 745 static void
 746 ice_res_pool_destroy(struct ice_res_pool_info *pool)
 747 {
 748         struct pool_entry *entry, *next_entry;
 749
 750         if (!pool)
 751                 return;
 752
 753         for (entry = LIST_FIRST(&pool->alloc_list);
 754              entry && (next_entry = LIST_NEXT(entry, next), 1);
 755              entry = next_entry) {
 756                 LIST_REMOVE(entry, next);
 757                 rte_free(entry);
 758         }
 759
 760         for (entry = LIST_FIRST(&pool->free_list);
 761              entry && (next_entry = LIST_NEXT(entry, next), 1);
 762              entry = next_entry) {
 763                 LIST_REMOVE(entry, next);
 764                 rte_free(entry);
 765         }
 766
 767         pool->num_free = 0;
 768         pool->num_alloc = 0;
 769         pool->base = 0;
 770         LIST_INIT(&pool->alloc_list);
 771         LIST_INIT(&pool->free_list);
 772 }
 773
 774 static void
 775 ice_vsi_config_default_rss(struct ice_aqc_vsi_props *info)
 776 {
 777         /* Set VSI LUT selection */
 778         info->q_opt_rss = ICE_AQ_VSI_Q_OPT_RSS_LUT_VSI &
 779                           ICE_AQ_VSI_Q_OPT_RSS_LUT_M;
 780         /* Set Hash scheme */
 781         info->q_opt_rss |= ICE_AQ_VSI_Q_OPT_RSS_TPLZ &
 782                            ICE_AQ_VSI_Q_OPT_RSS_HASH_M;
 783         /* enable TC */
 784         info->q_opt_tc = ICE_AQ_VSI_Q_OPT_TC_OVR_M;
 785 }
 786
 787 static enum ice_status
 788 ice_vsi_config_tc_queue_mapping(struct ice_vsi *vsi,
 789                                 struct ice_aqc_vsi_props *info,
 790                                 uint8_t enabled_tcmap)
 791 {
 792         uint16_t bsf, qp_idx;
 793
 794         /* default tc 0 now. Multi-TC supporting need to be done later.
 795          * Configure TC and queue mapping parameters, for enabled TC,
 796          * allocate qpnum_per_tc queues to this traffic.
 797          */
 798         if (enabled_tcmap != 0x01) {
 799                 PMD_INIT_LOG(ERR, "only TC0 is supported");
 800                 return -ENOTSUP;
 801         }
 802
 803         vsi->nb_qps = RTE_MIN(vsi->nb_qps, ICE_MAX_Q_PER_TC);
 804         bsf = rte_bsf32(vsi->nb_qps);
 805         /* Adjust the queue number to actual queues that can be applied */
 806         vsi->nb_qps = 0x1 << bsf;
 807
 808         qp_idx = 0;
 809         /* Set tc and queue mapping with VSI */
 810         info->tc_mapping[0] = rte_cpu_to_le_16((qp_idx <<
 811                                                 ICE_AQ_VSI_TC_Q_OFFSET_S) |
 812                                                (bsf << ICE_AQ_VSI_TC_Q_NUM_S));
 813
 814         /* Associate queue number with VSI */
 815         info->mapping_flags |= rte_cpu_to_le_16(ICE_AQ_VSI_Q_MAP_CONTIG);
 816         info->q_mapping[0] = rte_cpu_to_le_16(vsi->base_queue);
 817         info->q_mapping[1] = rte_cpu_to_le_16(vsi->nb_qps);
 818         info->valid_sections |=
 819                 rte_cpu_to_le_16(ICE_AQ_VSI_PROP_RXQ_MAP_VALID);
 820         /* Set the info.ingress_table and info.egress_table
 821          * for UP translate table. Now just set it to 1:1 map by default
 822          * -- 0b 111 110 101 100 011 010 001 000 == 0xFAC688
 823          */
 824 #define ICE_TC_QUEUE_TABLE_DFLT 0x00FAC688
 825         info->ingress_table  = rte_cpu_to_le_32(ICE_TC_QUEUE_TABLE_DFLT);
 826         info->egress_table   = rte_cpu_to_le_32(ICE_TC_QUEUE_TABLE_DFLT);
 827         info->outer_up_table = rte_cpu_to_le_32(ICE_TC_QUEUE_TABLE_DFLT);
 828         return 0;
 829 }
 830
 831 static int
 832 ice_init_mac_address(struct rte_eth_dev *dev)
 833 {
 834         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 835
 836         if (!rte_is_unicast_ether_addr
 837                 ((struct rte_ether_addr *)hw->port_info[0].mac.lan_addr)) {
 838                 PMD_INIT_LOG(ERR, "Invalid MAC address");
 839                 return -EINVAL;
 840         }
 841
 842         rte_ether_addr_copy(
 843                 (struct rte_ether_addr *)hw->port_info[0].mac.lan_addr,
 844                 (struct rte_ether_addr *)hw->port_info[0].mac.perm_addr);
 845
 846         dev->data->mac_addrs =
 847                 rte_zmalloc(NULL, sizeof(struct rte_ether_addr) * ICE_NUM_MACADDR_MAX, 0);
 848         if (!dev->data->mac_addrs) {
 849                 PMD_INIT_LOG(ERR,
 850                              "Failed to allocate memory to store mac address");
 851                 return -ENOMEM;
 852         }
 853         /* store it to dev data */
 854         rte_ether_addr_copy(
 855                 (struct rte_ether_addr *)hw->port_info[0].mac.perm_addr,
 856                 &dev->data->mac_addrs[0]);
 857         return 0;
 858 }
 859
 860 /* Find out specific MAC filter */
 861 static struct ice_mac_filter *
 862 ice_find_mac_filter(struct ice_vsi *vsi, struct rte_ether_addr *macaddr)
 863 {
 864         struct ice_mac_filter *f;
 865
 866         TAILQ_FOREACH(f, &vsi->mac_list, next) {
 867                 if (rte_is_same_ether_addr(macaddr, &f->mac_info.mac_addr))
 868                         return f;
 869         }
 870
 871         return NULL;
 872 }
 873
 874 static int
 875 ice_add_mac_filter(struct ice_vsi *vsi, struct rte_ether_addr *mac_addr)
 876 {
 877         struct ice_fltr_list_entry *m_list_itr = NULL;
 878         struct ice_mac_filter *f;
 879         struct LIST_HEAD_TYPE list_head;
 880         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
 881         int ret = 0;
 882
 883         /* If it's added and configured, return */
 884         f = ice_find_mac_filter(vsi, mac_addr);
 885         if (f) {
 886                 PMD_DRV_LOG(INFO, "This MAC filter already exists.");
 887                 return 0;
 888         }
 889
 890         INIT_LIST_HEAD(&list_head);
 891
 892         m_list_itr = (struct ice_fltr_list_entry *)
 893                 ice_malloc(hw, sizeof(*m_list_itr));
 894         if (!m_list_itr) {
 895                 ret = -ENOMEM;
 896                 goto DONE;
 897         }
 898         ice_memcpy(m_list_itr->fltr_info.l_data.mac.mac_addr,
 899                    mac_addr, ETH_ALEN, ICE_NONDMA_TO_NONDMA);
 900         m_list_itr->fltr_info.src_id = ICE_SRC_ID_VSI;
 901         m_list_itr->fltr_info.fltr_act = ICE_FWD_TO_VSI;
 902         m_list_itr->fltr_info.lkup_type = ICE_SW_LKUP_MAC;
 903         m_list_itr->fltr_info.flag = ICE_FLTR_TX;
 904         m_list_itr->fltr_info.vsi_handle = vsi->idx;
 905
 906         LIST_ADD(&m_list_itr->list_entry, &list_head);
 907
 908         /* Add the mac */
 909         ret = ice_add_mac(hw, &list_head);
 910         if (ret != ICE_SUCCESS) {
 911                 PMD_DRV_LOG(ERR, "Failed to add MAC filter");
 912                 ret = -EINVAL;
 913                 goto DONE;
 914         }
 915         /* Add the mac addr into mac list */
 916         f = rte_zmalloc(NULL, sizeof(*f), 0);
 917         if (!f) {
 918                 PMD_DRV_LOG(ERR, "failed to allocate memory");
 919                 ret = -ENOMEM;
 920                 goto DONE;
 921         }
 922         rte_ether_addr_copy(mac_addr, &f->mac_info.mac_addr);
 923         TAILQ_INSERT_TAIL(&vsi->mac_list, f, next);
 924         vsi->mac_num++;
 925
 926         ret = 0;
 927
 928 DONE:
 929         rte_free(m_list_itr);
 930         return ret;
 931 }
 932
 933 static int
 934 ice_remove_mac_filter(struct ice_vsi *vsi, struct rte_ether_addr *mac_addr)
 935 {
 936         struct ice_fltr_list_entry *m_list_itr = NULL;
 937         struct ice_mac_filter *f;
 938         struct LIST_HEAD_TYPE list_head;
 939         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
 940         int ret = 0;
 941
 942         /* Can't find it, return an error */
 943         f = ice_find_mac_filter(vsi, mac_addr);
 944         if (!f)
 945                 return -EINVAL;
 946
 947         INIT_LIST_HEAD(&list_head);
 948
 949         m_list_itr = (struct ice_fltr_list_entry *)
 950                 ice_malloc(hw, sizeof(*m_list_itr));
 951         if (!m_list_itr) {
 952                 ret = -ENOMEM;
 953                 goto DONE;
 954         }
 955         ice_memcpy(m_list_itr->fltr_info.l_data.mac.mac_addr,
 956                    mac_addr, ETH_ALEN, ICE_NONDMA_TO_NONDMA);
 957         m_list_itr->fltr_info.src_id = ICE_SRC_ID_VSI;
 958         m_list_itr->fltr_info.fltr_act = ICE_FWD_TO_VSI;
 959         m_list_itr->fltr_info.lkup_type = ICE_SW_LKUP_MAC;
 960         m_list_itr->fltr_info.flag = ICE_FLTR_TX;
 961         m_list_itr->fltr_info.vsi_handle = vsi->idx;
 962
 963         LIST_ADD(&m_list_itr->list_entry, &list_head);
 964
 965         /* remove the mac filter */
 966         ret = ice_remove_mac(hw, &list_head);
 967         if (ret != ICE_SUCCESS) {
 968                 PMD_DRV_LOG(ERR, "Failed to remove MAC filter");
 969                 ret = -EINVAL;
 970                 goto DONE;
 971         }
 972
 973         /* Remove the mac addr from mac list */
 974         TAILQ_REMOVE(&vsi->mac_list, f, next);
 975         rte_free(f);
 976         vsi->mac_num--;
 977
 978         ret = 0;
 979 DONE:
 980         rte_free(m_list_itr);
 981         return ret;
 982 }
 983
 984 /* Find out specific VLAN filter */
 985 static struct ice_vlan_filter *
 986 ice_find_vlan_filter(struct ice_vsi *vsi, struct ice_vlan *vlan)
 987 {
 988         struct ice_vlan_filter *f;
 989
 990         TAILQ_FOREACH(f, &vsi->vlan_list, next) {
 991                 if (vlan->tpid == f->vlan_info.vlan.tpid &&
 992                     vlan->vid == f->vlan_info.vlan.vid)
 993                         return f;
 994         }
 995
 996         return NULL;
 997 }
 998
 999 static int
1000 ice_add_vlan_filter(struct ice_vsi *vsi, struct ice_vlan *vlan)
1001 {
1002         struct ice_fltr_list_entry *v_list_itr = NULL;
1003         struct ice_vlan_filter *f;
1004         struct LIST_HEAD_TYPE list_head;
1005         struct ice_hw *hw;
1006         int ret = 0;
1007
1008         if (!vsi || vlan->vid > RTE_ETHER_MAX_VLAN_ID)
1009                 return -EINVAL;
1010
1011         hw = ICE_VSI_TO_HW(vsi);
1012
1013         /* If it's added and configured, return. */
1014         f = ice_find_vlan_filter(vsi, vlan);
1015         if (f) {
1016                 PMD_DRV_LOG(INFO, "This VLAN filter already exists.");
1017                 return 0;
1018         }
1019
1020         if (!vsi->vlan_anti_spoof_on && !vsi->vlan_filter_on)
1021                 return 0;
1022
1023         INIT_LIST_HEAD(&list_head);
1024
1025         v_list_itr = (struct ice_fltr_list_entry *)
1026                       ice_malloc(hw, sizeof(*v_list_itr));
1027         if (!v_list_itr) {
1028                 ret = -ENOMEM;
1029                 goto DONE;
1030         }
1031         v_list_itr->fltr_info.l_data.vlan.vlan_id = vlan->vid;
1032         v_list_itr->fltr_info.l_data.vlan.tpid = vlan->tpid;
1033         v_list_itr->fltr_info.l_data.vlan.tpid_valid = true;
1034         v_list_itr->fltr_info.src_id = ICE_SRC_ID_VSI;
1035         v_list_itr->fltr_info.fltr_act = ICE_FWD_TO_VSI;
1036         v_list_itr->fltr_info.lkup_type = ICE_SW_LKUP_VLAN;
1037         v_list_itr->fltr_info.flag = ICE_FLTR_TX;
1038         v_list_itr->fltr_info.vsi_handle = vsi->idx;
1039
1040         LIST_ADD(&v_list_itr->list_entry, &list_head);
1041
1042         /* Add the vlan */
1043         ret = ice_add_vlan(hw, &list_head);
1044         if (ret != ICE_SUCCESS) {
1045                 PMD_DRV_LOG(ERR, "Failed to add VLAN filter");
1046                 ret = -EINVAL;
1047                 goto DONE;
1048         }
1049
1050         /* Add vlan into vlan list */
1051         f = rte_zmalloc(NULL, sizeof(*f), 0);
1052         if (!f) {
1053                 PMD_DRV_LOG(ERR, "failed to allocate memory");
1054                 ret = -ENOMEM;
1055                 goto DONE;
1056         }
1057         f->vlan_info.vlan.tpid = vlan->tpid;
1058         f->vlan_info.vlan.vid = vlan->vid;
1059         TAILQ_INSERT_TAIL(&vsi->vlan_list, f, next);
1060         vsi->vlan_num++;
1061
1062         ret = 0;
1063
1064 DONE:
1065         rte_free(v_list_itr);
1066         return ret;
1067 }
1068
1069 static int
1070 ice_remove_vlan_filter(struct ice_vsi *vsi, struct ice_vlan *vlan)
1071 {
1072         struct ice_fltr_list_entry *v_list_itr = NULL;
1073         struct ice_vlan_filter *f;
1074         struct LIST_HEAD_TYPE list_head;
1075         struct ice_hw *hw;
1076         int ret = 0;
1077
1078         if (!vsi || vlan->vid > RTE_ETHER_MAX_VLAN_ID)
1079                 return -EINVAL;
1080
1081         hw = ICE_VSI_TO_HW(vsi);
1082
1083         /* Can't find it, return an error */
1084         f = ice_find_vlan_filter(vsi, vlan);
1085         if (!f)
1086                 return -EINVAL;
1087
1088         INIT_LIST_HEAD(&list_head);
1089
1090         v_list_itr = (struct ice_fltr_list_entry *)
1091                       ice_malloc(hw, sizeof(*v_list_itr));
1092         if (!v_list_itr) {
1093                 ret = -ENOMEM;
1094                 goto DONE;
1095         }
1096
1097         v_list_itr->fltr_info.l_data.vlan.vlan_id = vlan->vid;
1098         v_list_itr->fltr_info.l_data.vlan.tpid = vlan->tpid;
1099         v_list_itr->fltr_info.l_data.vlan.tpid_valid = true;
1100         v_list_itr->fltr_info.src_id = ICE_SRC_ID_VSI;
1101         v_list_itr->fltr_info.fltr_act = ICE_FWD_TO_VSI;
1102         v_list_itr->fltr_info.lkup_type = ICE_SW_LKUP_VLAN;
1103         v_list_itr->fltr_info.flag = ICE_FLTR_TX;
1104         v_list_itr->fltr_info.vsi_handle = vsi->idx;
1105
1106         LIST_ADD(&v_list_itr->list_entry, &list_head);
1107
1108         /* remove the vlan filter */
1109         ret = ice_remove_vlan(hw, &list_head);
1110         if (ret != ICE_SUCCESS) {
1111                 PMD_DRV_LOG(ERR, "Failed to remove VLAN filter");
1112                 ret = -EINVAL;
1113                 goto DONE;
1114         }
1115
1116         /* Remove the vlan id from vlan list */
1117         TAILQ_REMOVE(&vsi->vlan_list, f, next);
1118         rte_free(f);
1119         vsi->vlan_num--;
1120
1121         ret = 0;
1122 DONE:
1123         rte_free(v_list_itr);
1124         return ret;
1125 }
1126
1127 static int
1128 ice_remove_all_mac_vlan_filters(struct ice_vsi *vsi)
1129 {
1130         struct ice_mac_filter *m_f;
1131         struct ice_vlan_filter *v_f;
1132         void *temp;
1133         int ret = 0;
1134
1135         if (!vsi || !vsi->mac_num)
1136                 return -EINVAL;
1137
1138         RTE_TAILQ_FOREACH_SAFE(m_f, &vsi->mac_list, next, temp) {
1139                 ret = ice_remove_mac_filter(vsi, &m_f->mac_info.mac_addr);
1140                 if (ret != ICE_SUCCESS) {
1141                         ret = -EINVAL;
1142                         goto DONE;
1143                 }
1144         }
1145
1146         if (vsi->vlan_num == 0)
1147                 return 0;
1148
1149         RTE_TAILQ_FOREACH_SAFE(v_f, &vsi->vlan_list, next, temp) {
1150                 ret = ice_remove_vlan_filter(vsi, &v_f->vlan_info.vlan);
1151                 if (ret != ICE_SUCCESS) {
1152                         ret = -EINVAL;
1153                         goto DONE;
1154                 }
1155         }
1156
1157 DONE:
1158         return ret;
1159 }
1160
1161 /* Enable IRQ0 */
1162 static void
1163 ice_pf_enable_irq0(struct ice_hw *hw)
1164 {
1165         /* reset the registers */
1166         ICE_WRITE_REG(hw, PFINT_OICR_ENA, 0);
1167         ICE_READ_REG(hw, PFINT_OICR);
1168
1169 #ifdef ICE_LSE_SPT
1170         ICE_WRITE_REG(hw, PFINT_OICR_ENA,
1171                       (uint32_t)(PFINT_OICR_ENA_INT_ENA_M &
1172                                  (~PFINT_OICR_LINK_STAT_CHANGE_M)));
1173
1174         ICE_WRITE_REG(hw, PFINT_OICR_CTL,
1175                       (0 & PFINT_OICR_CTL_MSIX_INDX_M) |
1176                       ((0 << PFINT_OICR_CTL_ITR_INDX_S) &
1177                        PFINT_OICR_CTL_ITR_INDX_M) |
1178                       PFINT_OICR_CTL_CAUSE_ENA_M);
1179
1180         ICE_WRITE_REG(hw, PFINT_FW_CTL,
1181                       (0 & PFINT_FW_CTL_MSIX_INDX_M) |
1182                       ((0 << PFINT_FW_CTL_ITR_INDX_S) &
1183                        PFINT_FW_CTL_ITR_INDX_M) |
1184                       PFINT_FW_CTL_CAUSE_ENA_M);
1185 #else
1186         ICE_WRITE_REG(hw, PFINT_OICR_ENA, PFINT_OICR_ENA_INT_ENA_M);
1187 #endif
1188
1189         ICE_WRITE_REG(hw, GLINT_DYN_CTL(0),
1190                       GLINT_DYN_CTL_INTENA_M |
1191                       GLINT_DYN_CTL_CLEARPBA_M |
1192                       GLINT_DYN_CTL_ITR_INDX_M);
1193
1194         ice_flush(hw);
1195 }
1196
1197 /* Disable IRQ0 */
1198 static void
1199 ice_pf_disable_irq0(struct ice_hw *hw)
1200 {
1201         /* Disable all interrupt types */
1202         ICE_WRITE_REG(hw, GLINT_DYN_CTL(0), GLINT_DYN_CTL_WB_ON_ITR_M);
1203         ice_flush(hw);
1204 }
1205
1206 #ifdef ICE_LSE_SPT
1207 static void
1208 ice_handle_aq_msg(struct rte_eth_dev *dev)
1209 {
1210         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1211         struct ice_ctl_q_info *cq = &hw->adminq;
1212         struct ice_rq_event_info event;
1213         uint16_t pending, opcode;
1214         int ret;
1215
1216         event.buf_len = ICE_AQ_MAX_BUF_LEN;
1217         event.msg_buf = rte_zmalloc(NULL, event.buf_len, 0);
1218         if (!event.msg_buf) {
1219                 PMD_DRV_LOG(ERR, "Failed to allocate mem");
1220                 return;
1221         }
1222
1223         pending = 1;
1224         while (pending) {
1225                 ret = ice_clean_rq_elem(hw, cq, &event, &pending);
1226
1227                 if (ret != ICE_SUCCESS) {
1228                         PMD_DRV_LOG(INFO,
1229                                     "Failed to read msg from AdminQ, "
1230                                     "adminq_err: %u",
1231                                     hw->adminq.sq_last_status);
1232                         break;
1233                 }
1234                 opcode = rte_le_to_cpu_16(event.desc.opcode);
1235
1236                 switch (opcode) {
1237                 case ice_aqc_opc_get_link_status:
1238                         ret = ice_link_update(dev, 0);
1239                         if (!ret)
1240                                 rte_eth_dev_callback_process
1241                                         (dev, RTE_ETH_EVENT_INTR_LSC, NULL);
1242                         break;
1243                 default:
1244                         PMD_DRV_LOG(DEBUG, "Request %u is not supported yet",
1245                                     opcode);
1246                         break;
1247                 }
1248         }
1249         rte_free(event.msg_buf);
1250 }
1251 #endif
1252
1253 /**
1254  * Interrupt handler triggered by NIC for handling
1255  * specific interrupt.
1256  *
1257  * @param handle
1258  *  Pointer to interrupt handle.
1259  * @param param
1260  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1261  *
1262  * @return
1263  *  void
1264  */
1265 static void
1266 ice_interrupt_handler(void *param)
1267 {
1268         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1269         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1270         uint32_t oicr;
1271         uint32_t reg;
1272         uint8_t pf_num;
1273         uint8_t event;
1274         uint16_t queue;
1275         int ret;
1276 #ifdef ICE_LSE_SPT
1277         uint32_t int_fw_ctl;
1278 #endif
1279
1280         /* Disable interrupt */
1281         ice_pf_disable_irq0(hw);
1282
1283         /* read out interrupt causes */
1284         oicr = ICE_READ_REG(hw, PFINT_OICR);
1285 #ifdef ICE_LSE_SPT
1286         int_fw_ctl = ICE_READ_REG(hw, PFINT_FW_CTL);
1287 #endif
1288
1289         /* No interrupt event indicated */
1290         if (!(oicr & PFINT_OICR_INTEVENT_M)) {
1291                 PMD_DRV_LOG(INFO, "No interrupt event");
1292                 goto done;
1293         }
1294
1295 #ifdef ICE_LSE_SPT
1296         if (int_fw_ctl & PFINT_FW_CTL_INTEVENT_M) {
1297                 PMD_DRV_LOG(INFO, "FW_CTL: link state change event");
1298                 ice_handle_aq_msg(dev);
1299         }
1300 #else
1301         if (oicr & PFINT_OICR_LINK_STAT_CHANGE_M) {
1302                 PMD_DRV_LOG(INFO, "OICR: link state change event");
1303                 ret = ice_link_update(dev, 0);
1304                 if (!ret)
1305                         rte_eth_dev_callback_process
1306                                 (dev, RTE_ETH_EVENT_INTR_LSC, NULL);
1307         }
1308 #endif
1309
1310         if (oicr & PFINT_OICR_MAL_DETECT_M) {
1311                 PMD_DRV_LOG(WARNING, "OICR: MDD event");
1312                 reg = ICE_READ_REG(hw, GL_MDET_TX_PQM);
1313                 if (reg & GL_MDET_TX_PQM_VALID_M) {
1314                         pf_num = (reg & GL_MDET_TX_PQM_PF_NUM_M) >>
1315                                  GL_MDET_TX_PQM_PF_NUM_S;
1316                         event = (reg & GL_MDET_TX_PQM_MAL_TYPE_M) >>
1317                                 GL_MDET_TX_PQM_MAL_TYPE_S;
1318                         queue = (reg & GL_MDET_TX_PQM_QNUM_M) >>
1319                                 GL_MDET_TX_PQM_QNUM_S;
1320
1321                         PMD_DRV_LOG(WARNING, "Malicious Driver Detection event "
1322                                     "%d by PQM on TX queue %d PF# %d",
1323                                     event, queue, pf_num);
1324                 }
1325
1326                 reg = ICE_READ_REG(hw, GL_MDET_TX_TCLAN);
1327                 if (reg & GL_MDET_TX_TCLAN_VALID_M) {
1328                         pf_num = (reg & GL_MDET_TX_TCLAN_PF_NUM_M) >>
1329                                  GL_MDET_TX_TCLAN_PF_NUM_S;
1330                         event = (reg & GL_MDET_TX_TCLAN_MAL_TYPE_M) >>
1331                                 GL_MDET_TX_TCLAN_MAL_TYPE_S;
1332                         queue = (reg & GL_MDET_TX_TCLAN_QNUM_M) >>
1333                                 GL_MDET_TX_TCLAN_QNUM_S;
1334
1335                         PMD_DRV_LOG(WARNING, "Malicious Driver Detection event "
1336                                     "%d by TCLAN on TX queue %d PF# %d",
1337                                     event, queue, pf_num);
1338                 }
1339         }
1340 done:
1341         /* Enable interrupt */
1342         ice_pf_enable_irq0(hw);
1343         rte_intr_ack(dev->intr_handle);
1344 }
1345
1346 static void
1347 ice_init_proto_xtr(struct rte_eth_dev *dev)
1348 {
1349         struct ice_adapter *ad =
1350                         ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1351         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1352         struct ice_hw *hw = ICE_PF_TO_HW(pf);
1353         const struct proto_xtr_ol_flag *ol_flag;
1354         bool proto_xtr_enable = false;
1355         int offset;
1356         uint16_t i;
1357
1358         pf->proto_xtr = rte_zmalloc(NULL, pf->lan_nb_qps, 0);
1359         if (unlikely(pf->proto_xtr == NULL)) {
1360                 PMD_DRV_LOG(ERR, "No memory for setting up protocol extraction table");
1361                 return;
1362         }
1363
1364         for (i = 0; i < pf->lan_nb_qps; i++) {
1365                 pf->proto_xtr[i] = ad->devargs.proto_xtr[i] != PROTO_XTR_NONE ?
1366                                    ad->devargs.proto_xtr[i] :
1367                                    ad->devargs.proto_xtr_dflt;
1368
1369                 if (pf->proto_xtr[i] != PROTO_XTR_NONE) {
1370                         uint8_t type = pf->proto_xtr[i];
1371
1372                         ice_proto_xtr_ol_flag_params[type].required = true;
1373                         proto_xtr_enable = true;
1374                 }
1375         }
1376
1377         if (likely(!proto_xtr_enable))
1378                 return;
1379
1380         ice_check_proto_xtr_support(hw);
1381
1382         offset = rte_mbuf_dynfield_register(&ice_proto_xtr_metadata_param);
1383         if (unlikely(offset == -1)) {
1384                 PMD_DRV_LOG(ERR,
1385                             "Protocol extraction metadata is disabled in mbuf with error %d",
1386                             -rte_errno);
1387                 return;
1388         }
1389
1390         PMD_DRV_LOG(DEBUG,
1391                     "Protocol extraction metadata offset in mbuf is : %d",
1392                     offset);
1393         rte_net_ice_dynfield_proto_xtr_metadata_offs = offset;
1394
1395         for (i = 0; i < RTE_DIM(ice_proto_xtr_ol_flag_params); i++) {
1396                 ol_flag = &ice_proto_xtr_ol_flag_params[i];
1397
1398                 if (!ol_flag->required)
1399                         continue;
1400
1401                 if (!ice_proto_xtr_hw_support[i]) {
1402                         PMD_DRV_LOG(ERR,
1403                                     "Protocol extraction type %u is not supported in hardware",
1404                                     i);
1405                         rte_net_ice_dynfield_proto_xtr_metadata_offs = -1;
1406                         break;
1407                 }
1408
1409                 offset = rte_mbuf_dynflag_register(&ol_flag->param);
1410                 if (unlikely(offset == -1)) {
1411                         PMD_DRV_LOG(ERR,
1412                                     "Protocol extraction offload '%s' failed to register with error %d",
1413                                     ol_flag->param.name, -rte_errno);
1414
1415                         rte_net_ice_dynfield_proto_xtr_metadata_offs = -1;
1416                         break;
1417                 }
1418
1419                 PMD_DRV_LOG(DEBUG,
1420                             "Protocol extraction offload '%s' offset in mbuf is : %d",
1421                             ol_flag->param.name, offset);
1422                 *ol_flag->ol_flag = 1ULL << offset;
1423         }
1424 }
1425
1426 /*  Initialize SW parameters of PF */
1427 static int
1428 ice_pf_sw_init(struct rte_eth_dev *dev)
1429 {
1430         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1431         struct ice_hw *hw = ICE_PF_TO_HW(pf);
1432
1433         pf->lan_nb_qp_max =
1434                 (uint16_t)RTE_MIN(hw->func_caps.common_cap.num_txq,
1435                                   hw->func_caps.common_cap.num_rxq);
1436
1437         pf->lan_nb_qps = pf->lan_nb_qp_max;
1438
1439         ice_init_proto_xtr(dev);
1440
1441         if (hw->func_caps.fd_fltr_guar > 0 ||
1442             hw->func_caps.fd_fltr_best_effort > 0) {
1443                 pf->flags |= ICE_FLAG_FDIR;
1444                 pf->fdir_nb_qps = ICE_DEFAULT_QP_NUM_FDIR;
1445                 pf->lan_nb_qps = pf->lan_nb_qp_max - pf->fdir_nb_qps;
1446         } else {
1447                 pf->fdir_nb_qps = 0;
1448         }
1449         pf->fdir_qp_offset = 0;
1450
1451         return 0;
1452 }
1453
1454 struct ice_vsi *
1455 ice_setup_vsi(struct ice_pf *pf, enum ice_vsi_type type)
1456 {
1457         struct ice_hw *hw = ICE_PF_TO_HW(pf);
1458         struct ice_vsi *vsi = NULL;
1459         struct ice_vsi_ctx vsi_ctx;
1460         int ret;
1461         struct rte_ether_addr broadcast = {
1462                 .addr_bytes = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff} };
1463         struct rte_ether_addr mac_addr;
1464         uint16_t max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 };
1465         uint8_t tc_bitmap = 0x1;
1466         uint16_t cfg;
1467
1468         /* hw->num_lports = 1 in NIC mode */
1469         vsi = rte_zmalloc(NULL, sizeof(struct ice_vsi), 0);
1470         if (!vsi)
1471                 return NULL;
1472
1473         vsi->idx = pf->next_vsi_idx;
1474         pf->next_vsi_idx++;
1475         vsi->type = type;
1476         vsi->adapter = ICE_PF_TO_ADAPTER(pf);
1477         vsi->max_macaddrs = ICE_NUM_MACADDR_MAX;
1478         vsi->vlan_anti_spoof_on = 0;
1479         vsi->vlan_filter_on = 1;
1480         TAILQ_INIT(&vsi->mac_list);
1481         TAILQ_INIT(&vsi->vlan_list);
1482
1483         /* Be sync with ETH_RSS_RETA_SIZE_x maximum value definition */
1484         pf->hash_lut_size = hw->func_caps.common_cap.rss_table_size >
1485                         ETH_RSS_RETA_SIZE_512 ? ETH_RSS_RETA_SIZE_512 :
1486                         hw->func_caps.common_cap.rss_table_size;
1487         pf->flags |= ICE_FLAG_RSS_AQ_CAPABLE;
1488
1489         memset(&vsi_ctx, 0, sizeof(vsi_ctx));
1490         switch (type) {
1491         case ICE_VSI_PF:
1492                 vsi->nb_qps = pf->lan_nb_qps;
1493                 vsi->base_queue = 1;
1494                 ice_vsi_config_default_rss(&vsi_ctx.info);
1495                 vsi_ctx.alloc_from_pool = true;
1496                 vsi_ctx.flags = ICE_AQ_VSI_TYPE_PF;
1497                 /* switch_id is queried by get_switch_config aq, which is done
1498                  * by ice_init_hw
1499                  */
1500                 vsi_ctx.info.sw_id = hw->port_info->sw_id;
1501                 vsi_ctx.info.sw_flags2 = ICE_AQ_VSI_SW_FLAG_LAN_ENA;
1502                 /* Allow all untagged or tagged packets */
1503                 vsi_ctx.info.inner_vlan_flags = ICE_AQ_VSI_INNER_VLAN_TX_MODE_ALL;
1504                 vsi_ctx.info.inner_vlan_flags |= ICE_AQ_VSI_INNER_VLAN_EMODE_NOTHING;
1505                 vsi_ctx.info.q_opt_rss = ICE_AQ_VSI_Q_OPT_RSS_LUT_PF |
1506                                          ICE_AQ_VSI_Q_OPT_RSS_TPLZ;
1507                 if (ice_is_dvm_ena(hw)) {
1508                         vsi_ctx.info.outer_vlan_flags =
1509                                 (ICE_AQ_VSI_OUTER_VLAN_TX_MODE_ALL <<
1510                                  ICE_AQ_VSI_OUTER_VLAN_TX_MODE_S) &
1511                                 ICE_AQ_VSI_OUTER_VLAN_TX_MODE_M;
1512                         vsi_ctx.info.outer_vlan_flags |=
1513                                 (ICE_AQ_VSI_OUTER_TAG_VLAN_8100 <<
1514                                  ICE_AQ_VSI_OUTER_TAG_TYPE_S) &
1515                                 ICE_AQ_VSI_OUTER_TAG_TYPE_M;
1516                 }
1517
1518                 /* FDIR */
1519                 cfg = ICE_AQ_VSI_PROP_SECURITY_VALID |
1520                         ICE_AQ_VSI_PROP_FLOW_DIR_VALID;
1521                 vsi_ctx.info.valid_sections |= rte_cpu_to_le_16(cfg);
1522                 cfg = ICE_AQ_VSI_FD_ENABLE;
1523                 vsi_ctx.info.fd_options = rte_cpu_to_le_16(cfg);
1524                 vsi_ctx.info.max_fd_fltr_dedicated =
1525                         rte_cpu_to_le_16(hw->func_caps.fd_fltr_guar);
1526                 vsi_ctx.info.max_fd_fltr_shared =
1527                         rte_cpu_to_le_16(hw->func_caps.fd_fltr_best_effort);
1528
1529                 /* Enable VLAN/UP trip */
1530                 ret = ice_vsi_config_tc_queue_mapping(vsi,
1531                                                       &vsi_ctx.info,
1532                                                       ICE_DEFAULT_TCMAP);
1533                 if (ret) {
1534                         PMD_INIT_LOG(ERR,
1535                                      "tc queue mapping with vsi failed, "
1536                                      "err = %d",
1537                                      ret);
1538                         goto fail_mem;
1539                 }
1540
1541                 break;
1542         case ICE_VSI_CTRL:
1543                 vsi->nb_qps = pf->fdir_nb_qps;
1544                 vsi->base_queue = ICE_FDIR_QUEUE_ID;
1545                 vsi_ctx.alloc_from_pool = true;
1546                 vsi_ctx.flags = ICE_AQ_VSI_TYPE_PF;
1547
1548                 cfg = ICE_AQ_VSI_PROP_FLOW_DIR_VALID;
1549                 vsi_ctx.info.valid_sections |= rte_cpu_to_le_16(cfg);
1550                 cfg = ICE_AQ_VSI_FD_PROG_ENABLE;
1551                 vsi_ctx.info.fd_options = rte_cpu_to_le_16(cfg);
1552                 vsi_ctx.info.sw_id = hw->port_info->sw_id;
1553                 vsi_ctx.info.sw_flags2 = ICE_AQ_VSI_SW_FLAG_LAN_ENA;
1554                 ret = ice_vsi_config_tc_queue_mapping(vsi,
1555                                                       &vsi_ctx.info,
1556                                                       ICE_DEFAULT_TCMAP);
1557                 if (ret) {
1558                         PMD_INIT_LOG(ERR,
1559                                      "tc queue mapping with vsi failed, "
1560                                      "err = %d",
1561                                      ret);
1562                         goto fail_mem;
1563                 }
1564                 break;
1565         default:
1566                 /* for other types of VSI */
1567                 PMD_INIT_LOG(ERR, "other types of VSI not supported");
1568                 goto fail_mem;
1569         }
1570
1571         /* VF has MSIX interrupt in VF range, don't allocate here */
1572         if (type == ICE_VSI_PF) {
1573                 ret = ice_res_pool_alloc(&pf->msix_pool,
1574                                          RTE_MIN(vsi->nb_qps,
1575                                                  RTE_MAX_RXTX_INTR_VEC_ID));
1576                 if (ret < 0) {
1577                         PMD_INIT_LOG(ERR, "VSI MAIN %d get heap failed %d",
1578                                      vsi->vsi_id, ret);
1579                 }
1580                 vsi->msix_intr = ret;
1581                 vsi->nb_msix = RTE_MIN(vsi->nb_qps, RTE_MAX_RXTX_INTR_VEC_ID);
1582         } else if (type == ICE_VSI_CTRL) {
1583                 ret = ice_res_pool_alloc(&pf->msix_pool, 1);
1584                 if (ret < 0) {
1585                         PMD_DRV_LOG(ERR, "VSI %d get heap failed %d",
1586                                     vsi->vsi_id, ret);
1587                 }
1588                 vsi->msix_intr = ret;
1589                 vsi->nb_msix = 1;
1590         } else {
1591                 vsi->msix_intr = 0;
1592                 vsi->nb_msix = 0;
1593         }
1594         ret = ice_add_vsi(hw, vsi->idx, &vsi_ctx, NULL);
1595         if (ret != ICE_SUCCESS) {
1596                 PMD_INIT_LOG(ERR, "add vsi failed, err = %d", ret);
1597                 goto fail_mem;
1598         }
1599         /* store vsi information is SW structure */
1600         vsi->vsi_id = vsi_ctx.vsi_num;
1601         vsi->info = vsi_ctx.info;
1602         pf->vsis_allocated = vsi_ctx.vsis_allocd;
1603         pf->vsis_unallocated = vsi_ctx.vsis_unallocated;
1604
1605         if (type == ICE_VSI_PF) {
1606                 /* MAC configuration */
1607                 rte_ether_addr_copy((struct rte_ether_addr *)
1608                                         hw->port_info->mac.perm_addr,
1609                                     &pf->dev_addr);
1610
1611                 rte_ether_addr_copy(&pf->dev_addr, &mac_addr);
1612                 ret = ice_add_mac_filter(vsi, &mac_addr);
1613                 if (ret != ICE_SUCCESS)
1614                         PMD_INIT_LOG(ERR, "Failed to add dflt MAC filter");
1615
1616                 rte_ether_addr_copy(&broadcast, &mac_addr);
1617                 ret = ice_add_mac_filter(vsi, &mac_addr);
1618                 if (ret != ICE_SUCCESS)
1619                         PMD_INIT_LOG(ERR, "Failed to add MAC filter");
1620         }
1621
1622         /* At the beginning, only TC0. */
1623         /* What we need here is the maximam number of the TX queues.
1624          * Currently vsi->nb_qps means it.
1625          * Correct it if any change.
1626          */
1627         max_txqs[0] = vsi->nb_qps;
1628         ret = ice_cfg_vsi_lan(hw->port_info, vsi->idx,
1629                               tc_bitmap, max_txqs);
1630         if (ret != ICE_SUCCESS)
1631                 PMD_INIT_LOG(ERR, "Failed to config vsi sched");
1632
1633         return vsi;
1634 fail_mem:
1635         rte_free(vsi);
1636         pf->next_vsi_idx--;
1637         return NULL;
1638 }
1639
1640 static int
1641 ice_send_driver_ver(struct ice_hw *hw)
1642 {
1643         struct ice_driver_ver dv;
1644
1645         /* we don't have driver version use 0 for dummy */
1646         dv.major_ver = 0;
1647         dv.minor_ver = 0;
1648         dv.build_ver = 0;
1649         dv.subbuild_ver = 0;
1650         strncpy((char *)dv.driver_string, "dpdk", sizeof(dv.driver_string));
1651
1652         return ice_aq_send_driver_ver(hw, &dv, NULL);
1653 }
1654
1655 static int
1656 ice_pf_setup(struct ice_pf *pf)
1657 {
1658         struct ice_hw *hw = ICE_PF_TO_HW(pf);
1659         struct ice_vsi *vsi;
1660         uint16_t unused;
1661
1662         /* Clear all stats counters */
1663         pf->offset_loaded = false;
1664         memset(&pf->stats, 0, sizeof(struct ice_hw_port_stats));
1665         memset(&pf->stats_offset, 0, sizeof(struct ice_hw_port_stats));
1666         memset(&pf->internal_stats, 0, sizeof(struct ice_eth_stats));
1667         memset(&pf->internal_stats_offset, 0, sizeof(struct ice_eth_stats));
1668
1669         /* force guaranteed filter pool for PF */
1670         ice_alloc_fd_guar_item(hw, &unused,
1671                                hw->func_caps.fd_fltr_guar);
1672         /* force shared filter pool for PF */
1673         ice_alloc_fd_shrd_item(hw, &unused,
1674                                hw->func_caps.fd_fltr_best_effort);
1675
1676         vsi = ice_setup_vsi(pf, ICE_VSI_PF);
1677         if (!vsi) {
1678                 PMD_INIT_LOG(ERR, "Failed to add vsi for PF");
1679                 return -EINVAL;
1680         }
1681
1682         pf->main_vsi = vsi;
1683
1684         return 0;
1685 }
1686
1687 static enum ice_pkg_type
1688 ice_load_pkg_type(struct ice_hw *hw)
1689 {
1690         enum ice_pkg_type package_type;
1691
1692         /* store the activated package type (OS default or Comms) */
1693         if (!strncmp((char *)hw->active_pkg_name, ICE_OS_DEFAULT_PKG_NAME,
1694                 ICE_PKG_NAME_SIZE))
1695                 package_type = ICE_PKG_TYPE_OS_DEFAULT;
1696         else if (!strncmp((char *)hw->active_pkg_name, ICE_COMMS_PKG_NAME,
1697                 ICE_PKG_NAME_SIZE))
1698                 package_type = ICE_PKG_TYPE_COMMS;
1699         else
1700                 package_type = ICE_PKG_TYPE_UNKNOWN;
1701
1702         PMD_INIT_LOG(NOTICE, "Active package is: %d.%d.%d.%d, %s (%s VLAN mode)",
1703                 hw->active_pkg_ver.major, hw->active_pkg_ver.minor,
1704                 hw->active_pkg_ver.update, hw->active_pkg_ver.draft,
1705                 hw->active_pkg_name,
1706                 ice_is_dvm_ena(hw) ? "double" : "single");
1707
1708         return package_type;
1709 }
1710
1711 int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
1712 {
1713         struct ice_hw *hw = &adapter->hw;
1714         char pkg_file[ICE_MAX_PKG_FILENAME_SIZE];
1715         char opt_ddp_filename[ICE_MAX_PKG_FILENAME_SIZE];
1716         void *buf;
1717         size_t bufsz;
1718         int err;
1719
1720         if (!use_dsn)
1721                 goto no_dsn;
1722
1723         memset(opt_ddp_filename, 0, ICE_MAX_PKG_FILENAME_SIZE);
1724         snprintf(opt_ddp_filename, ICE_MAX_PKG_FILENAME_SIZE,
1725                 "ice-%016" PRIx64 ".pkg", dsn);
1726         strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_UPDATES,
1727                 ICE_MAX_PKG_FILENAME_SIZE);
1728         strcat(pkg_file, opt_ddp_filename);
1729         if (rte_firmware_read(pkg_file, &buf, &bufsz) == 0)
1730                 goto load_fw;
1731
1732         strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_DEFAULT,
1733                 ICE_MAX_PKG_FILENAME_SIZE);
1734         strcat(pkg_file, opt_ddp_filename);
1735         if (rte_firmware_read(pkg_file, &buf, &bufsz) == 0)
1736                 goto load_fw;
1737
1738 no_dsn:
1739         strncpy(pkg_file, ICE_PKG_FILE_UPDATES, ICE_MAX_PKG_FILENAME_SIZE);
1740         if (rte_firmware_read(pkg_file, &buf, &bufsz) == 0)
1741                 goto load_fw;
1742
1743         strncpy(pkg_file, ICE_PKG_FILE_DEFAULT, ICE_MAX_PKG_FILENAME_SIZE);
1744         if (rte_firmware_read(pkg_file, &buf, &bufsz) < 0) {
1745                 PMD_INIT_LOG(ERR, "failed to search file path\n");
1746                 return -1;
1747         }
1748
1749 load_fw:
1750         PMD_INIT_LOG(DEBUG, "DDP package name: %s", pkg_file);
1751
1752         err = ice_copy_and_init_pkg(hw, buf, bufsz);
1753         if (err) {
1754                 PMD_INIT_LOG(ERR, "ice_copy_and_init_hw failed: %d\n", err);
1755                 goto out;
1756         }
1757
1758         /* store the loaded pkg type info */
1759         adapter->active_pkg_type = ice_load_pkg_type(hw);
1760
1761 out:
1762         free(buf);
1763         return err;
1764 }
1765
1766 static void
1767 ice_base_queue_get(struct ice_pf *pf)
1768 {
1769         uint32_t reg;
1770         struct ice_hw *hw = ICE_PF_TO_HW(pf);
1771
1772         reg = ICE_READ_REG(hw, PFLAN_RX_QALLOC);
1773         if (reg & PFLAN_RX_QALLOC_VALID_M) {
1774                 pf->base_queue = reg & PFLAN_RX_QALLOC_FIRSTQ_M;
1775         } else {
1776                 PMD_INIT_LOG(WARNING, "Failed to get Rx base queue"
1777                                         " index");
1778         }
1779 }
1780
1781 static int
1782 parse_bool(const char *key, const char *value, void *args)
1783 {
1784         int *i = (int *)args;
1785         char *end;
1786         int num;
1787
1788         num = strtoul(value, &end, 10);
1789
1790         if (num != 0 && num != 1) {
1791                 PMD_DRV_LOG(WARNING, "invalid value:\"%s\" for key:\"%s\", "
1792                         "value must be 0 or 1",
1793                         value, key);
1794                 return -1;
1795         }
1796
1797         *i = num;
1798         return 0;
1799 }
1800
1801 static int
1802 parse_u64(const char *key, const char *value, void *args)
1803 {
1804         u64 *num = (u64 *)args;
1805         u64 tmp;
1806
1807         errno = 0;
1808         tmp = strtoull(value, NULL, 16);
1809         if (errno) {
1810                 PMD_DRV_LOG(WARNING, "%s: \"%s\" is not a valid u64",
1811                             key, value);
1812                 return -1;
1813         }
1814
1815         *num = tmp;
1816
1817         return 0;
1818 }
1819
1820 static int
1821 lookup_pps_type(const char *pps_name)
1822 {
1823         static struct {
1824                 const char *name;
1825                 enum pps_type type;
1826         } pps_type_map[] = {
1827                 { "pin",  PPS_PIN  },
1828         };
1829
1830         uint32_t i;
1831
1832         for (i = 0; i < RTE_DIM(pps_type_map); i++) {
1833                 if (strcmp(pps_name, pps_type_map[i].name) == 0)
1834                         return pps_type_map[i].type;
1835         }
1836
1837         return -1;
1838 }
1839
1840 static int
1841 parse_pin_set(const char *input, int pps_type, struct ice_devargs *devargs)
1842 {
1843         const char *str = input;
1844         char *end = NULL;
1845         uint32_t idx;
1846
1847         while (isblank(*str))
1848                 str++;
1849
1850         if (!isdigit(*str))
1851                 return -1;
1852
1853         if (pps_type == PPS_PIN) {
1854                 idx = strtoul(str, &end, 10);
1855                 if (end == NULL || idx >= ICE_MAX_PIN_NUM)
1856                         return -1;
1857                 while (isblank(*end))
1858                         end++;
1859                 if (*end != ']')
1860                         return -1;
1861
1862                 devargs->pin_idx = idx;
1863                 devargs->pps_out_ena = 1;
1864
1865                 return 0;
1866         }
1867
1868         return -1;
1869 }
1870
1871 static int
1872 parse_pps_out_parameter(const char *pins, struct ice_devargs *devargs)
1873 {
1874         const char *pin_start;
1875         uint32_t idx;
1876         int pps_type;
1877         char pps_name[32];
1878
1879         while (isblank(*pins))
1880                 pins++;
1881
1882         pins++;
1883         while (isblank(*pins))
1884                 pins++;
1885         if (*pins == '\0')
1886                 return -1;
1887
1888         for (idx = 0; ; idx++) {
1889                 if (isblank(pins[idx]) ||
1890                     pins[idx] == ':' ||
1891                     pins[idx] == '\0')
1892                         break;
1893
1894                 pps_name[idx] = pins[idx];
1895         }
1896         pps_name[idx] = '\0';
1897         pps_type = lookup_pps_type(pps_name);
1898         if (pps_type < 0)
1899                 return -1;
1900
1901         pins += idx;
1902
1903         pins += strcspn(pins, ":");
1904         if (*pins++ != ':')
1905                 return -1;
1906         while (isblank(*pins))
1907                 pins++;
1908
1909         pin_start = pins;
1910
1911         while (isblank(*pins))
1912                 pins++;
1913
1914         if (parse_pin_set(pin_start, pps_type, devargs) < 0)
1915                 return -1;
1916
1917         return 0;
1918 }
1919
1920 static int
1921 handle_pps_out_arg(__rte_unused const char *key, const char *value,
1922                    void *extra_args)
1923 {
1924         struct ice_devargs *devargs = extra_args;
1925
1926         if (value == NULL || extra_args == NULL)
1927                 return -EINVAL;
1928
1929         if (parse_pps_out_parameter(value, devargs) < 0) {
1930                 PMD_DRV_LOG(ERR,
1931                             "The GPIO pin parameter is wrong : '%s'",
1932                             value);
1933                 return -1;
1934         }
1935
1936         return 0;
1937 }
1938
1939 static int ice_parse_devargs(struct rte_eth_dev *dev)
1940 {
1941         struct ice_adapter *ad =
1942                 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1943         struct rte_devargs *devargs = dev->device->devargs;
1944         struct rte_kvargs *kvlist;
1945         int ret;
1946
1947         if (devargs == NULL)
1948                 return 0;
1949
1950         kvlist = rte_kvargs_parse(devargs->args, ice_valid_args);
1951         if (kvlist == NULL) {
1952                 PMD_INIT_LOG(ERR, "Invalid kvargs key\n");
1953                 return -EINVAL;
1954         }
1955
1956         ad->devargs.proto_xtr_dflt = PROTO_XTR_NONE;
1957         memset(ad->devargs.proto_xtr, PROTO_XTR_NONE,
1958                sizeof(ad->devargs.proto_xtr));
1959
1960         ret = rte_kvargs_process(kvlist, ICE_PROTO_XTR_ARG,
1961                                  &handle_proto_xtr_arg, &ad->devargs);
1962         if (ret)
1963                 goto bail;
1964
1965         ret = rte_kvargs_process(kvlist, ICE_SAFE_MODE_SUPPORT_ARG,
1966                                  &parse_bool, &ad->devargs.safe_mode_support);
1967         if (ret)
1968                 goto bail;
1969
1970         ret = rte_kvargs_process(kvlist, ICE_PIPELINE_MODE_SUPPORT_ARG,
1971                                  &parse_bool, &ad->devargs.pipe_mode_support);
1972         if (ret)
1973                 goto bail;
1974
1975         ret = rte_kvargs_process(kvlist, ICE_HW_DEBUG_MASK_ARG,
1976                                  &parse_u64, &ad->hw.debug_mask);
1977         if (ret)
1978                 goto bail;
1979
1980         ret = rte_kvargs_process(kvlist, ICE_ONE_PPS_OUT_ARG,
1981                                  &handle_pps_out_arg, &ad->devargs);
1982         if (ret)
1983                 goto bail;
1984
1985         ret = rte_kvargs_process(kvlist, ICE_RX_LOW_LATENCY_ARG,
1986                                  &parse_bool, &ad->devargs.rx_low_latency);
1987
1988 bail:
1989         rte_kvargs_free(kvlist);
1990         return ret;
1991 }
1992
1993 /* Forward LLDP packets to default VSI by set switch rules */
1994 static int
1995 ice_vsi_config_sw_lldp(struct ice_vsi *vsi,  bool on)
1996 {
1997         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
1998         struct ice_fltr_list_entry *s_list_itr = NULL;
1999         struct LIST_HEAD_TYPE list_head;
2000         int ret = 0;
2001
2002         INIT_LIST_HEAD(&list_head);
2003
2004         s_list_itr = (struct ice_fltr_list_entry *)
2005                         ice_malloc(hw, sizeof(*s_list_itr));
2006         if (!s_list_itr)
2007                 return -ENOMEM;
2008         s_list_itr->fltr_info.lkup_type = ICE_SW_LKUP_ETHERTYPE;
2009         s_list_itr->fltr_info.vsi_handle = vsi->idx;
2010         s_list_itr->fltr_info.l_data.ethertype_mac.ethertype =
2011                         RTE_ETHER_TYPE_LLDP;
2012         s_list_itr->fltr_info.fltr_act = ICE_FWD_TO_VSI;
2013         s_list_itr->fltr_info.flag = ICE_FLTR_RX;
2014         s_list_itr->fltr_info.src_id = ICE_SRC_ID_LPORT;
2015         LIST_ADD(&s_list_itr->list_entry, &list_head);
2016         if (on)
2017                 ret = ice_add_eth_mac(hw, &list_head);
2018         else
2019                 ret = ice_remove_eth_mac(hw, &list_head);
2020
2021         rte_free(s_list_itr);
2022         return ret;
2023 }
2024
2025 static enum ice_status
2026 ice_get_hw_res(struct ice_hw *hw, uint16_t res_type,
2027                 uint16_t num, uint16_t desc_id,
2028                 uint16_t *prof_buf, uint16_t *num_prof)
2029 {
2030         struct ice_aqc_res_elem *resp_buf;
2031         int ret;
2032         uint16_t buf_len;
2033         bool res_shared = 1;
2034         struct ice_aq_desc aq_desc;
2035         struct ice_sq_cd *cd = NULL;
2036         struct ice_aqc_get_allocd_res_desc *cmd =
2037                         &aq_desc.params.get_res_desc;
2038
2039         buf_len = sizeof(*resp_buf) * num;
2040         resp_buf = ice_malloc(hw, buf_len);
2041         if (!resp_buf)
2042                 return -ENOMEM;
2043
2044         ice_fill_dflt_direct_cmd_desc(&aq_desc,
2045                         ice_aqc_opc_get_allocd_res_desc);
2046
2047         cmd->ops.cmd.res = CPU_TO_LE16(((res_type << ICE_AQC_RES_TYPE_S) &
2048                                 ICE_AQC_RES_TYPE_M) | (res_shared ?
2049                                 ICE_AQC_RES_TYPE_FLAG_SHARED : 0));
2050         cmd->ops.cmd.first_desc = CPU_TO_LE16(desc_id);
2051
2052         ret = ice_aq_send_cmd(hw, &aq_desc, resp_buf, buf_len, cd);
2053         if (!ret)
2054                 *num_prof = LE16_TO_CPU(cmd->ops.resp.num_desc);
2055         else
2056                 goto exit;
2057
2058         ice_memcpy(prof_buf, resp_buf, sizeof(*resp_buf) *
2059                         (*num_prof), ICE_NONDMA_TO_NONDMA);
2060
2061 exit:
2062         rte_free(resp_buf);
2063         return ret;
2064 }
2065 static int
2066 ice_cleanup_resource(struct ice_hw *hw, uint16_t res_type)
2067 {
2068         int ret;
2069         uint16_t prof_id;
2070         uint16_t prof_buf[ICE_MAX_RES_DESC_NUM];
2071         uint16_t first_desc = 1;
2072         uint16_t num_prof = 0;
2073
2074         ret = ice_get_hw_res(hw, res_type, ICE_MAX_RES_DESC_NUM,
2075                         first_desc, prof_buf, &num_prof);
2076         if (ret) {
2077                 PMD_INIT_LOG(ERR, "Failed to get fxp resource");
2078                 return ret;
2079         }
2080
2081         for (prof_id = 0; prof_id < num_prof; prof_id++) {
2082                 ret = ice_free_hw_res(hw, res_type, 1, &prof_buf[prof_id]);
2083                 if (ret) {
2084                         PMD_INIT_LOG(ERR, "Failed to free fxp resource");
2085                         return ret;
2086                 }
2087         }
2088         return 0;
2089 }
2090
2091 static int
2092 ice_reset_fxp_resource(struct ice_hw *hw)
2093 {
2094         int ret;
2095
2096         ret = ice_cleanup_resource(hw, ICE_AQC_RES_TYPE_FD_PROF_BLDR_PROFID);
2097         if (ret) {
2098                 PMD_INIT_LOG(ERR, "Failed to clearup fdir resource");
2099                 return ret;
2100         }
2101
2102         ret = ice_cleanup_resource(hw, ICE_AQC_RES_TYPE_HASH_PROF_BLDR_PROFID);
2103         if (ret) {
2104                 PMD_INIT_LOG(ERR, "Failed to clearup rss resource");
2105                 return ret;
2106         }
2107
2108         return 0;
2109 }
2110
2111 static void
2112 ice_rss_ctx_init(struct ice_pf *pf)
2113 {
2114         memset(&pf->hash_ctx, 0, sizeof(pf->hash_ctx));
2115 }
2116
2117 static uint64_t
2118 ice_get_supported_rxdid(struct ice_hw *hw)
2119 {
2120         uint64_t supported_rxdid = 0; /* bitmap for supported RXDID */
2121         uint32_t regval;
2122         int i;
2123
2124         supported_rxdid |= BIT(ICE_RXDID_LEGACY_1);
2125
2126         for (i = ICE_RXDID_FLEX_NIC; i < ICE_FLEX_DESC_RXDID_MAX_NUM; i++) {
2127                 regval = ICE_READ_REG(hw, GLFLXP_RXDID_FLAGS(i, 0));
2128                 if ((regval >> GLFLXP_RXDID_FLAGS_FLEXIFLAG_4N_S)
2129                         & GLFLXP_RXDID_FLAGS_FLEXIFLAG_4N_M)
2130                         supported_rxdid |= BIT(i);
2131         }
2132         return supported_rxdid;
2133 }
2134
2135 static int
2136 ice_dev_init(struct rte_eth_dev *dev)
2137 {
2138         struct rte_pci_device *pci_dev;
2139         struct rte_intr_handle *intr_handle;
2140         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2141         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2142         struct ice_adapter *ad =
2143                 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
2144         struct ice_vsi *vsi;
2145         int ret;
2146 #ifndef RTE_EXEC_ENV_WINDOWS
2147         off_t pos;
2148         uint32_t dsn_low, dsn_high;
2149         uint64_t dsn;
2150         bool use_dsn;
2151 #endif
2152
2153         dev->dev_ops = &ice_eth_dev_ops;
2154         dev->rx_queue_count = ice_rx_queue_count;
2155         dev->rx_descriptor_status = ice_rx_descriptor_status;
2156         dev->tx_descriptor_status = ice_tx_descriptor_status;
2157         dev->rx_pkt_burst = ice_recv_pkts;
2158         dev->tx_pkt_burst = ice_xmit_pkts;
2159         dev->tx_pkt_prepare = ice_prep_pkts;
2160
2161         /* for secondary processes, we don't initialise any further as primary
2162          * has already done this work.
2163          */
2164         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2165                 ice_set_rx_function(dev);
2166                 ice_set_tx_function(dev);
2167                 return 0;
2168         }
2169
2170         dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
2171
2172         ice_set_default_ptype_table(dev);
2173         pci_dev = RTE_DEV_TO_PCI(dev->device);
2174         intr_handle = &pci_dev->intr_handle;
2175
2176         pf->adapter = ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
2177         pf->dev_data = dev->data;
2178         hw->back = pf->adapter;
2179         hw->hw_addr = (uint8_t *)pci_dev->mem_resource[0].addr;
2180         hw->vendor_id = pci_dev->id.vendor_id;
2181         hw->device_id = pci_dev->id.device_id;
2182         hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
2183         hw->subsystem_device_id = pci_dev->id.subsystem_device_id;
2184         hw->bus.device = pci_dev->addr.devid;
2185         hw->bus.func = pci_dev->addr.function;
2186
2187         ret = ice_parse_devargs(dev);
2188         if (ret) {
2189                 PMD_INIT_LOG(ERR, "Failed to parse devargs");
2190                 return -EINVAL;
2191         }
2192
2193         ice_init_controlq_parameter(hw);
2194
2195         ret = ice_init_hw(hw);
2196         if (ret) {
2197                 PMD_INIT_LOG(ERR, "Failed to initialize HW");
2198                 return -EINVAL;
2199         }
2200
2201 #ifndef RTE_EXEC_ENV_WINDOWS
2202         use_dsn = false;
2203         dsn = 0;
2204         pos = rte_pci_find_ext_capability(pci_dev, RTE_PCI_EXT_CAP_ID_DSN);
2205         if (pos) {
2206                 if (rte_pci_read_config(pci_dev, &dsn_low, 4, pos + 4) < 0 ||
2207                                 rte_pci_read_config(pci_dev, &dsn_high, 4, pos + 8) < 0) {
2208                         PMD_INIT_LOG(ERR, "Failed to read pci config space\n");
2209                 } else {
2210                         use_dsn = true;
2211                         dsn = (uint64_t)dsn_high << 32 | dsn_low;
2212                 }
2213         } else {
2214                 PMD_INIT_LOG(ERR, "Failed to read device serial number\n");
2215         }
2216
2217         ret = ice_load_pkg(pf->adapter, use_dsn, dsn);
2218         if (ret == 0) {
2219                 ret = ice_init_hw_tbls(hw);
2220                 if (ret) {
2221                         PMD_INIT_LOG(ERR, "ice_init_hw_tbls failed: %d\n", ret);
2222                         rte_free(hw->pkg_copy);
2223                 }
2224         }
2225
2226         if (ret) {
2227                 if (ad->devargs.safe_mode_support == 0) {
2228                         PMD_INIT_LOG(ERR, "Failed to load the DDP package,"
2229                                         "Use safe-mode-support=1 to enter Safe Mode");
2230                         goto err_init_fw;
2231                 }
2232
2233                 PMD_INIT_LOG(WARNING, "Failed to load the DDP package,"
2234                                         "Entering Safe Mode");
2235                 ad->is_safe_mode = 1;
2236         }
2237 #endif
2238
2239         PMD_INIT_LOG(INFO, "FW %d.%d.%05d API %d.%d",
2240                      hw->fw_maj_ver, hw->fw_min_ver, hw->fw_build,
2241                      hw->api_maj_ver, hw->api_min_ver);
2242
2243         ice_pf_sw_init(dev);
2244         ret = ice_init_mac_address(dev);
2245         if (ret) {
2246                 PMD_INIT_LOG(ERR, "Failed to initialize mac address");
2247                 goto err_init_mac;
2248         }
2249
2250         ret = ice_res_pool_init(&pf->msix_pool, 1,
2251                                 hw->func_caps.common_cap.num_msix_vectors - 1);
2252         if (ret) {
2253                 PMD_INIT_LOG(ERR, "Failed to init MSIX pool");
2254                 goto err_msix_pool_init;
2255         }
2256
2257         ret = ice_pf_setup(pf);
2258         if (ret) {
2259                 PMD_INIT_LOG(ERR, "Failed to setup PF");
2260                 goto err_pf_setup;
2261         }
2262
2263         ret = ice_send_driver_ver(hw);
2264         if (ret) {
2265                 PMD_INIT_LOG(ERR, "Failed to send driver version");
2266                 goto err_pf_setup;
2267         }
2268
2269         vsi = pf->main_vsi;
2270
2271         ret = ice_aq_stop_lldp(hw, true, false, NULL);
2272         if (ret != ICE_SUCCESS)
2273                 PMD_INIT_LOG(DEBUG, "lldp has already stopped\n");
2274         ret = ice_init_dcb(hw, true);
2275         if (ret != ICE_SUCCESS)
2276                 PMD_INIT_LOG(DEBUG, "Failed to init DCB\n");
2277         /* Forward LLDP packets to default VSI */
2278         ret = ice_vsi_config_sw_lldp(vsi, true);
2279         if (ret != ICE_SUCCESS)
2280                 PMD_INIT_LOG(DEBUG, "Failed to cfg lldp\n");
2281         /* register callback func to eal lib */
2282         rte_intr_callback_register(intr_handle,
2283                                    ice_interrupt_handler, dev);
2284
2285         ice_pf_enable_irq0(hw);
2286
2287         /* enable uio intr after callback register */
2288         rte_intr_enable(intr_handle);
2289
2290         /* get base queue pairs index  in the device */
2291         ice_base_queue_get(pf);
2292
2293         /* Initialize RSS context for gtpu_eh */
2294         ice_rss_ctx_init(pf);
2295
2296         if (!ad->is_safe_mode) {
2297                 ret = ice_flow_init(ad);
2298                 if (ret) {
2299                         PMD_INIT_LOG(ERR, "Failed to initialize flow");
2300                         goto err_flow_init;
2301                 }
2302         }
2303
2304         ret = ice_reset_fxp_resource(hw);
2305         if (ret) {
2306                 PMD_INIT_LOG(ERR, "Failed to reset fxp resource");
2307                 goto err_flow_init;
2308         }
2309
2310         pf->supported_rxdid = ice_get_supported_rxdid(hw);
2311
2312         return 0;
2313
2314 err_flow_init:
2315         ice_flow_uninit(ad);
2316         rte_intr_disable(intr_handle);
2317         ice_pf_disable_irq0(hw);
2318         rte_intr_callback_unregister(intr_handle,
2319                                      ice_interrupt_handler, dev);
2320 err_pf_setup:
2321         ice_res_pool_destroy(&pf->msix_pool);
2322 err_msix_pool_init:
2323         rte_free(dev->data->mac_addrs);
2324         dev->data->mac_addrs = NULL;
2325 err_init_mac:
2326         rte_free(pf->proto_xtr);
2327 #ifndef RTE_EXEC_ENV_WINDOWS
2328 err_init_fw:
2329 #endif
2330         ice_deinit_hw(hw);
2331
2332         return ret;
2333 }
2334
2335 int
2336 ice_release_vsi(struct ice_vsi *vsi)
2337 {
2338         struct ice_hw *hw;
2339         struct ice_vsi_ctx vsi_ctx;
2340         enum ice_status ret;
2341         int error = 0;
2342
2343         if (!vsi)
2344                 return error;
2345
2346         hw = ICE_VSI_TO_HW(vsi);
2347
2348         ice_remove_all_mac_vlan_filters(vsi);
2349
2350         memset(&vsi_ctx, 0, sizeof(vsi_ctx));
2351
2352         vsi_ctx.vsi_num = vsi->vsi_id;
2353         vsi_ctx.info = vsi->info;
2354         ret = ice_free_vsi(hw, vsi->idx, &vsi_ctx, false, NULL);
2355         if (ret != ICE_SUCCESS) {
2356                 PMD_INIT_LOG(ERR, "Failed to free vsi by aq, %u", vsi->vsi_id);
2357                 error = -1;
2358         }
2359
2360         rte_free(vsi->rss_lut);
2361         rte_free(vsi->rss_key);
2362         rte_free(vsi);
2363         return error;
2364 }
2365
2366 void
2367 ice_vsi_disable_queues_intr(struct ice_vsi *vsi)
2368 {
2369         struct rte_eth_dev *dev = &rte_eth_devices[vsi->adapter->pf.dev_data->port_id];
2370         struct rte_pci_device *pci_dev = ICE_DEV_TO_PCI(dev);
2371         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
2372         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
2373         uint16_t msix_intr, i;
2374
2375         /* disable interrupt and also clear all the exist config */
2376         for (i = 0; i < vsi->nb_qps; i++) {
2377                 ICE_WRITE_REG(hw, QINT_TQCTL(vsi->base_queue + i), 0);
2378                 ICE_WRITE_REG(hw, QINT_RQCTL(vsi->base_queue + i), 0);
2379                 rte_wmb();
2380         }
2381
2382         if (rte_intr_allow_others(intr_handle))
2383                 /* vfio-pci */
2384                 for (i = 0; i < vsi->nb_msix; i++) {
2385                         msix_intr = vsi->msix_intr + i;
2386                         ICE_WRITE_REG(hw, GLINT_DYN_CTL(msix_intr),
2387                                       GLINT_DYN_CTL_WB_ON_ITR_M);
2388                 }
2389         else
2390                 /* igb_uio */
2391                 ICE_WRITE_REG(hw, GLINT_DYN_CTL(0), GLINT_DYN_CTL_WB_ON_ITR_M);
2392 }
2393
2394 static int
2395 ice_dev_stop(struct rte_eth_dev *dev)
2396 {
2397         struct rte_eth_dev_data *data = dev->data;
2398         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2399         struct ice_vsi *main_vsi = pf->main_vsi;
2400         struct rte_pci_device *pci_dev = ICE_DEV_TO_PCI(dev);
2401         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
2402         uint16_t i;
2403
2404         /* avoid stopping again */
2405         if (pf->adapter_stopped)
2406                 return 0;
2407
2408         /* stop and clear all Rx queues */
2409         for (i = 0; i < data->nb_rx_queues; i++)
2410                 ice_rx_queue_stop(dev, i);
2411
2412         /* stop and clear all Tx queues */
2413         for (i = 0; i < data->nb_tx_queues; i++)
2414                 ice_tx_queue_stop(dev, i);
2415
2416         /* disable all queue interrupts */
2417         ice_vsi_disable_queues_intr(main_vsi);
2418
2419         if (pf->init_link_up)
2420                 ice_dev_set_link_up(dev);
2421         else
2422                 ice_dev_set_link_down(dev);
2423
2424         /* Clean datapath event and queue/vec mapping */
2425         rte_intr_efd_disable(intr_handle);
2426         if (intr_handle->intr_vec) {
2427                 rte_free(intr_handle->intr_vec);
2428                 intr_handle->intr_vec = NULL;
2429         }
2430
2431         pf->adapter_stopped = true;
2432         dev->data->dev_started = 0;
2433
2434         return 0;
2435 }
2436
2437 static int
2438 ice_dev_close(struct rte_eth_dev *dev)
2439 {
2440         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2441         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2442         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2443         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
2444         struct ice_adapter *ad =
2445                 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
2446         int ret;
2447         uint32_t val;
2448         uint8_t timer = hw->func_caps.ts_func_info.tmr_index_owned;
2449         uint32_t pin_idx = ad->devargs.pin_idx;
2450
2451         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2452                 return 0;
2453
2454         /* Since stop will make link down, then the link event will be
2455          * triggered, disable the irq firstly to avoid the port_infoe etc
2456          * resources deallocation causing the interrupt service thread
2457          * crash.
2458          */
2459         ice_pf_disable_irq0(hw);
2460
2461         ret = ice_dev_stop(dev);
2462
2463         if (!ad->is_safe_mode)
2464                 ice_flow_uninit(ad);
2465
2466         /* release all queue resource */
2467         ice_free_queues(dev);
2468
2469         ice_res_pool_destroy(&pf->msix_pool);
2470         ice_release_vsi(pf->main_vsi);
2471         ice_sched_cleanup_all(hw);
2472         ice_free_hw_tbls(hw);
2473         rte_free(hw->port_info);
2474         hw->port_info = NULL;
2475         ice_shutdown_all_ctrlq(hw);
2476         rte_free(pf->proto_xtr);
2477         pf->proto_xtr = NULL;
2478
2479         if (ad->devargs.pps_out_ena) {
2480                 ICE_WRITE_REG(hw, GLTSYN_AUX_OUT(pin_idx, timer), 0);
2481                 ICE_WRITE_REG(hw, GLTSYN_CLKO(pin_idx, timer), 0);
2482                 ICE_WRITE_REG(hw, GLTSYN_TGT_L(pin_idx, timer), 0);
2483                 ICE_WRITE_REG(hw, GLTSYN_TGT_H(pin_idx, timer), 0);
2484
2485                 val = GLGEN_GPIO_CTL_PIN_DIR_M;
2486                 ICE_WRITE_REG(hw, GLGEN_GPIO_CTL(pin_idx), val);
2487         }
2488
2489         /* disable uio intr before callback unregister */
2490         rte_intr_disable(intr_handle);
2491
2492         /* unregister callback func from eal lib */
2493         rte_intr_callback_unregister(intr_handle,
2494                                      ice_interrupt_handler, dev);
2495
2496         return ret;
2497 }
2498
2499 static int
2500 ice_dev_uninit(struct rte_eth_dev *dev)
2501 {
2502         ice_dev_close(dev);
2503
2504         return 0;
2505 }
2506
2507 static bool
2508 is_hash_cfg_valid(struct ice_rss_hash_cfg *cfg)
2509 {
2510         return (cfg->hash_flds != 0 && cfg->addl_hdrs != 0) ? true : false;
2511 }
2512
2513 static void
2514 hash_cfg_reset(struct ice_rss_hash_cfg *cfg)
2515 {
2516         cfg->hash_flds = 0;
2517         cfg->addl_hdrs = 0;
2518         cfg->symm = 0;
2519         cfg->hdr_type = ICE_RSS_OUTER_HEADERS;
2520 }
2521
2522 static int
2523 ice_hash_moveout(struct ice_pf *pf, struct ice_rss_hash_cfg *cfg)
2524 {
2525         enum ice_status status = ICE_SUCCESS;
2526         struct ice_hw *hw = ICE_PF_TO_HW(pf);
2527         struct ice_vsi *vsi = pf->main_vsi;
2528
2529         if (!is_hash_cfg_valid(cfg))
2530                 return -ENOENT;
2531
2532         status = ice_rem_rss_cfg(hw, vsi->idx, cfg);
2533         if (status && status != ICE_ERR_DOES_NOT_EXIST) {
2534                 PMD_DRV_LOG(ERR,
2535                             "ice_rem_rss_cfg failed for VSI:%d, error:%d\n",
2536                             vsi->idx, status);
2537                 return -EBUSY;
2538         }
2539
2540         return 0;
2541 }
2542
2543 static int
2544 ice_hash_moveback(struct ice_pf *pf, struct ice_rss_hash_cfg *cfg)
2545 {
2546         enum ice_status status = ICE_SUCCESS;
2547         struct ice_hw *hw = ICE_PF_TO_HW(pf);
2548         struct ice_vsi *vsi = pf->main_vsi;
2549
2550         if (!is_hash_cfg_valid(cfg))
2551                 return -ENOENT;
2552
2553         status = ice_add_rss_cfg(hw, vsi->idx, cfg);
2554         if (status) {
2555                 PMD_DRV_LOG(ERR,
2556                             "ice_add_rss_cfg failed for VSI:%d, error:%d\n",
2557                             vsi->idx, status);
2558                 return -EBUSY;
2559         }
2560
2561         return 0;
2562 }
2563
2564 static int
2565 ice_hash_remove(struct ice_pf *pf, struct ice_rss_hash_cfg *cfg)
2566 {
2567         int ret;
2568
2569         ret = ice_hash_moveout(pf, cfg);
2570         if (ret && (ret != -ENOENT))
2571                 return ret;
2572
2573         hash_cfg_reset(cfg);
2574
2575         return 0;
2576 }
2577
2578 static int
2579 ice_add_rss_cfg_pre_gtpu(struct ice_pf *pf, struct ice_hash_gtpu_ctx *ctx,
2580                          u8 ctx_idx)
2581 {
2582         int ret;
2583
2584         switch (ctx_idx) {
2585         case ICE_HASH_GTPU_CTX_EH_IP:
2586                 ret = ice_hash_remove(pf,
2587                                       &ctx->ctx[ICE_HASH_GTPU_CTX_EH_IP_UDP]);
2588                 if (ret && (ret != -ENOENT))
2589                         return ret;
2590
2591                 ret = ice_hash_remove(pf,
2592                                       &ctx->ctx[ICE_HASH_GTPU_CTX_EH_IP_TCP]);
2593                 if (ret && (ret != -ENOENT))
2594                         return ret;
2595
2596                 ret = ice_hash_remove(pf,
2597                                       &ctx->ctx[ICE_HASH_GTPU_CTX_UP_IP]);
2598                 if (ret && (ret != -ENOENT))
2599                         return ret;
2600
2601                 ret = ice_hash_remove(pf,
2602                                       &ctx->ctx[ICE_HASH_GTPU_CTX_UP_IP_UDP]);
2603                 if (ret && (ret != -ENOENT))
2604                         return ret;
2605
2606                 ret = ice_hash_remove(pf,
2607                                       &ctx->ctx[ICE_HASH_GTPU_CTX_UP_IP_TCP]);
2608                 if (ret && (ret != -ENOENT))
2609                         return ret;
2610
2611                 ret = ice_hash_remove(pf,
2612                                       &ctx->ctx[ICE_HASH_GTPU_CTX_DW_IP]);
2613                 if (ret && (ret != -ENOENT))
2614                         return ret;
2615
2616                 ret = ice_hash_remove(pf,
2617                                       &ctx->ctx[ICE_HASH_GTPU_CTX_DW_IP_UDP]);
2618                 if (ret && (ret != -ENOENT))
2619                         return ret;
2620
2621                 ret = ice_hash_remove(pf,
2622                                       &ctx->ctx[ICE_HASH_GTPU_CTX_DW_IP_TCP]);
2623                 if (ret && (ret != -ENOENT))
2624                         return ret;
2625
2626                 break;
2627         case ICE_HASH_GTPU_CTX_EH_IP_UDP:
2628                 ret = ice_hash_remove(pf,
2629                                       &ctx->ctx[ICE_HASH_GTPU_CTX_UP_IP_UDP]);
2630                 if (ret && (ret != -ENOENT))
2631                         return ret;
2632
2633                 ret = ice_hash_remove(pf,
2634                                       &ctx->ctx[ICE_HASH_GTPU_CTX_DW_IP_UDP]);
2635                 if (ret && (ret != -ENOENT))
2636                         return ret;
2637
2638                 ret = ice_hash_moveout(pf,
2639                                        &ctx->ctx[ICE_HASH_GTPU_CTX_UP_IP]);
2640                 if (ret && (ret != -ENOENT))
2641                         return ret;
2642
2643                 ret = ice_hash_moveout(pf,
2644                                        &ctx->ctx[ICE_HASH_GTPU_CTX_UP_IP_TCP]);
2645                 if (ret && (ret != -ENOENT))
2646                         return ret;
2647
2648                 ret = ice_hash_moveout(pf,
2649                                        &ctx->ctx[ICE_HASH_GTPU_CTX_DW_IP]);
2650                 if (ret && (ret != -ENOENT))
2651                         return ret;
2652
2653                 ret = ice_hash_moveout(pf,
2654                                        &ctx->ctx[ICE_HASH_GTPU_CTX_DW_IP_TCP]);
2655                 if (ret && (ret != -ENOENT))
2656                         return ret;
2657
2658                 break;
2659         case ICE_HASH_GTPU_CTX_EH_IP_TCP:
2660                 ret = ice_hash_remove(pf,
2661                                       &ctx->ctx[ICE_HASH_GTPU_CTX_UP_IP_TCP]);
2662                 if (ret && (ret != -ENOENT))
2663                         return ret;
2664
2665                 ret = ice_hash_remove(pf,
2666                                       &ctx->ctx[ICE_HASH_GTPU_CTX_DW_IP_TCP]);
2667                 if (ret && (ret != -ENOENT))
2668                         return ret;
2669
2670                 ret = ice_hash_moveout(pf,
2671                                        &ctx->ctx[ICE_HASH_GTPU_CTX_UP_IP]);
2672                 if (ret && (ret != -ENOENT))
2673                         return ret;
2674
2675                 ret = ice_hash_moveout(pf,
2676                                        &ctx->ctx[ICE_HASH_GTPU_CTX_UP_IP_UDP]);
2677                 if (ret && (ret != -ENOENT))
2678                         return ret;
2679
2680                 ret = ice_hash_moveout(pf,
2681                                        &ctx->ctx[ICE_HASH_GTPU_CTX_DW_IP]);
2682                 if (ret && (ret != -ENOENT))
2683                         return ret;
2684
2685                 ret = ice_hash_moveout(pf,
2686                                        &ctx->ctx[ICE_HASH_GTPU_CTX_DW_IP_UDP]);
2687                 if (ret && (ret != -ENOENT))
2688                         return ret;
2689
2690                 break;
2691         case ICE_HASH_GTPU_CTX_UP_IP:
2692                 ret = ice_hash_remove(pf,
2693                                       &ctx->ctx[ICE_HASH_GTPU_CTX_UP_IP_UDP]);
2694                 if (ret && (ret != -ENOENT))
2695                         return ret;
2696
2697                 ret = ice_hash_remove(pf,
2698                                       &ctx->ctx[ICE_HASH_GTPU_CTX_UP_IP_TCP]);
2699                 if (ret && (ret != -ENOENT))
2700                         return ret;
2701
2702                 ret = ice_hash_moveout(pf,
2703                                        &ctx->ctx[ICE_HASH_GTPU_CTX_EH_IP]);
2704                 if (ret && (ret != -ENOENT))
2705                         return ret;
2706
2707                 ret = ice_hash_moveout(pf,
2708                                        &ctx->ctx[ICE_HASH_GTPU_CTX_EH_IP_UDP]);
2709                 if (ret && (ret != -ENOENT))
2710                         return ret;
2711
2712                 ret = ice_hash_moveout(pf,
2713                                        &ctx->ctx[ICE_HASH_GTPU_CTX_EH_IP_TCP]);
2714                 if (ret && (ret != -ENOENT))
2715                         return ret;
2716
2717                 break;
2718         case ICE_HASH_GTPU_CTX_UP_IP_UDP:
2719         case ICE_HASH_GTPU_CTX_UP_IP_TCP:
2720                 ret = ice_hash_moveout(pf,
2721                                        &ctx->ctx[ICE_HASH_GTPU_CTX_EH_IP]);
2722                 if (ret && (ret != -ENOENT))
2723                         return ret;
2724
2725                 ret = ice_hash_moveout(pf,
2726                                        &ctx->ctx[ICE_HASH_GTPU_CTX_EH_IP_UDP]);
2727                 if (ret && (ret != -ENOENT))
2728                         return ret;
2729
2730                 ret = ice_hash_moveout(pf,
2731                                        &ctx->ctx[ICE_HASH_GTPU_CTX_EH_IP_TCP]);
2732                 if (ret && (ret != -ENOENT))
2733                         return ret;
2734
2735                 break;
2736         case ICE_HASH_GTPU_CTX_DW_IP:
2737                 ret = ice_hash_remove(pf,
2738                                       &ctx->ctx[ICE_HASH_GTPU_CTX_DW_IP_UDP]);
2739                 if (ret && (ret != -ENOENT))
2740                         return ret;
2741
2742                 ret = ice_hash_remove(pf,
2743                                       &ctx->ctx[ICE_HASH_GTPU_CTX_DW_IP_TCP]);
2744                 if (ret && (ret != -ENOENT))
2745                         return ret;
2746
2747                 ret = ice_hash_moveout(pf,
2748                                        &ctx->ctx[ICE_HASH_GTPU_CTX_EH_IP]);
2749                 if (ret && (ret != -ENOENT))
2750                         return ret;
2751
2752                 ret = ice_hash_moveout(pf,
2753                                        &ctx->ctx[ICE_HASH_GTPU_CTX_EH_IP_UDP]);
2754                 if (ret && (ret != -ENOENT))
2755                         return ret;
2756
2757                 ret = ice_hash_moveout(pf,
2758                                        &ctx->ctx[ICE_HASH_GTPU_CTX_EH_IP_TCP]);
2759                 if (ret && (ret != -ENOENT))
2760                         return ret;
2761
2762                 break;
2763         case ICE_HASH_GTPU_CTX_DW_IP_UDP:
2764         case ICE_HASH_GTPU_CTX_DW_IP_TCP:
2765                 ret = ice_hash_moveout(pf,
2766                                        &ctx->ctx[ICE_HASH_GTPU_CTX_EH_IP]);
2767                 if (ret && (ret != -ENOENT))
2768                         return ret;
2769
2770                 ret = ice_hash_moveout(pf,
2771                                        &ctx->ctx[ICE_HASH_GTPU_CTX_EH_IP_UDP]);
2772                 if (ret && (ret != -ENOENT))
2773                         return ret;
2774
2775                 ret = ice_hash_moveout(pf,
2776                                        &ctx->ctx[ICE_HASH_GTPU_CTX_EH_IP_TCP]);
2777                 if (ret && (ret != -ENOENT))
2778                         return ret;
2779
2780                 break;
2781         default:
2782                 break;
2783         }
2784
2785         return 0;
2786 }
2787
2788 static u8 calc_gtpu_ctx_idx(uint32_t hdr)
2789 {
2790         u8 eh_idx, ip_idx;
2791
2792         if (hdr & ICE_FLOW_SEG_HDR_GTPU_EH)
2793                 eh_idx = 0;
2794         else if (hdr & ICE_FLOW_SEG_HDR_GTPU_UP)
2795                 eh_idx = 1;
2796         else if (hdr & ICE_FLOW_SEG_HDR_GTPU_DWN)
2797                 eh_idx = 2;
2798         else
2799                 return ICE_HASH_GTPU_CTX_MAX;
2800
2801         ip_idx = 0;
2802         if (hdr & ICE_FLOW_SEG_HDR_UDP)
2803                 ip_idx = 1;
2804         else if (hdr & ICE_FLOW_SEG_HDR_TCP)
2805                 ip_idx = 2;
2806
2807         if (hdr & (ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV6))
2808                 return eh_idx * 3 + ip_idx;
2809         else
2810                 return ICE_HASH_GTPU_CTX_MAX;
2811 }
2812
2813 static int
2814 ice_add_rss_cfg_pre(struct ice_pf *pf, uint32_t hdr)
2815 {
2816         u8 gtpu_ctx_idx = calc_gtpu_ctx_idx(hdr);
2817
2818         if (hdr & ICE_FLOW_SEG_HDR_IPV4)
2819                 return ice_add_rss_cfg_pre_gtpu(pf, &pf->hash_ctx.gtpu4,
2820                                                 gtpu_ctx_idx);
2821         else if (hdr & ICE_FLOW_SEG_HDR_IPV6)
2822                 return ice_add_rss_cfg_pre_gtpu(pf, &pf->hash_ctx.gtpu6,
2823                                                 gtpu_ctx_idx);
2824
2825         return 0;
2826 }
2827
2828 static int
2829 ice_add_rss_cfg_post_gtpu(struct ice_pf *pf, struct ice_hash_gtpu_ctx *ctx,
2830                           u8 ctx_idx, struct ice_rss_hash_cfg *cfg)
2831 {
2832         int ret;
2833
2834         if (ctx_idx < ICE_HASH_GTPU_CTX_MAX)
2835                 ctx->ctx[ctx_idx] = *cfg;
2836
2837         switch (ctx_idx) {
2838         case ICE_HASH_GTPU_CTX_EH_IP:
2839                 break;
2840         case ICE_HASH_GTPU_CTX_EH_IP_UDP:
2841                 ret = ice_hash_moveback(pf,
2842                                         &ctx->ctx[ICE_HASH_GTPU_CTX_UP_IP]);
2843                 if (ret && (ret != -ENOENT))
2844                         return ret;
2845
2846                 ret = ice_hash_moveback(pf,
2847                                         &ctx->ctx[ICE_HASH_GTPU_CTX_UP_IP_TCP]);
2848                 if (ret && (ret != -ENOENT))
2849                         return ret;
2850
2851                 ret = ice_hash_moveback(pf,
2852                                         &ctx->ctx[ICE_HASH_GTPU_CTX_DW_IP]);
2853                 if (ret && (ret != -ENOENT))
2854                         return ret;
2855
2856                 ret = ice_hash_moveback(pf,
2857                                         &ctx->ctx[ICE_HASH_GTPU_CTX_DW_IP_TCP]);
2858                 if (ret && (ret != -ENOENT))
2859                         return ret;
2860
2861                 break;
2862         case ICE_HASH_GTPU_CTX_EH_IP_TCP:
2863                 ret = ice_hash_moveback(pf,
2864                                         &ctx->ctx[ICE_HASH_GTPU_CTX_UP_IP]);
2865                 if (ret && (ret != -ENOENT))
2866                         return ret;
2867
2868                 ret = ice_hash_moveback(pf,
2869                                         &ctx->ctx[ICE_HASH_GTPU_CTX_UP_IP_UDP]);
2870                 if (ret && (ret != -ENOENT))
2871                         return ret;
2872
2873                 ret = ice_hash_moveback(pf,
2874                                         &ctx->ctx[ICE_HASH_GTPU_CTX_DW_IP]);
2875                 if (ret && (ret != -ENOENT))
2876                         return ret;
2877
2878                 ret = ice_hash_moveback(pf,
2879                                         &ctx->ctx[ICE_HASH_GTPU_CTX_DW_IP_UDP]);
2880                 if (ret && (ret != -ENOENT))
2881                         return ret;
2882
2883                 break;
2884         case ICE_HASH_GTPU_CTX_UP_IP:
2885         case ICE_HASH_GTPU_CTX_UP_IP_UDP:
2886         case ICE_HASH_GTPU_CTX_UP_IP_TCP:
2887         case ICE_HASH_GTPU_CTX_DW_IP:
2888         case ICE_HASH_GTPU_CTX_DW_IP_UDP:
2889         case ICE_HASH_GTPU_CTX_DW_IP_TCP:
2890                 ret = ice_hash_moveback(pf,
2891                                         &ctx->ctx[ICE_HASH_GTPU_CTX_EH_IP]);
2892                 if (ret && (ret != -ENOENT))
2893                         return ret;
2894
2895                 ret = ice_hash_moveback(pf,
2896                                         &ctx->ctx[ICE_HASH_GTPU_CTX_EH_IP_UDP]);
2897                 if (ret && (ret != -ENOENT))
2898                         return ret;
2899
2900                 ret = ice_hash_moveback(pf,
2901                                         &ctx->ctx[ICE_HASH_GTPU_CTX_EH_IP_TCP]);
2902                 if (ret && (ret != -ENOENT))
2903                         return ret;
2904
2905                 break;
2906         default:
2907                 break;
2908         }
2909
2910         return 0;
2911 }
2912
2913 static int
2914 ice_add_rss_cfg_post(struct ice_pf *pf, struct ice_rss_hash_cfg *cfg)
2915 {
2916         u8 gtpu_ctx_idx = calc_gtpu_ctx_idx(cfg->addl_hdrs);
2917
2918         if (cfg->addl_hdrs & ICE_FLOW_SEG_HDR_IPV4)
2919                 return ice_add_rss_cfg_post_gtpu(pf, &pf->hash_ctx.gtpu4,
2920                                                  gtpu_ctx_idx, cfg);
2921         else if (cfg->addl_hdrs & ICE_FLOW_SEG_HDR_IPV6)
2922                 return ice_add_rss_cfg_post_gtpu(pf, &pf->hash_ctx.gtpu6,
2923                                                  gtpu_ctx_idx, cfg);
2924
2925         return 0;
2926 }
2927
2928 static void
2929 ice_rem_rss_cfg_post(struct ice_pf *pf, uint32_t hdr)
2930 {
2931         u8 gtpu_ctx_idx = calc_gtpu_ctx_idx(hdr);
2932
2933         if (gtpu_ctx_idx >= ICE_HASH_GTPU_CTX_MAX)
2934                 return;
2935
2936         if (hdr & ICE_FLOW_SEG_HDR_IPV4)
2937                 hash_cfg_reset(&pf->hash_ctx.gtpu4.ctx[gtpu_ctx_idx]);
2938         else if (hdr & ICE_FLOW_SEG_HDR_IPV6)
2939                 hash_cfg_reset(&pf->hash_ctx.gtpu6.ctx[gtpu_ctx_idx]);
2940 }
2941
2942 int
2943 ice_rem_rss_cfg_wrap(struct ice_pf *pf, uint16_t vsi_id,
2944                      struct ice_rss_hash_cfg *cfg)
2945 {
2946         struct ice_hw *hw = ICE_PF_TO_HW(pf);
2947         int ret;
2948
2949         ret = ice_rem_rss_cfg(hw, vsi_id, cfg);
2950         if (ret && ret != ICE_ERR_DOES_NOT_EXIST)
2951                 PMD_DRV_LOG(ERR, "remove rss cfg failed\n");
2952
2953         ice_rem_rss_cfg_post(pf, cfg->addl_hdrs);
2954
2955         return 0;
2956 }
2957
2958 int
2959 ice_add_rss_cfg_wrap(struct ice_pf *pf, uint16_t vsi_id,
2960                      struct ice_rss_hash_cfg *cfg)
2961 {
2962         struct ice_hw *hw = ICE_PF_TO_HW(pf);
2963         int ret;
2964
2965         ret = ice_add_rss_cfg_pre(pf, cfg->addl_hdrs);
2966         if (ret)
2967                 PMD_DRV_LOG(ERR, "add rss cfg pre failed\n");
2968
2969         ret = ice_add_rss_cfg(hw, vsi_id, cfg);
2970         if (ret)
2971                 PMD_DRV_LOG(ERR, "add rss cfg failed\n");
2972
2973         ret = ice_add_rss_cfg_post(pf, cfg);
2974         if (ret)
2975                 PMD_DRV_LOG(ERR, "add rss cfg post failed\n");
2976
2977         return 0;
2978 }
2979
2980 static void
2981 ice_rss_hash_set(struct ice_pf *pf, uint64_t rss_hf)
2982 {
2983         struct ice_hw *hw = ICE_PF_TO_HW(pf);
2984         struct ice_vsi *vsi = pf->main_vsi;
2985         struct ice_rss_hash_cfg cfg;
2986         int ret;
2987
2988 #define ICE_RSS_HF_ALL ( \
2989         ETH_RSS_IPV4 | \
2990         ETH_RSS_IPV6 | \
2991         ETH_RSS_NONFRAG_IPV4_UDP | \
2992         ETH_RSS_NONFRAG_IPV6_UDP | \
2993         ETH_RSS_NONFRAG_IPV4_TCP | \
2994         ETH_RSS_NONFRAG_IPV6_TCP | \
2995         ETH_RSS_NONFRAG_IPV4_SCTP | \
2996         ETH_RSS_NONFRAG_IPV6_SCTP)
2997
2998         ret = ice_rem_vsi_rss_cfg(hw, vsi->idx);
2999         if (ret)
3000                 PMD_DRV_LOG(ERR, "%s Remove rss vsi fail %d",
3001                             __func__, ret);
3002
3003         cfg.symm = 0;
3004         cfg.hdr_type = ICE_RSS_OUTER_HEADERS;
3005         /* Configure RSS for IPv4 with src/dst addr as input set */
3006         if (rss_hf & ETH_RSS_IPV4) {
3007                 cfg.addl_hdrs = ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER;
3008                 cfg.hash_flds = ICE_FLOW_HASH_IPV4;
3009                 ret = ice_add_rss_cfg_wrap(pf, vsi->idx, &cfg);
3010                 if (ret)
3011                         PMD_DRV_LOG(ERR, "%s IPV4 rss flow fail %d",
3012                                     __func__, ret);
3013         }
3014
3015         /* Configure RSS for IPv6 with src/dst addr as input set */
3016         if (rss_hf & ETH_RSS_IPV6) {
3017                 cfg.addl_hdrs = ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER;
3018                 cfg.hash_flds = ICE_FLOW_HASH_IPV6;
3019                 ret = ice_add_rss_cfg_wrap(pf, vsi->idx, &cfg);
3020                 if (ret)
3021                         PMD_DRV_LOG(ERR, "%s IPV6 rss flow fail %d",
3022                                     __func__, ret);
3023         }
3024
3025         /* Configure RSS for udp4 with src/dst addr and port as input set */
3026         if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
3027                 cfg.addl_hdrs = ICE_FLOW_SEG_HDR_UDP | ICE_FLOW_SEG_HDR_IPV4 |
3028                                 ICE_FLOW_SEG_HDR_IPV_OTHER;
3029                 cfg.hash_flds = ICE_HASH_UDP_IPV4;
3030                 ret = ice_add_rss_cfg_wrap(pf, vsi->idx, &cfg);
3031                 if (ret)
3032                         PMD_DRV_LOG(ERR, "%s UDP_IPV4 rss flow fail %d",
3033                                     __func__, ret);
3034         }
3035
3036         /* Configure RSS for udp6 with src/dst addr and port as input set */
3037         if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) {
3038                 cfg.addl_hdrs = ICE_FLOW_SEG_HDR_UDP | ICE_FLOW_SEG_HDR_IPV6 |
3039                                 ICE_FLOW_SEG_HDR_IPV_OTHER;
3040                 cfg.hash_flds = ICE_HASH_UDP_IPV6;
3041                 ret = ice_add_rss_cfg_wrap(pf, vsi->idx, &cfg);
3042                 if (ret)
3043                         PMD_DRV_LOG(ERR, "%s UDP_IPV6 rss flow fail %d",
3044                                     __func__, ret);
3045         }
3046
3047         /* Configure RSS for tcp4 with src/dst addr and port as input set */
3048         if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
3049                 cfg.addl_hdrs = ICE_FLOW_SEG_HDR_TCP | ICE_FLOW_SEG_HDR_IPV4 |
3050                                 ICE_FLOW_SEG_HDR_IPV_OTHER;
3051                 cfg.hash_flds = ICE_HASH_TCP_IPV4;
3052                 ret = ice_add_rss_cfg_wrap(pf, vsi->idx, &cfg);
3053                 if (ret)
3054                         PMD_DRV_LOG(ERR, "%s TCP_IPV4 rss flow fail %d",
3055                                     __func__, ret);
3056         }
3057
3058         /* Configure RSS for tcp6 with src/dst addr and port as input set */
3059         if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) {
3060                 cfg.addl_hdrs = ICE_FLOW_SEG_HDR_TCP | ICE_FLOW_SEG_HDR_IPV6 |
3061                                 ICE_FLOW_SEG_HDR_IPV_OTHER;
3062                 cfg.hash_flds = ICE_HASH_TCP_IPV6;
3063                 ret = ice_add_rss_cfg_wrap(pf, vsi->idx, &cfg);
3064                 if (ret)
3065                         PMD_DRV_LOG(ERR, "%s TCP_IPV6 rss flow fail %d",
3066                                     __func__, ret);
3067         }
3068
3069         /* Configure RSS for sctp4 with src/dst addr and port as input set */
3070         if (rss_hf & ETH_RSS_NONFRAG_IPV4_SCTP) {
3071                 cfg.addl_hdrs = ICE_FLOW_SEG_HDR_SCTP | ICE_FLOW_SEG_HDR_IPV4 |
3072                                 ICE_FLOW_SEG_HDR_IPV_OTHER;
3073                 cfg.hash_flds = ICE_HASH_SCTP_IPV4;
3074                 ret = ice_add_rss_cfg_wrap(pf, vsi->idx, &cfg);
3075                 if (ret)
3076                         PMD_DRV_LOG(ERR, "%s SCTP_IPV4 rss flow fail %d",
3077                                     __func__, ret);
3078         }
3079
3080         /* Configure RSS for sctp6 with src/dst addr and port as input set */
3081         if (rss_hf & ETH_RSS_NONFRAG_IPV6_SCTP) {
3082                 cfg.addl_hdrs = ICE_FLOW_SEG_HDR_SCTP | ICE_FLOW_SEG_HDR_IPV6 |
3083                                 ICE_FLOW_SEG_HDR_IPV_OTHER;
3084                 cfg.hash_flds = ICE_HASH_SCTP_IPV6;
3085                 ret = ice_add_rss_cfg_wrap(pf, vsi->idx, &cfg);
3086                 if (ret)
3087                         PMD_DRV_LOG(ERR, "%s SCTP_IPV6 rss flow fail %d",
3088                                     __func__, ret);
3089         }
3090
3091         if (rss_hf & ETH_RSS_IPV4) {
3092                 cfg.addl_hdrs = ICE_FLOW_SEG_HDR_PPPOE | ICE_FLOW_SEG_HDR_IPV4 |
3093                                 ICE_FLOW_SEG_HDR_IPV_OTHER;
3094                 cfg.hash_flds = ICE_FLOW_HASH_IPV4;
3095                 ret = ice_add_rss_cfg_wrap(pf, vsi->idx, &cfg);
3096                 if (ret)
3097                         PMD_DRV_LOG(ERR, "%s PPPoE_IPV4 rss flow fail %d",
3098                                     __func__, ret);
3099         }
3100
3101         if (rss_hf & ETH_RSS_IPV6) {
3102                 cfg.addl_hdrs = ICE_FLOW_SEG_HDR_PPPOE | ICE_FLOW_SEG_HDR_IPV6 |
3103                                 ICE_FLOW_SEG_HDR_IPV_OTHER;
3104                 cfg.hash_flds = ICE_FLOW_HASH_IPV6;
3105                 ret = ice_add_rss_cfg_wrap(pf, vsi->idx, &cfg);
3106                 if (ret)
3107                         PMD_DRV_LOG(ERR, "%s PPPoE_IPV6 rss flow fail %d",
3108                                     __func__, ret);
3109         }
3110
3111         if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
3112                 cfg.addl_hdrs = ICE_FLOW_SEG_HDR_PPPOE | ICE_FLOW_SEG_HDR_UDP |
3113                                 ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER;
3114                 cfg.hash_flds = ICE_HASH_UDP_IPV4;
3115                 ret = ice_add_rss_cfg_wrap(pf, vsi->idx, &cfg);
3116                 if (ret)
3117                         PMD_DRV_LOG(ERR, "%s PPPoE_IPV4_UDP rss flow fail %d",
3118                                     __func__, ret);
3119         }
3120
3121         if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) {
3122                 cfg.addl_hdrs = ICE_FLOW_SEG_HDR_PPPOE | ICE_FLOW_SEG_HDR_UDP |
3123                                 ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER;
3124                 cfg.hash_flds = ICE_HASH_UDP_IPV6;
3125                 ret = ice_add_rss_cfg_wrap(pf, vsi->idx, &cfg);
3126                 if (ret)
3127                         PMD_DRV_LOG(ERR, "%s PPPoE_IPV6_UDP rss flow fail %d",
3128                                     __func__, ret);
3129         }
3130
3131         if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
3132                 cfg.addl_hdrs = ICE_FLOW_SEG_HDR_PPPOE | ICE_FLOW_SEG_HDR_TCP |
3133                                 ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_IPV_OTHER;
3134                 cfg.hash_flds = ICE_HASH_TCP_IPV4;
3135                 ret = ice_add_rss_cfg_wrap(pf, vsi->idx, &cfg);
3136                 if (ret)
3137                         PMD_DRV_LOG(ERR, "%s PPPoE_IPV4_TCP rss flow fail %d",
3138                                     __func__, ret);
3139         }
3140
3141         if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) {
3142                 cfg.addl_hdrs = ICE_FLOW_SEG_HDR_PPPOE | ICE_FLOW_SEG_HDR_TCP |
3143                                 ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_IPV_OTHER;
3144                 cfg.hash_flds = ICE_HASH_TCP_IPV6;
3145                 ret = ice_add_rss_cfg_wrap(pf, vsi->idx, &cfg);
3146                 if (ret)
3147                         PMD_DRV_LOG(ERR, "%s PPPoE_IPV6_TCP rss flow fail %d",
3148                                     __func__, ret);
3149         }
3150
3151         pf->rss_hf = rss_hf & ICE_RSS_HF_ALL;
3152 }
3153
3154 static void
3155 ice_get_default_rss_key(uint8_t *rss_key, uint32_t rss_key_size)
3156 {
3157         static struct ice_aqc_get_set_rss_keys default_key;
3158         static bool default_key_done;
3159         uint8_t *key = (uint8_t *)&default_key;
3160         size_t i;
3161
3162         if (rss_key_size > sizeof(default_key)) {
3163                 PMD_DRV_LOG(WARNING,
3164                             "requested size %u is larger than default %zu, "
3165                             "only %zu bytes are gotten for key\n",
3166                             rss_key_size, sizeof(default_key),
3167                             sizeof(default_key));
3168         }
3169
3170         if (!default_key_done) {
3171                 /* Calculate the default hash key */
3172                 for (i = 0; i < sizeof(default_key); i++)
3173                         key[i] = (uint8_t)rte_rand();
3174                 default_key_done = true;
3175         }
3176         rte_memcpy(rss_key, key, RTE_MIN(rss_key_size, sizeof(default_key)));
3177 }
3178
3179 static int ice_init_rss(struct ice_pf *pf)
3180 {
3181         struct ice_hw *hw = ICE_PF_TO_HW(pf);
3182         struct ice_vsi *vsi = pf->main_vsi;
3183         struct rte_eth_dev_data *dev_data = pf->dev_data;
3184         struct ice_aq_get_set_rss_lut_params lut_params;
3185         struct rte_eth_rss_conf *rss_conf;
3186         struct ice_aqc_get_set_rss_keys key;
3187         uint16_t i, nb_q;
3188         int ret = 0;
3189         bool is_safe_mode = pf->adapter->is_safe_mode;
3190         uint32_t reg;
3191
3192         rss_conf = &dev_data->dev_conf.rx_adv_conf.rss_conf;
3193         nb_q = dev_data->nb_rx_queues;
3194         vsi->rss_key_size = ICE_AQC_GET_SET_RSS_KEY_DATA_RSS_KEY_SIZE;
3195         vsi->rss_lut_size = pf->hash_lut_size;
3196
3197         if (nb_q == 0) {
3198                 PMD_DRV_LOG(WARNING,
3199                         "RSS is not supported as rx queues number is zero\n");
3200                 return 0;
3201         }
3202
3203         if (is_safe_mode) {
3204                 PMD_DRV_LOG(WARNING, "RSS is not supported in safe mode\n");
3205                 return 0;
3206         }
3207
3208         if (!vsi->rss_key) {
3209                 vsi->rss_key = rte_zmalloc(NULL,
3210                                            vsi->rss_key_size, 0);
3211                 if (vsi->rss_key == NULL) {
3212                         PMD_DRV_LOG(ERR, "Failed to allocate memory for rss_key");
3213                         return -ENOMEM;
3214                 }
3215         }
3216         if (!vsi->rss_lut) {
3217                 vsi->rss_lut = rte_zmalloc(NULL,
3218                                            vsi->rss_lut_size, 0);
3219                 if (vsi->rss_lut == NULL) {
3220                         PMD_DRV_LOG(ERR, "Failed to allocate memory for rss_key");
3221                         rte_free(vsi->rss_key);
3222                         vsi->rss_key = NULL;
3223                         return -ENOMEM;
3224                 }
3225         }
3226         /* configure RSS key */
3227         if (!rss_conf->rss_key)
3228                 ice_get_default_rss_key(vsi->rss_key, vsi->rss_key_size);
3229         else
3230                 rte_memcpy(vsi->rss_key, rss_conf->rss_key,
3231                            RTE_MIN(rss_conf->rss_key_len,
3232                                    vsi->rss_key_size));
3233
3234         rte_memcpy(key.standard_rss_key, vsi->rss_key, vsi->rss_key_size);
3235         ret = ice_aq_set_rss_key(hw, vsi->idx, &key);
3236         if (ret)
3237                 goto out;
3238
3239         /* init RSS LUT table */
3240         for (i = 0; i < vsi->rss_lut_size; i++)
3241                 vsi->rss_lut[i] = i % nb_q;
3242
3243         lut_params.vsi_handle = vsi->idx;
3244         lut_params.lut_size = vsi->rss_lut_size;
3245         lut_params.lut_type = ICE_AQC_GSET_RSS_LUT_TABLE_TYPE_PF;
3246         lut_params.lut = vsi->rss_lut;
3247         lut_params.global_lut_id = 0;
3248         ret = ice_aq_set_rss_lut(hw, &lut_params);
3249         if (ret)
3250                 goto out;
3251
3252         /* Enable registers for symmetric_toeplitz function. */
3253         reg = ICE_READ_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id));
3254         reg = (reg & (~VSIQF_HASH_CTL_HASH_SCHEME_M)) |
3255                 (1 << VSIQF_HASH_CTL_HASH_SCHEME_S);
3256         ICE_WRITE_REG(hw, VSIQF_HASH_CTL(vsi->vsi_id), reg);
3257
3258         /* RSS hash configuration */
3259         ice_rss_hash_set(pf, rss_conf->rss_hf);
3260
3261         return 0;
3262 out:
3263         rte_free(vsi->rss_key);
3264         vsi->rss_key = NULL;
3265         rte_free(vsi->rss_lut);
3266         vsi->rss_lut = NULL;
3267         return -EINVAL;
3268 }
3269
3270 static int
3271 ice_dev_configure(struct rte_eth_dev *dev)
3272 {
3273         struct ice_adapter *ad =
3274                 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
3275         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3276         int ret;
3277
3278         /* Initialize to TRUE. If any of Rx queues doesn't meet the
3279          * bulk allocation or vector Rx preconditions we will reset it.
3280          */
3281         ad->rx_bulk_alloc_allowed = true;
3282         ad->tx_simple_allowed = true;
3283
3284         if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)
3285                 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH;
3286
3287         if (dev->data->nb_rx_queues) {
3288                 ret = ice_init_rss(pf);
3289                 if (ret) {
3290                         PMD_DRV_LOG(ERR, "Failed to enable rss for PF");
3291                         return ret;
3292                 }
3293         }
3294
3295         return 0;
3296 }
3297
3298 static void
3299 __vsi_queues_bind_intr(struct ice_vsi *vsi, uint16_t msix_vect,
3300                        int base_queue, int nb_queue)
3301 {
3302         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
3303         uint32_t val, val_tx;
3304         int rx_low_latency, i;
3305
3306         rx_low_latency = vsi->adapter->devargs.rx_low_latency;
3307         for (i = 0; i < nb_queue; i++) {
3308                 /*do actual bind*/
3309                 val = (msix_vect & QINT_RQCTL_MSIX_INDX_M) |
3310                       (0 << QINT_RQCTL_ITR_INDX_S) | QINT_RQCTL_CAUSE_ENA_M;
3311                 val_tx = (msix_vect & QINT_TQCTL_MSIX_INDX_M) |
3312                          (0 << QINT_TQCTL_ITR_INDX_S) | QINT_TQCTL_CAUSE_ENA_M;
3313
3314                 PMD_DRV_LOG(INFO, "queue %d is binding to vect %d",
3315                             base_queue + i, msix_vect);
3316
3317                 /* set ITR0 value */
3318                 if (rx_low_latency) {
3319                         /**
3320                          * Empirical configuration for optimal real time
3321                          * latency reduced interrupt throttling to 2us
3322                          */
3323                         ICE_WRITE_REG(hw, GLINT_ITR(0, msix_vect), 0x1);
3324                         ICE_WRITE_REG(hw, QRX_ITR(base_queue + i),
3325                                       QRX_ITR_NO_EXPR_M);
3326                 } else {
3327                         ICE_WRITE_REG(hw, GLINT_ITR(0, msix_vect), 0x2);
3328                         ICE_WRITE_REG(hw, QRX_ITR(base_queue + i), 0);
3329                 }
3330
3331                 ICE_WRITE_REG(hw, QINT_RQCTL(base_queue + i), val);
3332                 ICE_WRITE_REG(hw, QINT_TQCTL(base_queue + i), val_tx);
3333         }
3334 }
3335
3336 void
3337 ice_vsi_queues_bind_intr(struct ice_vsi *vsi)
3338 {
3339         struct rte_eth_dev *dev = &rte_eth_devices[vsi->adapter->pf.dev_data->port_id];
3340         struct rte_pci_device *pci_dev = ICE_DEV_TO_PCI(dev);
3341         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
3342         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
3343         uint16_t msix_vect = vsi->msix_intr;
3344         uint16_t nb_msix = RTE_MIN(vsi->nb_msix, intr_handle->nb_efd);
3345         uint16_t queue_idx = 0;
3346         int record = 0;
3347         int i;
3348
3349         /* clear Rx/Tx queue interrupt */
3350         for (i = 0; i < vsi->nb_used_qps; i++) {
3351                 ICE_WRITE_REG(hw, QINT_TQCTL(vsi->base_queue + i), 0);
3352                 ICE_WRITE_REG(hw, QINT_RQCTL(vsi->base_queue + i), 0);
3353         }
3354
3355         /* PF bind interrupt */
3356         if (rte_intr_dp_is_en(intr_handle)) {
3357                 queue_idx = 0;
3358                 record = 1;
3359         }
3360
3361         for (i = 0; i < vsi->nb_used_qps; i++) {
3362                 if (nb_msix <= 1) {
3363                         if (!rte_intr_allow_others(intr_handle))
3364                                 msix_vect = ICE_MISC_VEC_ID;
3365
3366                         /* uio mapping all queue to one msix_vect */
3367                         __vsi_queues_bind_intr(vsi, msix_vect,
3368                                                vsi->base_queue + i,
3369                                                vsi->nb_used_qps - i);
3370
3371                         for (; !!record && i < vsi->nb_used_qps; i++)
3372                                 intr_handle->intr_vec[queue_idx + i] =
3373                                         msix_vect;
3374                         break;
3375                 }
3376
3377                 /* vfio 1:1 queue/msix_vect mapping */
3378                 __vsi_queues_bind_intr(vsi, msix_vect,
3379                                        vsi->base_queue + i, 1);
3380
3381                 if (!!record)
3382                         intr_handle->intr_vec[queue_idx + i] = msix_vect;
3383
3384                 msix_vect++;
3385                 nb_msix--;
3386         }
3387 }
3388
3389 void
3390 ice_vsi_enable_queues_intr(struct ice_vsi *vsi)
3391 {
3392         struct rte_eth_dev *dev = &rte_eth_devices[vsi->adapter->pf.dev_data->port_id];
3393         struct rte_pci_device *pci_dev = ICE_DEV_TO_PCI(dev);
3394         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
3395         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
3396         uint16_t msix_intr, i;
3397
3398         if (rte_intr_allow_others(intr_handle))
3399                 for (i = 0; i < vsi->nb_used_qps; i++) {
3400                         msix_intr = vsi->msix_intr + i;
3401                         ICE_WRITE_REG(hw, GLINT_DYN_CTL(msix_intr),
3402                                       GLINT_DYN_CTL_INTENA_M |
3403                                       GLINT_DYN_CTL_CLEARPBA_M |
3404                                       GLINT_DYN_CTL_ITR_INDX_M |
3405                                       GLINT_DYN_CTL_WB_ON_ITR_M);
3406                 }
3407         else
3408                 ICE_WRITE_REG(hw, GLINT_DYN_CTL(0),
3409                               GLINT_DYN_CTL_INTENA_M |
3410                               GLINT_DYN_CTL_CLEARPBA_M |
3411                               GLINT_DYN_CTL_ITR_INDX_M |
3412                               GLINT_DYN_CTL_WB_ON_ITR_M);
3413 }
3414
3415 static int
3416 ice_rxq_intr_setup(struct rte_eth_dev *dev)
3417 {
3418         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3419         struct rte_pci_device *pci_dev = ICE_DEV_TO_PCI(dev);
3420         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
3421         struct ice_vsi *vsi = pf->main_vsi;
3422         uint32_t intr_vector = 0;
3423
3424         rte_intr_disable(intr_handle);
3425
3426         /* check and configure queue intr-vector mapping */
3427         if ((rte_intr_cap_multiple(intr_handle) ||
3428              !RTE_ETH_DEV_SRIOV(dev).active) &&
3429             dev->data->dev_conf.intr_conf.rxq != 0) {
3430                 intr_vector = dev->data->nb_rx_queues;
3431                 if (intr_vector > ICE_MAX_INTR_QUEUE_NUM) {
3432                         PMD_DRV_LOG(ERR, "At most %d intr queues supported",
3433                                     ICE_MAX_INTR_QUEUE_NUM);
3434                         return -ENOTSUP;
3435                 }
3436                 if (rte_intr_efd_enable(intr_handle, intr_vector))
3437                         return -1;
3438         }
3439
3440         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
3441                 intr_handle->intr_vec =
3442                 rte_zmalloc(NULL, dev->data->nb_rx_queues * sizeof(int),
3443                             0);
3444                 if (!intr_handle->intr_vec) {
3445                         PMD_DRV_LOG(ERR,
3446                                     "Failed to allocate %d rx_queues intr_vec",
3447                                     dev->data->nb_rx_queues);
3448                         return -ENOMEM;
3449                 }
3450         }
3451
3452         /* Map queues with MSIX interrupt */
3453         vsi->nb_used_qps = dev->data->nb_rx_queues;
3454         ice_vsi_queues_bind_intr(vsi);
3455
3456         /* Enable interrupts for all the queues */
3457         ice_vsi_enable_queues_intr(vsi);
3458
3459         rte_intr_enable(intr_handle);
3460
3461         return 0;
3462 }
3463
3464 static void
3465 ice_get_init_link_status(struct rte_eth_dev *dev)
3466 {
3467         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3468         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3469         bool enable_lse = dev->data->dev_conf.intr_conf.lsc ? true : false;
3470         struct ice_link_status link_status;
3471         int ret;
3472
3473         ret = ice_aq_get_link_info(hw->port_info, enable_lse,
3474                                    &link_status, NULL);
3475         if (ret != ICE_SUCCESS) {
3476                 PMD_DRV_LOG(ERR, "Failed to get link info");
3477                 pf->init_link_up = false;
3478                 return;
3479         }
3480
3481         if (link_status.link_info & ICE_AQ_LINK_UP)
3482                 pf->init_link_up = true;
3483 }
3484
3485 static int
3486 ice_pps_out_cfg(struct ice_hw *hw, int idx, int timer)
3487 {
3488         uint64_t current_time, start_time;
3489         uint32_t hi, lo, lo2, func, val;
3490
3491         lo = ICE_READ_REG(hw, GLTSYN_TIME_L(timer));
3492         hi = ICE_READ_REG(hw, GLTSYN_TIME_H(timer));
3493         lo2 = ICE_READ_REG(hw, GLTSYN_TIME_L(timer));
3494
3495         if (lo2 < lo) {
3496                 lo = ICE_READ_REG(hw, GLTSYN_TIME_L(timer));
3497                 hi = ICE_READ_REG(hw, GLTSYN_TIME_H(timer));
3498         }
3499
3500         current_time = ((uint64_t)hi << 32) | lo;
3501
3502         start_time = (current_time + NSEC_PER_SEC) /
3503                         NSEC_PER_SEC * NSEC_PER_SEC;
3504         start_time = start_time - PPS_OUT_DELAY_NS;
3505
3506         func = 8 + idx + timer * 4;
3507         val = GLGEN_GPIO_CTL_PIN_DIR_M |
3508                 ((func << GLGEN_GPIO_CTL_PIN_FUNC_S) &
3509                 GLGEN_GPIO_CTL_PIN_FUNC_M);
3510
3511         /* Write clkout with half of period value */
3512         ICE_WRITE_REG(hw, GLTSYN_CLKO(idx, timer), NSEC_PER_SEC / 2);
3513
3514         /* Write TARGET time register */
3515         ICE_WRITE_REG(hw, GLTSYN_TGT_L(idx, timer), start_time & 0xffffffff);
3516         ICE_WRITE_REG(hw, GLTSYN_TGT_H(idx, timer), start_time >> 32);
3517
3518         /* Write AUX_OUT register */
3519         ICE_WRITE_REG(hw, GLTSYN_AUX_OUT(idx, timer),
3520                       GLTSYN_AUX_OUT_0_OUT_ENA_M | GLTSYN_AUX_OUT_0_OUTMOD_M);
3521
3522         /* Write GPIO CTL register */
3523         ICE_WRITE_REG(hw, GLGEN_GPIO_CTL(idx), val);
3524
3525         return 0;
3526 }
3527
3528 static int
3529 ice_dev_start(struct rte_eth_dev *dev)
3530 {
3531         struct rte_eth_dev_data *data = dev->data;
3532         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3533         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3534         struct ice_vsi *vsi = pf->main_vsi;
3535         struct ice_adapter *ad =
3536                         ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
3537         uint16_t nb_rxq = 0;
3538         uint16_t nb_txq, i;
3539         uint16_t max_frame_size;
3540         int mask, ret;
3541         uint8_t timer = hw->func_caps.ts_func_info.tmr_index_owned;
3542         uint32_t pin_idx = ad->devargs.pin_idx;
3543
3544         /* program Tx queues' context in hardware */
3545         for (nb_txq = 0; nb_txq < data->nb_tx_queues; nb_txq++) {
3546                 ret = ice_tx_queue_start(dev, nb_txq);
3547                 if (ret) {
3548                         PMD_DRV_LOG(ERR, "fail to start Tx queue %u", nb_txq);
3549                         goto tx_err;
3550                 }
3551         }
3552
3553         /* program Rx queues' context in hardware*/
3554         for (nb_rxq = 0; nb_rxq < data->nb_rx_queues; nb_rxq++) {
3555                 ret = ice_rx_queue_start(dev, nb_rxq);
3556                 if (ret) {
3557                         PMD_DRV_LOG(ERR, "fail to start Rx queue %u", nb_rxq);
3558                         goto rx_err;
3559                 }
3560         }
3561
3562         ice_set_rx_function(dev);
3563         ice_set_tx_function(dev);
3564
3565         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK |
3566                         ETH_VLAN_EXTEND_MASK;
3567         ret = ice_vlan_offload_set(dev, mask);
3568         if (ret) {
3569                 PMD_INIT_LOG(ERR, "Unable to set VLAN offload");
3570                 goto rx_err;
3571         }
3572
3573         /* enable Rx interrput and mapping Rx queue to interrupt vector */
3574         if (ice_rxq_intr_setup(dev))
3575                 return -EIO;
3576
3577         /* Enable receiving broadcast packets and transmitting packets */
3578         ret = ice_set_vsi_promisc(hw, vsi->idx,
3579                                   ICE_PROMISC_BCAST_RX | ICE_PROMISC_BCAST_TX |
3580                                   ICE_PROMISC_UCAST_TX | ICE_PROMISC_MCAST_TX,
3581                                   0);
3582         if (ret != ICE_SUCCESS)
3583                 PMD_DRV_LOG(INFO, "fail to set vsi broadcast");
3584
3585         ret = ice_aq_set_event_mask(hw, hw->port_info->lport,
3586                                     ((u16)(ICE_AQ_LINK_EVENT_LINK_FAULT |
3587                                      ICE_AQ_LINK_EVENT_PHY_TEMP_ALARM |
3588                                      ICE_AQ_LINK_EVENT_EXCESSIVE_ERRORS |
3589                                      ICE_AQ_LINK_EVENT_SIGNAL_DETECT |
3590                                      ICE_AQ_LINK_EVENT_AN_COMPLETED |
3591                                      ICE_AQ_LINK_EVENT_PORT_TX_SUSPENDED)),
3592                                      NULL);
3593         if (ret != ICE_SUCCESS)
3594                 PMD_DRV_LOG(WARNING, "Fail to set phy mask");
3595
3596         ice_get_init_link_status(dev);
3597
3598         ice_dev_set_link_up(dev);
3599
3600         /* Call get_link_info aq commond to enable/disable LSE */
3601         ice_link_update(dev, 0);
3602
3603         pf->adapter_stopped = false;
3604
3605         /* Set the max frame size to default value*/
3606         max_frame_size = pf->dev_data->mtu ?
3607                 pf->dev_data->mtu + ICE_ETH_OVERHEAD :
3608                 ICE_FRAME_SIZE_MAX;
3609
3610         /* Set the max frame size to HW*/
3611         ice_aq_set_mac_cfg(hw, max_frame_size, NULL);
3612
3613         if (ad->devargs.pps_out_ena) {
3614                 ret = ice_pps_out_cfg(hw, pin_idx, timer);
3615                 if (ret) {
3616                         PMD_DRV_LOG(ERR, "Fail to configure 1pps out");
3617                         goto rx_err;
3618                 }
3619         }
3620
3621         return 0;
3622
3623         /* stop the started queues if failed to start all queues */
3624 rx_err:
3625         for (i = 0; i < nb_rxq; i++)
3626                 ice_rx_queue_stop(dev, i);
3627 tx_err:
3628         for (i = 0; i < nb_txq; i++)
3629                 ice_tx_queue_stop(dev, i);
3630
3631         return -EIO;
3632 }
3633
3634 static int
3635 ice_dev_reset(struct rte_eth_dev *dev)
3636 {
3637         int ret;
3638
3639         if (dev->data->sriov.active)
3640                 return -ENOTSUP;
3641
3642         ret = ice_dev_uninit(dev);
3643         if (ret) {
3644                 PMD_INIT_LOG(ERR, "failed to uninit device, status = %d", ret);
3645                 return -ENXIO;
3646         }
3647
3648         ret = ice_dev_init(dev);
3649         if (ret) {
3650                 PMD_INIT_LOG(ERR, "failed to init device, status = %d", ret);
3651                 return -ENXIO;
3652         }
3653
3654         return 0;
3655 }
3656
3657 static int
3658 ice_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
3659 {
3660         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3661         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3662         struct ice_vsi *vsi = pf->main_vsi;
3663         struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device);
3664         bool is_safe_mode = pf->adapter->is_safe_mode;
3665         u64 phy_type_low;
3666         u64 phy_type_high;
3667
3668         dev_info->min_rx_bufsize = ICE_BUF_SIZE_MIN;
3669         dev_info->max_rx_pktlen = ICE_FRAME_SIZE_MAX;
3670         dev_info->max_rx_queues = vsi->nb_qps;
3671         dev_info->max_tx_queues = vsi->nb_qps;
3672         dev_info->max_mac_addrs = vsi->max_macaddrs;
3673         dev_info->max_vfs = pci_dev->max_vfs;
3674         dev_info->max_mtu = dev_info->max_rx_pktlen - ICE_ETH_OVERHEAD;
3675         dev_info->min_mtu = RTE_ETHER_MIN_MTU;
3676
3677         dev_info->rx_offload_capa =
3678                 DEV_RX_OFFLOAD_VLAN_STRIP |
3679                 DEV_RX_OFFLOAD_JUMBO_FRAME |
3680                 DEV_RX_OFFLOAD_KEEP_CRC |
3681                 DEV_RX_OFFLOAD_SCATTER |
3682                 DEV_RX_OFFLOAD_VLAN_FILTER;
3683         dev_info->tx_offload_capa =
3684                 DEV_TX_OFFLOAD_VLAN_INSERT |
3685                 DEV_TX_OFFLOAD_TCP_TSO |
3686                 DEV_TX_OFFLOAD_MULTI_SEGS |
3687                 DEV_TX_OFFLOAD_MBUF_FAST_FREE;
3688         dev_info->flow_type_rss_offloads = 0;
3689
3690         if (!is_safe_mode) {
3691                 dev_info->rx_offload_capa |=
3692                         DEV_RX_OFFLOAD_IPV4_CKSUM |
3693                         DEV_RX_OFFLOAD_UDP_CKSUM |
3694                         DEV_RX_OFFLOAD_TCP_CKSUM |
3695                         DEV_RX_OFFLOAD_QINQ_STRIP |
3696                         DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
3697                         DEV_RX_OFFLOAD_VLAN_EXTEND |
3698                         DEV_RX_OFFLOAD_RSS_HASH |
3699                         DEV_RX_OFFLOAD_TIMESTAMP;
3700                 dev_info->tx_offload_capa |=
3701                         DEV_TX_OFFLOAD_QINQ_INSERT |
3702                         DEV_TX_OFFLOAD_IPV4_CKSUM |
3703                         DEV_TX_OFFLOAD_UDP_CKSUM |
3704                         DEV_TX_OFFLOAD_TCP_CKSUM |
3705                         DEV_TX_OFFLOAD_SCTP_CKSUM |
3706                         DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
3707                         DEV_TX_OFFLOAD_OUTER_UDP_CKSUM;
3708                 dev_info->flow_type_rss_offloads |= ICE_RSS_OFFLOAD_ALL;
3709         }
3710
3711         dev_info->rx_queue_offload_capa = 0;
3712         dev_info->tx_queue_offload_capa = DEV_TX_OFFLOAD_MBUF_FAST_FREE;
3713
3714         dev_info->reta_size = pf->hash_lut_size;
3715         dev_info->hash_key_size = (VSIQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t);
3716
3717         dev_info->default_rxconf = (struct rte_eth_rxconf) {
3718                 .rx_thresh = {
3719                         .pthresh = ICE_DEFAULT_RX_PTHRESH,
3720                         .hthresh = ICE_DEFAULT_RX_HTHRESH,
3721                         .wthresh = ICE_DEFAULT_RX_WTHRESH,
3722                 },
3723                 .rx_free_thresh = ICE_DEFAULT_RX_FREE_THRESH,
3724                 .rx_drop_en = 0,
3725                 .offloads = 0,
3726         };
3727
3728         dev_info->default_txconf = (struct rte_eth_txconf) {
3729                 .tx_thresh = {
3730                         .pthresh = ICE_DEFAULT_TX_PTHRESH,
3731                         .hthresh = ICE_DEFAULT_TX_HTHRESH,
3732                         .wthresh = ICE_DEFAULT_TX_WTHRESH,
3733                 },
3734                 .tx_free_thresh = ICE_DEFAULT_TX_FREE_THRESH,
3735                 .tx_rs_thresh = ICE_DEFAULT_TX_RSBIT_THRESH,
3736                 .offloads = 0,
3737         };
3738
3739         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
3740                 .nb_max = ICE_MAX_RING_DESC,
3741                 .nb_min = ICE_MIN_RING_DESC,
3742                 .nb_align = ICE_ALIGN_RING_DESC,
3743         };
3744
3745         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
3746                 .nb_max = ICE_MAX_RING_DESC,
3747                 .nb_min = ICE_MIN_RING_DESC,
3748                 .nb_align = ICE_ALIGN_RING_DESC,
3749         };
3750
3751         dev_info->speed_capa = ETH_LINK_SPEED_10M |
3752                                ETH_LINK_SPEED_100M |
3753                                ETH_LINK_SPEED_1G |
3754                                ETH_LINK_SPEED_2_5G |
3755                                ETH_LINK_SPEED_5G |
3756                                ETH_LINK_SPEED_10G |
3757                                ETH_LINK_SPEED_20G |
3758                                ETH_LINK_SPEED_25G;
3759
3760         phy_type_low = hw->port_info->phy.phy_type_low;
3761         phy_type_high = hw->port_info->phy.phy_type_high;
3762
3763         if (ICE_PHY_TYPE_SUPPORT_50G(phy_type_low))
3764                 dev_info->speed_capa |= ETH_LINK_SPEED_50G;
3765
3766         if (ICE_PHY_TYPE_SUPPORT_100G_LOW(phy_type_low) ||
3767                         ICE_PHY_TYPE_SUPPORT_100G_HIGH(phy_type_high))
3768                 dev_info->speed_capa |= ETH_LINK_SPEED_100G;
3769
3770         dev_info->nb_rx_queues = dev->data->nb_rx_queues;
3771         dev_info->nb_tx_queues = dev->data->nb_tx_queues;
3772
3773         dev_info->default_rxportconf.burst_size = ICE_RX_MAX_BURST;
3774         dev_info->default_txportconf.burst_size = ICE_TX_MAX_BURST;
3775         dev_info->default_rxportconf.nb_queues = 1;
3776         dev_info->default_txportconf.nb_queues = 1;
3777         dev_info->default_rxportconf.ring_size = ICE_BUF_SIZE_MIN;
3778         dev_info->default_txportconf.ring_size = ICE_BUF_SIZE_MIN;
3779
3780         return 0;
3781 }
3782
3783 static inline int
3784 ice_atomic_read_link_status(struct rte_eth_dev *dev,
3785                             struct rte_eth_link *link)
3786 {
3787         struct rte_eth_link *dst = link;
3788         struct rte_eth_link *src = &dev->data->dev_link;
3789
3790         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
3791                                 *(uint64_t *)src) == 0)
3792                 return -1;
3793
3794         return 0;
3795 }
3796
3797 static inline int
3798 ice_atomic_write_link_status(struct rte_eth_dev *dev,
3799                              struct rte_eth_link *link)
3800 {
3801         struct rte_eth_link *dst = &dev->data->dev_link;
3802         struct rte_eth_link *src = link;
3803
3804         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
3805                                 *(uint64_t *)src) == 0)
3806                 return -1;
3807
3808         return 0;
3809 }
3810
3811 static int
3812 ice_link_update(struct rte_eth_dev *dev, int wait_to_complete)
3813 {
3814 #define CHECK_INTERVAL 100  /* 100ms */
3815 #define MAX_REPEAT_TIME 10  /* 1s (10 * 100ms) in total */
3816         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3817         struct ice_link_status link_status;
3818         struct rte_eth_link link, old;
3819         int status;
3820         unsigned int rep_cnt = MAX_REPEAT_TIME;
3821         bool enable_lse = dev->data->dev_conf.intr_conf.lsc ? true : false;
3822
3823         memset(&link, 0, sizeof(link));
3824         memset(&old, 0, sizeof(old));
3825         memset(&link_status, 0, sizeof(link_status));
3826         ice_atomic_read_link_status(dev, &old);
3827
3828         do {
3829                 /* Get link status information from hardware */
3830                 status = ice_aq_get_link_info(hw->port_info, enable_lse,
3831                                               &link_status, NULL);
3832                 if (status != ICE_SUCCESS) {
3833                         link.link_speed = ETH_SPEED_NUM_100M;
3834                         link.link_duplex = ETH_LINK_FULL_DUPLEX;
3835                         PMD_DRV_LOG(ERR, "Failed to get link info");
3836                         goto out;
3837                 }
3838
3839                 link.link_status = link_status.link_info & ICE_AQ_LINK_UP;
3840                 if (!wait_to_complete || link.link_status)
3841                         break;
3842
3843                 rte_delay_ms(CHECK_INTERVAL);
3844         } while (--rep_cnt);
3845
3846         if (!link.link_status)
3847                 goto out;
3848
3849         /* Full-duplex operation at all supported speeds */
3850         link.link_duplex = ETH_LINK_FULL_DUPLEX;
3851
3852         /* Parse the link status */
3853         switch (link_status.link_speed) {
3854         case ICE_AQ_LINK_SPEED_10MB:
3855                 link.link_speed = ETH_SPEED_NUM_10M;
3856                 break;
3857         case ICE_AQ_LINK_SPEED_100MB:
3858                 link.link_speed = ETH_SPEED_NUM_100M;
3859                 break;
3860         case ICE_AQ_LINK_SPEED_1000MB:
3861                 link.link_speed = ETH_SPEED_NUM_1G;
3862                 break;
3863         case ICE_AQ_LINK_SPEED_2500MB:
3864                 link.link_speed = ETH_SPEED_NUM_2_5G;
3865                 break;
3866         case ICE_AQ_LINK_SPEED_5GB:
3867                 link.link_speed = ETH_SPEED_NUM_5G;
3868                 break;
3869         case ICE_AQ_LINK_SPEED_10GB:
3870                 link.link_speed = ETH_SPEED_NUM_10G;
3871                 break;
3872         case ICE_AQ_LINK_SPEED_20GB:
3873                 link.link_speed = ETH_SPEED_NUM_20G;
3874                 break;
3875         case ICE_AQ_LINK_SPEED_25GB:
3876                 link.link_speed = ETH_SPEED_NUM_25G;
3877                 break;
3878         case ICE_AQ_LINK_SPEED_40GB:
3879                 link.link_speed = ETH_SPEED_NUM_40G;
3880                 break;
3881         case ICE_AQ_LINK_SPEED_50GB:
3882                 link.link_speed = ETH_SPEED_NUM_50G;
3883                 break;
3884         case ICE_AQ_LINK_SPEED_100GB:
3885                 link.link_speed = ETH_SPEED_NUM_100G;
3886                 break;
3887         case ICE_AQ_LINK_SPEED_UNKNOWN:
3888                 PMD_DRV_LOG(ERR, "Unknown link speed");
3889                 link.link_speed = ETH_SPEED_NUM_UNKNOWN;
3890                 break;
3891         default:
3892                 PMD_DRV_LOG(ERR, "None link speed");
3893                 link.link_speed = ETH_SPEED_NUM_NONE;
3894                 break;
3895         }
3896
3897         link.link_autoneg = !(dev->data->dev_conf.link_speeds &
3898                               ETH_LINK_SPEED_FIXED);
3899
3900 out:
3901         ice_atomic_write_link_status(dev, &link);
3902         if (link.link_status == old.link_status)
3903                 return -1;
3904
3905         return 0;
3906 }
3907
3908 /* Force the physical link state by getting the current PHY capabilities from
3909  * hardware and setting the PHY config based on the determined capabilities. If
3910  * link changes, link event will be triggered because both the Enable Automatic
3911  * Link Update and LESM Enable bits are set when setting the PHY capabilities.
3912  */
3913 static enum ice_status
3914 ice_force_phys_link_state(struct ice_hw *hw, bool link_up)
3915 {
3916         struct ice_aqc_set_phy_cfg_data cfg = { 0 };
3917         struct ice_aqc_get_phy_caps_data *pcaps;
3918         struct ice_port_info *pi;
3919         enum ice_status status;
3920
3921         if (!hw || !hw->port_info)
3922                 return ICE_ERR_PARAM;
3923
3924         pi = hw->port_info;
3925
3926         pcaps = (struct ice_aqc_get_phy_caps_data *)
3927                 ice_malloc(hw, sizeof(*pcaps));
3928         if (!pcaps)
3929                 return ICE_ERR_NO_MEMORY;
3930
3931         status = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_ACTIVE_CFG,
3932                                      pcaps, NULL);
3933         if (status)
3934                 goto out;
3935
3936         /* No change in link */
3937         if (link_up == !!(pcaps->caps & ICE_AQC_PHY_EN_LINK) &&
3938             link_up == !!(pi->phy.link_info.link_info & ICE_AQ_LINK_UP))
3939                 goto out;
3940
3941         cfg.phy_type_low = pcaps->phy_type_low;
3942         cfg.phy_type_high = pcaps->phy_type_high;
3943         cfg.caps = pcaps->caps | ICE_AQ_PHY_ENA_AUTO_LINK_UPDT;
3944         cfg.low_power_ctrl_an = pcaps->low_power_ctrl_an;
3945         cfg.eee_cap = pcaps->eee_cap;
3946         cfg.eeer_value = pcaps->eeer_value;
3947         cfg.link_fec_opt = pcaps->link_fec_options;
3948         if (link_up)
3949                 cfg.caps |= ICE_AQ_PHY_ENA_LINK;
3950         else
3951                 cfg.caps &= ~ICE_AQ_PHY_ENA_LINK;
3952
3953         status = ice_aq_set_phy_cfg(hw, pi, &cfg, NULL);
3954
3955 out:
3956         ice_free(hw, pcaps);
3957         return status;
3958 }
3959
3960 static int
3961 ice_dev_set_link_up(struct rte_eth_dev *dev)
3962 {
3963         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3964
3965         return ice_force_phys_link_state(hw, true);
3966 }
3967
3968 static int
3969 ice_dev_set_link_down(struct rte_eth_dev *dev)
3970 {
3971         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3972
3973         return ice_force_phys_link_state(hw, false);
3974 }
3975
3976 static int
3977 ice_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
3978 {
3979         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3980         struct rte_eth_dev_data *dev_data = pf->dev_data;
3981         uint32_t frame_size = mtu + ICE_ETH_OVERHEAD;
3982
3983         /* check if mtu is within the allowed range */
3984         if (mtu < RTE_ETHER_MIN_MTU || frame_size > ICE_FRAME_SIZE_MAX)
3985                 return -EINVAL;
3986
3987         /* mtu setting is forbidden if port is start */
3988         if (dev_data->dev_started) {
3989                 PMD_DRV_LOG(ERR,
3990                             "port %d must be stopped before configuration",
3991                             dev_data->port_id);
3992                 return -EBUSY;
3993         }
3994
3995         return 0;
3996 }
3997
3998 static int ice_macaddr_set(struct rte_eth_dev *dev,
3999                            struct rte_ether_addr *mac_addr)
4000 {
4001         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4002         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
4003         struct ice_vsi *vsi = pf->main_vsi;
4004         struct ice_mac_filter *f;
4005         uint8_t flags = 0;
4006         int ret;
4007
4008         if (!rte_is_valid_assigned_ether_addr(mac_addr)) {
4009                 PMD_DRV_LOG(ERR, "Tried to set invalid MAC address.");
4010                 return -EINVAL;
4011         }
4012
4013         TAILQ_FOREACH(f, &vsi->mac_list, next) {
4014                 if (rte_is_same_ether_addr(&pf->dev_addr, &f->mac_info.mac_addr))
4015                         break;
4016         }
4017
4018         if (!f) {
4019                 PMD_DRV_LOG(ERR, "Failed to find filter for default mac");
4020                 return -EIO;
4021         }
4022
4023         ret = ice_remove_mac_filter(vsi, &f->mac_info.mac_addr);
4024         if (ret != ICE_SUCCESS) {
4025                 PMD_DRV_LOG(ERR, "Failed to delete mac filter");
4026                 return -EIO;
4027         }
4028         ret = ice_add_mac_filter(vsi, mac_addr);
4029         if (ret != ICE_SUCCESS) {
4030                 PMD_DRV_LOG(ERR, "Failed to add mac filter");
4031                 return -EIO;
4032         }
4033         rte_ether_addr_copy(mac_addr, &pf->dev_addr);
4034
4035         flags = ICE_AQC_MAN_MAC_UPDATE_LAA_WOL;
4036         ret = ice_aq_manage_mac_write(hw, mac_addr->addr_bytes, flags, NULL);
4037         if (ret != ICE_SUCCESS)
4038                 PMD_DRV_LOG(ERR, "Failed to set manage mac");
4039
4040         return 0;
4041 }
4042
4043 /* Add a MAC address, and update filters */
4044 static int
4045 ice_macaddr_add(struct rte_eth_dev *dev,
4046                 struct rte_ether_addr *mac_addr,
4047                 __rte_unused uint32_t index,
4048                 __rte_unused uint32_t pool)
4049 {
4050         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
4051         struct ice_vsi *vsi = pf->main_vsi;
4052         int ret;
4053
4054         ret = ice_add_mac_filter(vsi, mac_addr);
4055         if (ret != ICE_SUCCESS) {
4056                 PMD_DRV_LOG(ERR, "Failed to add MAC filter");
4057                 return -EINVAL;
4058         }
4059
4060         return ICE_SUCCESS;
4061 }
4062
4063 /* Remove a MAC address, and update filters */
4064 static void
4065 ice_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
4066 {
4067         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
4068         struct ice_vsi *vsi = pf->main_vsi;
4069         struct rte_eth_dev_data *data = dev->data;
4070         struct rte_ether_addr *macaddr;
4071         int ret;
4072
4073         macaddr = &data->mac_addrs[index];
4074         ret = ice_remove_mac_filter(vsi, macaddr);
4075         if (ret) {
4076                 PMD_DRV_LOG(ERR, "Failed to remove MAC filter");
4077                 return;
4078         }
4079 }
4080
4081 static int
4082 ice_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
4083 {
4084         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
4085         struct ice_vlan vlan = ICE_VLAN(RTE_ETHER_TYPE_VLAN, vlan_id);
4086         struct ice_vsi *vsi = pf->main_vsi;
4087         int ret;
4088
4089         PMD_INIT_FUNC_TRACE();
4090
4091         /**
4092          * Vlan 0 is the generic filter for untagged packets
4093          * and can't be removed or added by user.
4094          */
4095         if (vlan_id == 0)
4096                 return 0;
4097
4098         if (on) {
4099                 ret = ice_add_vlan_filter(vsi, &vlan);
4100                 if (ret < 0) {
4101                         PMD_DRV_LOG(ERR, "Failed to add vlan filter");
4102                         return -EINVAL;
4103                 }
4104         } else {
4105                 ret = ice_remove_vlan_filter(vsi, &vlan);
4106                 if (ret < 0) {
4107                         PMD_DRV_LOG(ERR, "Failed to remove vlan filter");
4108                         return -EINVAL;
4109                 }
4110         }
4111
4112         return 0;
4113 }
4114
4115 /* In Single VLAN Mode (SVM), single VLAN filters via ICE_SW_LKUP_VLAN are
4116  * based on the inner VLAN ID, so the VLAN TPID (i.e. 0x8100 or 0x888a8)
4117  * doesn't matter. In Double VLAN Mode (DVM), outer/single VLAN filters via
4118  * ICE_SW_LKUP_VLAN are based on the outer/single VLAN ID + VLAN TPID.
4119  *
4120  * For both modes add a VLAN 0 + no VLAN TPID filter to handle untagged traffic
4121  * when VLAN pruning is enabled. Also, this handles VLAN 0 priority tagged
4122  * traffic in SVM, since the VLAN TPID isn't part of filtering.
4123  *
4124  * If DVM is enabled then an explicit VLAN 0 + VLAN TPID filter needs to be
4125  * added to allow VLAN 0 priority tagged traffic in DVM, since the VLAN TPID is
4126  * part of filtering.
4127  */
4128 static int
4129 ice_vsi_add_vlan_zero(struct ice_vsi *vsi)
4130 {
4131         struct ice_vlan vlan;
4132         int err;
4133
4134         vlan = ICE_VLAN(0, 0);
4135         err = ice_add_vlan_filter(vsi, &vlan);
4136         if (err) {
4137                 PMD_DRV_LOG(DEBUG, "Failed to add VLAN ID 0");
4138                 return err;
4139         }
4140
4141         /* in SVM both VLAN 0 filters are identical */
4142         if (!ice_is_dvm_ena(&vsi->adapter->hw))
4143                 return 0;
4144
4145         vlan = ICE_VLAN(RTE_ETHER_TYPE_VLAN, 0);
4146         err = ice_add_vlan_filter(vsi, &vlan);
4147         if (err) {
4148                 PMD_DRV_LOG(DEBUG, "Failed to add VLAN ID 0 in double VLAN mode");
4149                 return err;
4150         }
4151
4152         return 0;
4153 }
4154
4155 /*
4156  * Delete the VLAN 0 filters in the same manner that they were added in
4157  * ice_vsi_add_vlan_zero.
4158  */
4159 static int
4160 ice_vsi_del_vlan_zero(struct ice_vsi *vsi)
4161 {
4162         struct ice_vlan vlan;
4163         int err;
4164
4165         vlan = ICE_VLAN(0, 0);
4166         err = ice_remove_vlan_filter(vsi, &vlan);
4167         if (err) {
4168                 PMD_DRV_LOG(DEBUG, "Failed to remove VLAN ID 0");
4169                 return err;
4170         }
4171
4172         /* in SVM both VLAN 0 filters are identical */
4173         if (!ice_is_dvm_ena(&vsi->adapter->hw))
4174                 return 0;
4175
4176         vlan = ICE_VLAN(RTE_ETHER_TYPE_VLAN, 0);
4177         err = ice_remove_vlan_filter(vsi, &vlan);
4178         if (err) {
4179                 PMD_DRV_LOG(DEBUG, "Failed to remove VLAN ID 0 in double VLAN mode");
4180                 return err;
4181         }
4182
4183         return 0;
4184 }
4185
4186 /* Configure vlan filter on or off */
4187 static int
4188 ice_vsi_config_vlan_filter(struct ice_vsi *vsi, bool on)
4189 {
4190         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
4191         struct ice_vsi_ctx ctxt;
4192         uint8_t sw_flags2;
4193         int ret = 0;
4194
4195         sw_flags2 = ICE_AQ_VSI_SW_FLAG_RX_VLAN_PRUNE_ENA;
4196
4197         if (on)
4198                 vsi->info.sw_flags2 |= sw_flags2;
4199         else
4200                 vsi->info.sw_flags2 &= ~sw_flags2;
4201
4202         vsi->info.sw_id = hw->port_info->sw_id;
4203         (void)rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
4204         ctxt.info.valid_sections =
4205                 rte_cpu_to_le_16(ICE_AQ_VSI_PROP_SW_VALID |
4206                                  ICE_AQ_VSI_PROP_SECURITY_VALID);
4207         ctxt.vsi_num = vsi->vsi_id;
4208
4209         ret = ice_update_vsi(hw, vsi->idx, &ctxt, NULL);
4210         if (ret) {
4211                 PMD_DRV_LOG(INFO, "Update VSI failed to %s vlan rx pruning",
4212                             on ? "enable" : "disable");
4213                 return -EINVAL;
4214         } else {
4215                 vsi->info.valid_sections |=
4216                         rte_cpu_to_le_16(ICE_AQ_VSI_PROP_SW_VALID |
4217                                          ICE_AQ_VSI_PROP_SECURITY_VALID);
4218         }
4219
4220         /* consist with other drivers, allow untagged packet when vlan filter on */
4221         if (on)
4222                 ret = ice_vsi_add_vlan_zero(vsi);
4223         else
4224                 ret = ice_vsi_del_vlan_zero(vsi);
4225
4226         return 0;
4227 }
4228
4229 /* Manage VLAN stripping for the VSI for Rx */
4230 static int
4231 ice_vsi_manage_vlan_stripping(struct ice_vsi *vsi, bool ena)
4232 {
4233         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
4234         struct ice_vsi_ctx ctxt;
4235         enum ice_status status;
4236         int err = 0;
4237
4238         /* do not allow modifying VLAN stripping when a port VLAN is configured
4239          * on this VSI
4240          */
4241         if (vsi->info.port_based_inner_vlan)
4242                 return 0;
4243
4244         memset(&ctxt, 0, sizeof(ctxt));
4245
4246         if (ena)
4247                 /* Strip VLAN tag from Rx packet and put it in the desc */
4248                 ctxt.info.inner_vlan_flags =
4249                                         ICE_AQ_VSI_INNER_VLAN_EMODE_STR_BOTH;
4250         else
4251                 /* Disable stripping. Leave tag in packet */
4252                 ctxt.info.inner_vlan_flags =
4253                                         ICE_AQ_VSI_INNER_VLAN_EMODE_NOTHING;
4254
4255         /* Allow all packets untagged/tagged */
4256         ctxt.info.inner_vlan_flags |= ICE_AQ_VSI_INNER_VLAN_TX_MODE_ALL;
4257
4258         ctxt.info.valid_sections = rte_cpu_to_le_16(ICE_AQ_VSI_PROP_VLAN_VALID);
4259
4260         status = ice_update_vsi(hw, vsi->idx, &ctxt, NULL);
4261         if (status) {
4262                 PMD_DRV_LOG(ERR, "Update VSI failed to %s vlan stripping",
4263                             ena ? "enable" : "disable");
4264                 err = -EIO;
4265         } else {
4266                 vsi->info.inner_vlan_flags = ctxt.info.inner_vlan_flags;
4267         }
4268
4269         return err;
4270 }
4271
4272 static int
4273 ice_vsi_ena_inner_stripping(struct ice_vsi *vsi)
4274 {
4275         return ice_vsi_manage_vlan_stripping(vsi, true);
4276 }
4277
4278 static int
4279 ice_vsi_dis_inner_stripping(struct ice_vsi *vsi)
4280 {
4281         return ice_vsi_manage_vlan_stripping(vsi, false);
4282 }
4283
4284 static int ice_vsi_ena_outer_stripping(struct ice_vsi *vsi)
4285 {
4286         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
4287         struct ice_vsi_ctx ctxt;
4288         enum ice_status status;
4289         int err = 0;
4290
4291         /* do not allow modifying VLAN stripping when a port VLAN is configured
4292          * on this VSI
4293          */
4294         if (vsi->info.port_based_outer_vlan)
4295                 return 0;
4296
4297         memset(&ctxt, 0, sizeof(ctxt));
4298
4299         ctxt.info.valid_sections =
4300                 rte_cpu_to_le_16(ICE_AQ_VSI_PROP_OUTER_TAG_VALID);
4301         /* clear current outer VLAN strip settings */
4302         ctxt.info.outer_vlan_flags = vsi->info.outer_vlan_flags &
4303                 ~(ICE_AQ_VSI_OUTER_VLAN_EMODE_M | ICE_AQ_VSI_OUTER_TAG_TYPE_M);
4304         ctxt.info.outer_vlan_flags |=
4305                 (ICE_AQ_VSI_OUTER_VLAN_EMODE_SHOW_BOTH <<
4306                  ICE_AQ_VSI_OUTER_VLAN_EMODE_S) |
4307                 (ICE_AQ_VSI_OUTER_TAG_VLAN_8100 <<
4308                  ICE_AQ_VSI_OUTER_TAG_TYPE_S);
4309
4310         status = ice_update_vsi(hw, vsi->idx, &ctxt, NULL);
4311         if (status) {
4312                 PMD_DRV_LOG(ERR, "Update VSI failed to enable outer VLAN stripping");
4313                 err = -EIO;
4314         } else {
4315                 vsi->info.outer_vlan_flags = ctxt.info.outer_vlan_flags;
4316         }
4317
4318         return err;
4319 }
4320
4321 static int
4322 ice_vsi_dis_outer_stripping(struct ice_vsi *vsi)
4323 {
4324         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
4325         struct ice_vsi_ctx ctxt;
4326         enum ice_status status;
4327         int err = 0;
4328
4329         if (vsi->info.port_based_outer_vlan)
4330                 return 0;
4331
4332         memset(&ctxt, 0, sizeof(ctxt));
4333
4334         ctxt.info.valid_sections =
4335                 rte_cpu_to_le_16(ICE_AQ_VSI_PROP_OUTER_TAG_VALID);
4336         /* clear current outer VLAN strip settings */
4337         ctxt.info.outer_vlan_flags = vsi->info.outer_vlan_flags &
4338                 ~ICE_AQ_VSI_OUTER_VLAN_EMODE_M;
4339         ctxt.info.outer_vlan_flags |= ICE_AQ_VSI_OUTER_VLAN_EMODE_NOTHING <<
4340                 ICE_AQ_VSI_OUTER_VLAN_EMODE_S;
4341
4342         status = ice_update_vsi(hw, vsi->idx, &ctxt, NULL);
4343         if (status) {
4344                 PMD_DRV_LOG(ERR, "Update VSI failed to disable outer VLAN stripping");
4345                 err = -EIO;
4346         } else {
4347                 vsi->info.outer_vlan_flags = ctxt.info.outer_vlan_flags;
4348         }
4349
4350         return err;
4351 }
4352
4353 static int
4354 ice_vsi_config_vlan_stripping(struct ice_vsi *vsi, bool ena)
4355 {
4356         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
4357         int ret;
4358
4359         if (ice_is_dvm_ena(hw)) {
4360                 if (ena)
4361                         ret = ice_vsi_ena_outer_stripping(vsi);
4362                 else
4363                         ret = ice_vsi_dis_outer_stripping(vsi);
4364         } else {
4365                 if (ena)
4366                         ret = ice_vsi_ena_inner_stripping(vsi);
4367                 else
4368                         ret = ice_vsi_dis_inner_stripping(vsi);
4369         }
4370
4371         return ret;
4372 }
4373
4374 static int
4375 ice_vlan_offload_set(struct rte_eth_dev *dev, int mask)
4376 {
4377         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
4378         struct ice_vsi *vsi = pf->main_vsi;
4379         struct rte_eth_rxmode *rxmode;
4380
4381         rxmode = &dev->data->dev_conf.rxmode;
4382         if (mask & ETH_VLAN_FILTER_MASK) {
4383                 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_FILTER)
4384                         ice_vsi_config_vlan_filter(vsi, true);
4385                 else
4386                         ice_vsi_config_vlan_filter(vsi, false);
4387         }
4388
4389         if (mask & ETH_VLAN_STRIP_MASK) {
4390                 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
4391                         ice_vsi_config_vlan_stripping(vsi, true);
4392                 else
4393                         ice_vsi_config_vlan_stripping(vsi, false);
4394         }
4395
4396         return 0;
4397 }
4398
4399 static int
4400 ice_get_rss_lut(struct ice_vsi *vsi, uint8_t *lut, uint16_t lut_size)
4401 {
4402         struct ice_aq_get_set_rss_lut_params lut_params;
4403         struct ice_pf *pf = ICE_VSI_TO_PF(vsi);
4404         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
4405         int ret;
4406
4407         if (!lut)
4408                 return -EINVAL;
4409
4410         if (pf->flags & ICE_FLAG_RSS_AQ_CAPABLE) {
4411                 lut_params.vsi_handle = vsi->idx;
4412                 lut_params.lut_size = lut_size;
4413                 lut_params.lut_type = ICE_AQC_GSET_RSS_LUT_TABLE_TYPE_PF;
4414                 lut_params.lut = lut;
4415                 lut_params.global_lut_id = 0;
4416                 ret = ice_aq_get_rss_lut(hw, &lut_params);
4417                 if (ret) {
4418                         PMD_DRV_LOG(ERR, "Failed to get RSS lookup table");
4419                         return -EINVAL;
4420                 }
4421         } else {
4422                 uint64_t *lut_dw = (uint64_t *)lut;
4423                 uint16_t i, lut_size_dw = lut_size / 4;
4424
4425                 for (i = 0; i < lut_size_dw; i++)
4426                         lut_dw[i] = ICE_READ_REG(hw, PFQF_HLUT(i));
4427         }
4428
4429         return 0;
4430 }
4431
4432 static int
4433 ice_set_rss_lut(struct ice_vsi *vsi, uint8_t *lut, uint16_t lut_size)
4434 {
4435         struct ice_aq_get_set_rss_lut_params lut_params;
4436         struct ice_pf *pf;
4437         struct ice_hw *hw;
4438         int ret;
4439
4440         if (!vsi || !lut)
4441                 return -EINVAL;
4442
4443         pf = ICE_VSI_TO_PF(vsi);
4444         hw = ICE_VSI_TO_HW(vsi);
4445
4446         if (pf->flags & ICE_FLAG_RSS_AQ_CAPABLE) {
4447                 lut_params.vsi_handle = vsi->idx;
4448                 lut_params.lut_size = lut_size;
4449                 lut_params.lut_type = ICE_AQC_GSET_RSS_LUT_TABLE_TYPE_PF;
4450                 lut_params.lut = lut;
4451                 lut_params.global_lut_id = 0;
4452                 ret = ice_aq_set_rss_lut(hw, &lut_params);
4453                 if (ret) {
4454                         PMD_DRV_LOG(ERR, "Failed to set RSS lookup table");
4455                         return -EINVAL;
4456                 }
4457         } else {
4458                 uint64_t *lut_dw = (uint64_t *)lut;
4459                 uint16_t i, lut_size_dw = lut_size / 4;
4460
4461                 for (i = 0; i < lut_size_dw; i++)
4462                         ICE_WRITE_REG(hw, PFQF_HLUT(i), lut_dw[i]);
4463
4464                 ice_flush(hw);
4465         }
4466
4467         return 0;
4468 }
4469
4470 static int
4471 ice_rss_reta_update(struct rte_eth_dev *dev,
4472                     struct rte_eth_rss_reta_entry64 *reta_conf,
4473                     uint16_t reta_size)
4474 {
4475         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
4476         uint16_t i, lut_size = pf->hash_lut_size;
4477         uint16_t idx, shift;
4478         uint8_t *lut;
4479         int ret;
4480
4481         if (reta_size != ICE_AQC_GSET_RSS_LUT_TABLE_SIZE_128 &&
4482             reta_size != ICE_AQC_GSET_RSS_LUT_TABLE_SIZE_512 &&
4483             reta_size != ICE_AQC_GSET_RSS_LUT_TABLE_SIZE_2K) {
4484                 PMD_DRV_LOG(ERR,
4485                             "The size of hash lookup table configured (%d)"
4486                             "doesn't match the number hardware can "
4487                             "supported (128, 512, 2048)",
4488                             reta_size);
4489                 return -EINVAL;
4490         }
4491
4492         /* It MUST use the current LUT size to get the RSS lookup table,
4493          * otherwise if will fail with -100 error code.
4494          */
4495         lut = rte_zmalloc(NULL,  RTE_MAX(reta_size, lut_size), 0);
4496         if (!lut) {
4497                 PMD_DRV_LOG(ERR, "No memory can be allocated");
4498                 return -ENOMEM;
4499         }
4500         ret = ice_get_rss_lut(pf->main_vsi, lut, lut_size);
4501         if (ret)
4502                 goto out;
4503
4504         for (i = 0; i < reta_size; i++) {
4505                 idx = i / RTE_RETA_GROUP_SIZE;
4506                 shift = i % RTE_RETA_GROUP_SIZE;
4507                 if (reta_conf[idx].mask & (1ULL << shift))
4508                         lut[i] = reta_conf[idx].reta[shift];
4509         }
4510         ret = ice_set_rss_lut(pf->main_vsi, lut, reta_size);
4511         if (ret == 0 && lut_size != reta_size) {
4512                 PMD_DRV_LOG(INFO,
4513                             "The size of hash lookup table is changed from (%d) to (%d)",
4514                             lut_size, reta_size);
4515                 pf->hash_lut_size = reta_size;
4516         }
4517
4518 out:
4519         rte_free(lut);
4520
4521         return ret;
4522 }
4523
4524 static int
4525 ice_rss_reta_query(struct rte_eth_dev *dev,
4526                    struct rte_eth_rss_reta_entry64 *reta_conf,
4527                    uint16_t reta_size)
4528 {
4529         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
4530         uint16_t i, lut_size = pf->hash_lut_size;
4531         uint16_t idx, shift;
4532         uint8_t *lut;
4533         int ret;
4534
4535         if (reta_size != lut_size) {
4536                 PMD_DRV_LOG(ERR,
4537                             "The size of hash lookup table configured (%d)"
4538                             "doesn't match the number hardware can "
4539                             "supported (%d)",
4540                             reta_size, lut_size);
4541                 return -EINVAL;
4542         }
4543
4544         lut = rte_zmalloc(NULL, reta_size, 0);
4545         if (!lut) {
4546                 PMD_DRV_LOG(ERR, "No memory can be allocated");
4547                 return -ENOMEM;
4548         }
4549
4550         ret = ice_get_rss_lut(pf->main_vsi, lut, reta_size);
4551         if (ret)
4552                 goto out;
4553
4554         for (i = 0; i < reta_size; i++) {
4555                 idx = i / RTE_RETA_GROUP_SIZE;
4556                 shift = i % RTE_RETA_GROUP_SIZE;
4557                 if (reta_conf[idx].mask & (1ULL << shift))
4558                         reta_conf[idx].reta[shift] = lut[i];
4559         }
4560
4561 out:
4562         rte_free(lut);
4563
4564         return ret;
4565 }
4566
4567 static int
4568 ice_set_rss_key(struct ice_vsi *vsi, uint8_t *key, uint8_t key_len)
4569 {
4570         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
4571         int ret = 0;
4572
4573         if (!key || key_len == 0) {
4574                 PMD_DRV_LOG(DEBUG, "No key to be configured");
4575                 return 0;
4576         } else if (key_len != (VSIQF_HKEY_MAX_INDEX + 1) *
4577                    sizeof(uint32_t)) {
4578                 PMD_DRV_LOG(ERR, "Invalid key length %u", key_len);
4579                 return -EINVAL;
4580         }
4581
4582         struct ice_aqc_get_set_rss_keys *key_dw =
4583                 (struct ice_aqc_get_set_rss_keys *)key;
4584
4585         ret = ice_aq_set_rss_key(hw, vsi->idx, key_dw);
4586         if (ret) {
4587                 PMD_DRV_LOG(ERR, "Failed to configure RSS key via AQ");
4588                 ret = -EINVAL;
4589         }
4590
4591         return ret;
4592 }
4593
4594 static int
4595 ice_get_rss_key(struct ice_vsi *vsi, uint8_t *key, uint8_t *key_len)
4596 {
4597         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
4598         int ret;
4599
4600         if (!key || !key_len)
4601                 return -EINVAL;
4602
4603         ret = ice_aq_get_rss_key
4604                 (hw, vsi->idx,
4605                  (struct ice_aqc_get_set_rss_keys *)key);
4606         if (ret) {
4607                 PMD_DRV_LOG(ERR, "Failed to get RSS key via AQ");
4608                 return -EINVAL;
4609         }
4610         *key_len = (VSIQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t);
4611
4612         return 0;
4613 }
4614
4615 static int
4616 ice_rss_hash_update(struct rte_eth_dev *dev,
4617                     struct rte_eth_rss_conf *rss_conf)
4618 {
4619         enum ice_status status = ICE_SUCCESS;
4620         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
4621         struct ice_vsi *vsi = pf->main_vsi;
4622
4623         /* set hash key */
4624         status = ice_set_rss_key(vsi, rss_conf->rss_key, rss_conf->rss_key_len);
4625         if (status)
4626                 return status;
4627
4628         if (rss_conf->rss_hf == 0) {
4629                 pf->rss_hf = 0;
4630                 return 0;
4631         }
4632
4633         /* RSS hash configuration */
4634         ice_rss_hash_set(pf, rss_conf->rss_hf);
4635
4636         return 0;
4637 }
4638
4639 static int
4640 ice_rss_hash_conf_get(struct rte_eth_dev *dev,
4641                       struct rte_eth_rss_conf *rss_conf)
4642 {
4643         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
4644         struct ice_vsi *vsi = pf->main_vsi;
4645
4646         ice_get_rss_key(vsi, rss_conf->rss_key,
4647                         &rss_conf->rss_key_len);
4648
4649         rss_conf->rss_hf = pf->rss_hf;
4650         return 0;
4651 }
4652
4653 static int
4654 ice_promisc_enable(struct rte_eth_dev *dev)
4655 {
4656         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
4657         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4658         struct ice_vsi *vsi = pf->main_vsi;
4659         enum ice_status status;
4660         uint8_t pmask;
4661         int ret = 0;
4662
4663         pmask = ICE_PROMISC_UCAST_RX | ICE_PROMISC_UCAST_TX |
4664                 ICE_PROMISC_MCAST_RX | ICE_PROMISC_MCAST_TX;
4665
4666         status = ice_set_vsi_promisc(hw, vsi->idx, pmask, 0);
4667         switch (status) {
4668         case ICE_ERR_ALREADY_EXISTS:
4669                 PMD_DRV_LOG(DEBUG, "Promisc mode has already been enabled");
4670         case ICE_SUCCESS:
4671                 break;
4672         default:
4673                 PMD_DRV_LOG(ERR, "Failed to enable promisc, err=%d", status);
4674                 ret = -EAGAIN;
4675         }
4676
4677         return ret;
4678 }
4679
4680 static int
4681 ice_promisc_disable(struct rte_eth_dev *dev)
4682 {
4683         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
4684         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4685         struct ice_vsi *vsi = pf->main_vsi;
4686         enum ice_status status;
4687         uint8_t pmask;
4688         int ret = 0;
4689
4690         if (dev->data->all_multicast == 1)
4691                 pmask = ICE_PROMISC_UCAST_RX | ICE_PROMISC_UCAST_TX;
4692         else
4693                 pmask = ICE_PROMISC_UCAST_RX | ICE_PROMISC_UCAST_TX |
4694                         ICE_PROMISC_MCAST_RX | ICE_PROMISC_MCAST_TX;
4695
4696         status = ice_clear_vsi_promisc(hw, vsi->idx, pmask, 0);
4697         if (status != ICE_SUCCESS) {
4698                 PMD_DRV_LOG(ERR, "Failed to clear promisc, err=%d", status);
4699                 ret = -EAGAIN;
4700         }
4701
4702         return ret;
4703 }
4704
4705 static int
4706 ice_allmulti_enable(struct rte_eth_dev *dev)
4707 {
4708         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
4709         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4710         struct ice_vsi *vsi = pf->main_vsi;
4711         enum ice_status status;
4712         uint8_t pmask;
4713         int ret = 0;
4714
4715         pmask = ICE_PROMISC_MCAST_RX | ICE_PROMISC_MCAST_TX;
4716
4717         status = ice_set_vsi_promisc(hw, vsi->idx, pmask, 0);
4718
4719         switch (status) {
4720         case ICE_ERR_ALREADY_EXISTS:
4721                 PMD_DRV_LOG(DEBUG, "Allmulti has already been enabled");
4722         case ICE_SUCCESS:
4723                 break;
4724         default:
4725                 PMD_DRV_LOG(ERR, "Failed to enable allmulti, err=%d", status);
4726                 ret = -EAGAIN;
4727         }
4728
4729         return ret;
4730 }
4731
4732 static int
4733 ice_allmulti_disable(struct rte_eth_dev *dev)
4734 {
4735         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
4736         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4737         struct ice_vsi *vsi = pf->main_vsi;
4738         enum ice_status status;
4739         uint8_t pmask;
4740         int ret = 0;
4741
4742         if (dev->data->promiscuous == 1)
4743                 return 0; /* must remain in all_multicast mode */
4744
4745         pmask = ICE_PROMISC_MCAST_RX | ICE_PROMISC_MCAST_TX;
4746
4747         status = ice_clear_vsi_promisc(hw, vsi->idx, pmask, 0);
4748         if (status != ICE_SUCCESS) {
4749                 PMD_DRV_LOG(ERR, "Failed to clear allmulti, err=%d", status);
4750                 ret = -EAGAIN;
4751         }
4752
4753         return ret;
4754 }
4755
4756 static int ice_rx_queue_intr_enable(struct rte_eth_dev *dev,
4757                                     uint16_t queue_id)
4758 {
4759         struct rte_pci_device *pci_dev = ICE_DEV_TO_PCI(dev);
4760         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
4761         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4762         uint32_t val;
4763         uint16_t msix_intr;
4764
4765         msix_intr = intr_handle->intr_vec[queue_id];
4766
4767         val = GLINT_DYN_CTL_INTENA_M | GLINT_DYN_CTL_CLEARPBA_M |
4768               GLINT_DYN_CTL_ITR_INDX_M;
4769         val &= ~GLINT_DYN_CTL_WB_ON_ITR_M;
4770
4771         ICE_WRITE_REG(hw, GLINT_DYN_CTL(msix_intr), val);
4772         rte_intr_ack(&pci_dev->intr_handle);
4773
4774         return 0;
4775 }
4776
4777 static int ice_rx_queue_intr_disable(struct rte_eth_dev *dev,
4778                                      uint16_t queue_id)
4779 {
4780         struct rte_pci_device *pci_dev = ICE_DEV_TO_PCI(dev);
4781         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
4782         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4783         uint16_t msix_intr;
4784
4785         msix_intr = intr_handle->intr_vec[queue_id];
4786
4787         ICE_WRITE_REG(hw, GLINT_DYN_CTL(msix_intr), GLINT_DYN_CTL_WB_ON_ITR_M);
4788
4789         return 0;
4790 }
4791
4792 static int
4793 ice_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size)
4794 {
4795         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4796         u8 ver, patch;
4797         u16 build;
4798         int ret;
4799
4800         ver = hw->flash.orom.major;
4801         patch = hw->flash.orom.patch;
4802         build = hw->flash.orom.build;
4803
4804         ret = snprintf(fw_version, fw_size,
4805                         "%x.%02x 0x%08x %d.%d.%d",
4806                         hw->flash.nvm.major,
4807                         hw->flash.nvm.minor,
4808                         hw->flash.nvm.eetrack,
4809                         ver, build, patch);
4810         if (ret < 0)
4811                 return -EINVAL;
4812
4813         /* add the size of '\0' */
4814         ret += 1;
4815         if (fw_size < (size_t)ret)
4816                 return ret;
4817         else
4818                 return 0;
4819 }
4820
4821 static int
4822 ice_vsi_vlan_pvid_set(struct ice_vsi *vsi, struct ice_vsi_vlan_pvid_info *info)
4823 {
4824         struct ice_hw *hw;
4825         struct ice_vsi_ctx ctxt;
4826         uint8_t vlan_flags = 0;
4827         int ret;
4828
4829         if (!vsi || !info) {
4830                 PMD_DRV_LOG(ERR, "invalid parameters");
4831                 return -EINVAL;
4832         }
4833
4834         if (info->on) {
4835                 vsi->info.port_based_inner_vlan = info->config.pvid;
4836                 /**
4837                  * If insert pvid is enabled, only tagged pkts are
4838                  * allowed to be sent out.
4839                  */
4840                 vlan_flags = ICE_AQ_VSI_INNER_VLAN_INSERT_PVID |
4841                              ICE_AQ_VSI_INNER_VLAN_TX_MODE_ACCEPTUNTAGGED;
4842         } else {
4843                 vsi->info.port_based_inner_vlan = 0;
4844                 if (info->config.reject.tagged == 0)
4845                         vlan_flags |= ICE_AQ_VSI_INNER_VLAN_TX_MODE_ACCEPTTAGGED;
4846
4847                 if (info->config.reject.untagged == 0)
4848                         vlan_flags |= ICE_AQ_VSI_INNER_VLAN_TX_MODE_ACCEPTUNTAGGED;
4849         }
4850         vsi->info.inner_vlan_flags &= ~(ICE_AQ_VSI_INNER_VLAN_INSERT_PVID |
4851                                   ICE_AQ_VSI_INNER_VLAN_EMODE_M);
4852         vsi->info.inner_vlan_flags |= vlan_flags;
4853         memset(&ctxt, 0, sizeof(ctxt));
4854         rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
4855         ctxt.info.valid_sections =
4856                 rte_cpu_to_le_16(ICE_AQ_VSI_PROP_VLAN_VALID);
4857         ctxt.vsi_num = vsi->vsi_id;
4858
4859         hw = ICE_VSI_TO_HW(vsi);
4860         ret = ice_update_vsi(hw, vsi->idx, &ctxt, NULL);
4861         if (ret != ICE_SUCCESS) {
4862                 PMD_DRV_LOG(ERR,
4863                             "update VSI for VLAN insert failed, err %d",
4864                             ret);
4865                 return -EINVAL;
4866         }
4867
4868         vsi->info.valid_sections |=
4869                 rte_cpu_to_le_16(ICE_AQ_VSI_PROP_VLAN_VALID);
4870
4871         return ret;
4872 }
4873
4874 static int
4875 ice_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid, int on)
4876 {
4877         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
4878         struct ice_vsi *vsi = pf->main_vsi;
4879         struct rte_eth_dev_data *data = pf->dev_data;
4880         struct ice_vsi_vlan_pvid_info info;
4881         int ret;
4882
4883         memset(&info, 0, sizeof(info));
4884         info.on = on;
4885         if (info.on) {
4886                 info.config.pvid = pvid;
4887         } else {
4888                 info.config.reject.tagged =
4889                         data->dev_conf.txmode.hw_vlan_reject_tagged;
4890                 info.config.reject.untagged =
4891                         data->dev_conf.txmode.hw_vlan_reject_untagged;
4892         }
4893
4894         ret = ice_vsi_vlan_pvid_set(vsi, &info);
4895         if (ret < 0) {
4896                 PMD_DRV_LOG(ERR, "Failed to set pvid.");
4897                 return -EINVAL;
4898         }
4899
4900         return 0;
4901 }
4902
4903 static int
4904 ice_get_eeprom_length(struct rte_eth_dev *dev)
4905 {
4906         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4907
4908         return hw->flash.flash_size;
4909 }
4910
4911 static int
4912 ice_get_eeprom(struct rte_eth_dev *dev,
4913                struct rte_dev_eeprom_info *eeprom)
4914 {
4915         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4916         enum ice_status status = ICE_SUCCESS;
4917         uint8_t *data = eeprom->data;
4918
4919         eeprom->magic = hw->vendor_id | (hw->device_id << 16);
4920
4921         status = ice_acquire_nvm(hw, ICE_RES_READ);
4922         if (status) {
4923                 PMD_DRV_LOG(ERR, "acquire nvm failed.");
4924                 return -EIO;
4925         }
4926
4927         status = ice_read_flat_nvm(hw, eeprom->offset, &eeprom->length,
4928                                    data, false);
4929
4930         ice_release_nvm(hw);
4931
4932         if (status) {
4933                 PMD_DRV_LOG(ERR, "EEPROM read failed.");
4934                 return -EIO;
4935         }
4936
4937         return 0;
4938 }
4939
4940 static void
4941 ice_stat_update_32(struct ice_hw *hw,
4942                    uint32_t reg,
4943                    bool offset_loaded,
4944                    uint64_t *offset,
4945                    uint64_t *stat)
4946 {
4947         uint64_t new_data;
4948
4949         new_data = (uint64_t)ICE_READ_REG(hw, reg);
4950         if (!offset_loaded)
4951                 *offset = new_data;
4952
4953         if (new_data >= *offset)
4954                 *stat = (uint64_t)(new_data - *offset);
4955         else
4956                 *stat = (uint64_t)((new_data +
4957                                     ((uint64_t)1 << ICE_32_BIT_WIDTH))
4958                                    - *offset);
4959 }
4960
4961 static void
4962 ice_stat_update_40(struct ice_hw *hw,
4963                    uint32_t hireg,
4964                    uint32_t loreg,
4965                    bool offset_loaded,
4966                    uint64_t *offset,
4967                    uint64_t *stat)
4968 {
4969         uint64_t new_data;
4970
4971         new_data = (uint64_t)ICE_READ_REG(hw, loreg);
4972         new_data |= (uint64_t)(ICE_READ_REG(hw, hireg) & ICE_8_BIT_MASK) <<
4973                     ICE_32_BIT_WIDTH;
4974
4975         if (!offset_loaded)
4976                 *offset = new_data;
4977
4978         if (new_data >= *offset)
4979                 *stat = new_data - *offset;
4980         else
4981                 *stat = (uint64_t)((new_data +
4982                                     ((uint64_t)1 << ICE_40_BIT_WIDTH)) -
4983                                    *offset);
4984
4985         *stat &= ICE_40_BIT_MASK;
4986 }
4987
4988 /* Get all the statistics of a VSI */
4989 static void
4990 ice_update_vsi_stats(struct ice_vsi *vsi)
4991 {
4992         struct ice_eth_stats *oes = &vsi->eth_stats_offset;
4993         struct ice_eth_stats *nes = &vsi->eth_stats;
4994         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
4995         int idx = rte_le_to_cpu_16(vsi->vsi_id);
4996
4997         ice_stat_update_40(hw, GLV_GORCH(idx), GLV_GORCL(idx),
4998                            vsi->offset_loaded, &oes->rx_bytes,
4999                            &nes->rx_bytes);
5000         ice_stat_update_40(hw, GLV_UPRCH(idx), GLV_UPRCL(idx),
5001                            vsi->offset_loaded, &oes->rx_unicast,
5002                            &nes->rx_unicast);
5003         ice_stat_update_40(hw, GLV_MPRCH(idx), GLV_MPRCL(idx),
5004                            vsi->offset_loaded, &oes->rx_multicast,
5005                            &nes->rx_multicast);
5006         ice_stat_update_40(hw, GLV_BPRCH(idx), GLV_BPRCL(idx),
5007                            vsi->offset_loaded, &oes->rx_broadcast,
5008                            &nes->rx_broadcast);
5009         /* enlarge the limitation when rx_bytes overflowed */
5010         if (vsi->offset_loaded) {
5011                 if (ICE_RXTX_BYTES_LOW(vsi->old_rx_bytes) > nes->rx_bytes)
5012                         nes->rx_bytes += (uint64_t)1 << ICE_40_BIT_WIDTH;
5013                 nes->rx_bytes += ICE_RXTX_BYTES_HIGH(vsi->old_rx_bytes);
5014         }
5015         vsi->old_rx_bytes = nes->rx_bytes;
5016         /* exclude CRC bytes */
5017         nes->rx_bytes -= (nes->rx_unicast + nes->rx_multicast +
5018                           nes->rx_broadcast) * RTE_ETHER_CRC_LEN;
5019
5020         ice_stat_update_32(hw, GLV_RDPC(idx), vsi->offset_loaded,
5021                            &oes->rx_discards, &nes->rx_discards);
5022         /* GLV_REPC not supported */
5023         /* GLV_RMPC not supported */
5024         ice_stat_update_32(hw, GLSWID_RUPP(idx), vsi->offset_loaded,
5025                            &oes->rx_unknown_protocol,
5026                            &nes->rx_unknown_protocol);
5027         ice_stat_update_40(hw, GLV_GOTCH(idx), GLV_GOTCL(idx),
5028                            vsi->offset_loaded, &oes->tx_bytes,
5029                            &nes->tx_bytes);
5030         ice_stat_update_40(hw, GLV_UPTCH(idx), GLV_UPTCL(idx),
5031                            vsi->offset_loaded, &oes->tx_unicast,
5032                            &nes->tx_unicast);
5033         ice_stat_update_40(hw, GLV_MPTCH(idx), GLV_MPTCL(idx),
5034                            vsi->offset_loaded, &oes->tx_multicast,
5035                            &nes->tx_multicast);
5036         ice_stat_update_40(hw, GLV_BPTCH(idx), GLV_BPTCL(idx),
5037                            vsi->offset_loaded,  &oes->tx_broadcast,
5038                            &nes->tx_broadcast);
5039         /* GLV_TDPC not supported */
5040         ice_stat_update_32(hw, GLV_TEPC(idx), vsi->offset_loaded,
5041                            &oes->tx_errors, &nes->tx_errors);
5042         /* enlarge the limitation when tx_bytes overflowed */
5043         if (vsi->offset_loaded) {
5044                 if (ICE_RXTX_BYTES_LOW(vsi->old_tx_bytes) > nes->tx_bytes)
5045                         nes->tx_bytes += (uint64_t)1 << ICE_40_BIT_WIDTH;
5046                 nes->tx_bytes += ICE_RXTX_BYTES_HIGH(vsi->old_tx_bytes);
5047         }
5048         vsi->old_tx_bytes = nes->tx_bytes;
5049         vsi->offset_loaded = true;
5050
5051         PMD_DRV_LOG(DEBUG, "************** VSI[%u] stats start **************",
5052                     vsi->vsi_id);
5053         PMD_DRV_LOG(DEBUG, "rx_bytes:            %"PRIu64"", nes->rx_bytes);
5054         PMD_DRV_LOG(DEBUG, "rx_unicast:          %"PRIu64"", nes->rx_unicast);
5055         PMD_DRV_LOG(DEBUG, "rx_multicast:        %"PRIu64"", nes->rx_multicast);
5056         PMD_DRV_LOG(DEBUG, "rx_broadcast:        %"PRIu64"", nes->rx_broadcast);
5057         PMD_DRV_LOG(DEBUG, "rx_discards:         %"PRIu64"", nes->rx_discards);
5058         PMD_DRV_LOG(DEBUG, "rx_unknown_protocol: %"PRIu64"",
5059                     nes->rx_unknown_protocol);
5060         PMD_DRV_LOG(DEBUG, "tx_bytes:            %"PRIu64"", nes->tx_bytes);
5061         PMD_DRV_LOG(DEBUG, "tx_unicast:          %"PRIu64"", nes->tx_unicast);
5062         PMD_DRV_LOG(DEBUG, "tx_multicast:        %"PRIu64"", nes->tx_multicast);
5063         PMD_DRV_LOG(DEBUG, "tx_broadcast:        %"PRIu64"", nes->tx_broadcast);
5064         PMD_DRV_LOG(DEBUG, "tx_discards:         %"PRIu64"", nes->tx_discards);
5065         PMD_DRV_LOG(DEBUG, "tx_errors:           %"PRIu64"", nes->tx_errors);
5066         PMD_DRV_LOG(DEBUG, "************** VSI[%u] stats end ****************",
5067                     vsi->vsi_id);
5068 }
5069
5070 static void
5071 ice_read_stats_registers(struct ice_pf *pf, struct ice_hw *hw)
5072 {
5073         struct ice_hw_port_stats *ns = &pf->stats; /* new stats */
5074         struct ice_hw_port_stats *os = &pf->stats_offset; /* old stats */
5075
5076         /* Get statistics of struct ice_eth_stats */
5077         ice_stat_update_40(hw, GLPRT_GORCH(hw->port_info->lport),
5078                            GLPRT_GORCL(hw->port_info->lport),
5079                            pf->offset_loaded, &os->eth.rx_bytes,
5080                            &ns->eth.rx_bytes);
5081         ice_stat_update_40(hw, GLPRT_UPRCH(hw->port_info->lport),
5082                            GLPRT_UPRCL(hw->port_info->lport),
5083                            pf->offset_loaded, &os->eth.rx_unicast,
5084                            &ns->eth.rx_unicast);
5085         ice_stat_update_40(hw, GLPRT_MPRCH(hw->port_info->lport),
5086                            GLPRT_MPRCL(hw->port_info->lport),
5087                            pf->offset_loaded, &os->eth.rx_multicast,
5088                            &ns->eth.rx_multicast);
5089         ice_stat_update_40(hw, GLPRT_BPRCH(hw->port_info->lport),
5090                            GLPRT_BPRCL(hw->port_info->lport),
5091                            pf->offset_loaded, &os->eth.rx_broadcast,
5092                            &ns->eth.rx_broadcast);
5093         ice_stat_update_32(hw, PRTRPB_RDPC,
5094                            pf->offset_loaded, &os->eth.rx_discards,
5095                            &ns->eth.rx_discards);
5096         /* enlarge the limitation when rx_bytes overflowed */
5097         if (pf->offset_loaded) {
5098                 if (ICE_RXTX_BYTES_LOW(pf->old_rx_bytes) > ns->eth.rx_bytes)
5099                         ns->eth.rx_bytes += (uint64_t)1 << ICE_40_BIT_WIDTH;
5100                 ns->eth.rx_bytes += ICE_RXTX_BYTES_HIGH(pf->old_rx_bytes);
5101         }
5102         pf->old_rx_bytes = ns->eth.rx_bytes;
5103
5104         /* Workaround: CRC size should not be included in byte statistics,
5105          * so subtract RTE_ETHER_CRC_LEN from the byte counter for each rx
5106          * packet.
5107          */
5108         ns->eth.rx_bytes -= (ns->eth.rx_unicast + ns->eth.rx_multicast +
5109                              ns->eth.rx_broadcast) * RTE_ETHER_CRC_LEN;
5110
5111         /* GLPRT_REPC not supported */
5112         /* GLPRT_RMPC not supported */
5113         ice_stat_update_32(hw, GLSWID_RUPP(hw->port_info->lport),
5114                            pf->offset_loaded,
5115                            &os->eth.rx_unknown_protocol,
5116                            &ns->eth.rx_unknown_protocol);
5117         ice_stat_update_40(hw, GLPRT_GOTCH(hw->port_info->lport),
5118                            GLPRT_GOTCL(hw->port_info->lport),
5119                            pf->offset_loaded, &os->eth.tx_bytes,
5120                            &ns->eth.tx_bytes);
5121         ice_stat_update_40(hw, GLPRT_UPTCH(hw->port_info->lport),
5122                            GLPRT_UPTCL(hw->port_info->lport),
5123                            pf->offset_loaded, &os->eth.tx_unicast,
5124                            &ns->eth.tx_unicast);
5125         ice_stat_update_40(hw, GLPRT_MPTCH(hw->port_info->lport),
5126                            GLPRT_MPTCL(hw->port_info->lport),
5127                            pf->offset_loaded, &os->eth.tx_multicast,
5128                            &ns->eth.tx_multicast);
5129         ice_stat_update_40(hw, GLPRT_BPTCH(hw->port_info->lport),
5130                            GLPRT_BPTCL(hw->port_info->lport),
5131                            pf->offset_loaded, &os->eth.tx_broadcast,
5132                            &ns->eth.tx_broadcast);
5133         /* enlarge the limitation when tx_bytes overflowed */
5134         if (pf->offset_loaded) {
5135                 if (ICE_RXTX_BYTES_LOW(pf->old_tx_bytes) > ns->eth.tx_bytes)
5136                         ns->eth.tx_bytes += (uint64_t)1 << ICE_40_BIT_WIDTH;
5137                 ns->eth.tx_bytes += ICE_RXTX_BYTES_HIGH(pf->old_tx_bytes);
5138         }
5139         pf->old_tx_bytes = ns->eth.tx_bytes;
5140         ns->eth.tx_bytes -= (ns->eth.tx_unicast + ns->eth.tx_multicast +
5141                              ns->eth.tx_broadcast) * RTE_ETHER_CRC_LEN;
5142
5143         /* GLPRT_TEPC not supported */
5144
5145         /* additional port specific stats */
5146         ice_stat_update_32(hw, GLPRT_TDOLD(hw->port_info->lport),
5147                            pf->offset_loaded, &os->tx_dropped_link_down,
5148                            &ns->tx_dropped_link_down);
5149         ice_stat_update_32(hw, GLPRT_CRCERRS(hw->port_info->lport),
5150                            pf->offset_loaded, &os->crc_errors,
5151                            &ns->crc_errors);
5152         ice_stat_update_32(hw, GLPRT_ILLERRC(hw->port_info->lport),
5153                            pf->offset_loaded, &os->illegal_bytes,
5154                            &ns->illegal_bytes);
5155         /* GLPRT_ERRBC not supported */
5156         ice_stat_update_32(hw, GLPRT_MLFC(hw->port_info->lport),
5157                            pf->offset_loaded, &os->mac_local_faults,
5158                            &ns->mac_local_faults);
5159         ice_stat_update_32(hw, GLPRT_MRFC(hw->port_info->lport),
5160                            pf->offset_loaded, &os->mac_remote_faults,
5161                            &ns->mac_remote_faults);
5162
5163         ice_stat_update_32(hw, GLPRT_RLEC(hw->port_info->lport),
5164                            pf->offset_loaded, &os->rx_len_errors,
5165                            &ns->rx_len_errors);
5166
5167         ice_stat_update_32(hw, GLPRT_LXONRXC(hw->port_info->lport),
5168                            pf->offset_loaded, &os->link_xon_rx,
5169                            &ns->link_xon_rx);
5170         ice_stat_update_32(hw, GLPRT_LXOFFRXC(hw->port_info->lport),
5171                            pf->offset_loaded, &os->link_xoff_rx,
5172                            &ns->link_xoff_rx);
5173         ice_stat_update_32(hw, GLPRT_LXONTXC(hw->port_info->lport),
5174                            pf->offset_loaded, &os->link_xon_tx,
5175                            &ns->link_xon_tx);
5176         ice_stat_update_32(hw, GLPRT_LXOFFTXC(hw->port_info->lport),
5177                            pf->offset_loaded, &os->link_xoff_tx,
5178                            &ns->link_xoff_tx);
5179         ice_stat_update_40(hw, GLPRT_PRC64H(hw->port_info->lport),
5180                            GLPRT_PRC64L(hw->port_info->lport),
5181                            pf->offset_loaded, &os->rx_size_64,
5182                            &ns->rx_size_64);
5183         ice_stat_update_40(hw, GLPRT_PRC127H(hw->port_info->lport),
5184                            GLPRT_PRC127L(hw->port_info->lport),
5185                            pf->offset_loaded, &os->rx_size_127,
5186                            &ns->rx_size_127);
5187         ice_stat_update_40(hw, GLPRT_PRC255H(hw->port_info->lport),
5188                            GLPRT_PRC255L(hw->port_info->lport),
5189                            pf->offset_loaded, &os->rx_size_255,
5190                            &ns->rx_size_255);
5191         ice_stat_update_40(hw, GLPRT_PRC511H(hw->port_info->lport),
5192                            GLPRT_PRC511L(hw->port_info->lport),
5193                            pf->offset_loaded, &os->rx_size_511,
5194                            &ns->rx_size_511);
5195         ice_stat_update_40(hw, GLPRT_PRC1023H(hw->port_info->lport),
5196                            GLPRT_PRC1023L(hw->port_info->lport),
5197                            pf->offset_loaded, &os->rx_size_1023,
5198                            &ns->rx_size_1023);
5199         ice_stat_update_40(hw, GLPRT_PRC1522H(hw->port_info->lport),
5200                            GLPRT_PRC1522L(hw->port_info->lport),
5201                            pf->offset_loaded, &os->rx_size_1522,
5202                            &ns->rx_size_1522);
5203         ice_stat_update_40(hw, GLPRT_PRC9522H(hw->port_info->lport),
5204                            GLPRT_PRC9522L(hw->port_info->lport),
5205                            pf->offset_loaded, &os->rx_size_big,
5206                            &ns->rx_size_big);
5207         ice_stat_update_32(hw, GLPRT_RUC(hw->port_info->lport),
5208                            pf->offset_loaded, &os->rx_undersize,
5209                            &ns->rx_undersize);
5210         ice_stat_update_32(hw, GLPRT_RFC(hw->port_info->lport),
5211                            pf->offset_loaded, &os->rx_fragments,
5212                            &ns->rx_fragments);
5213         ice_stat_update_32(hw, GLPRT_ROC(hw->port_info->lport),
5214                            pf->offset_loaded, &os->rx_oversize,
5215                            &ns->rx_oversize);
5216         ice_stat_update_32(hw, GLPRT_RJC(hw->port_info->lport),
5217                            pf->offset_loaded, &os->rx_jabber,
5218                            &ns->rx_jabber);
5219         ice_stat_update_40(hw, GLPRT_PTC64H(hw->port_info->lport),
5220                            GLPRT_PTC64L(hw->port_info->lport),
5221                            pf->offset_loaded, &os->tx_size_64,
5222                            &ns->tx_size_64);
5223         ice_stat_update_40(hw, GLPRT_PTC127H(hw->port_info->lport),
5224                            GLPRT_PTC127L(hw->port_info->lport),
5225                            pf->offset_loaded, &os->tx_size_127,
5226                            &ns->tx_size_127);
5227         ice_stat_update_40(hw, GLPRT_PTC255H(hw->port_info->lport),
5228                            GLPRT_PTC255L(hw->port_info->lport),
5229                            pf->offset_loaded, &os->tx_size_255,
5230                            &ns->tx_size_255);
5231         ice_stat_update_40(hw, GLPRT_PTC511H(hw->port_info->lport),
5232                            GLPRT_PTC511L(hw->port_info->lport),
5233                            pf->offset_loaded, &os->tx_size_511,
5234                            &ns->tx_size_511);
5235         ice_stat_update_40(hw, GLPRT_PTC1023H(hw->port_info->lport),
5236                            GLPRT_PTC1023L(hw->port_info->lport),
5237                            pf->offset_loaded, &os->tx_size_1023,
5238                            &ns->tx_size_1023);
5239         ice_stat_update_40(hw, GLPRT_PTC1522H(hw->port_info->lport),
5240                            GLPRT_PTC1522L(hw->port_info->lport),
5241                            pf->offset_loaded, &os->tx_size_1522,
5242                            &ns->tx_size_1522);
5243         ice_stat_update_40(hw, GLPRT_PTC9522H(hw->port_info->lport),
5244                            GLPRT_PTC9522L(hw->port_info->lport),
5245                            pf->offset_loaded, &os->tx_size_big,
5246                            &ns->tx_size_big);
5247
5248         /* GLPRT_MSPDC not supported */
5249         /* GLPRT_XEC not supported */
5250
5251         pf->offset_loaded = true;
5252
5253         if (pf->main_vsi)
5254                 ice_update_vsi_stats(pf->main_vsi);
5255 }
5256
5257 /* Get all statistics of a port */
5258 static int
5259 ice_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
5260 {
5261         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
5262         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5263         struct ice_hw_port_stats *ns = &pf->stats; /* new stats */
5264
5265         /* call read registers - updates values, now write them to struct */
5266         ice_read_stats_registers(pf, hw);
5267
5268         stats->ipackets = pf->main_vsi->eth_stats.rx_unicast +
5269                           pf->main_vsi->eth_stats.rx_multicast +
5270                           pf->main_vsi->eth_stats.rx_broadcast -
5271                           pf->main_vsi->eth_stats.rx_discards;
5272         stats->opackets = ns->eth.tx_unicast +
5273                           ns->eth.tx_multicast +
5274                           ns->eth.tx_broadcast;
5275         stats->ibytes   = pf->main_vsi->eth_stats.rx_bytes;
5276         stats->obytes   = ns->eth.tx_bytes;
5277         stats->oerrors  = ns->eth.tx_errors +
5278                           pf->main_vsi->eth_stats.tx_errors;
5279
5280         /* Rx Errors */
5281         stats->imissed  = ns->eth.rx_discards +
5282                           pf->main_vsi->eth_stats.rx_discards;
5283         stats->ierrors  = ns->crc_errors +
5284                           ns->rx_undersize +
5285                           ns->rx_oversize + ns->rx_fragments + ns->rx_jabber;
5286
5287         PMD_DRV_LOG(DEBUG, "*************** PF stats start *****************");
5288         PMD_DRV_LOG(DEBUG, "rx_bytes:   %"PRIu64"", ns->eth.rx_bytes);
5289         PMD_DRV_LOG(DEBUG, "rx_unicast: %"PRIu64"", ns->eth.rx_unicast);
5290         PMD_DRV_LOG(DEBUG, "rx_multicast:%"PRIu64"", ns->eth.rx_multicast);
5291         PMD_DRV_LOG(DEBUG, "rx_broadcast:%"PRIu64"", ns->eth.rx_broadcast);
5292         PMD_DRV_LOG(DEBUG, "rx_discards:%"PRIu64"", ns->eth.rx_discards);
5293         PMD_DRV_LOG(DEBUG, "vsi rx_discards:%"PRIu64"",
5294                     pf->main_vsi->eth_stats.rx_discards);
5295         PMD_DRV_LOG(DEBUG, "rx_unknown_protocol:  %"PRIu64"",
5296                     ns->eth.rx_unknown_protocol);
5297         PMD_DRV_LOG(DEBUG, "tx_bytes:   %"PRIu64"", ns->eth.tx_bytes);
5298         PMD_DRV_LOG(DEBUG, "tx_unicast: %"PRIu64"", ns->eth.tx_unicast);
5299         PMD_DRV_LOG(DEBUG, "tx_multicast:%"PRIu64"", ns->eth.tx_multicast);
5300         PMD_DRV_LOG(DEBUG, "tx_broadcast:%"PRIu64"", ns->eth.tx_broadcast);
5301         PMD_DRV_LOG(DEBUG, "tx_discards:%"PRIu64"", ns->eth.tx_discards);
5302         PMD_DRV_LOG(DEBUG, "vsi tx_discards:%"PRIu64"",
5303                     pf->main_vsi->eth_stats.tx_discards);
5304         PMD_DRV_LOG(DEBUG, "tx_errors:          %"PRIu64"", ns->eth.tx_errors);
5305
5306         PMD_DRV_LOG(DEBUG, "tx_dropped_link_down:       %"PRIu64"",
5307                     ns->tx_dropped_link_down);
5308         PMD_DRV_LOG(DEBUG, "crc_errors: %"PRIu64"", ns->crc_errors);
5309         PMD_DRV_LOG(DEBUG, "illegal_bytes:      %"PRIu64"",
5310                     ns->illegal_bytes);
5311         PMD_DRV_LOG(DEBUG, "error_bytes:        %"PRIu64"", ns->error_bytes);
5312         PMD_DRV_LOG(DEBUG, "mac_local_faults:   %"PRIu64"",
5313                     ns->mac_local_faults);
5314         PMD_DRV_LOG(DEBUG, "mac_remote_faults:  %"PRIu64"",
5315                     ns->mac_remote_faults);
5316         PMD_DRV_LOG(DEBUG, "link_xon_rx:        %"PRIu64"", ns->link_xon_rx);
5317         PMD_DRV_LOG(DEBUG, "link_xoff_rx:       %"PRIu64"", ns->link_xoff_rx);
5318         PMD_DRV_LOG(DEBUG, "link_xon_tx:        %"PRIu64"", ns->link_xon_tx);
5319         PMD_DRV_LOG(DEBUG, "link_xoff_tx:       %"PRIu64"", ns->link_xoff_tx);
5320         PMD_DRV_LOG(DEBUG, "rx_size_64:         %"PRIu64"", ns->rx_size_64);
5321         PMD_DRV_LOG(DEBUG, "rx_size_127:        %"PRIu64"", ns->rx_size_127);
5322         PMD_DRV_LOG(DEBUG, "rx_size_255:        %"PRIu64"", ns->rx_size_255);
5323         PMD_DRV_LOG(DEBUG, "rx_size_511:        %"PRIu64"", ns->rx_size_511);
5324         PMD_DRV_LOG(DEBUG, "rx_size_1023:       %"PRIu64"", ns->rx_size_1023);
5325         PMD_DRV_LOG(DEBUG, "rx_size_1522:       %"PRIu64"", ns->rx_size_1522);
5326         PMD_DRV_LOG(DEBUG, "rx_size_big:        %"PRIu64"", ns->rx_size_big);
5327         PMD_DRV_LOG(DEBUG, "rx_undersize:       %"PRIu64"", ns->rx_undersize);
5328         PMD_DRV_LOG(DEBUG, "rx_fragments:       %"PRIu64"", ns->rx_fragments);
5329         PMD_DRV_LOG(DEBUG, "rx_oversize:        %"PRIu64"", ns->rx_oversize);
5330         PMD_DRV_LOG(DEBUG, "rx_jabber:          %"PRIu64"", ns->rx_jabber);
5331         PMD_DRV_LOG(DEBUG, "tx_size_64:         %"PRIu64"", ns->tx_size_64);
5332         PMD_DRV_LOG(DEBUG, "tx_size_127:        %"PRIu64"", ns->tx_size_127);
5333         PMD_DRV_LOG(DEBUG, "tx_size_255:        %"PRIu64"", ns->tx_size_255);
5334         PMD_DRV_LOG(DEBUG, "tx_size_511:        %"PRIu64"", ns->tx_size_511);
5335         PMD_DRV_LOG(DEBUG, "tx_size_1023:       %"PRIu64"", ns->tx_size_1023);
5336         PMD_DRV_LOG(DEBUG, "tx_size_1522:       %"PRIu64"", ns->tx_size_1522);
5337         PMD_DRV_LOG(DEBUG, "tx_size_big:        %"PRIu64"", ns->tx_size_big);
5338         PMD_DRV_LOG(DEBUG, "rx_len_errors:      %"PRIu64"", ns->rx_len_errors);
5339         PMD_DRV_LOG(DEBUG, "************* PF stats end ****************");
5340         return 0;
5341 }
5342
5343 /* Reset the statistics */
5344 static int
5345 ice_stats_reset(struct rte_eth_dev *dev)
5346 {
5347         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
5348         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5349
5350         /* Mark PF and VSI stats to update the offset, aka "reset" */
5351         pf->offset_loaded = false;
5352         if (pf->main_vsi)
5353                 pf->main_vsi->offset_loaded = false;
5354
5355         /* read the stats, reading current register values into offset */
5356         ice_read_stats_registers(pf, hw);
5357
5358         return 0;
5359 }
5360
5361 static uint32_t
5362 ice_xstats_calc_num(void)
5363 {
5364         uint32_t num;
5365
5366         num = ICE_NB_ETH_XSTATS + ICE_NB_HW_PORT_XSTATS;
5367
5368         return num;
5369 }
5370
5371 static int
5372 ice_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
5373                unsigned int n)
5374 {
5375         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
5376         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5377         unsigned int i;
5378         unsigned int count;
5379         struct ice_hw_port_stats *hw_stats = &pf->stats;
5380
5381         count = ice_xstats_calc_num();
5382         if (n < count)
5383                 return count;
5384
5385         ice_read_stats_registers(pf, hw);
5386
5387         if (!xstats)
5388                 return 0;
5389
5390         count = 0;
5391
5392         /* Get stats from ice_eth_stats struct */
5393         for (i = 0; i < ICE_NB_ETH_XSTATS; i++) {
5394                 xstats[count].value =
5395                         *(uint64_t *)((char *)&hw_stats->eth +
5396                                       ice_stats_strings[i].offset);
5397                 xstats[count].id = count;
5398                 count++;
5399         }
5400
5401         /* Get individiual stats from ice_hw_port struct */
5402         for (i = 0; i < ICE_NB_HW_PORT_XSTATS; i++) {
5403                 xstats[count].value =
5404                         *(uint64_t *)((char *)hw_stats +
5405                                       ice_hw_port_strings[i].offset);
5406                 xstats[count].id = count;
5407                 count++;
5408         }
5409
5410         return count;
5411 }
5412
5413 static int ice_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
5414                                 struct rte_eth_xstat_name *xstats_names,
5415                                 __rte_unused unsigned int limit)
5416 {
5417         unsigned int count = 0;
5418         unsigned int i;
5419
5420         if (!xstats_names)
5421                 return ice_xstats_calc_num();
5422
5423         /* Note: limit checked in rte_eth_xstats_names() */
5424
5425         /* Get stats from ice_eth_stats struct */
5426         for (i = 0; i < ICE_NB_ETH_XSTATS; i++) {
5427                 strlcpy(xstats_names[count].name, ice_stats_strings[i].name,
5428                         sizeof(xstats_names[count].name));
5429                 count++;
5430         }
5431
5432         /* Get individiual stats from ice_hw_port struct */
5433         for (i = 0; i < ICE_NB_HW_PORT_XSTATS; i++) {
5434                 strlcpy(xstats_names[count].name, ice_hw_port_strings[i].name,
5435                         sizeof(xstats_names[count].name));
5436                 count++;
5437         }
5438
5439         return count;
5440 }
5441
5442 static int
5443 ice_dev_flow_ops_get(struct rte_eth_dev *dev,
5444                      const struct rte_flow_ops **ops)
5445 {
5446         if (!dev)
5447                 return -EINVAL;
5448
5449         *ops = &ice_flow_ops;
5450         return 0;
5451 }
5452
5453 /* Add UDP tunneling port */
5454 static int
5455 ice_dev_udp_tunnel_port_add(struct rte_eth_dev *dev,
5456                              struct rte_eth_udp_tunnel *udp_tunnel)
5457 {
5458         int ret = 0;
5459         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5460
5461         if (udp_tunnel == NULL)
5462                 return -EINVAL;
5463
5464         switch (udp_tunnel->prot_type) {
5465         case RTE_TUNNEL_TYPE_VXLAN:
5466                 ret = ice_create_tunnel(hw, TNL_VXLAN, udp_tunnel->udp_port);
5467                 break;
5468         default:
5469                 PMD_DRV_LOG(ERR, "Invalid tunnel type");
5470                 ret = -EINVAL;
5471                 break;
5472         }
5473
5474         return ret;
5475 }
5476
5477 /* Delete UDP tunneling port */
5478 static int
5479 ice_dev_udp_tunnel_port_del(struct rte_eth_dev *dev,
5480                              struct rte_eth_udp_tunnel *udp_tunnel)
5481 {
5482         int ret = 0;
5483         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5484
5485         if (udp_tunnel == NULL)
5486                 return -EINVAL;
5487
5488         switch (udp_tunnel->prot_type) {
5489         case RTE_TUNNEL_TYPE_VXLAN:
5490                 ret = ice_destroy_tunnel(hw, udp_tunnel->udp_port, 0);
5491                 break;
5492         default:
5493                 PMD_DRV_LOG(ERR, "Invalid tunnel type");
5494                 ret = -EINVAL;
5495                 break;
5496         }
5497
5498         return ret;
5499 }
5500
5501 static int
5502 ice_timesync_enable(struct rte_eth_dev *dev)
5503 {
5504         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5505         struct ice_adapter *ad =
5506                         ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
5507         int ret;
5508
5509         if (dev->data->dev_started && !(dev->data->dev_conf.rxmode.offloads &
5510             DEV_RX_OFFLOAD_TIMESTAMP)) {
5511                 PMD_DRV_LOG(ERR, "Rx timestamp offload not configured");
5512                 return -1;
5513         }
5514
5515         if (hw->func_caps.ts_func_info.src_tmr_owned) {
5516                 ret = ice_ptp_init_phc(hw);
5517                 if (ret) {
5518                         PMD_DRV_LOG(ERR, "Failed to initialize PHC");
5519                         return -1;
5520                 }
5521
5522                 ret = ice_ptp_write_incval(hw, ICE_PTP_NOMINAL_INCVAL_E810);
5523                 if (ret) {
5524                         PMD_DRV_LOG(ERR,
5525                                 "Failed to write PHC increment time value");
5526                         return -1;
5527                 }
5528         }
5529
5530         /* Initialize cycle counters for system time/RX/TX timestamp */
5531         memset(&ad->systime_tc, 0, sizeof(struct rte_timecounter));
5532         memset(&ad->rx_tstamp_tc, 0, sizeof(struct rte_timecounter));
5533         memset(&ad->tx_tstamp_tc, 0, sizeof(struct rte_timecounter));
5534
5535         ad->systime_tc.cc_mask = ICE_CYCLECOUNTER_MASK;
5536         ad->systime_tc.cc_shift = 0;
5537         ad->systime_tc.nsec_mask = 0;
5538
5539         ad->rx_tstamp_tc.cc_mask = ICE_CYCLECOUNTER_MASK;
5540         ad->rx_tstamp_tc.cc_shift = 0;
5541         ad->rx_tstamp_tc.nsec_mask = 0;
5542
5543         ad->tx_tstamp_tc.cc_mask = ICE_CYCLECOUNTER_MASK;
5544         ad->tx_tstamp_tc.cc_shift = 0;
5545         ad->tx_tstamp_tc.nsec_mask = 0;
5546
5547         ad->ptp_ena = 1;
5548
5549         return 0;
5550 }
5551
5552 static int
5553 ice_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
5554                                struct timespec *timestamp, uint32_t flags)
5555 {
5556         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5557         struct ice_adapter *ad =
5558                         ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
5559         struct ice_rx_queue *rxq;
5560         uint32_t ts_high;
5561         uint64_t ts_ns, ns;
5562
5563         rxq = dev->data->rx_queues[flags];
5564
5565         ts_high = rxq->time_high;
5566         ts_ns = ice_tstamp_convert_32b_64b(hw, ts_high);
5567         ns = rte_timecounter_update(&ad->rx_tstamp_tc, ts_ns);
5568         *timestamp = rte_ns_to_timespec(ns);
5569
5570         return 0;
5571 }
5572
5573 static int
5574 ice_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
5575                                struct timespec *timestamp)
5576 {
5577         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5578         struct ice_adapter *ad =
5579                         ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
5580         uint8_t lport;
5581         uint64_t ts_ns, ns, tstamp;
5582         const uint64_t mask = 0xFFFFFFFF;
5583         int ret;
5584
5585         lport = hw->port_info->lport;
5586
5587         ret = ice_read_phy_tstamp(hw, lport, 0, &tstamp);
5588         if (ret) {
5589                 PMD_DRV_LOG(ERR, "Failed to read phy timestamp");
5590                 return -1;
5591         }
5592
5593         ts_ns = ice_tstamp_convert_32b_64b(hw, (tstamp >> 8) & mask);
5594         ns = rte_timecounter_update(&ad->tx_tstamp_tc, ts_ns);
5595         *timestamp = rte_ns_to_timespec(ns);
5596
5597         return 0;
5598 }
5599
5600 static int
5601 ice_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta)
5602 {
5603         struct ice_adapter *ad =
5604                         ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
5605
5606         ad->systime_tc.nsec += delta;
5607         ad->rx_tstamp_tc.nsec += delta;
5608         ad->tx_tstamp_tc.nsec += delta;
5609
5610         return 0;
5611 }
5612
5613 static int
5614 ice_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts)
5615 {
5616         struct ice_adapter *ad =
5617                         ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
5618         uint64_t ns;
5619
5620         ns = rte_timespec_to_ns(ts);
5621
5622         ad->systime_tc.nsec = ns;
5623         ad->rx_tstamp_tc.nsec = ns;
5624         ad->tx_tstamp_tc.nsec = ns;
5625
5626         return 0;
5627 }
5628
5629 static int
5630 ice_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts)
5631 {
5632         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5633         struct ice_adapter *ad =
5634                         ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
5635         uint32_t hi, lo, lo2;
5636         uint64_t time, ns;
5637
5638         lo = ICE_READ_REG(hw, GLTSYN_TIME_L(0));
5639         hi = ICE_READ_REG(hw, GLTSYN_TIME_H(0));
5640         lo2 = ICE_READ_REG(hw, GLTSYN_TIME_L(0));
5641
5642         if (lo2 < lo) {
5643                 lo = ICE_READ_REG(hw, GLTSYN_TIME_L(0));
5644                 hi = ICE_READ_REG(hw, GLTSYN_TIME_H(0));
5645         }
5646
5647         time = ((uint64_t)hi << 32) | lo;
5648         ns = rte_timecounter_update(&ad->systime_tc, time);
5649         *ts = rte_ns_to_timespec(ns);
5650
5651         return 0;
5652 }
5653
5654 static int
5655 ice_timesync_disable(struct rte_eth_dev *dev)
5656 {
5657         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5658         struct ice_adapter *ad =
5659                         ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
5660         uint64_t val;
5661         uint8_t lport;
5662
5663         lport = hw->port_info->lport;
5664
5665         ice_clear_phy_tstamp(hw, lport, 0);
5666
5667         val = ICE_READ_REG(hw, GLTSYN_ENA(0));
5668         val &= ~GLTSYN_ENA_TSYN_ENA_M;
5669         ICE_WRITE_REG(hw, GLTSYN_ENA(0), val);
5670
5671         ICE_WRITE_REG(hw, GLTSYN_INCVAL_L(0), 0);
5672         ICE_WRITE_REG(hw, GLTSYN_INCVAL_H(0), 0);
5673
5674         ad->ptp_ena = 0;
5675
5676         return 0;
5677 }
5678
5679 static int
5680 ice_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
5681               struct rte_pci_device *pci_dev)
5682 {
5683         return rte_eth_dev_pci_generic_probe(pci_dev,
5684                                              sizeof(struct ice_adapter),
5685                                              ice_dev_init);
5686 }
5687
5688 static int
5689 ice_pci_remove(struct rte_pci_device *pci_dev)
5690 {
5691         return rte_eth_dev_pci_generic_remove(pci_dev, ice_dev_uninit);
5692 }
5693
5694 static struct rte_pci_driver rte_ice_pmd = {
5695         .id_table = pci_id_ice_map,
5696         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
5697         .probe = ice_pci_probe,
5698         .remove = ice_pci_remove,
5699 };
5700
5701 /**
5702  * Driver initialization routine.
5703  * Invoked once at EAL init time.
5704  * Register itself as the [Poll Mode] Driver of PCI devices.
5705  */
5706 RTE_PMD_REGISTER_PCI(net_ice, rte_ice_pmd);
5707 RTE_PMD_REGISTER_PCI_TABLE(net_ice, pci_id_ice_map);
5708 RTE_PMD_REGISTER_KMOD_DEP(net_ice, "* igb_uio | uio_pci_generic | vfio-pci");
5709 RTE_PMD_REGISTER_PARAM_STRING(net_ice,
5710                               ICE_HW_DEBUG_MASK_ARG "=0xXXX"
5711                               ICE_PROTO_XTR_ARG "=[queue:]<vlan|ipv4|ipv6|ipv6_flow|tcp|ip_offset>"
5712                               ICE_SAFE_MODE_SUPPORT_ARG "=<0|1>"
5713                               ICE_PIPELINE_MODE_SUPPORT_ARG "=<0|1>"
5714                               ICE_RX_LOW_LATENCY_ARG "=<0|1>");
5715
5716 RTE_LOG_REGISTER_SUFFIX(ice_logtype_init, init, NOTICE);
5717 RTE_LOG_REGISTER_SUFFIX(ice_logtype_driver, driver, NOTICE);
5718 #ifdef RTE_ETHDEV_DEBUG_RX
5719 RTE_LOG_REGISTER_SUFFIX(ice_logtype_rx, rx, DEBUG);
5720 #endif
5721 #ifdef RTE_ETHDEV_DEBUG_TX
5722 RTE_LOG_REGISTER_SUFFIX(ice_logtype_tx, tx, DEBUG);
5723 #endif