lib/librte_pmd_i40e/i40e_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <inttypes.h>
  42
  43 #include <rte_string_fns.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_memzone.h>
  48 #include <rte_malloc.h>
  49 #include <rte_memcpy.h>
  50 #include <rte_dev.h>
  51
  52 #include "i40e_logs.h"
  53 #include "i40e/i40e_register_x710_int.h"
  54 #include "i40e/i40e_prototype.h"
  55 #include "i40e/i40e_adminq_cmd.h"
  56 #include "i40e/i40e_type.h"
  57 #include "i40e_ethdev.h"
  58 #include "i40e_rxtx.h"
  59 #include "i40e_pf.h"
  60
  61 /* Maximun number of MAC addresses */
  62 #define I40E_NUM_MACADDR_MAX       64
  63 #define I40E_CLEAR_PXE_WAIT_MS     200
  64
  65 /* Maximun number of capability elements */
  66 #define I40E_MAX_CAP_ELE_NUM       128
  67
  68 /* Wait count and inteval */
  69 #define I40E_CHK_Q_ENA_COUNT       1000
  70 #define I40E_CHK_Q_ENA_INTERVAL_US 1000
  71
  72 /* Maximun number of VSI */
  73 #define I40E_MAX_NUM_VSIS          (384UL)
  74
  75 /* Bit shift and mask */
  76 #define I40E_16_BIT_SHIFT 16
  77 #define I40E_16_BIT_MASK  0xFFFF
  78 #define I40E_32_BIT_SHIFT 32
  79 #define I40E_32_BIT_MASK  0xFFFFFFFF
  80 #define I40E_48_BIT_SHIFT 48
  81 #define I40E_48_BIT_MASK  0xFFFFFFFFFFFFULL
  82
  83 /* Default queue interrupt throttling time in microseconds*/
  84 #define I40E_ITR_INDEX_DEFAULT          0
  85 #define I40E_QUEUE_ITR_INTERVAL_DEFAULT 32 /* 32 us */
  86 #define I40E_QUEUE_ITR_INTERVAL_MAX     8160 /* 8160 us */
  87
  88 #define I40E_RSS_OFFLOAD_ALL ( \
  89         ETH_RSS_NONF_IPV4_UDP | \
  90         ETH_RSS_NONF_IPV4_TCP | \
  91         ETH_RSS_NONF_IPV4_SCTP | \
  92         ETH_RSS_NONF_IPV4_OTHER | \
  93         ETH_RSS_FRAG_IPV4 | \
  94         ETH_RSS_NONF_IPV6_UDP | \
  95         ETH_RSS_NONF_IPV6_TCP | \
  96         ETH_RSS_NONF_IPV6_SCTP | \
  97         ETH_RSS_NONF_IPV6_OTHER | \
  98         ETH_RSS_FRAG_IPV6 | \
  99         ETH_RSS_L2_PAYLOAD)
 100
 101 /* All bits of RSS hash enable */
 102 #define I40E_RSS_HENA_ALL ( \
 103         (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | \
 104         (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | \
 105         (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | \
 106         (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | \
 107         (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV4) | \
 108         (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | \
 109         (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | \
 110         (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | \
 111         (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | \
 112         (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV6) | \
 113         (1ULL << I40E_FILTER_PCTYPE_FCOE_OX) | \
 114         (1ULL << I40E_FILTER_PCTYPE_FCOE_RX) | \
 115         (1ULL << I40E_FILTER_PCTYPE_FCOE_OTHER) | \
 116         (1ULL << I40E_FILTER_PCTYPE_L2_PAYLOAD))
 117
 118 static int eth_i40e_dev_init(\
 119                         __attribute__((unused)) struct eth_driver *eth_drv,
 120                         struct rte_eth_dev *eth_dev);
 121 static int i40e_dev_configure(struct rte_eth_dev *dev);
 122 static int i40e_dev_start(struct rte_eth_dev *dev);
 123 static void i40e_dev_stop(struct rte_eth_dev *dev);
 124 static void i40e_dev_close(struct rte_eth_dev *dev);
 125 static void i40e_dev_promiscuous_enable(struct rte_eth_dev *dev);
 126 static void i40e_dev_promiscuous_disable(struct rte_eth_dev *dev);
 127 static void i40e_dev_allmulticast_enable(struct rte_eth_dev *dev);
 128 static void i40e_dev_allmulticast_disable(struct rte_eth_dev *dev);
 129 static void i40e_dev_stats_get(struct rte_eth_dev *dev,
 130                                struct rte_eth_stats *stats);
 131 static void i40e_dev_stats_reset(struct rte_eth_dev *dev);
 132 static int i40e_dev_queue_stats_mapping_set(struct rte_eth_dev *dev,
 133                                             uint16_t queue_id,
 134                                             uint8_t stat_idx,
 135                                             uint8_t is_rx);
 136 static void i40e_dev_info_get(struct rte_eth_dev *dev,
 137                               struct rte_eth_dev_info *dev_info);
 138 static int i40e_vlan_filter_set(struct rte_eth_dev *dev,
 139                                 uint16_t vlan_id,
 140                                 int on);
 141 static void i40e_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid);
 142 static void i40e_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 143 static void i40e_vlan_strip_queue_set(struct rte_eth_dev *dev,
 144                                       uint16_t queue,
 145                                       int on);
 146 static int i40e_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid, int on);
 147 static int i40e_dev_led_on(struct rte_eth_dev *dev);
 148 static int i40e_dev_led_off(struct rte_eth_dev *dev);
 149 static int i40e_flow_ctrl_set(struct rte_eth_dev *dev,
 150                               struct rte_eth_fc_conf *fc_conf);
 151 static int i40e_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 152                                        struct rte_eth_pfc_conf *pfc_conf);
 153 static void i40e_macaddr_add(struct rte_eth_dev *dev,
 154                           struct ether_addr *mac_addr,
 155                           uint32_t index,
 156                           uint32_t pool);
 157 static void i40e_macaddr_remove(struct rte_eth_dev *dev, uint32_t index);
 158 static int i40e_dev_rss_reta_update(struct rte_eth_dev *dev,
 159                                     struct rte_eth_rss_reta *reta_conf);
 160 static int i40e_dev_rss_reta_query(struct rte_eth_dev *dev,
 161                                    struct rte_eth_rss_reta *reta_conf);
 162
 163 static int i40e_get_cap(struct i40e_hw *hw);
 164 static int i40e_pf_parameter_init(struct rte_eth_dev *dev);
 165 static int i40e_pf_setup(struct i40e_pf *pf);
 166 static int i40e_vsi_init(struct i40e_vsi *vsi);
 167 static void i40e_stat_update_32(struct i40e_hw *hw, uint32_t reg,
 168                 bool offset_loaded, uint64_t *offset, uint64_t *stat);
 169 static void i40e_stat_update_48(struct i40e_hw *hw,
 170                                uint32_t hireg,
 171                                uint32_t loreg,
 172                                bool offset_loaded,
 173                                uint64_t *offset,
 174                                uint64_t *stat);
 175 static void i40e_pf_config_irq0(struct i40e_hw *hw);
 176 static void i40e_dev_interrupt_handler(
 177                 __rte_unused struct rte_intr_handle *handle, void *param);
 178 static int i40e_res_pool_init(struct i40e_res_pool_info *pool,
 179                                 uint32_t base, uint32_t num);
 180 static void i40e_res_pool_destroy(struct i40e_res_pool_info *pool);
 181 static int i40e_res_pool_free(struct i40e_res_pool_info *pool,
 182                         uint32_t base);
 183 static int i40e_res_pool_alloc(struct i40e_res_pool_info *pool,
 184                         uint16_t num);
 185 static int i40e_vsi_init_vlan(struct i40e_vsi *vsi);
 186 static int i40e_veb_release(struct i40e_veb *veb);
 187 static struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf,
 188                                                 struct i40e_vsi *vsi);
 189 static int i40e_vsi_config_vlan_stripping(struct i40e_vsi *vsi, bool on);
 190 static int i40e_pf_config_mq_rx(struct i40e_pf *pf);
 191 static int i40e_vsi_config_double_vlan(struct i40e_vsi *vsi, int on);
 192 static int i40e_pf_disable_all_queues(struct i40e_hw *hw);
 193 static inline int i40e_find_all_vlan_for_mac(struct i40e_vsi *vsi,
 194                                              struct i40e_macvlan_filter *mv_f,
 195                                              int num,
 196                                              struct ether_addr *addr);
 197 static inline int i40e_find_all_mac_for_vlan(struct i40e_vsi *vsi,
 198                                              struct i40e_macvlan_filter *mv_f,
 199                                              int num,
 200                                              uint16_t vlan);
 201 static int i40e_vsi_remove_all_macvlan_filter(struct i40e_vsi *vsi);
 202 static int i40e_dev_rss_hash_update(struct rte_eth_dev *dev,
 203                                     struct rte_eth_rss_conf *rss_conf);
 204 static int i40e_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
 205                                       struct rte_eth_rss_conf *rss_conf);
 206
 207 /* Default hash key buffer for RSS */
 208 static uint32_t rss_key_default[I40E_PFQF_HKEY_MAX_INDEX + 1];
 209
 210 static struct rte_pci_id pci_id_i40e_map[] = {
 211 #define RTE_PCI_DEV_ID_DECL_I40E(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 212 #include "rte_pci_dev_ids.h"
 213 { .vendor_id = 0, /* sentinel */ },
 214 };
 215
 216 static struct eth_dev_ops i40e_eth_dev_ops = {
 217         .dev_configure                = i40e_dev_configure,
 218         .dev_start                    = i40e_dev_start,
 219         .dev_stop                     = i40e_dev_stop,
 220         .dev_close                    = i40e_dev_close,
 221         .promiscuous_enable           = i40e_dev_promiscuous_enable,
 222         .promiscuous_disable          = i40e_dev_promiscuous_disable,
 223         .allmulticast_enable          = i40e_dev_allmulticast_enable,
 224         .allmulticast_disable         = i40e_dev_allmulticast_disable,
 225         .link_update                  = i40e_dev_link_update,
 226         .stats_get                    = i40e_dev_stats_get,
 227         .stats_reset                  = i40e_dev_stats_reset,
 228         .queue_stats_mapping_set      = i40e_dev_queue_stats_mapping_set,
 229         .dev_infos_get                = i40e_dev_info_get,
 230         .vlan_filter_set              = i40e_vlan_filter_set,
 231         .vlan_tpid_set                = i40e_vlan_tpid_set,
 232         .vlan_offload_set             = i40e_vlan_offload_set,
 233         .vlan_strip_queue_set         = i40e_vlan_strip_queue_set,
 234         .vlan_pvid_set                = i40e_vlan_pvid_set,
 235         .rx_queue_setup               = i40e_dev_rx_queue_setup,
 236         .rx_queue_release             = i40e_dev_rx_queue_release,
 237         .rx_queue_count               = i40e_dev_rx_queue_count,
 238         .rx_descriptor_done           = i40e_dev_rx_descriptor_done,
 239         .tx_queue_setup               = i40e_dev_tx_queue_setup,
 240         .tx_queue_release             = i40e_dev_tx_queue_release,
 241         .dev_led_on                   = i40e_dev_led_on,
 242         .dev_led_off                  = i40e_dev_led_off,
 243         .flow_ctrl_set                = i40e_flow_ctrl_set,
 244         .priority_flow_ctrl_set       = i40e_priority_flow_ctrl_set,
 245         .mac_addr_add                 = i40e_macaddr_add,
 246         .mac_addr_remove              = i40e_macaddr_remove,
 247         .reta_update                  = i40e_dev_rss_reta_update,
 248         .reta_query                   = i40e_dev_rss_reta_query,
 249         .rss_hash_update              = i40e_dev_rss_hash_update,
 250         .rss_hash_conf_get            = i40e_dev_rss_hash_conf_get,
 251 };
 252
 253 static struct eth_driver rte_i40e_pmd = {
 254         {
 255                 .name = "rte_i40e_pmd",
 256                 .id_table = pci_id_i40e_map,
 257                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
 258         },
 259         .eth_dev_init = eth_i40e_dev_init,
 260         .dev_private_size = sizeof(struct i40e_adapter),
 261 };
 262
 263 static inline int
 264 i40e_prev_power_of_2(int n)
 265 {
 266        int p = n;
 267
 268        --p;
 269        p |= p >> 1;
 270        p |= p >> 2;
 271        p |= p >> 4;
 272        p |= p >> 8;
 273        p |= p >> 16;
 274        if (p == (n - 1))
 275                return n;
 276        p >>= 1;
 277
 278        return ++p;
 279 }
 280
 281 static inline int
 282 rte_i40e_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 283                                      struct rte_eth_link *link)
 284 {
 285         struct rte_eth_link *dst = link;
 286         struct rte_eth_link *src = &(dev->data->dev_link);
 287
 288         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 289                                         *(uint64_t *)src) == 0)
 290                 return -1;
 291
 292         return 0;
 293 }
 294
 295 static inline int
 296 rte_i40e_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 297                                       struct rte_eth_link *link)
 298 {
 299         struct rte_eth_link *dst = &(dev->data->dev_link);
 300         struct rte_eth_link *src = link;
 301
 302         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 303                                         *(uint64_t *)src) == 0)
 304                 return -1;
 305
 306         return 0;
 307 }
 308
 309 /*
 310  * Driver initialization routine.
 311  * Invoked once at EAL init time.
 312  * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
 313  */
 314 static int
 315 rte_i40e_pmd_init(const char *name __rte_unused,
 316                   const char *params __rte_unused)
 317 {
 318         PMD_INIT_FUNC_TRACE();
 319         rte_eth_driver_register(&rte_i40e_pmd);
 320
 321         return 0;
 322 }
 323
 324 static struct rte_driver rte_i40e_driver = {
 325         .type = PMD_PDEV,
 326         .init = rte_i40e_pmd_init,
 327 };
 328
 329 PMD_REGISTER_DRIVER(rte_i40e_driver);
 330
 331 static int
 332 eth_i40e_dev_init(__rte_unused struct eth_driver *eth_drv,
 333                   struct rte_eth_dev *dev)
 334 {
 335         struct rte_pci_device *pci_dev;
 336         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 337         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 338         struct i40e_vsi *vsi;
 339         int ret;
 340         uint32_t len;
 341         uint8_t aq_fail = 0;
 342
 343         PMD_INIT_FUNC_TRACE();
 344
 345         dev->dev_ops = &i40e_eth_dev_ops;
 346         dev->rx_pkt_burst = i40e_recv_pkts;
 347         dev->tx_pkt_burst = i40e_xmit_pkts;
 348
 349         /* for secondary processes, we don't initialise any further as primary
 350          * has already done this work. Only check we don't need a different
 351          * RX function */
 352         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 353                 if (dev->data->scattered_rx)
 354                         dev->rx_pkt_burst = i40e_recv_scattered_pkts;
 355                 return 0;
 356         }
 357         pci_dev = dev->pci_dev;
 358         pf->adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 359         pf->adapter->eth_dev = dev;
 360         pf->dev_data = dev->data;
 361
 362         hw->back = I40E_PF_TO_ADAPTER(pf);
 363         hw->hw_addr = (uint8_t *)(pci_dev->mem_resource[0].addr);
 364         if (!hw->hw_addr) {
 365                 PMD_INIT_LOG(ERR, "Hardware is not available, "
 366                                         "as address is NULL\n");
 367                 return -ENODEV;
 368         }
 369
 370         hw->vendor_id = pci_dev->id.vendor_id;
 371         hw->device_id = pci_dev->id.device_id;
 372         hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
 373         hw->subsystem_device_id = pci_dev->id.subsystem_device_id;
 374         hw->bus.device = pci_dev->addr.devid;
 375         hw->bus.func = pci_dev->addr.function;
 376
 377         /* Disable all queues before PF reset, as required */
 378         ret = i40e_pf_disable_all_queues(hw);
 379         if (ret != I40E_SUCCESS) {
 380                 PMD_INIT_LOG(ERR, "Failed to disable queues %u\n", ret);
 381                 return ret;
 382         }
 383
 384         /* Reset here to make sure all is clean for each PF */
 385         ret = i40e_pf_reset(hw);
 386         if (ret) {
 387                 PMD_INIT_LOG(ERR, "Failed to reset pf: %d", ret);
 388                 return ret;
 389         }
 390
 391         /* Initialize the shared code */
 392         ret = i40e_init_shared_code(hw);
 393         if (ret) {
 394                 PMD_INIT_LOG(ERR, "Failed to init shared code: %d", ret);
 395                 return ret;
 396         }
 397
 398         /* Initialize the parameters for adminq */
 399         i40e_init_adminq_parameter(hw);
 400         ret = i40e_init_adminq(hw);
 401         if (ret != I40E_SUCCESS) {
 402                 PMD_INIT_LOG(ERR, "Failed to init adminq: %d", ret);
 403                 return -EIO;
 404         }
 405         PMD_INIT_LOG(INFO, "FW %d.%d API %d.%d NVM "
 406                         "%02d.%02d.%02d eetrack %04x\n",
 407                         hw->aq.fw_maj_ver, hw->aq.fw_min_ver,
 408                         hw->aq.api_maj_ver, hw->aq.api_min_ver,
 409                         ((hw->nvm.version >> 12) & 0xf),
 410                         ((hw->nvm.version >> 4) & 0xff),
 411                         (hw->nvm.version & 0xf), hw->nvm.eetrack);
 412
 413         /* Disable LLDP */
 414         ret = i40e_aq_stop_lldp(hw, true, NULL);
 415         if (ret != I40E_SUCCESS) /* Its failure can be ignored */
 416                 PMD_INIT_LOG(INFO, "Failed to stop lldp\n");
 417
 418         /* Clear PXE mode */
 419         i40e_clear_pxe_mode(hw);
 420
 421         /* Get hw capabilities */
 422         ret = i40e_get_cap(hw);
 423         if (ret != I40E_SUCCESS) {
 424                 PMD_INIT_LOG(ERR, "Failed to get capabilities: %d", ret);
 425                 goto err_get_capabilities;
 426         }
 427
 428         /* Initialize parameters for PF */
 429         ret = i40e_pf_parameter_init(dev);
 430         if (ret != 0) {
 431                 PMD_INIT_LOG(ERR, "Failed to do parameter init: %d", ret);
 432                 goto err_parameter_init;
 433         }
 434
 435         /* Initialize the queue management */
 436         ret = i40e_res_pool_init(&pf->qp_pool, 0, hw->func_caps.num_tx_qp);
 437         if (ret < 0) {
 438                 PMD_INIT_LOG(ERR, "Failed to init queue pool\n");
 439                 goto err_qp_pool_init;
 440         }
 441         ret = i40e_res_pool_init(&pf->msix_pool, 1,
 442                                 hw->func_caps.num_msix_vectors - 1);
 443         if (ret < 0) {
 444                 PMD_INIT_LOG(ERR, "Failed to init MSIX pool\n");
 445                 goto err_msix_pool_init;
 446         }
 447
 448         /* Initialize lan hmc */
 449         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
 450                                 hw->func_caps.num_rx_qp, 0, 0);
 451         if (ret != I40E_SUCCESS) {
 452                 PMD_INIT_LOG(ERR, "Failed to init lan hmc: %d", ret);
 453                 goto err_init_lan_hmc;
 454         }
 455
 456         /* Configure lan hmc */
 457         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
 458         if (ret != I40E_SUCCESS) {
 459                 PMD_INIT_LOG(ERR, "Failed to configure lan hmc: %d", ret);
 460                 goto err_configure_lan_hmc;
 461         }
 462
 463         /* Get and check the mac address */
 464         i40e_get_mac_addr(hw, hw->mac.addr);
 465         if (i40e_validate_mac_addr(hw->mac.addr) != I40E_SUCCESS) {
 466                 PMD_INIT_LOG(ERR, "mac address is not valid");
 467                 ret = -EIO;
 468                 goto err_get_mac_addr;
 469         }
 470         /* Copy the permanent MAC address */
 471         ether_addr_copy((struct ether_addr *) hw->mac.addr,
 472                         (struct ether_addr *) hw->mac.perm_addr);
 473
 474         /* Disable flow control */
 475         hw->fc.requested_mode = I40E_FC_NONE;
 476         i40e_set_fc(hw, &aq_fail, TRUE);
 477
 478         /* PF setup, which includes VSI setup */
 479         ret = i40e_pf_setup(pf);
 480         if (ret) {
 481                 PMD_INIT_LOG(ERR, "Failed to setup pf switch: %d", ret);
 482                 goto err_setup_pf_switch;
 483         }
 484
 485         vsi = pf->main_vsi;
 486         if (!vsi->max_macaddrs)
 487                 len = ETHER_ADDR_LEN;
 488         else
 489                 len = ETHER_ADDR_LEN * vsi->max_macaddrs;
 490
 491         /* Should be after VSI initialized */
 492         dev->data->mac_addrs = rte_zmalloc("i40e", len, 0);
 493         if (!dev->data->mac_addrs) {
 494                 PMD_INIT_LOG(ERR, "Failed to allocated memory "
 495                                         "for storing mac address");
 496                 goto err_get_mac_addr;
 497         }
 498         ether_addr_copy((struct ether_addr *)hw->mac.perm_addr,
 499                                         &dev->data->mac_addrs[0]);
 500
 501         /* initialize pf host driver to setup SRIOV resource if applicable */
 502         i40e_pf_host_init(dev);
 503
 504         /* register callback func to eal lib */
 505         rte_intr_callback_register(&(pci_dev->intr_handle),
 506                 i40e_dev_interrupt_handler, (void *)dev);
 507
 508         /* configure and enable device interrupt */
 509         i40e_pf_config_irq0(hw);
 510         i40e_pf_enable_irq0(hw);
 511
 512         /* enable uio intr after callback register */
 513         rte_intr_enable(&(pci_dev->intr_handle));
 514
 515         return 0;
 516
 517 err_setup_pf_switch:
 518         rte_free(pf->main_vsi);
 519 err_get_mac_addr:
 520 err_configure_lan_hmc:
 521         (void)i40e_shutdown_lan_hmc(hw);
 522 err_init_lan_hmc:
 523         i40e_res_pool_destroy(&pf->msix_pool);
 524 err_msix_pool_init:
 525         i40e_res_pool_destroy(&pf->qp_pool);
 526 err_qp_pool_init:
 527 err_parameter_init:
 528 err_get_capabilities:
 529         (void)i40e_shutdown_adminq(hw);
 530
 531         return ret;
 532 }
 533
 534 static int
 535 i40e_dev_configure(struct rte_eth_dev *dev)
 536 {
 537         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 538         struct i40e_vsi *vsi = pf->main_vsi;
 539         int ret;
 540
 541         ret = i40e_vsi_init_vlan(vsi);
 542
 543         return ret;
 544 }
 545
 546 void
 547 i40e_vsi_queues_unbind_intr(struct i40e_vsi *vsi)
 548 {
 549         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
 550         uint16_t msix_vect = vsi->msix_intr;
 551         uint16_t i;
 552
 553         for (i = 0; i < vsi->nb_qps; i++) {
 554                 I40E_WRITE_REG(hw, I40E_QINT_TQCTL(vsi->base_queue + i), 0);
 555                 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(vsi->base_queue + i), 0);
 556                 rte_wmb();
 557         }
 558
 559         if (vsi->type != I40E_VSI_SRIOV) {
 560                 I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(msix_vect - 1), 0);
 561                 I40E_WRITE_REG(hw, I40E_PFINT_ITRN(I40E_ITR_INDEX_DEFAULT,
 562                                 msix_vect - 1), 0);
 563         } else {
 564                 uint32_t reg;
 565                 reg = (hw->func_caps.num_msix_vectors_vf - 1) *
 566                         vsi->user_param + (msix_vect - 1);
 567
 568                 I40E_WRITE_REG(hw, I40E_VPINT_LNKLSTN(reg), 0);
 569         }
 570         I40E_WRITE_FLUSH(hw);
 571 }
 572
 573 static inline uint16_t
 574 i40e_calc_itr_interval(int16_t interval)
 575 {
 576         if (interval < 0 || interval > I40E_QUEUE_ITR_INTERVAL_MAX)
 577                 interval = I40E_QUEUE_ITR_INTERVAL_DEFAULT;
 578
 579         /* Convert to hardware count, as writing each 1 represents 2 us */
 580         return (interval/2);
 581 }
 582
 583 void
 584 i40e_vsi_queues_bind_intr(struct i40e_vsi *vsi)
 585 {
 586         uint32_t val;
 587         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
 588         uint16_t msix_vect = vsi->msix_intr;
 589         uint16_t interval = i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL);
 590         int i;
 591
 592         for (i = 0; i < vsi->nb_qps; i++)
 593                 I40E_WRITE_REG(hw, I40E_QINT_TQCTL(vsi->base_queue + i), 0);
 594
 595         /* Bind all RX queues to allocated MSIX interrupt */
 596         for (i = 0; i < vsi->nb_qps; i++) {
 597                 val = (msix_vect << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
 598                         (interval << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
 599                         ((vsi->base_queue + i + 1) <<
 600                         I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
 601                         (0 << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) |
 602                         I40E_QINT_RQCTL_CAUSE_ENA_MASK;
 603
 604                 if (i == vsi->nb_qps - 1)
 605                         val |= I40E_QINT_RQCTL_NEXTQ_INDX_MASK;
 606                 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(vsi->base_queue + i), val);
 607         }
 608
 609         /* Write first RX queue to Link list register as the head element */
 610         if (vsi->type != I40E_VSI_SRIOV) {
 611                 I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(msix_vect - 1),
 612                         (vsi->base_queue << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) |
 613                         (0x0 << I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT));
 614
 615                 I40E_WRITE_REG(hw, I40E_PFINT_ITRN(I40E_ITR_INDEX_DEFAULT,
 616                                 msix_vect - 1), interval);
 617
 618                 /* Disable auto-mask on enabling of all none-zero  interrupt */
 619                 I40E_WRITE_REG(hw, I40E_GLINT_CTL,
 620                                 I40E_GLINT_CTL_DIS_AUTOMASK_N_MASK);
 621         }
 622         else {
 623                 uint32_t reg;
 624                 /* num_msix_vectors_vf needs to minus irq0 */
 625                 reg = (hw->func_caps.num_msix_vectors_vf - 1) *
 626                         vsi->user_param + (msix_vect - 1);
 627
 628                 I40E_WRITE_REG(hw, I40E_VPINT_LNKLSTN(reg),
 629                         (vsi->base_queue << I40E_VPINT_LNKLSTN_FIRSTQ_INDX_SHIFT) |
 630                         (0x0 << I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_SHIFT));
 631         }
 632
 633         I40E_WRITE_FLUSH(hw);
 634 }
 635
 636 static void
 637 i40e_vsi_enable_queues_intr(struct i40e_vsi *vsi)
 638 {
 639         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
 640         uint16_t interval = i40e_calc_itr_interval(\
 641                         RTE_LIBRTE_I40E_ITR_INTERVAL);
 642
 643         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(vsi->msix_intr - 1),
 644                                         I40E_PFINT_DYN_CTLN_INTENA_MASK |
 645                                         I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
 646                                 (0 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT) |
 647                         (interval << I40E_PFINT_DYN_CTLN_INTERVAL_SHIFT));
 648 }
 649
 650 static void
 651 i40e_vsi_disable_queues_intr(struct i40e_vsi *vsi)
 652 {
 653         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
 654
 655         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(vsi->msix_intr - 1), 0);
 656 }
 657
 658 static int
 659 i40e_dev_start(struct rte_eth_dev *dev)
 660 {
 661         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 662         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 663         struct i40e_vsi *vsi = pf->main_vsi;
 664         int ret;
 665
 666         /* Initialize VSI */
 667         ret = i40e_vsi_init(vsi);
 668         if (ret != I40E_SUCCESS) {
 669                 PMD_DRV_LOG(ERR, "Failed to init VSI\n");
 670                 goto err_up;
 671         }
 672
 673         /* Map queues with MSIX interrupt */
 674         i40e_vsi_queues_bind_intr(vsi);
 675         i40e_vsi_enable_queues_intr(vsi);
 676
 677         /* Enable all queues which have been configured */
 678         ret = i40e_vsi_switch_queues(vsi, TRUE);
 679         if (ret != I40E_SUCCESS) {
 680                 PMD_DRV_LOG(ERR, "Failed to enable VSI\n");
 681                 goto err_up;
 682         }
 683
 684         /* Enable receiving broadcast packets */
 685         if ((vsi->type == I40E_VSI_MAIN) || (vsi->type == I40E_VSI_VMDQ2)) {
 686                 ret = i40e_aq_set_vsi_broadcast(hw, vsi->seid, true, NULL);
 687                 if (ret != I40E_SUCCESS)
 688                         PMD_DRV_LOG(INFO, "fail to set vsi broadcast\n");
 689         }
 690
 691         return I40E_SUCCESS;
 692
 693 err_up:
 694         i40e_vsi_switch_queues(vsi, FALSE);
 695         i40e_dev_clear_queues(dev);
 696
 697         return ret;
 698 }
 699
 700 static void
 701 i40e_dev_stop(struct rte_eth_dev *dev)
 702 {
 703         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 704         struct i40e_vsi *vsi = pf->main_vsi;
 705
 706         /* Disable all queues */
 707         i40e_vsi_switch_queues(vsi, FALSE);
 708
 709         /* Clear all queues and release memory */
 710         i40e_dev_clear_queues(dev);
 711
 712         /* un-map queues with interrupt registers */
 713         i40e_vsi_disable_queues_intr(vsi);
 714         i40e_vsi_queues_unbind_intr(vsi);
 715 }
 716
 717 static void
 718 i40e_dev_close(struct rte_eth_dev *dev)
 719 {
 720         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 721         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 722         uint32_t reg;
 723
 724         PMD_INIT_FUNC_TRACE();
 725
 726         i40e_dev_stop(dev);
 727
 728         /* Disable interrupt */
 729         i40e_pf_disable_irq0(hw);
 730         rte_intr_disable(&(dev->pci_dev->intr_handle));
 731
 732         /* shutdown and destroy the HMC */
 733         i40e_shutdown_lan_hmc(hw);
 734
 735         /* release all the existing VSIs and VEBs */
 736         i40e_vsi_release(pf->main_vsi);
 737
 738         /* shutdown the adminq */
 739         i40e_aq_queue_shutdown(hw, true);
 740         i40e_shutdown_adminq(hw);
 741
 742         i40e_res_pool_destroy(&pf->qp_pool);
 743         i40e_res_pool_destroy(&pf->msix_pool);
 744
 745         /* force a PF reset to clean anything leftover */
 746         reg = I40E_READ_REG(hw, I40E_PFGEN_CTRL);
 747         I40E_WRITE_REG(hw, I40E_PFGEN_CTRL,
 748                         (reg | I40E_PFGEN_CTRL_PFSWR_MASK));
 749         I40E_WRITE_FLUSH(hw);
 750 }
 751
 752 static void
 753 i40e_dev_promiscuous_enable(struct rte_eth_dev *dev)
 754 {
 755         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 756         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 757         struct i40e_vsi *vsi = pf->main_vsi;
 758         int status;
 759
 760         status = i40e_aq_set_vsi_unicast_promiscuous(hw, vsi->seid,
 761                                                         true, NULL);
 762         if (status != I40E_SUCCESS)
 763                 PMD_DRV_LOG(ERR, "Failed to enable unicast promiscuous\n");
 764 }
 765
 766 static void
 767 i40e_dev_promiscuous_disable(struct rte_eth_dev *dev)
 768 {
 769         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 770         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 771         struct i40e_vsi *vsi = pf->main_vsi;
 772         int status;
 773
 774         status = i40e_aq_set_vsi_unicast_promiscuous(hw, vsi->seid,
 775                                                         false, NULL);
 776         if (status != I40E_SUCCESS)
 777                 PMD_DRV_LOG(ERR, "Failed to disable unicast promiscuous\n");
 778 }
 779
 780 static void
 781 i40e_dev_allmulticast_enable(struct rte_eth_dev *dev)
 782 {
 783         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 784         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 785         struct i40e_vsi *vsi = pf->main_vsi;
 786         int ret;
 787
 788         ret = i40e_aq_set_vsi_multicast_promiscuous(hw, vsi->seid, TRUE, NULL);
 789         if (ret != I40E_SUCCESS)
 790                 PMD_DRV_LOG(ERR, "Failed to enable multicast promiscuous\n");
 791 }
 792
 793 static void
 794 i40e_dev_allmulticast_disable(struct rte_eth_dev *dev)
 795 {
 796         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 797         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 798         struct i40e_vsi *vsi = pf->main_vsi;
 799         int ret;
 800
 801         ret = i40e_aq_set_vsi_multicast_promiscuous(hw,
 802                                 vsi->seid, FALSE, NULL);
 803         if (ret != I40E_SUCCESS)
 804                 PMD_DRV_LOG(ERR, "Failed to disable multicast promiscuous\n");
 805 }
 806
 807 int
 808 i40e_dev_link_update(struct rte_eth_dev *dev,
 809                      __rte_unused int wait_to_complete)
 810 {
 811         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 812         struct i40e_link_status link_status;
 813         struct rte_eth_link link, old;
 814         int status;
 815
 816         memset(&link, 0, sizeof(link));
 817         memset(&old, 0, sizeof(old));
 818         memset(&link_status, 0, sizeof(link_status));
 819         rte_i40e_dev_atomic_read_link_status(dev, &old);
 820
 821         /* Get link status information from hardware */
 822         status = i40e_aq_get_link_info(hw, false, &link_status, NULL);
 823         if (status != I40E_SUCCESS) {
 824                 link.link_speed = ETH_LINK_SPEED_100;
 825                 link.link_duplex = ETH_LINK_FULL_DUPLEX;
 826                 PMD_DRV_LOG(ERR, "Failed to get link info\n");
 827                 goto out;
 828         }
 829
 830         link.link_status = link_status.link_info & I40E_AQ_LINK_UP;
 831
 832         if (!link.link_status)
 833                 goto out;
 834
 835         /* i40e uses full duplex only */
 836         link.link_duplex = ETH_LINK_FULL_DUPLEX;
 837
 838         /* Parse the link status */
 839         switch (link_status.link_speed) {
 840         case I40E_LINK_SPEED_100MB:
 841                 link.link_speed = ETH_LINK_SPEED_100;
 842                 break;
 843         case I40E_LINK_SPEED_1GB:
 844                 link.link_speed = ETH_LINK_SPEED_1000;
 845                 break;
 846         case I40E_LINK_SPEED_10GB:
 847                 link.link_speed = ETH_LINK_SPEED_10G;
 848                 break;
 849         case I40E_LINK_SPEED_20GB:
 850                 link.link_speed = ETH_LINK_SPEED_20G;
 851                 break;
 852         case I40E_LINK_SPEED_40GB:
 853                 link.link_speed = ETH_LINK_SPEED_40G;
 854                 break;
 855         default:
 856                 link.link_speed = ETH_LINK_SPEED_100;
 857                 break;
 858         }
 859
 860 out:
 861         rte_i40e_dev_atomic_write_link_status(dev, &link);
 862         if (link.link_status == old.link_status)
 863                 return -1;
 864
 865         return 0;
 866 }
 867
 868 /* Get all the statistics of a VSI */
 869 void
 870 i40e_update_vsi_stats(struct i40e_vsi *vsi)
 871 {
 872         struct i40e_eth_stats *oes = &vsi->eth_stats_offset;
 873         struct i40e_eth_stats *nes = &vsi->eth_stats;
 874         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
 875         int idx = rte_le_to_cpu_16(vsi->info.stat_counter_idx);
 876
 877         i40e_stat_update_48(hw, I40E_GLV_GORCH(idx), I40E_GLV_GORCL(idx),
 878                             vsi->offset_loaded, &oes->rx_bytes,
 879                             &nes->rx_bytes);
 880         i40e_stat_update_48(hw, I40E_GLV_UPRCH(idx), I40E_GLV_UPRCL(idx),
 881                             vsi->offset_loaded, &oes->rx_unicast,
 882                             &nes->rx_unicast);
 883         i40e_stat_update_48(hw, I40E_GLV_MPRCH(idx), I40E_GLV_MPRCL(idx),
 884                             vsi->offset_loaded, &oes->rx_multicast,
 885                             &nes->rx_multicast);
 886         i40e_stat_update_48(hw, I40E_GLV_BPRCH(idx), I40E_GLV_BPRCL(idx),
 887                             vsi->offset_loaded, &oes->rx_broadcast,
 888                             &nes->rx_broadcast);
 889         i40e_stat_update_32(hw, I40E_GLV_RDPC(idx), vsi->offset_loaded,
 890                             &oes->rx_discards, &nes->rx_discards);
 891         /* GLV_REPC not supported */
 892         /* GLV_RMPC not supported */
 893         i40e_stat_update_32(hw, I40E_GLV_RUPP(idx), vsi->offset_loaded,
 894                             &oes->rx_unknown_protocol,
 895                             &nes->rx_unknown_protocol);
 896         i40e_stat_update_48(hw, I40E_GLV_GOTCH(idx), I40E_GLV_GOTCL(idx),
 897                             vsi->offset_loaded, &oes->tx_bytes,
 898                             &nes->tx_bytes);
 899         i40e_stat_update_48(hw, I40E_GLV_UPTCH(idx), I40E_GLV_UPTCL(idx),
 900                             vsi->offset_loaded, &oes->tx_unicast,
 901                             &nes->tx_unicast);
 902         i40e_stat_update_48(hw, I40E_GLV_MPTCH(idx), I40E_GLV_MPTCL(idx),
 903                             vsi->offset_loaded, &oes->tx_multicast,
 904                             &nes->tx_multicast);
 905         i40e_stat_update_48(hw, I40E_GLV_BPTCH(idx), I40E_GLV_BPTCL(idx),
 906                             vsi->offset_loaded,  &oes->tx_broadcast,
 907                             &nes->tx_broadcast);
 908         /* GLV_TDPC not supported */
 909         i40e_stat_update_32(hw, I40E_GLV_TEPC(idx), vsi->offset_loaded,
 910                             &oes->tx_errors, &nes->tx_errors);
 911         vsi->offset_loaded = true;
 912
 913 #ifdef RTE_LIBRTE_I40E_DEBUG_DRIVER
 914         printf("***************** VSI[%u] stats start *******************\n",
 915                                                                 vsi->vsi_id);
 916         printf("rx_bytes:            %lu\n", nes->rx_bytes);
 917         printf("rx_unicast:          %lu\n", nes->rx_unicast);
 918         printf("rx_multicast:        %lu\n", nes->rx_multicast);
 919         printf("rx_broadcast:        %lu\n", nes->rx_broadcast);
 920         printf("rx_discards:         %lu\n", nes->rx_discards);
 921         printf("rx_unknown_protocol: %lu\n", nes->rx_unknown_protocol);
 922         printf("tx_bytes:            %lu\n", nes->tx_bytes);
 923         printf("tx_unicast:          %lu\n", nes->tx_unicast);
 924         printf("tx_multicast:        %lu\n", nes->tx_multicast);
 925         printf("tx_broadcast:        %lu\n", nes->tx_broadcast);
 926         printf("tx_discards:         %lu\n", nes->tx_discards);
 927         printf("tx_errors:           %lu\n", nes->tx_errors);
 928         printf("***************** VSI[%u] stats end *******************\n",
 929                                                                 vsi->vsi_id);
 930 #endif /* RTE_LIBRTE_I40E_DEBUG_DRIVER */
 931 }
 932
 933 /* Get all statistics of a port */
 934 static void
 935 i40e_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 936 {
 937         uint32_t i;
 938         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 939         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 940         struct i40e_hw_port_stats *ns = &pf->stats; /* new stats */
 941         struct i40e_hw_port_stats *os = &pf->stats_offset; /* old stats */
 942
 943         /* Get statistics of struct i40e_eth_stats */
 944         i40e_stat_update_48(hw, I40E_GLPRT_GORCH(hw->port),
 945                             I40E_GLPRT_GORCL(hw->port),
 946                             pf->offset_loaded, &os->eth.rx_bytes,
 947                             &ns->eth.rx_bytes);
 948         i40e_stat_update_48(hw, I40E_GLPRT_UPRCH(hw->port),
 949                             I40E_GLPRT_UPRCL(hw->port),
 950                             pf->offset_loaded, &os->eth.rx_unicast,
 951                             &ns->eth.rx_unicast);
 952         i40e_stat_update_48(hw, I40E_GLPRT_MPRCH(hw->port),
 953                             I40E_GLPRT_MPRCL(hw->port),
 954                             pf->offset_loaded, &os->eth.rx_multicast,
 955                             &ns->eth.rx_multicast);
 956         i40e_stat_update_48(hw, I40E_GLPRT_BPRCH(hw->port),
 957                             I40E_GLPRT_BPRCL(hw->port),
 958                             pf->offset_loaded, &os->eth.rx_broadcast,
 959                             &ns->eth.rx_broadcast);
 960         i40e_stat_update_32(hw, I40E_GLPRT_RDPC(hw->port),
 961                             pf->offset_loaded, &os->eth.rx_discards,
 962                             &ns->eth.rx_discards);
 963         /* GLPRT_REPC not supported */
 964         /* GLPRT_RMPC not supported */
 965         i40e_stat_update_32(hw, I40E_GLPRT_RUPP(hw->port),
 966                             pf->offset_loaded,
 967                             &os->eth.rx_unknown_protocol,
 968                             &ns->eth.rx_unknown_protocol);
 969         i40e_stat_update_48(hw, I40E_GLPRT_GOTCH(hw->port),
 970                             I40E_GLPRT_GOTCL(hw->port),
 971                             pf->offset_loaded, &os->eth.tx_bytes,
 972                             &ns->eth.tx_bytes);
 973         i40e_stat_update_48(hw, I40E_GLPRT_UPTCH(hw->port),
 974                             I40E_GLPRT_UPTCL(hw->port),
 975                             pf->offset_loaded, &os->eth.tx_unicast,
 976                             &ns->eth.tx_unicast);
 977         i40e_stat_update_48(hw, I40E_GLPRT_MPTCH(hw->port),
 978                             I40E_GLPRT_MPTCL(hw->port),
 979                             pf->offset_loaded, &os->eth.tx_multicast,
 980                             &ns->eth.tx_multicast);
 981         i40e_stat_update_48(hw, I40E_GLPRT_BPTCH(hw->port),
 982                             I40E_GLPRT_BPTCL(hw->port),
 983                             pf->offset_loaded, &os->eth.tx_broadcast,
 984                             &ns->eth.tx_broadcast);
 985         i40e_stat_update_32(hw, I40E_GLPRT_TDPC(hw->port),
 986                             pf->offset_loaded, &os->eth.tx_discards,
 987                             &ns->eth.tx_discards);
 988         /* GLPRT_TEPC not supported */
 989
 990         /* additional port specific stats */
 991         i40e_stat_update_32(hw, I40E_GLPRT_TDOLD(hw->port),
 992                             pf->offset_loaded, &os->tx_dropped_link_down,
 993                             &ns->tx_dropped_link_down);
 994         i40e_stat_update_32(hw, I40E_GLPRT_CRCERRS(hw->port),
 995                             pf->offset_loaded, &os->crc_errors,
 996                             &ns->crc_errors);
 997         i40e_stat_update_32(hw, I40E_GLPRT_ILLERRC(hw->port),
 998                             pf->offset_loaded, &os->illegal_bytes,
 999                             &ns->illegal_bytes);
1000         /* GLPRT_ERRBC not supported */
1001         i40e_stat_update_32(hw, I40E_GLPRT_MLFC(hw->port),
1002                             pf->offset_loaded, &os->mac_local_faults,
1003                             &ns->mac_local_faults);
1004         i40e_stat_update_32(hw, I40E_GLPRT_MRFC(hw->port),
1005                             pf->offset_loaded, &os->mac_remote_faults,
1006                             &ns->mac_remote_faults);
1007         i40e_stat_update_32(hw, I40E_GLPRT_RLEC(hw->port),
1008                             pf->offset_loaded, &os->rx_length_errors,
1009                             &ns->rx_length_errors);
1010         i40e_stat_update_32(hw, I40E_GLPRT_LXONRXC(hw->port),
1011                             pf->offset_loaded, &os->link_xon_rx,
1012                             &ns->link_xon_rx);
1013         i40e_stat_update_32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
1014                             pf->offset_loaded, &os->link_xoff_rx,
1015                             &ns->link_xoff_rx);
1016         for (i = 0; i < 8; i++) {
1017                 i40e_stat_update_32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
1018                                     pf->offset_loaded,
1019                                     &os->priority_xon_rx[i],
1020                                     &ns->priority_xon_rx[i]);
1021                 i40e_stat_update_32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
1022                                     pf->offset_loaded,
1023                                     &os->priority_xoff_rx[i],
1024                                     &ns->priority_xoff_rx[i]);
1025         }
1026         i40e_stat_update_32(hw, I40E_GLPRT_LXONTXC(hw->port),
1027                             pf->offset_loaded, &os->link_xon_tx,
1028                             &ns->link_xon_tx);
1029         i40e_stat_update_32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
1030                             pf->offset_loaded, &os->link_xoff_tx,
1031                             &ns->link_xoff_tx);
1032         for (i = 0; i < 8; i++) {
1033                 i40e_stat_update_32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
1034                                     pf->offset_loaded,
1035                                     &os->priority_xon_tx[i],
1036                                     &ns->priority_xon_tx[i]);
1037                 i40e_stat_update_32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
1038                                     pf->offset_loaded,
1039                                     &os->priority_xoff_tx[i],
1040                                     &ns->priority_xoff_tx[i]);
1041                 i40e_stat_update_32(hw, I40E_GLPRT_RXON2OFFCNT(hw->port, i),
1042                                     pf->offset_loaded,
1043                                     &os->priority_xon_2_xoff[i],
1044                                     &ns->priority_xon_2_xoff[i]);
1045         }
1046         i40e_stat_update_48(hw, I40E_GLPRT_PRC64H(hw->port),
1047                             I40E_GLPRT_PRC64L(hw->port),
1048                             pf->offset_loaded, &os->rx_size_64,
1049                             &ns->rx_size_64);
1050         i40e_stat_update_48(hw, I40E_GLPRT_PRC127H(hw->port),
1051                             I40E_GLPRT_PRC127L(hw->port),
1052                             pf->offset_loaded, &os->rx_size_127,
1053                             &ns->rx_size_127);
1054         i40e_stat_update_48(hw, I40E_GLPRT_PRC255H(hw->port),
1055                             I40E_GLPRT_PRC255L(hw->port),
1056                             pf->offset_loaded, &os->rx_size_255,
1057                             &ns->rx_size_255);
1058         i40e_stat_update_48(hw, I40E_GLPRT_PRC511H(hw->port),
1059                             I40E_GLPRT_PRC511L(hw->port),
1060                             pf->offset_loaded, &os->rx_size_511,
1061                             &ns->rx_size_511);
1062         i40e_stat_update_48(hw, I40E_GLPRT_PRC1023H(hw->port),
1063                             I40E_GLPRT_PRC1023L(hw->port),
1064                             pf->offset_loaded, &os->rx_size_1023,
1065                             &ns->rx_size_1023);
1066         i40e_stat_update_48(hw, I40E_GLPRT_PRC1522H(hw->port),
1067                             I40E_GLPRT_PRC1522L(hw->port),
1068                             pf->offset_loaded, &os->rx_size_1522,
1069                             &ns->rx_size_1522);
1070         i40e_stat_update_48(hw, I40E_GLPRT_PRC9522H(hw->port),
1071                             I40E_GLPRT_PRC9522L(hw->port),
1072                             pf->offset_loaded, &os->rx_size_big,
1073                             &ns->rx_size_big);
1074         i40e_stat_update_32(hw, I40E_GLPRT_RUC(hw->port),
1075                             pf->offset_loaded, &os->rx_undersize,
1076                             &ns->rx_undersize);
1077         i40e_stat_update_32(hw, I40E_GLPRT_RFC(hw->port),
1078                             pf->offset_loaded, &os->rx_fragments,
1079                             &ns->rx_fragments);
1080         i40e_stat_update_32(hw, I40E_GLPRT_ROC(hw->port),
1081                             pf->offset_loaded, &os->rx_oversize,
1082                             &ns->rx_oversize);
1083         i40e_stat_update_32(hw, I40E_GLPRT_RJC(hw->port),
1084                             pf->offset_loaded, &os->rx_jabber,
1085                             &ns->rx_jabber);
1086         i40e_stat_update_48(hw, I40E_GLPRT_PTC64H(hw->port),
1087                             I40E_GLPRT_PTC64L(hw->port),
1088                             pf->offset_loaded, &os->tx_size_64,
1089                             &ns->tx_size_64);
1090         i40e_stat_update_48(hw, I40E_GLPRT_PTC127H(hw->port),
1091                             I40E_GLPRT_PTC127L(hw->port),
1092                             pf->offset_loaded, &os->tx_size_127,
1093                             &ns->tx_size_127);
1094         i40e_stat_update_48(hw, I40E_GLPRT_PTC255H(hw->port),
1095                             I40E_GLPRT_PTC255L(hw->port),
1096                             pf->offset_loaded, &os->tx_size_255,
1097                             &ns->tx_size_255);
1098         i40e_stat_update_48(hw, I40E_GLPRT_PTC511H(hw->port),
1099                             I40E_GLPRT_PTC511L(hw->port),
1100                             pf->offset_loaded, &os->tx_size_511,
1101                             &ns->tx_size_511);
1102         i40e_stat_update_48(hw, I40E_GLPRT_PTC1023H(hw->port),
1103                             I40E_GLPRT_PTC1023L(hw->port),
1104                             pf->offset_loaded, &os->tx_size_1023,
1105                             &ns->tx_size_1023);
1106         i40e_stat_update_48(hw, I40E_GLPRT_PTC1522H(hw->port),
1107                             I40E_GLPRT_PTC1522L(hw->port),
1108                             pf->offset_loaded, &os->tx_size_1522,
1109                             &ns->tx_size_1522);
1110         i40e_stat_update_48(hw, I40E_GLPRT_PTC9522H(hw->port),
1111                             I40E_GLPRT_PTC9522L(hw->port),
1112                             pf->offset_loaded, &os->tx_size_big,
1113                             &ns->tx_size_big);
1114         /* GLPRT_MSPDC not supported */
1115         /* GLPRT_XEC not supported */
1116
1117         pf->offset_loaded = true;
1118
1119         stats->ipackets = ns->eth.rx_unicast + ns->eth.rx_multicast +
1120                                                 ns->eth.rx_broadcast;
1121         stats->opackets = ns->eth.tx_unicast + ns->eth.tx_multicast +
1122                                                 ns->eth.tx_broadcast;
1123         stats->ibytes   = ns->eth.rx_bytes;
1124         stats->obytes   = ns->eth.tx_bytes;
1125         stats->oerrors  = ns->eth.tx_errors;
1126         stats->imcasts  = ns->eth.rx_multicast;
1127
1128         if (pf->main_vsi)
1129                 i40e_update_vsi_stats(pf->main_vsi);
1130
1131 #ifdef RTE_LIBRTE_I40E_DEBUG_DRIVER
1132         printf("***************** PF stats start *******************\n");
1133         printf("rx_bytes:            %lu\n", ns->eth.rx_bytes);
1134         printf("rx_unicast:          %lu\n", ns->eth.rx_unicast);
1135         printf("rx_multicast:        %lu\n", ns->eth.rx_multicast);
1136         printf("rx_broadcast:        %lu\n", ns->eth.rx_broadcast);
1137         printf("rx_discards:         %lu\n", ns->eth.rx_discards);
1138         printf("rx_unknown_protocol: %lu\n", ns->eth.rx_unknown_protocol);
1139         printf("tx_bytes:            %lu\n", ns->eth.tx_bytes);
1140         printf("tx_unicast:          %lu\n", ns->eth.tx_unicast);
1141         printf("tx_multicast:        %lu\n", ns->eth.tx_multicast);
1142         printf("tx_broadcast:        %lu\n", ns->eth.tx_broadcast);
1143         printf("tx_discards:         %lu\n", ns->eth.tx_discards);
1144         printf("tx_errors:           %lu\n", ns->eth.tx_errors);
1145
1146         printf("tx_dropped_link_down:     %lu\n", ns->tx_dropped_link_down);
1147         printf("crc_errors:               %lu\n", ns->crc_errors);
1148         printf("illegal_bytes:            %lu\n", ns->illegal_bytes);
1149         printf("error_bytes:              %lu\n", ns->error_bytes);
1150         printf("mac_local_faults:         %lu\n", ns->mac_local_faults);
1151         printf("mac_remote_faults:        %lu\n", ns->mac_remote_faults);
1152         printf("rx_length_errors:         %lu\n", ns->rx_length_errors);
1153         printf("link_xon_rx:              %lu\n", ns->link_xon_rx);
1154         printf("link_xoff_rx:             %lu\n", ns->link_xoff_rx);
1155         for (i = 0; i < 8; i++) {
1156                 printf("priority_xon_rx[%d]:      %lu\n",
1157                                 i, ns->priority_xon_rx[i]);
1158                 printf("priority_xoff_rx[%d]:     %lu\n",
1159                                 i, ns->priority_xoff_rx[i]);
1160         }
1161         printf("link_xon_tx:              %lu\n", ns->link_xon_tx);
1162         printf("link_xoff_tx:             %lu\n", ns->link_xoff_tx);
1163         for (i = 0; i < 8; i++) {
1164                 printf("priority_xon_tx[%d]:      %lu\n",
1165                                 i, ns->priority_xon_tx[i]);
1166                 printf("priority_xoff_tx[%d]:     %lu\n",
1167                                 i, ns->priority_xoff_tx[i]);
1168                 printf("priority_xon_2_xoff[%d]:  %lu\n",
1169                                 i, ns->priority_xon_2_xoff[i]);
1170         }
1171         printf("rx_size_64:               %lu\n", ns->rx_size_64);
1172         printf("rx_size_127:              %lu\n", ns->rx_size_127);
1173         printf("rx_size_255:              %lu\n", ns->rx_size_255);
1174         printf("rx_size_511:              %lu\n", ns->rx_size_511);
1175         printf("rx_size_1023:             %lu\n", ns->rx_size_1023);
1176         printf("rx_size_1522:             %lu\n", ns->rx_size_1522);
1177         printf("rx_size_big:              %lu\n", ns->rx_size_big);
1178         printf("rx_undersize:             %lu\n", ns->rx_undersize);
1179         printf("rx_fragments:             %lu\n", ns->rx_fragments);
1180         printf("rx_oversize:              %lu\n", ns->rx_oversize);
1181         printf("rx_jabber:                %lu\n", ns->rx_jabber);
1182         printf("tx_size_64:               %lu\n", ns->tx_size_64);
1183         printf("tx_size_127:              %lu\n", ns->tx_size_127);
1184         printf("tx_size_255:              %lu\n", ns->tx_size_255);
1185         printf("tx_size_511:              %lu\n", ns->tx_size_511);
1186         printf("tx_size_1023:             %lu\n", ns->tx_size_1023);
1187         printf("tx_size_1522:             %lu\n", ns->tx_size_1522);
1188         printf("tx_size_big:              %lu\n", ns->tx_size_big);
1189         printf("mac_short_packet_dropped: %lu\n",
1190                         ns->mac_short_packet_dropped);
1191         printf("checksum_error:           %lu\n", ns->checksum_error);
1192         printf("***************** PF stats end ********************\n");
1193 #endif /* RTE_LIBRTE_I40E_DEBUG_DRIVER */
1194 }
1195
1196 /* Reset the statistics */
1197 static void
1198 i40e_dev_stats_reset(struct rte_eth_dev *dev)
1199 {
1200         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1201
1202         /* It results in reloading the start point of each counter */
1203         pf->offset_loaded = false;
1204 }
1205
1206 static int
1207 i40e_dev_queue_stats_mapping_set(__rte_unused struct rte_eth_dev *dev,
1208                                  __rte_unused uint16_t queue_id,
1209                                  __rte_unused uint8_t stat_idx,
1210                                  __rte_unused uint8_t is_rx)
1211 {
1212         PMD_INIT_FUNC_TRACE();
1213
1214         return -ENOSYS;
1215 }
1216
1217 static void
1218 i40e_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1219 {
1220         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1221         struct i40e_vsi *vsi = pf->main_vsi;
1222
1223         dev_info->max_rx_queues = vsi->nb_qps;
1224         dev_info->max_tx_queues = vsi->nb_qps;
1225         dev_info->min_rx_bufsize = I40E_BUF_SIZE_MIN;
1226         dev_info->max_rx_pktlen = I40E_FRAME_SIZE_MAX;
1227         dev_info->max_mac_addrs = vsi->max_macaddrs;
1228         dev_info->max_vfs = dev->pci_dev->max_vfs;
1229         dev_info->rx_offload_capa =
1230                 DEV_RX_OFFLOAD_VLAN_STRIP |
1231                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1232                 DEV_RX_OFFLOAD_UDP_CKSUM |
1233                 DEV_RX_OFFLOAD_TCP_CKSUM;
1234         dev_info->tx_offload_capa =
1235                 DEV_TX_OFFLOAD_VLAN_INSERT |
1236                 DEV_TX_OFFLOAD_IPV4_CKSUM |
1237                 DEV_TX_OFFLOAD_UDP_CKSUM |
1238                 DEV_TX_OFFLOAD_TCP_CKSUM |
1239                 DEV_TX_OFFLOAD_SCTP_CKSUM;
1240 }
1241
1242 static int
1243 i40e_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1244 {
1245         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1246         struct i40e_vsi *vsi = pf->main_vsi;
1247         PMD_INIT_FUNC_TRACE();
1248
1249         if (on)
1250                 return i40e_vsi_add_vlan(vsi, vlan_id);
1251         else
1252                 return i40e_vsi_delete_vlan(vsi, vlan_id);
1253 }
1254
1255 static void
1256 i40e_vlan_tpid_set(__rte_unused struct rte_eth_dev *dev,
1257                    __rte_unused uint16_t tpid)
1258 {
1259         PMD_INIT_FUNC_TRACE();
1260 }
1261
1262 static void
1263 i40e_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1264 {
1265         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1266         struct i40e_vsi *vsi = pf->main_vsi;
1267
1268         if (mask & ETH_VLAN_STRIP_MASK) {
1269                 /* Enable or disable VLAN stripping */
1270                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1271                         i40e_vsi_config_vlan_stripping(vsi, TRUE);
1272                 else
1273                         i40e_vsi_config_vlan_stripping(vsi, FALSE);
1274         }
1275
1276         if (mask & ETH_VLAN_EXTEND_MASK) {
1277                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1278                         i40e_vsi_config_double_vlan(vsi, TRUE);
1279                 else
1280                         i40e_vsi_config_double_vlan(vsi, FALSE);
1281         }
1282 }
1283
1284 static void
1285 i40e_vlan_strip_queue_set(__rte_unused struct rte_eth_dev *dev,
1286                           __rte_unused uint16_t queue,
1287                           __rte_unused int on)
1288 {
1289         PMD_INIT_FUNC_TRACE();
1290 }
1291
1292 static int
1293 i40e_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid, int on)
1294 {
1295         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1296         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
1297         struct i40e_vsi *vsi = pf->main_vsi;
1298         struct rte_eth_dev_data *data = I40E_VSI_TO_DEV_DATA(vsi);
1299         struct i40e_vsi_context ctxt;
1300         uint8_t vlan_flags = 0;
1301         int ret;
1302
1303         if (on) {
1304                 /**
1305                  * If insert pvid is enabled, only tagged pkts are
1306                  * allowed to be sent out.
1307                  */
1308                 vlan_flags |= I40E_AQ_VSI_PVLAN_INSERT_PVID |
1309                                 I40E_AQ_VSI_PVLAN_MODE_TAGGED;
1310         } else {
1311                 if (data->dev_conf.txmode.hw_vlan_reject_tagged == 0)
1312                         vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_TAGGED;
1313                 if (data->dev_conf.txmode.hw_vlan_reject_untagged == 0)
1314                         vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_UNTAGGED;
1315         }
1316         vsi->info.port_vlan_flags &= ~(I40E_AQ_VSI_PVLAN_INSERT_PVID |
1317                                         I40E_AQ_VSI_PVLAN_MODE_MASK);
1318         vsi->info.port_vlan_flags |= vlan_flags;
1319         vsi->info.pvid = pvid;
1320         vsi->info.valid_sections =
1321                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
1322         (void)rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
1323         ctxt.seid = vsi->seid;
1324         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
1325         if (ret != I40E_SUCCESS)
1326                 PMD_DRV_LOG(INFO, "Failed to update VSI params\n");
1327
1328         return ret;
1329 }
1330
1331 static int
1332 i40e_dev_led_on(struct rte_eth_dev *dev)
1333 {
1334         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1335         uint32_t mode = i40e_led_get(hw);
1336
1337         if (mode == 0)
1338                 i40e_led_set(hw, 0xf, true); /* 0xf means led always true */
1339
1340         return 0;
1341 }
1342
1343 static int
1344 i40e_dev_led_off(struct rte_eth_dev *dev)
1345 {
1346         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1347         uint32_t mode = i40e_led_get(hw);
1348
1349         if (mode != 0)
1350                 i40e_led_set(hw, 0, false);
1351
1352         return 0;
1353 }
1354
1355 static int
1356 i40e_flow_ctrl_set(__rte_unused struct rte_eth_dev *dev,
1357                    __rte_unused struct rte_eth_fc_conf *fc_conf)
1358 {
1359         PMD_INIT_FUNC_TRACE();
1360
1361         return -ENOSYS;
1362 }
1363
1364 static int
1365 i40e_priority_flow_ctrl_set(__rte_unused struct rte_eth_dev *dev,
1366                             __rte_unused struct rte_eth_pfc_conf *pfc_conf)
1367 {
1368         PMD_INIT_FUNC_TRACE();
1369
1370         return -ENOSYS;
1371 }
1372
1373 /* Add a MAC address, and update filters */
1374 static void
1375 i40e_macaddr_add(struct rte_eth_dev *dev,
1376                  struct ether_addr *mac_addr,
1377                  __attribute__((unused)) uint32_t index,
1378                  __attribute__((unused)) uint32_t pool)
1379 {
1380         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1381         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1382         struct i40e_vsi *vsi = pf->main_vsi;
1383         struct ether_addr old_mac;
1384         int ret;
1385
1386         if (!is_valid_assigned_ether_addr(mac_addr)) {
1387                 PMD_DRV_LOG(ERR, "Invalid ethernet address\n");
1388                 return;
1389         }
1390
1391         if (is_same_ether_addr(mac_addr, &(pf->dev_addr))) {
1392                 PMD_DRV_LOG(INFO, "Ignore adding permanent mac address\n");
1393                 return;
1394         }
1395
1396         /* Write mac address */
1397         ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_ONLY,
1398                                         mac_addr->addr_bytes, NULL);
1399         if (ret != I40E_SUCCESS) {
1400                 PMD_DRV_LOG(ERR, "Failed to write mac address\n");
1401                 return;
1402         }
1403
1404         (void)rte_memcpy(&old_mac, hw->mac.addr, ETHER_ADDR_LEN);
1405         (void)rte_memcpy(hw->mac.addr, mac_addr->addr_bytes,
1406                         ETHER_ADDR_LEN);
1407
1408         ret = i40e_vsi_add_mac(vsi, mac_addr);
1409         if (ret != I40E_SUCCESS) {
1410                 PMD_DRV_LOG(ERR, "Failed to add MACVLAN filter\n");
1411                 return;
1412         }
1413
1414         ether_addr_copy(mac_addr, &pf->dev_addr);
1415         i40e_vsi_delete_mac(vsi, &old_mac);
1416 }
1417
1418 /* Remove a MAC address, and update filters */
1419 static void
1420 i40e_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1421 {
1422         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1423         struct i40e_vsi *vsi = pf->main_vsi;
1424         struct rte_eth_dev_data *data = I40E_VSI_TO_DEV_DATA(vsi);
1425         struct ether_addr *macaddr;
1426         int ret;
1427         struct i40e_hw *hw =
1428                 I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1429
1430         if (index >= vsi->max_macaddrs)
1431                 return;
1432
1433         macaddr = &(data->mac_addrs[index]);
1434         if (!is_valid_assigned_ether_addr(macaddr))
1435                 return;
1436
1437         ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_ONLY,
1438                                         hw->mac.perm_addr, NULL);
1439         if (ret != I40E_SUCCESS) {
1440                 PMD_DRV_LOG(ERR, "Failed to write mac address\n");
1441                 return;
1442         }
1443
1444         (void)rte_memcpy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN);
1445
1446         ret = i40e_vsi_delete_mac(vsi, macaddr);
1447         if (ret != I40E_SUCCESS)
1448                 return;
1449
1450         /* Clear device address as it has been removed */
1451         if (is_same_ether_addr(&(pf->dev_addr), macaddr))
1452                 memset(&pf->dev_addr, 0, sizeof(struct ether_addr));
1453 }
1454
1455 static int
1456 i40e_dev_rss_reta_update(struct rte_eth_dev *dev,
1457                          struct rte_eth_rss_reta *reta_conf)
1458 {
1459         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1460         uint32_t lut, l;
1461         uint8_t i, j, mask, max = ETH_RSS_RETA_NUM_ENTRIES / 2;
1462
1463         for (i = 0; i < ETH_RSS_RETA_NUM_ENTRIES; i += 4) {
1464                 if (i < max)
1465                         mask = (uint8_t)((reta_conf->mask_lo >> i) & 0xF);
1466                 else
1467                         mask = (uint8_t)((reta_conf->mask_hi >>
1468                                                 (i - max)) & 0xF);
1469
1470                 if (!mask)
1471                         continue;
1472
1473                 if (mask == 0xF)
1474                         l = 0;
1475                 else
1476                         l = I40E_READ_REG(hw, I40E_PFQF_HLUT(i >> 2));
1477
1478                 for (j = 0, lut = 0; j < 4; j++) {
1479                         if (mask & (0x1 < j))
1480                                 lut |= reta_conf->reta[i + j] << (8 * j);
1481                         else
1482                                 lut |= l & (0xFF << (8 * j));
1483                 }
1484                 I40E_WRITE_REG(hw, I40E_PFQF_HLUT(i >> 2), lut);
1485         }
1486
1487         return 0;
1488 }
1489
1490 static int
1491 i40e_dev_rss_reta_query(struct rte_eth_dev *dev,
1492                         struct rte_eth_rss_reta *reta_conf)
1493 {
1494         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1495         uint32_t lut;
1496         uint8_t i, j, mask, max = ETH_RSS_RETA_NUM_ENTRIES / 2;
1497
1498         for (i = 0; i < ETH_RSS_RETA_NUM_ENTRIES; i += 4) {
1499                 if (i < max)
1500                         mask = (uint8_t)((reta_conf->mask_lo >> i) & 0xF);
1501                 else
1502                         mask = (uint8_t)((reta_conf->mask_hi >>
1503                                                 (i - max)) & 0xF);
1504
1505                 if (!mask)
1506                         continue;
1507
1508                 lut = I40E_READ_REG(hw, I40E_PFQF_HLUT(i >> 2));
1509                 for (j = 0; j < 4; j++) {
1510                         if (mask & (0x1 << j))
1511                                 reta_conf->reta[i + j] =
1512                                         (uint8_t)((lut >> (8 * j)) & 0xFF);
1513                 }
1514         }
1515
1516         return 0;
1517 }
1518
1519 /**
1520  * i40e_allocate_dma_mem_d - specific memory alloc for shared code
1521  * @hw:   pointer to the HW structure
1522  * @mem:  pointer to mem struct to fill out
1523  * @size: size of memory requested
1524  * @alignment: what to align the allocation to
1525  **/
1526 enum i40e_status_code
1527 i40e_allocate_dma_mem_d(__attribute__((unused)) struct i40e_hw *hw,
1528                         struct i40e_dma_mem *mem,
1529                         u64 size,
1530                         u32 alignment)
1531 {
1532         static uint64_t id = 0;
1533         const struct rte_memzone *mz = NULL;
1534         char z_name[RTE_MEMZONE_NAMESIZE];
1535
1536         if (!mem)
1537                 return I40E_ERR_PARAM;
1538
1539         id++;
1540         rte_snprintf(z_name, sizeof(z_name), "i40e_dma_%lu", id);
1541         mz = rte_memzone_reserve_aligned(z_name, size, 0, 0, alignment);
1542         if (!mz)
1543                 return I40E_ERR_NO_MEMORY;
1544
1545         mem->id = id;
1546         mem->size = size;
1547         mem->va = mz->addr;
1548         mem->pa = mz->phys_addr;
1549
1550         return I40E_SUCCESS;
1551 }
1552
1553 /**
1554  * i40e_free_dma_mem_d - specific memory free for shared code
1555  * @hw:   pointer to the HW structure
1556  * @mem:  ptr to mem struct to free
1557  **/
1558 enum i40e_status_code
1559 i40e_free_dma_mem_d(__attribute__((unused)) struct i40e_hw *hw,
1560                     struct i40e_dma_mem *mem)
1561 {
1562         if (!mem || !mem->va)
1563                 return I40E_ERR_PARAM;
1564
1565         mem->va = NULL;
1566         mem->pa = (u64)0;
1567
1568         return I40E_SUCCESS;
1569 }
1570
1571 /**
1572  * i40e_allocate_virt_mem_d - specific memory alloc for shared code
1573  * @hw:   pointer to the HW structure
1574  * @mem:  pointer to mem struct to fill out
1575  * @size: size of memory requested
1576  **/
1577 enum i40e_status_code
1578 i40e_allocate_virt_mem_d(__attribute__((unused)) struct i40e_hw *hw,
1579                          struct i40e_virt_mem *mem,
1580                          u32 size)
1581 {
1582         if (!mem)
1583                 return I40E_ERR_PARAM;
1584
1585         mem->size = size;
1586         mem->va = rte_zmalloc("i40e", size, 0);
1587
1588         if (mem->va)
1589                 return I40E_SUCCESS;
1590         else
1591                 return I40E_ERR_NO_MEMORY;
1592 }
1593
1594 /**
1595  * i40e_free_virt_mem_d - specific memory free for shared code
1596  * @hw:   pointer to the HW structure
1597  * @mem:  pointer to mem struct to free
1598  **/
1599 enum i40e_status_code
1600 i40e_free_virt_mem_d(__attribute__((unused)) struct i40e_hw *hw,
1601                      struct i40e_virt_mem *mem)
1602 {
1603         if (!mem)
1604                 return I40E_ERR_PARAM;
1605
1606         rte_free(mem->va);
1607         mem->va = NULL;
1608
1609         return I40E_SUCCESS;
1610 }
1611
1612 void
1613 i40e_init_spinlock_d(struct i40e_spinlock *sp)
1614 {
1615         rte_spinlock_init(&sp->spinlock);
1616 }
1617
1618 void
1619 i40e_acquire_spinlock_d(struct i40e_spinlock *sp)
1620 {
1621         rte_spinlock_lock(&sp->spinlock);
1622 }
1623
1624 void
1625 i40e_release_spinlock_d(struct i40e_spinlock *sp)
1626 {
1627         rte_spinlock_unlock(&sp->spinlock);
1628 }
1629
1630 void
1631 i40e_destroy_spinlock_d(__attribute__((unused)) struct i40e_spinlock *sp)
1632 {
1633         return;
1634 }
1635
1636 /**
1637  * Get the hardware capabilities, which will be parsed
1638  * and saved into struct i40e_hw.
1639  */
1640 static int
1641 i40e_get_cap(struct i40e_hw *hw)
1642 {
1643         struct i40e_aqc_list_capabilities_element_resp *buf;
1644         uint16_t len, size = 0;
1645         int ret;
1646
1647         /* Calculate a huge enough buff for saving response data temporarily */
1648         len = sizeof(struct i40e_aqc_list_capabilities_element_resp) *
1649                                                 I40E_MAX_CAP_ELE_NUM;
1650         buf = rte_zmalloc("i40e", len, 0);
1651         if (!buf) {
1652                 PMD_DRV_LOG(ERR, "Failed to allocate memory\n");
1653                 return I40E_ERR_NO_MEMORY;
1654         }
1655
1656         /* Get, parse the capabilities and save it to hw */
1657         ret = i40e_aq_discover_capabilities(hw, buf, len, &size,
1658                         i40e_aqc_opc_list_func_capabilities, NULL);
1659         if (ret != I40E_SUCCESS)
1660                 PMD_DRV_LOG(ERR, "Failed to discover capabilities\n");
1661
1662         /* Free the temporary buffer after being used */
1663         rte_free(buf);
1664
1665         return ret;
1666 }
1667
1668 static int
1669 i40e_pf_parameter_init(struct rte_eth_dev *dev)
1670 {
1671         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1672         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
1673         uint16_t sum_queues = 0, sum_vsis;
1674
1675         /* First check if FW support SRIOV */
1676         if (dev->pci_dev->max_vfs && !hw->func_caps.sr_iov_1_1) {
1677                 PMD_INIT_LOG(ERR, "HW configuration doesn't support SRIOV\n");
1678                 return -EINVAL;
1679         }
1680
1681         pf->flags = I40E_FLAG_HEADER_SPLIT_DISABLED;
1682         pf->max_num_vsi = RTE_MIN(hw->func_caps.num_vsis, I40E_MAX_NUM_VSIS);
1683         PMD_INIT_LOG(INFO, "Max supported VSIs:%u\n", pf->max_num_vsi);
1684         /* Allocate queues for pf */
1685         if (hw->func_caps.rss) {
1686                 pf->flags |= I40E_FLAG_RSS;
1687                 pf->lan_nb_qps = RTE_MIN(hw->func_caps.num_tx_qp,
1688                         (uint32_t)(1 << hw->func_caps.rss_table_entry_width));
1689                 pf->lan_nb_qps = i40e_prev_power_of_2(pf->lan_nb_qps);
1690         } else
1691                 pf->lan_nb_qps = 1;
1692         sum_queues = pf->lan_nb_qps;
1693         /* Default VSI is not counted in */
1694         sum_vsis = 0;
1695         PMD_INIT_LOG(INFO, "PF queue pairs:%u\n", pf->lan_nb_qps);
1696
1697         if (hw->func_caps.sr_iov_1_1 && dev->pci_dev->max_vfs) {
1698                 pf->flags |= I40E_FLAG_SRIOV;
1699                 pf->vf_nb_qps = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VF;
1700                 if (dev->pci_dev->max_vfs > hw->func_caps.num_vfs) {
1701                         PMD_INIT_LOG(ERR, "Config VF number %u, "
1702                                 "max supported %u.\n", dev->pci_dev->max_vfs,
1703                                                 hw->func_caps.num_vfs);
1704                         return -EINVAL;
1705                 }
1706                 if (pf->vf_nb_qps > I40E_MAX_QP_NUM_PER_VF) {
1707                         PMD_INIT_LOG(ERR, "FVL VF queue %u, "
1708                                 "max support %u queues.\n", pf->vf_nb_qps,
1709                                                 I40E_MAX_QP_NUM_PER_VF);
1710                         return -EINVAL;
1711                 }
1712                 pf->vf_num = dev->pci_dev->max_vfs;
1713                 sum_queues += pf->vf_nb_qps * pf->vf_num;
1714                 sum_vsis   += pf->vf_num;
1715                 PMD_INIT_LOG(INFO, "Max VF num:%u each has queue pairs:%u\n",
1716                                                 pf->vf_num, pf->vf_nb_qps);
1717         } else
1718                 pf->vf_num = 0;
1719
1720         if (hw->func_caps.vmdq) {
1721                 pf->flags |= I40E_FLAG_VMDQ;
1722                 pf->vmdq_nb_qps = I40E_DEFAULT_QP_NUM_VMDQ;
1723                 sum_queues += pf->vmdq_nb_qps;
1724                 sum_vsis += 1;
1725                 PMD_INIT_LOG(INFO, "VMDQ queue pairs:%u\n", pf->vmdq_nb_qps);
1726         }
1727
1728         if (hw->func_caps.fd) {
1729                 pf->flags |= I40E_FLAG_FDIR;
1730                 pf->fdir_nb_qps = I40E_DEFAULT_QP_NUM_FDIR;
1731                 /**
1732                  * Each flow director consumes one VSI and one queue,
1733                  * but can't calculate out predictably here.
1734                  */
1735         }
1736
1737         if (sum_vsis > pf->max_num_vsi ||
1738                 sum_queues > hw->func_caps.num_rx_qp) {
1739                 PMD_INIT_LOG(ERR, "VSI/QUEUE setting can't be satisfied\n");
1740                 PMD_INIT_LOG(ERR, "Max VSIs: %u, asked:%u\n",
1741                                 pf->max_num_vsi, sum_vsis);
1742                 PMD_INIT_LOG(ERR, "Total queue pairs:%u, asked:%u\n",
1743                                 hw->func_caps.num_rx_qp, sum_queues);
1744                 return -EINVAL;
1745         }
1746
1747         /* Each VSI occupy 1 MSIX interrupt at least, plus IRQ0 for misc intr cause */
1748         if (sum_vsis > hw->func_caps.num_msix_vectors - 1) {
1749                 PMD_INIT_LOG(ERR, "Too many VSIs(%u), MSIX intr(%u) not enough\n",
1750                                 sum_vsis, hw->func_caps.num_msix_vectors);
1751                 return -EINVAL;
1752         }
1753         return I40E_SUCCESS;
1754 }
1755
1756 static int
1757 i40e_pf_get_switch_config(struct i40e_pf *pf)
1758 {
1759         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
1760         struct i40e_aqc_get_switch_config_resp *switch_config;
1761         struct i40e_aqc_switch_config_element_resp *element;
1762         uint16_t start_seid = 0, num_reported;
1763         int ret;
1764
1765         switch_config = (struct i40e_aqc_get_switch_config_resp *)\
1766                         rte_zmalloc("i40e", I40E_AQ_LARGE_BUF, 0);
1767         if (!switch_config) {
1768                 PMD_DRV_LOG(ERR, "Failed to allocated memory\n");
1769                 return -ENOMEM;
1770         }
1771
1772         /* Get the switch configurations */
1773         ret = i40e_aq_get_switch_config(hw, switch_config,
1774                 I40E_AQ_LARGE_BUF, &start_seid, NULL);
1775         if (ret != I40E_SUCCESS) {
1776                 PMD_DRV_LOG(ERR, "Failed to get switch configurations\n");
1777                 goto fail;
1778         }
1779         num_reported = rte_le_to_cpu_16(switch_config->header.num_reported);
1780         if (num_reported != 1) { /* The number should be 1 */
1781                 PMD_DRV_LOG(ERR, "Wrong number of switch config reported\n");
1782                 goto fail;
1783         }
1784
1785         /* Parse the switch configuration elements */
1786         element = &(switch_config->element[0]);
1787         if (element->element_type == I40E_SWITCH_ELEMENT_TYPE_VSI) {
1788                 pf->mac_seid = rte_le_to_cpu_16(element->uplink_seid);
1789                 pf->main_vsi_seid = rte_le_to_cpu_16(element->seid);
1790         } else
1791                 PMD_DRV_LOG(INFO, "Unknown element type\n");
1792
1793 fail:
1794         rte_free(switch_config);
1795
1796         return ret;
1797 }
1798
1799 static int
1800 i40e_res_pool_init (struct i40e_res_pool_info *pool, uint32_t base,
1801                         uint32_t num)
1802 {
1803         struct pool_entry *entry;
1804
1805         if (pool == NULL || num == 0)
1806                 return -EINVAL;
1807
1808         entry = rte_zmalloc("i40e", sizeof(*entry), 0);
1809         if (entry == NULL) {
1810                 PMD_DRV_LOG(ERR, "Failed to allocate memory for "
1811                                                 "resource pool\n");
1812                 return -ENOMEM;
1813         }
1814
1815         /* queue heap initialize */
1816         pool->num_free = num;
1817         pool->num_alloc = 0;
1818         pool->base = base;
1819         LIST_INIT(&pool->alloc_list);
1820         LIST_INIT(&pool->free_list);
1821
1822         /* Initialize element  */
1823         entry->base = 0;
1824         entry->len = num;
1825
1826         LIST_INSERT_HEAD(&pool->free_list, entry, next);
1827         return 0;
1828 }
1829
1830 static void
1831 i40e_res_pool_destroy(struct i40e_res_pool_info *pool)
1832 {
1833         struct pool_entry *entry;
1834
1835         if (pool == NULL)
1836                 return;
1837
1838         LIST_FOREACH(entry, &pool->alloc_list, next) {
1839                 LIST_REMOVE(entry, next);
1840                 rte_free(entry);
1841         }
1842
1843         LIST_FOREACH(entry, &pool->free_list, next) {
1844                 LIST_REMOVE(entry, next);
1845                 rte_free(entry);
1846         }
1847
1848         pool->num_free = 0;
1849         pool->num_alloc = 0;
1850         pool->base = 0;
1851         LIST_INIT(&pool->alloc_list);
1852         LIST_INIT(&pool->free_list);
1853 }
1854
1855 static int
1856 i40e_res_pool_free(struct i40e_res_pool_info *pool,
1857                        uint32_t base)
1858 {
1859         struct pool_entry *entry, *next, *prev, *valid_entry = NULL;
1860         uint32_t pool_offset;
1861         int insert;
1862
1863         if (pool == NULL) {
1864                 PMD_DRV_LOG(ERR, "Invalid parameter\n");
1865                 return -EINVAL;
1866         }
1867
1868         pool_offset = base - pool->base;
1869         /* Lookup in alloc list */
1870         LIST_FOREACH(entry, &pool->alloc_list, next) {
1871                 if (entry->base == pool_offset) {
1872                         valid_entry = entry;
1873                         LIST_REMOVE(entry, next);
1874                         break;
1875                 }
1876         }
1877
1878         /* Not find, return */
1879         if (valid_entry == NULL) {
1880                 PMD_DRV_LOG(ERR, "Failed to find entry\n");
1881                 return -EINVAL;
1882         }
1883
1884         /**
1885          * Found it, move it to free list  and try to merge.
1886          * In order to make merge easier, always sort it by qbase.
1887          * Find adjacent prev and last entries.
1888          */
1889         prev = next = NULL;
1890         LIST_FOREACH(entry, &pool->free_list, next) {
1891                 if (entry->base > valid_entry->base) {
1892                         next = entry;
1893                         break;
1894                 }
1895                 prev = entry;
1896         }
1897
1898         insert = 0;
1899         /* Try to merge with next one*/
1900         if (next != NULL) {
1901                 /* Merge with next one */
1902                 if (valid_entry->base + valid_entry->len == next->base) {
1903                         next->base = valid_entry->base;
1904                         next->len += valid_entry->len;
1905                         rte_free(valid_entry);
1906                         valid_entry = next;
1907                         insert = 1;
1908                 }
1909         }
1910
1911         if (prev != NULL) {
1912                 /* Merge with previous one */
1913                 if (prev->base + prev->len == valid_entry->base) {
1914                         prev->len += valid_entry->len;
1915                         /* If it merge with next one, remove next node */
1916                         if (insert == 1) {
1917                                 LIST_REMOVE(valid_entry, next);
1918                                 rte_free(valid_entry);
1919                         } else {
1920                                 rte_free(valid_entry);
1921                                 insert = 1;
1922                         }
1923                 }
1924         }
1925
1926         /* Not find any entry to merge, insert */
1927         if (insert == 0) {
1928                 if (prev != NULL)
1929                         LIST_INSERT_AFTER(prev, valid_entry, next);
1930                 else if (next != NULL)
1931                         LIST_INSERT_BEFORE(next, valid_entry, next);
1932                 else /* It's empty list, insert to head */
1933                         LIST_INSERT_HEAD(&pool->free_list, valid_entry, next);
1934         }
1935
1936         pool->num_free += valid_entry->len;
1937         pool->num_alloc -= valid_entry->len;
1938
1939         return 0;
1940 }
1941
1942 static int
1943 i40e_res_pool_alloc(struct i40e_res_pool_info *pool,
1944                        uint16_t num)
1945 {
1946         struct pool_entry *entry, *valid_entry;
1947
1948         if (pool == NULL || num == 0) {
1949                 PMD_DRV_LOG(ERR, "Invalid parameter\n");
1950                 return -EINVAL;
1951         }
1952
1953         if (pool->num_free < num) {
1954                 PMD_DRV_LOG(ERR, "No resource. ask:%u, available:%u\n",
1955                                 num, pool->num_free);
1956                 return -ENOMEM;
1957         }
1958
1959         valid_entry = NULL;
1960         /* Lookup  in free list and find most fit one */
1961         LIST_FOREACH(entry, &pool->free_list, next) {
1962                 if (entry->len >= num) {
1963                         /* Find best one */
1964                         if (entry->len == num) {
1965                                 valid_entry = entry;
1966                                 break;
1967                         }
1968                         if (valid_entry == NULL || valid_entry->len > entry->len)
1969                                 valid_entry = entry;
1970                 }
1971         }
1972
1973         /* Not find one to satisfy the request, return */
1974         if (valid_entry == NULL) {
1975                 PMD_DRV_LOG(ERR, "No valid entry found\n");
1976                 return -ENOMEM;
1977         }
1978         /**
1979          * The entry have equal queue number as requested,
1980          * remove it from alloc_list.
1981          */
1982         if (valid_entry->len == num) {
1983                 LIST_REMOVE(valid_entry, next);
1984         } else {
1985                 /**
1986                  * The entry have more numbers than requested,
1987                  * create a new entry for alloc_list and minus its
1988                  * queue base and number in free_list.
1989                  */
1990                 entry = rte_zmalloc("res_pool", sizeof(*entry), 0);
1991                 if (entry == NULL) {
1992                         PMD_DRV_LOG(ERR, "Failed to allocate memory for "
1993                                         "resource pool\n");
1994                         return -ENOMEM;
1995                 }
1996                 entry->base = valid_entry->base;
1997                 entry->len = num;
1998                 valid_entry->base += num;
1999                 valid_entry->len -= num;
2000                 valid_entry = entry;
2001         }
2002
2003         /* Insert it into alloc list, not sorted */
2004         LIST_INSERT_HEAD(&pool->alloc_list, valid_entry, next);
2005
2006         pool->num_free -= valid_entry->len;
2007         pool->num_alloc += valid_entry->len;
2008
2009         return (valid_entry->base + pool->base);
2010 }
2011
2012 /**
2013  * bitmap_is_subset - Check whether src2 is subset of src1
2014  **/
2015 static inline int
2016 bitmap_is_subset(uint8_t src1, uint8_t src2)
2017 {
2018         return !((src1 ^ src2) & src2);
2019 }
2020
2021 static int
2022 validate_tcmap_parameter(struct i40e_vsi *vsi, uint8_t enabled_tcmap)
2023 {
2024         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2025
2026         /* If DCB is not supported, only default TC is supported */
2027         if (!hw->func_caps.dcb && enabled_tcmap != I40E_DEFAULT_TCMAP) {
2028                 PMD_DRV_LOG(ERR, "DCB is not enabled, "
2029                                 "only TC0 is supported\n");
2030                 return -EINVAL;
2031         }
2032
2033         if (!bitmap_is_subset(hw->func_caps.enabled_tcmap, enabled_tcmap)) {
2034                 PMD_DRV_LOG(ERR, "Enabled TC map 0x%x not applicable to "
2035                         "HW support 0x%x\n", hw->func_caps.enabled_tcmap,
2036                                                         enabled_tcmap);
2037                 return -EINVAL;
2038         }
2039         return I40E_SUCCESS;
2040 }
2041
2042 static int
2043 i40e_vsi_update_tc_bandwidth(struct i40e_vsi *vsi, uint8_t enabled_tcmap)
2044 {
2045         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2046         int i, ret;
2047         struct i40e_aqc_configure_vsi_tc_bw_data tc_bw_data;
2048
2049         ret = validate_tcmap_parameter(vsi, enabled_tcmap);
2050         if (ret != I40E_SUCCESS)
2051                 return ret;
2052
2053         if (!vsi->seid) {
2054                 PMD_DRV_LOG(ERR, "seid not valid\n");
2055                 return -EINVAL;
2056         }
2057
2058         memset(&tc_bw_data, 0, sizeof(tc_bw_data));
2059         tc_bw_data.tc_valid_bits = enabled_tcmap;
2060         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
2061                 tc_bw_data.tc_bw_credits[i] =
2062                         (enabled_tcmap & (1 << i)) ? 1 : 0;
2063
2064         ret = i40e_aq_config_vsi_tc_bw(hw, vsi->seid, &tc_bw_data, NULL);
2065         if (ret != I40E_SUCCESS) {
2066                 PMD_DRV_LOG(ERR, "Failed to configure TC BW\n");
2067                 return ret;
2068         }
2069
2070         (void)rte_memcpy(vsi->info.qs_handle, tc_bw_data.qs_handles,
2071                                         sizeof(vsi->info.qs_handle));
2072         return I40E_SUCCESS;
2073 }
2074
2075 static int
2076 i40e_vsi_config_tc_queue_mapping(struct i40e_vsi *vsi,
2077                                  struct i40e_aqc_vsi_properties_data *info,
2078                                  uint8_t enabled_tcmap)
2079 {
2080         int ret, total_tc = 0, i;
2081         uint16_t qpnum_per_tc, bsf, qp_idx;
2082
2083         ret = validate_tcmap_parameter(vsi, enabled_tcmap);
2084         if (ret != I40E_SUCCESS)
2085                 return ret;
2086
2087         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
2088                 if (enabled_tcmap & (1 << i))
2089                         total_tc++;
2090         vsi->enabled_tc = enabled_tcmap;
2091
2092         /* Number of queues per enabled TC */
2093         qpnum_per_tc = i40e_prev_power_of_2(vsi->nb_qps / total_tc);
2094         qpnum_per_tc = RTE_MIN(qpnum_per_tc, I40E_MAX_Q_PER_TC);
2095         bsf = rte_bsf32(qpnum_per_tc);
2096
2097         /* Adjust the queue number to actual queues that can be applied */
2098         vsi->nb_qps = qpnum_per_tc * total_tc;
2099
2100         /**
2101          * Configure TC and queue mapping parameters, for enabled TC,
2102          * allocate qpnum_per_tc queues to this traffic. For disabled TC,
2103          * default queue will serve it.
2104          */
2105         qp_idx = 0;
2106         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
2107                 if (vsi->enabled_tc & (1 << i)) {
2108                         info->tc_mapping[i] = rte_cpu_to_le_16((qp_idx <<
2109                                         I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
2110                                 (bsf << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT));
2111                         qp_idx += qpnum_per_tc;
2112                 } else
2113                         info->tc_mapping[i] = 0;
2114         }
2115
2116         /* Associate queue number with VSI */
2117         if (vsi->type == I40E_VSI_SRIOV) {
2118                 info->mapping_flags |=
2119                         rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
2120                 for (i = 0; i < vsi->nb_qps; i++)
2121                         info->queue_mapping[i] =
2122                                 rte_cpu_to_le_16(vsi->base_queue + i);
2123         } else {
2124                 info->mapping_flags |=
2125                         rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_CONTIG);
2126                 info->queue_mapping[0] = rte_cpu_to_le_16(vsi->base_queue);
2127         }
2128         info->valid_sections =
2129                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID);
2130
2131         return I40E_SUCCESS;
2132 }
2133
2134 static int
2135 i40e_veb_release(struct i40e_veb *veb)
2136 {
2137         struct i40e_vsi *vsi;
2138         struct i40e_hw *hw;
2139
2140         if (veb == NULL || veb->associate_vsi == NULL)
2141                 return -EINVAL;
2142
2143         if (!TAILQ_EMPTY(&veb->head)) {
2144                 PMD_DRV_LOG(ERR, "VEB still has VSI attached, can't remove\n");
2145                 return -EACCES;
2146         }
2147
2148         vsi = veb->associate_vsi;
2149         hw = I40E_VSI_TO_HW(vsi);
2150
2151         vsi->uplink_seid = veb->uplink_seid;
2152         i40e_aq_delete_element(hw, veb->seid, NULL);
2153         rte_free(veb);
2154         vsi->veb = NULL;
2155         return I40E_SUCCESS;
2156 }
2157
2158 /* Setup a veb */
2159 static struct i40e_veb *
2160 i40e_veb_setup(struct i40e_pf *pf, struct i40e_vsi *vsi)
2161 {
2162         struct i40e_veb *veb;
2163         int ret;
2164         struct i40e_hw *hw;
2165
2166         if (NULL == pf || vsi == NULL) {
2167                 PMD_DRV_LOG(ERR, "veb setup failed, "
2168                         "associated VSI shouldn't null\n");
2169                 return NULL;
2170         }
2171         hw = I40E_PF_TO_HW(pf);
2172
2173         veb = rte_zmalloc("i40e_veb", sizeof(struct i40e_veb), 0);
2174         if (!veb) {
2175                 PMD_DRV_LOG(ERR, "Failed to allocate memory for veb\n");
2176                 goto fail;
2177         }
2178
2179         veb->associate_vsi = vsi;
2180         TAILQ_INIT(&veb->head);
2181         veb->uplink_seid = vsi->uplink_seid;
2182
2183         ret = i40e_aq_add_veb(hw, veb->uplink_seid, vsi->seid,
2184                 I40E_DEFAULT_TCMAP, false, false, &veb->seid, NULL);
2185
2186         if (ret != I40E_SUCCESS) {
2187                 PMD_DRV_LOG(ERR, "Add veb failed, aq_err: %d\n",
2188                                         hw->aq.asq_last_status);
2189                 goto fail;
2190         }
2191
2192         /* get statistics index */
2193         ret = i40e_aq_get_veb_parameters(hw, veb->seid, NULL, NULL,
2194                                 &veb->stats_idx, NULL, NULL, NULL);
2195         if (ret != I40E_SUCCESS) {
2196                 PMD_DRV_LOG(ERR, "Get veb statics index failed, aq_err: %d\n",
2197                                                 hw->aq.asq_last_status);
2198                 goto fail;
2199         }
2200
2201         /* Get VEB bandwidth, to be implemented */
2202         /* Now associated vsi binding to the VEB, set uplink to this VEB */
2203         vsi->uplink_seid = veb->seid;
2204
2205         return veb;
2206 fail:
2207         rte_free(veb);
2208         return NULL;
2209 }
2210
2211 int
2212 i40e_vsi_release(struct i40e_vsi *vsi)
2213 {
2214         struct i40e_pf *pf;
2215         struct i40e_hw *hw;
2216         struct i40e_vsi_list *vsi_list;
2217         int ret;
2218         struct i40e_mac_filter *f;
2219
2220         if (!vsi)
2221                 return I40E_SUCCESS;
2222
2223         pf = I40E_VSI_TO_PF(vsi);
2224         hw = I40E_VSI_TO_HW(vsi);
2225
2226         /* VSI has child to attach, release child first */
2227         if (vsi->veb) {
2228                 TAILQ_FOREACH(vsi_list, &vsi->veb->head, list) {
2229                         if (i40e_vsi_release(vsi_list->vsi) != I40E_SUCCESS)
2230                                 return -1;
2231                         TAILQ_REMOVE(&vsi->veb->head, vsi_list, list);
2232                 }
2233                 i40e_veb_release(vsi->veb);
2234         }
2235
2236         /* Remove all macvlan filters of the VSI */
2237         i40e_vsi_remove_all_macvlan_filter(vsi);
2238         TAILQ_FOREACH(f, &vsi->mac_list, next)
2239                 rte_free(f);
2240
2241         if (vsi->type != I40E_VSI_MAIN) {
2242                 /* Remove vsi from parent's sibling list */
2243                 if (vsi->parent_vsi == NULL || vsi->parent_vsi->veb == NULL) {
2244                         PMD_DRV_LOG(ERR, "VSI's parent VSI is NULL\n");
2245                         return I40E_ERR_PARAM;
2246                 }
2247                 TAILQ_REMOVE(&vsi->parent_vsi->veb->head,
2248                                 &vsi->sib_vsi_list, list);
2249
2250                 /* Remove all switch element of the VSI */
2251                 ret = i40e_aq_delete_element(hw, vsi->seid, NULL);
2252                 if (ret != I40E_SUCCESS)
2253                         PMD_DRV_LOG(ERR, "Failed to delete element\n");
2254         }
2255         i40e_res_pool_free(&pf->qp_pool, vsi->base_queue);
2256
2257         if (vsi->type != I40E_VSI_SRIOV)
2258                 i40e_res_pool_free(&pf->msix_pool, vsi->msix_intr);
2259         rte_free(vsi);
2260
2261         return I40E_SUCCESS;
2262 }
2263
2264 static int
2265 i40e_update_default_filter_setting(struct i40e_vsi *vsi)
2266 {
2267         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2268         struct i40e_aqc_remove_macvlan_element_data def_filter;
2269         int ret;
2270
2271         if (vsi->type != I40E_VSI_MAIN)
2272                 return I40E_ERR_CONFIG;
2273         memset(&def_filter, 0, sizeof(def_filter));
2274         (void)rte_memcpy(def_filter.mac_addr, hw->mac.perm_addr,
2275                                         ETH_ADDR_LEN);
2276         def_filter.vlan_tag = 0;
2277         def_filter.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
2278                                 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
2279         ret = i40e_aq_remove_macvlan(hw, vsi->seid, &def_filter, 1, NULL);
2280         if (ret != I40E_SUCCESS)
2281                 return ret;
2282
2283         return i40e_vsi_add_mac(vsi, (struct ether_addr *)(hw->mac.perm_addr));
2284 }
2285
2286 static int
2287 i40e_vsi_dump_bw_config(struct i40e_vsi *vsi)
2288 {
2289         struct i40e_aqc_query_vsi_bw_config_resp bw_config;
2290         struct i40e_aqc_query_vsi_ets_sla_config_resp ets_sla_config;
2291         struct i40e_hw *hw = &vsi->adapter->hw;
2292         i40e_status ret;
2293         int i;
2294
2295         memset(&bw_config, 0, sizeof(bw_config));
2296         ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
2297         if (ret != I40E_SUCCESS) {
2298                 PMD_DRV_LOG(ERR, "VSI failed to get bandwidth "
2299                         "configuration %u\n", hw->aq.asq_last_status);
2300                 return ret;
2301         }
2302
2303         memset(&ets_sla_config, 0, sizeof(ets_sla_config));
2304         ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid,
2305                                         &ets_sla_config, NULL);
2306         if (ret != I40E_SUCCESS) {
2307                 PMD_DRV_LOG(ERR, "VSI failed to get TC bandwdith "
2308                         "configuration %u\n", hw->aq.asq_last_status);
2309                 return ret;
2310         }
2311
2312         /* Not store the info yet, just print out */
2313         PMD_DRV_LOG(INFO, "VSI bw limit:%u\n", bw_config.port_bw_limit);
2314         PMD_DRV_LOG(INFO, "VSI max_bw:%u\n", bw_config.max_bw);
2315         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
2316                 PMD_DRV_LOG(INFO, "\tVSI TC%u:share credits %u\n", i,
2317                                         ets_sla_config.share_credits[i]);
2318                 PMD_DRV_LOG(INFO, "\tVSI TC%u:credits %u\n", i,
2319                         rte_le_to_cpu_16(ets_sla_config.credits[i]));
2320                 PMD_DRV_LOG(INFO, "\tVSI TC%u: max credits: %u", i,
2321                         rte_le_to_cpu_16(ets_sla_config.credits[i / 4]) >>
2322                                                                 (i * 4));
2323         }
2324
2325         return 0;
2326 }
2327
2328 /* Setup a VSI */
2329 struct i40e_vsi *
2330 i40e_vsi_setup(struct i40e_pf *pf,
2331                enum i40e_vsi_type type,
2332                struct i40e_vsi *uplink_vsi,
2333                uint16_t user_param)
2334 {
2335         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
2336         struct i40e_vsi *vsi;
2337         int ret;
2338         struct i40e_vsi_context ctxt;
2339         struct ether_addr broadcast =
2340                 {.addr_bytes = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}};
2341
2342         if (type != I40E_VSI_MAIN && uplink_vsi == NULL) {
2343                 PMD_DRV_LOG(ERR, "VSI setup failed, "
2344                         "VSI link shouldn't be NULL\n");
2345                 return NULL;
2346         }
2347
2348         if (type == I40E_VSI_MAIN && uplink_vsi != NULL) {
2349                 PMD_DRV_LOG(ERR, "VSI setup failed, MAIN VSI "
2350                                 "uplink VSI should be NULL\n");
2351                 return NULL;
2352         }
2353
2354         /* If uplink vsi didn't setup VEB, create one first */
2355         if (type != I40E_VSI_MAIN && uplink_vsi->veb == NULL) {
2356                 uplink_vsi->veb = i40e_veb_setup(pf, uplink_vsi);
2357
2358                 if (NULL == uplink_vsi->veb) {
2359                         PMD_DRV_LOG(ERR, "VEB setup failed\n");
2360                         return NULL;
2361                 }
2362         }
2363
2364         vsi = rte_zmalloc("i40e_vsi", sizeof(struct i40e_vsi), 0);
2365         if (!vsi) {
2366                 PMD_DRV_LOG(ERR, "Failed to allocate memory for vsi\n");
2367                 return NULL;
2368         }
2369         TAILQ_INIT(&vsi->mac_list);
2370         vsi->type = type;
2371         vsi->adapter = I40E_PF_TO_ADAPTER(pf);
2372         vsi->max_macaddrs = I40E_NUM_MACADDR_MAX;
2373         vsi->parent_vsi = uplink_vsi;
2374         vsi->user_param = user_param;
2375         /* Allocate queues */
2376         switch (vsi->type) {
2377         case I40E_VSI_MAIN  :
2378                 vsi->nb_qps = pf->lan_nb_qps;
2379                 break;
2380         case I40E_VSI_SRIOV :
2381                 vsi->nb_qps = pf->vf_nb_qps;
2382                 break;
2383         default:
2384                 goto fail_mem;
2385         }
2386         ret = i40e_res_pool_alloc(&pf->qp_pool, vsi->nb_qps);
2387         if (ret < 0) {
2388                 PMD_DRV_LOG(ERR, "VSI %d allocate queue failed %d",
2389                                 vsi->seid, ret);
2390                 goto fail_mem;
2391         }
2392         vsi->base_queue = ret;
2393
2394         /* VF has MSIX interrupt in VF range, don't allocate here */
2395         if (type != I40E_VSI_SRIOV) {
2396                 ret = i40e_res_pool_alloc(&pf->msix_pool, 1);
2397                 if (ret < 0) {
2398                         PMD_DRV_LOG(ERR, "VSI %d get heap failed %d", vsi->seid, ret);
2399                         goto fail_queue_alloc;
2400                 }
2401                 vsi->msix_intr = ret;
2402         } else
2403                 vsi->msix_intr = 0;
2404         /* Add VSI */
2405         if (type == I40E_VSI_MAIN) {
2406                 /* For main VSI, no need to add since it's default one */
2407                 vsi->uplink_seid = pf->mac_seid;
2408                 vsi->seid = pf->main_vsi_seid;
2409                 /* Bind queues with specific MSIX interrupt */
2410                 /**
2411                  * Needs 2 interrupt at least, one for misc cause which will
2412                  * enabled from OS side, Another for queues binding the
2413                  * interrupt from device side only.
2414                  */
2415
2416                 /* Get default VSI parameters from hardware */
2417                 memset(&ctxt, 0, sizeof(ctxt));
2418                 ctxt.seid = vsi->seid;
2419                 ctxt.pf_num = hw->pf_id;
2420                 ctxt.uplink_seid = vsi->uplink_seid;
2421                 ctxt.vf_num = 0;
2422                 ret = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
2423                 if (ret != I40E_SUCCESS) {
2424                         PMD_DRV_LOG(ERR, "Failed to get VSI params\n");
2425                         goto fail_msix_alloc;
2426                 }
2427                 (void)rte_memcpy(&vsi->info, &ctxt.info,
2428                         sizeof(struct i40e_aqc_vsi_properties_data));
2429                 vsi->vsi_id = ctxt.vsi_number;
2430                 vsi->info.valid_sections = 0;
2431
2432                 /* Configure tc, enabled TC0 only */
2433                 if (i40e_vsi_update_tc_bandwidth(vsi, I40E_DEFAULT_TCMAP) !=
2434                         I40E_SUCCESS) {
2435                         PMD_DRV_LOG(ERR, "Failed to update TC bandwidth\n");
2436                         goto fail_msix_alloc;
2437                 }
2438
2439                 /* TC, queue mapping */
2440                 memset(&ctxt, 0, sizeof(ctxt));
2441                 vsi->info.valid_sections |=
2442                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
2443                 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2444                                         I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2445                 (void)rte_memcpy(&ctxt.info, &vsi->info,
2446                         sizeof(struct i40e_aqc_vsi_properties_data));
2447                 ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
2448                                                 I40E_DEFAULT_TCMAP);
2449                 if (ret != I40E_SUCCESS) {
2450                         PMD_DRV_LOG(ERR, "Failed to configure "
2451                                         "TC queue mapping\n");
2452                         goto fail_msix_alloc;
2453                 }
2454                 ctxt.seid = vsi->seid;
2455                 ctxt.pf_num = hw->pf_id;
2456                 ctxt.uplink_seid = vsi->uplink_seid;
2457                 ctxt.vf_num = 0;
2458
2459                 /* Update VSI parameters */
2460                 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
2461                 if (ret != I40E_SUCCESS) {
2462                         PMD_DRV_LOG(ERR, "Failed to update VSI params\n");
2463                         goto fail_msix_alloc;
2464                 }
2465
2466                 (void)rte_memcpy(&vsi->info.tc_mapping, &ctxt.info.tc_mapping,
2467                                                 sizeof(vsi->info.tc_mapping));
2468                 (void)rte_memcpy(&vsi->info.queue_mapping,
2469                                 &ctxt.info.queue_mapping,
2470                         sizeof(vsi->info.queue_mapping));
2471                 vsi->info.mapping_flags = ctxt.info.mapping_flags;
2472                 vsi->info.valid_sections = 0;
2473
2474                 (void)rte_memcpy(pf->dev_addr.addr_bytes, hw->mac.perm_addr,
2475                                 ETH_ADDR_LEN);
2476                 ret = i40e_update_default_filter_setting(vsi);
2477                 if (ret != I40E_SUCCESS) {
2478                         PMD_DRV_LOG(ERR, "Failed to remove default "
2479                                                 "filter setting\n");
2480                         goto fail_msix_alloc;
2481                 }
2482         }
2483         else if (type == I40E_VSI_SRIOV) {
2484                 memset(&ctxt, 0, sizeof(ctxt));
2485                 /**
2486                  * For other VSI, the uplink_seid equals to uplink VSI's
2487                  * uplink_seid since they share same VEB
2488                  */
2489                 vsi->uplink_seid = uplink_vsi->uplink_seid;
2490                 ctxt.pf_num = hw->pf_id;
2491                 ctxt.vf_num = hw->func_caps.vf_base_id + user_param;
2492                 ctxt.uplink_seid = vsi->uplink_seid;
2493                 ctxt.connection_type = 0x1;
2494                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
2495
2496                 /* Configure switch ID */
2497                 ctxt.info.valid_sections |=
2498                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SWITCH_VALID);
2499                 ctxt.info.switch_id =
2500                         rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
2501                 /* Configure port/vlan */
2502                 ctxt.info.valid_sections |=
2503                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
2504                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
2505                 ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
2506                                                 I40E_DEFAULT_TCMAP);
2507                 if (ret != I40E_SUCCESS) {
2508                         PMD_DRV_LOG(ERR, "Failed to configure "
2509                                         "TC queue mapping\n");
2510                         goto fail_msix_alloc;
2511                 }
2512                 ctxt.info.up_enable_bits = I40E_DEFAULT_TCMAP;
2513                 ctxt.info.valid_sections |=
2514                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SCHED_VALID);
2515                 /**
2516                  * Since VSI is not created yet, only configure parameter,
2517                  * will add vsi below.
2518                  */
2519         }
2520         else {
2521                 PMD_DRV_LOG(ERR, "VSI: Not support other type VSI yet\n");
2522                 goto fail_msix_alloc;
2523         }
2524
2525         if (vsi->type != I40E_VSI_MAIN) {
2526                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
2527                 if (ret) {
2528                         PMD_DRV_LOG(ERR, "add vsi failed, aq_err=%d\n",
2529                                  hw->aq.asq_last_status);
2530                         goto fail_msix_alloc;
2531                 }
2532                 memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
2533                 vsi->info.valid_sections = 0;
2534                 vsi->seid = ctxt.seid;
2535                 vsi->vsi_id = ctxt.vsi_number;
2536                 vsi->sib_vsi_list.vsi = vsi;
2537                 TAILQ_INSERT_TAIL(&uplink_vsi->veb->head,
2538                                 &vsi->sib_vsi_list, list);
2539         }
2540
2541         /* MAC/VLAN configuration */
2542         ret = i40e_vsi_add_mac(vsi, &broadcast);
2543         if (ret != I40E_SUCCESS) {
2544                 PMD_DRV_LOG(ERR, "Failed to add MACVLAN filter\n");
2545                 goto fail_msix_alloc;
2546         }
2547
2548         /* Get VSI BW information */
2549         i40e_vsi_dump_bw_config(vsi);
2550         return vsi;
2551 fail_msix_alloc:
2552         i40e_res_pool_free(&pf->msix_pool,vsi->msix_intr);
2553 fail_queue_alloc:
2554         i40e_res_pool_free(&pf->qp_pool,vsi->base_queue);
2555 fail_mem:
2556         rte_free(vsi);
2557         return NULL;
2558 }
2559
2560 /* Configure vlan stripping on or off */
2561 static int
2562 i40e_vsi_config_vlan_stripping(struct i40e_vsi *vsi, bool on)
2563 {
2564         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2565         struct i40e_vsi_context ctxt;
2566         uint8_t vlan_flags;
2567         int ret = I40E_SUCCESS;
2568
2569         /* Check if it has been already on or off */
2570         if (vsi->info.valid_sections &
2571                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID)) {
2572                 if (on) {
2573                         if ((vsi->info.port_vlan_flags &
2574                                 I40E_AQ_VSI_PVLAN_EMOD_MASK) == 0)
2575                                 return 0; /* already on */
2576                 } else {
2577                         if ((vsi->info.port_vlan_flags &
2578                                 I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
2579                                 I40E_AQ_VSI_PVLAN_EMOD_MASK)
2580                                 return 0; /* already off */
2581                 }
2582         }
2583
2584         if (on)
2585                 vlan_flags = I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2586         else
2587                 vlan_flags = I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
2588         vsi->info.valid_sections =
2589                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
2590         vsi->info.port_vlan_flags &= ~(I40E_AQ_VSI_PVLAN_EMOD_MASK);
2591         vsi->info.port_vlan_flags |= vlan_flags;
2592         ctxt.seid = vsi->seid;
2593         (void)rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
2594         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
2595         if (ret)
2596                 PMD_DRV_LOG(INFO, "Update VSI failed to %s vlan stripping\n",
2597                                                 on ? "enable" : "disable");
2598
2599         return ret;
2600 }
2601
2602 static int
2603 i40e_vsi_init_vlan(struct i40e_vsi *vsi)
2604 {
2605         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2606         struct rte_eth_dev_data *data = I40E_VSI_TO_DEV_DATA(vsi);
2607         struct i40e_vsi_context ctxt;
2608         uint8_t vlan_flags = 0;
2609         int ret;
2610
2611         /* Set PVID */
2612         if (data->dev_conf.txmode.hw_vlan_insert_pvid == 1) {
2613                 /**
2614                  * If insert pvid is enabled, only tagged pkts are
2615                  * allowed to be sent out.
2616                  */
2617                 vlan_flags |= I40E_AQ_VSI_PVLAN_INSERT_PVID |
2618                                 I40E_AQ_VSI_PVLAN_MODE_TAGGED;
2619         } else {
2620                 if (data->dev_conf.txmode.hw_vlan_reject_tagged == 0)
2621                         vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_TAGGED;
2622                 if (data->dev_conf.txmode.hw_vlan_reject_untagged == 0)
2623                         vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_UNTAGGED;
2624         }
2625
2626         /* Strip VLAN tag or not */
2627         if (data->dev_conf.rxmode.hw_vlan_strip == 0)
2628                 vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
2629
2630         vsi->info.port_vlan_flags &= ~(I40E_AQ_VSI_PVLAN_MODE_MASK |
2631                 I40E_AQ_VSI_PVLAN_INSERT_PVID | I40E_AQ_VSI_PVLAN_EMOD_MASK);
2632         vsi->info.port_vlan_flags |= vlan_flags;
2633         vsi->info.pvid = data->dev_conf.txmode.pvid;
2634         vsi->info.valid_sections =
2635                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
2636
2637         (void)rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
2638         ctxt.seid = vsi->seid;
2639         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
2640         if (ret != I40E_SUCCESS)
2641                 PMD_DRV_LOG(INFO, "Failed to update VSI params\n");
2642
2643         return ret;
2644 }
2645
2646 static int
2647 i40e_vsi_config_double_vlan(struct i40e_vsi *vsi, int on)
2648 {
2649         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2650
2651         return i40e_aq_set_port_parameters(hw, vsi->seid, 0, 1, on, NULL);
2652 }
2653
2654 static int
2655 i40e_update_flow_control(struct i40e_hw *hw)
2656 {
2657 #define I40E_LINK_PAUSE_RXTX (I40E_AQ_LINK_PAUSE_RX | I40E_AQ_LINK_PAUSE_TX)
2658         struct i40e_link_status link_status;
2659         uint32_t rxfc = 0, txfc = 0, reg;
2660         uint8_t an_info;
2661         int ret;
2662
2663         memset(&link_status, 0, sizeof(link_status));
2664         ret = i40e_aq_get_link_info(hw, FALSE, &link_status, NULL);
2665         if (ret != I40E_SUCCESS) {
2666                 PMD_DRV_LOG(ERR, "Failed to get link status information\n");
2667                 goto write_reg; /* Disable flow control */
2668         }
2669
2670         an_info = hw->phy.link_info.an_info;
2671         if (!(an_info & I40E_AQ_AN_COMPLETED)) {
2672                 PMD_DRV_LOG(INFO, "Link auto negotiation not completed\n");
2673                 ret = I40E_ERR_NOT_READY;
2674                 goto write_reg; /* Disable flow control */
2675         }
2676         /**
2677          * If link auto negotiation is enabled, flow control needs to
2678          * be configured according to it
2679          */
2680         switch (an_info & I40E_LINK_PAUSE_RXTX) {
2681         case I40E_LINK_PAUSE_RXTX:
2682                 rxfc = 1;
2683                 txfc = 1;
2684                 hw->fc.current_mode = I40E_FC_FULL;
2685                 break;
2686         case I40E_AQ_LINK_PAUSE_RX:
2687                 rxfc = 1;
2688                 hw->fc.current_mode = I40E_FC_RX_PAUSE;
2689                 break;
2690         case I40E_AQ_LINK_PAUSE_TX:
2691                 txfc = 1;
2692                 hw->fc.current_mode = I40E_FC_TX_PAUSE;
2693                 break;
2694         default:
2695                 hw->fc.current_mode = I40E_FC_NONE;
2696                 break;
2697         }
2698
2699 write_reg:
2700         I40E_WRITE_REG(hw, I40E_PRTDCB_FCCFG,
2701                 txfc << I40E_PRTDCB_FCCFG_TFCE_SHIFT);
2702         reg = I40E_READ_REG(hw, I40E_PRTDCB_MFLCN);
2703         reg &= ~I40E_PRTDCB_MFLCN_RFCE_MASK;
2704         reg |= rxfc << I40E_PRTDCB_MFLCN_RFCE_SHIFT;
2705         I40E_WRITE_REG(hw, I40E_PRTDCB_MFLCN, reg);
2706
2707         return ret;
2708 }
2709
2710 /* PF setup */
2711 static int
2712 i40e_pf_setup(struct i40e_pf *pf)
2713 {
2714         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
2715         struct i40e_filter_control_settings settings;
2716         struct rte_eth_dev_data *dev_data = pf->dev_data;
2717         struct i40e_vsi *vsi;
2718         int ret;
2719
2720         /* Clear all stats counters */
2721         pf->offset_loaded = FALSE;
2722         memset(&pf->stats, 0, sizeof(struct i40e_hw_port_stats));
2723         memset(&pf->stats_offset, 0, sizeof(struct i40e_hw_port_stats));
2724
2725         ret = i40e_pf_get_switch_config(pf);
2726         if (ret != I40E_SUCCESS) {
2727                 PMD_DRV_LOG(ERR, "Could not get switch config, err %d", ret);
2728                 return ret;
2729         }
2730
2731         /* VSI setup */
2732         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, NULL, 0);
2733         if (!vsi) {
2734                 PMD_DRV_LOG(ERR, "Setup of main vsi failed");
2735                 return I40E_ERR_NOT_READY;
2736         }
2737         pf->main_vsi = vsi;
2738         dev_data->nb_rx_queues = vsi->nb_qps;
2739         dev_data->nb_tx_queues = vsi->nb_qps;
2740
2741         /* Configure filter control */
2742         memset(&settings, 0, sizeof(settings));
2743         settings.hash_lut_size = I40E_HASH_LUT_SIZE_128;
2744         /* Enable ethtype and macvlan filters */
2745         settings.enable_ethtype = TRUE;
2746         settings.enable_macvlan = TRUE;
2747         ret = i40e_set_filter_control(hw, &settings);
2748         if (ret)
2749                 PMD_INIT_LOG(WARNING, "setup_pf_filter_control failed: %d",
2750                                                                 ret);
2751
2752         /* Update flow control according to the auto negotiation */
2753         i40e_update_flow_control(hw);
2754
2755         return I40E_SUCCESS;
2756 }
2757
2758 int
2759 i40e_switch_tx_queue(struct i40e_hw *hw, uint16_t q_idx, bool on)
2760 {
2761         uint32_t reg;
2762         uint16_t j;
2763
2764         /* Wait until the request is finished */
2765         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
2766                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
2767                 reg = I40E_READ_REG(hw, I40E_QTX_ENA(q_idx));
2768                 if (!(((reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 0x1) ^
2769                         ((reg >> I40E_QTX_ENA_QENA_STAT_SHIFT)
2770                                                         & 0x1))) {
2771                         break;
2772                 }
2773         }
2774         if (on) {
2775                 if (reg & I40E_QTX_ENA_QENA_STAT_MASK)
2776                         return I40E_SUCCESS; /* already on, skip next steps */
2777                 reg |= I40E_QTX_ENA_QENA_REQ_MASK;
2778         } else {
2779                 if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK))
2780                         return I40E_SUCCESS; /* already off, skip next steps */
2781                 reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
2782         }
2783         /* Write the register */
2784         I40E_WRITE_REG(hw, I40E_QTX_ENA(q_idx), reg);
2785         /* Check the result */
2786         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
2787                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
2788                 reg = I40E_READ_REG(hw, I40E_QTX_ENA(q_idx));
2789                 if (on) {
2790                         if ((reg & I40E_QTX_ENA_QENA_REQ_MASK) &&
2791                                 (reg & I40E_QTX_ENA_QENA_STAT_MASK))
2792                                 break;
2793                 } else {
2794                         if (!(reg & I40E_QTX_ENA_QENA_REQ_MASK) &&
2795                                 !(reg & I40E_QTX_ENA_QENA_STAT_MASK))
2796                                 break;
2797                 }
2798         }
2799         /* Check if it is timeout */
2800         if (j >= I40E_CHK_Q_ENA_COUNT) {
2801                 PMD_DRV_LOG(ERR, "Failed to %s tx queue[%u]\n",
2802                         (on ? "enable" : "disable"), q_idx);
2803                 return I40E_ERR_TIMEOUT;
2804         }
2805         return I40E_SUCCESS;
2806 }
2807 /* Swith on or off the tx queues */
2808 static int
2809 i40e_vsi_switch_tx_queues(struct i40e_vsi *vsi, bool on)
2810 {
2811         struct rte_eth_dev_data *dev_data = I40E_VSI_TO_DEV_DATA(vsi);
2812         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2813         struct i40e_tx_queue *txq;
2814         uint16_t i, pf_q;
2815         int ret;
2816
2817         pf_q = vsi->base_queue;
2818         for (i = 0; i < dev_data->nb_tx_queues; i++, pf_q++) {
2819                 txq = dev_data->tx_queues[i];
2820                 if (!txq->q_set)
2821                         continue; /* Queue not configured */
2822                 ret = i40e_switch_tx_queue(hw, pf_q, on);
2823                 if ( ret != I40E_SUCCESS)
2824                         return ret;
2825         }
2826
2827         return I40E_SUCCESS;
2828 }
2829
2830 int
2831 i40e_switch_rx_queue(struct i40e_hw *hw, uint16_t q_idx, bool on)
2832 {
2833         uint32_t reg;
2834         uint16_t j;
2835
2836         /* Wait until the request is finished */
2837         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
2838                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
2839                 reg = I40E_READ_REG(hw, I40E_QRX_ENA(q_idx));
2840                 if (!((reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 0x1) ^
2841                         ((reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 0x1))
2842                         break;
2843         }
2844
2845         if (on) {
2846                 if (reg & I40E_QRX_ENA_QENA_STAT_MASK)
2847                         return I40E_SUCCESS; /* Already on, skip next steps */
2848                 reg |= I40E_QRX_ENA_QENA_REQ_MASK;
2849         } else {
2850                 if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK))
2851                         return I40E_SUCCESS; /* Already off, skip next steps */
2852                 reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
2853         }
2854
2855         /* Write the register */
2856         I40E_WRITE_REG(hw, I40E_QRX_ENA(q_idx), reg);
2857         /* Check the result */
2858         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
2859                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
2860                 reg = I40E_READ_REG(hw, I40E_QRX_ENA(q_idx));
2861                 if (on) {
2862                         if ((reg & I40E_QRX_ENA_QENA_REQ_MASK) &&
2863                                 (reg & I40E_QRX_ENA_QENA_STAT_MASK))
2864                                 break;
2865                 } else {
2866                         if (!(reg & I40E_QRX_ENA_QENA_REQ_MASK) &&
2867                                 !(reg & I40E_QRX_ENA_QENA_STAT_MASK))
2868                                 break;
2869                 }
2870         }
2871
2872         /* Check if it is timeout */
2873         if (j >= I40E_CHK_Q_ENA_COUNT) {
2874                 PMD_DRV_LOG(ERR, "Failed to %s rx queue[%u]\n",
2875                         (on ? "enable" : "disable"), q_idx);
2876                 return I40E_ERR_TIMEOUT;
2877         }
2878
2879         return I40E_SUCCESS;
2880 }
2881 /* Switch on or off the rx queues */
2882 static int
2883 i40e_vsi_switch_rx_queues(struct i40e_vsi *vsi, bool on)
2884 {
2885         struct rte_eth_dev_data *dev_data = I40E_VSI_TO_DEV_DATA(vsi);
2886         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2887         struct i40e_rx_queue *rxq;
2888         uint16_t i, pf_q;
2889         int ret;
2890
2891         pf_q = vsi->base_queue;
2892         for (i = 0; i < dev_data->nb_rx_queues; i++, pf_q++) {
2893                 rxq = dev_data->rx_queues[i];
2894                 if (!rxq->q_set)
2895                         continue; /* Queue not configured */
2896                 ret = i40e_switch_rx_queue(hw, pf_q, on);
2897                 if ( ret != I40E_SUCCESS)
2898                         return ret;
2899         }
2900
2901         return I40E_SUCCESS;
2902 }
2903
2904 /* Switch on or off all the rx/tx queues */
2905 int
2906 i40e_vsi_switch_queues(struct i40e_vsi *vsi, bool on)
2907 {
2908         int ret;
2909
2910         if (on) {
2911                 /* enable rx queues before enabling tx queues */
2912                 ret = i40e_vsi_switch_rx_queues(vsi, on);
2913                 if (ret) {
2914                         PMD_DRV_LOG(ERR, "Failed to switch rx queues\n");
2915                         return ret;
2916                 }
2917                 ret = i40e_vsi_switch_tx_queues(vsi, on);
2918         } else {
2919                 /* Stop tx queues before stopping rx queues */
2920                 ret = i40e_vsi_switch_tx_queues(vsi, on);
2921                 if (ret) {
2922                         PMD_DRV_LOG(ERR, "Failed to switch tx queues\n");
2923                         return ret;
2924                 }
2925                 ret = i40e_vsi_switch_rx_queues(vsi, on);
2926         }
2927
2928         return ret;
2929 }
2930
2931 /* Initialize VSI for TX */
2932 static int
2933 i40e_vsi_tx_init(struct i40e_vsi *vsi)
2934 {
2935         struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
2936         struct rte_eth_dev_data *data = pf->dev_data;
2937         uint16_t i;
2938         uint32_t ret = I40E_SUCCESS;
2939
2940         for (i = 0; i < data->nb_tx_queues; i++) {
2941                 ret = i40e_tx_queue_init(data->tx_queues[i]);
2942                 if (ret != I40E_SUCCESS)
2943                         break;
2944         }
2945
2946         return ret;
2947 }
2948
2949 /* Initialize VSI for RX */
2950 static int
2951 i40e_vsi_rx_init(struct i40e_vsi *vsi)
2952 {
2953         struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
2954         struct rte_eth_dev_data *data = pf->dev_data;
2955         int ret = I40E_SUCCESS;
2956         uint16_t i;
2957
2958         i40e_pf_config_mq_rx(pf);
2959         for (i = 0; i < data->nb_rx_queues; i++) {
2960                 ret = i40e_rx_queue_init(data->rx_queues[i]);
2961                 if (ret != I40E_SUCCESS) {
2962                         PMD_DRV_LOG(ERR, "Failed to do RX queue "
2963                                         "initialization\n");
2964                         break;
2965                 }
2966         }
2967
2968         return ret;
2969 }
2970
2971 /* Initialize VSI */
2972 static int
2973 i40e_vsi_init(struct i40e_vsi *vsi)
2974 {
2975         int err;
2976
2977         err = i40e_vsi_tx_init(vsi);
2978         if (err) {
2979                 PMD_DRV_LOG(ERR, "Failed to do vsi TX initialization\n");
2980                 return err;
2981         }
2982         err = i40e_vsi_rx_init(vsi);
2983         if (err) {
2984                 PMD_DRV_LOG(ERR, "Failed to do vsi RX initialization\n");
2985                 return err;
2986         }
2987
2988         return err;
2989 }
2990
2991 static void
2992 i40e_stat_update_32(struct i40e_hw *hw,
2993                    uint32_t reg,
2994                    bool offset_loaded,
2995                    uint64_t *offset,
2996                    uint64_t *stat)
2997 {
2998         uint64_t new_data;
2999
3000         new_data = (uint64_t)I40E_READ_REG(hw, reg);
3001         if (!offset_loaded)
3002                 *offset = new_data;
3003
3004         if (new_data >= *offset)
3005                 *stat = (uint64_t)(new_data - *offset);
3006         else
3007                 *stat = (uint64_t)((new_data +
3008                         ((uint64_t)1 << I40E_32_BIT_SHIFT)) - *offset);
3009 }
3010
3011 static void
3012 i40e_stat_update_48(struct i40e_hw *hw,
3013                    uint32_t hireg,
3014                    uint32_t loreg,
3015                    bool offset_loaded,
3016                    uint64_t *offset,
3017                    uint64_t *stat)
3018 {
3019         uint64_t new_data;
3020
3021         new_data = (uint64_t)I40E_READ_REG(hw, loreg);
3022         new_data |= ((uint64_t)(I40E_READ_REG(hw, hireg) &
3023                         I40E_16_BIT_MASK)) << I40E_32_BIT_SHIFT;
3024
3025         if (!offset_loaded)
3026                 *offset = new_data;
3027
3028         if (new_data >= *offset)
3029                 *stat = new_data - *offset;
3030         else
3031                 *stat = (uint64_t)((new_data +
3032                         ((uint64_t)1 << I40E_48_BIT_SHIFT)) - *offset);
3033
3034         *stat &= I40E_48_BIT_MASK;
3035 }
3036
3037 /* Disable IRQ0 */
3038 void
3039 i40e_pf_disable_irq0(struct i40e_hw *hw)
3040 {
3041         /* Disable all interrupt types */
3042         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, 0);
3043         I40E_WRITE_FLUSH(hw);
3044 }
3045
3046 /* Enable IRQ0 */
3047 void
3048 i40e_pf_enable_irq0(struct i40e_hw *hw)
3049 {
3050         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0,
3051                 I40E_PFINT_DYN_CTL0_INTENA_MASK |
3052                 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
3053                 I40E_PFINT_DYN_CTL0_ITR_INDX_MASK);
3054         I40E_WRITE_FLUSH(hw);
3055 }
3056
3057 static void
3058 i40e_pf_config_irq0(struct i40e_hw *hw)
3059 {
3060         uint32_t enable;
3061
3062         /* read pending request and disable first */
3063         i40e_pf_disable_irq0(hw);
3064         /**
3065          * Enable all interrupt error options to detect possible errors,
3066          * other informative int are ignored
3067          */
3068         enable = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK |
3069                  I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK |
3070                  I40E_PFINT_ICR0_ENA_GRST_MASK |
3071                  I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
3072                  I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK |
3073                  I40E_PFINT_ICR0_ENA_HMC_ERR_MASK |
3074                  I40E_PFINT_ICR0_ENA_VFLR_MASK |
3075                  I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3076
3077         I40E_WRITE_REG(hw, I40E_PFINT_ICR0_ENA, enable);
3078         I40E_WRITE_REG(hw, I40E_PFINT_STAT_CTL0,
3079                 I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_MASK);
3080
3081         /* Link no queues with irq0 */
3082         I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0,
3083                 I40E_PFINT_LNKLST0_FIRSTQ_INDX_MASK);
3084 }
3085
3086 static void
3087 i40e_dev_handle_vfr_event(struct rte_eth_dev *dev)
3088 {
3089         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3090         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3091         int i;
3092         uint16_t abs_vf_id;
3093         uint32_t index, offset, val;
3094
3095         if (!pf->vfs)
3096                 return;
3097         /**
3098          * Try to find which VF trigger a reset, use absolute VF id to access
3099          * since the reg is global register.
3100          */
3101         for (i = 0; i < pf->vf_num; i++) {
3102                 abs_vf_id = hw->func_caps.vf_base_id + i;
3103                 index = abs_vf_id / I40E_UINT32_BIT_SIZE;
3104                 offset = abs_vf_id % I40E_UINT32_BIT_SIZE;
3105                 val = I40E_READ_REG(hw, I40E_GLGEN_VFLRSTAT(index));
3106                 /* VFR event occured */
3107                 if (val & (0x1 << offset)) {
3108                         int ret;
3109
3110                         /* Clear the event first */
3111                         I40E_WRITE_REG(hw, I40E_GLGEN_VFLRSTAT(index),
3112                                                         (0x1 << offset));
3113                         PMD_DRV_LOG(INFO, "VF %u reset occured\n", abs_vf_id);
3114                         /**
3115                          * Only notify a VF reset event occured,
3116                          * don't trigger another SW reset
3117                          */
3118                         ret = i40e_pf_host_vf_reset(&pf->vfs[i], 0);
3119                         if (ret != I40E_SUCCESS)
3120                                 PMD_DRV_LOG(ERR, "Failed to do VF reset\n");
3121                 }
3122         }
3123 }
3124
3125 static void
3126 i40e_dev_handle_aq_msg(struct rte_eth_dev *dev)
3127 {
3128         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3129         struct i40e_arq_event_info info;
3130         uint16_t pending, opcode;
3131         int ret;
3132
3133         info.msg_size = I40E_AQ_BUF_SZ;
3134         info.msg_buf = rte_zmalloc("msg_buffer", I40E_AQ_BUF_SZ, 0);
3135         if (!info.msg_buf) {
3136                 PMD_DRV_LOG(ERR, "Failed to allocate mem\n");
3137                 return;
3138         }
3139
3140         pending = 1;
3141         while (pending) {
3142                 ret = i40e_clean_arq_element(hw, &info, &pending);
3143
3144                 if (ret != I40E_SUCCESS) {
3145                         PMD_DRV_LOG(INFO, "Failed to read msg from AdminQ, "
3146                                 "aq_err: %u\n", hw->aq.asq_last_status);
3147                         break;
3148                 }
3149                 opcode = rte_le_to_cpu_16(info.desc.opcode);
3150
3151                 switch (opcode) {
3152                 case i40e_aqc_opc_send_msg_to_pf:
3153                         /* Refer to i40e_aq_send_msg_to_pf() for argument layout*/
3154                         i40e_pf_host_handle_vf_msg(dev,
3155                                         rte_le_to_cpu_16(info.desc.retval),
3156                                         rte_le_to_cpu_32(info.desc.cookie_high),
3157                                         rte_le_to_cpu_32(info.desc.cookie_low),
3158                                         info.msg_buf,
3159                                         info.msg_size);
3160                         break;
3161                 default:
3162                         PMD_DRV_LOG(ERR, "Request %u is not supported yet\n",
3163                                 opcode);
3164                         break;
3165                 }
3166                 /* Reset the buffer after processing one */
3167                 info.msg_size = I40E_AQ_BUF_SZ;
3168         }
3169         rte_free(info.msg_buf);
3170 }
3171
3172 /**
3173  * Interrupt handler triggered by NIC  for handling
3174  * specific interrupt.
3175  *
3176  * @param handle
3177  *  Pointer to interrupt handle.
3178  * @param param
3179  *  The address of parameter (struct rte_eth_dev *) regsitered before.
3180  *
3181  * @return
3182  *  void
3183  */
3184 static void
3185 i40e_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
3186                            void *param)
3187 {
3188         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
3189         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3190         uint32_t cause, enable;
3191
3192         i40e_pf_disable_irq0(hw);
3193
3194         cause = I40E_READ_REG(hw, I40E_PFINT_ICR0);
3195         enable = I40E_READ_REG(hw, I40E_PFINT_ICR0_ENA);
3196
3197         /* Shared IRQ case, return */
3198         if (!(cause & I40E_PFINT_ICR0_INTEVENT_MASK)) {
3199                 PMD_DRV_LOG(INFO, "Port%d INT0:share IRQ case, "
3200                         "no INT event to process\n", hw->pf_id);
3201                 goto done;
3202         }
3203
3204         if (cause & I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK) {
3205                 PMD_DRV_LOG(INFO, "INT:Link status changed\n");
3206                 i40e_dev_link_update(dev, 0);
3207         }
3208
3209         if (cause & I40E_PFINT_ICR0_ECC_ERR_MASK)
3210                 PMD_DRV_LOG(INFO, "INT:Unrecoverable ECC Error\n");
3211
3212         if (cause & I40E_PFINT_ICR0_MAL_DETECT_MASK)
3213                 PMD_DRV_LOG(INFO, "INT:Malicious programming detected\n");
3214
3215         if (cause & I40E_PFINT_ICR0_GRST_MASK)
3216                 PMD_DRV_LOG(INFO, "INT:Global Resets Requested\n");
3217
3218         if (cause & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)
3219                 PMD_DRV_LOG(INFO, "INT:PCI EXCEPTION occured\n");
3220
3221         if (cause & I40E_PFINT_ICR0_HMC_ERR_MASK)
3222                 PMD_DRV_LOG(INFO, "INT:HMC error occured\n");
3223
3224         /* Add processing func to deal with VF reset vent */
3225         if (cause & I40E_PFINT_ICR0_VFLR_MASK) {
3226                 PMD_DRV_LOG(INFO, "INT:VF reset detected\n");
3227                 i40e_dev_handle_vfr_event(dev);
3228         }
3229         /* Find admin queue event */
3230         if (cause & I40E_PFINT_ICR0_ADMINQ_MASK) {
3231                 PMD_DRV_LOG(INFO, "INT:ADMINQ event\n");
3232                 i40e_dev_handle_aq_msg(dev);
3233         }
3234
3235 done:
3236         I40E_WRITE_REG(hw, I40E_PFINT_ICR0_ENA, enable);
3237         /* Re-enable interrupt from device side */
3238         i40e_pf_enable_irq0(hw);
3239         /* Re-enable interrupt from host side */
3240         rte_intr_enable(&(dev->pci_dev->intr_handle));
3241 }
3242
3243 static int
3244 i40e_add_macvlan_filters(struct i40e_vsi *vsi,
3245                          struct i40e_macvlan_filter *filter,
3246                          int total)
3247 {
3248         int ele_num, ele_buff_size;
3249         int num, actual_num, i;
3250         int ret = I40E_SUCCESS;
3251         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
3252         struct i40e_aqc_add_macvlan_element_data *req_list;
3253
3254         if (filter == NULL  || total == 0)
3255                 return I40E_ERR_PARAM;
3256         ele_num = hw->aq.asq_buf_size / sizeof(*req_list);
3257         ele_buff_size = hw->aq.asq_buf_size;
3258
3259         req_list = rte_zmalloc("macvlan_add", ele_buff_size, 0);
3260         if (req_list == NULL) {
3261                 PMD_DRV_LOG(ERR, "Fail to allocate memory\n");
3262                 return I40E_ERR_NO_MEMORY;
3263         }
3264
3265         num = 0;
3266         do {
3267                 actual_num = (num + ele_num > total) ? (total - num) : ele_num;
3268                 memset(req_list, 0, ele_buff_size);
3269
3270                 for (i = 0; i < actual_num; i++) {
3271                         (void)rte_memcpy(req_list[i].mac_addr,
3272                                 &filter[num + i].macaddr, ETH_ADDR_LEN);
3273                         req_list[i].vlan_tag =
3274                                 rte_cpu_to_le_16(filter[num + i].vlan_id);
3275                         req_list[i].flags = rte_cpu_to_le_16(\
3276                                 I40E_AQC_MACVLAN_ADD_PERFECT_MATCH);
3277                         req_list[i].queue_number = 0;
3278                 }
3279
3280                 ret = i40e_aq_add_macvlan(hw, vsi->seid, req_list,
3281                                                 actual_num, NULL);
3282                 if (ret != I40E_SUCCESS) {
3283                         PMD_DRV_LOG(ERR, "Failed to add macvlan filter\n");
3284                         goto DONE;
3285                 }
3286                 num += actual_num;
3287         } while (num < total);
3288
3289 DONE:
3290         rte_free(req_list);
3291         return ret;
3292 }
3293
3294 static int
3295 i40e_remove_macvlan_filters(struct i40e_vsi *vsi,
3296                             struct i40e_macvlan_filter *filter,
3297                             int total)
3298 {
3299         int ele_num, ele_buff_size;
3300         int num, actual_num, i;
3301         int ret = I40E_SUCCESS;
3302         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
3303         struct i40e_aqc_remove_macvlan_element_data *req_list;
3304
3305         if (filter == NULL  || total == 0)
3306                 return I40E_ERR_PARAM;
3307
3308         ele_num = hw->aq.asq_buf_size / sizeof(*req_list);
3309         ele_buff_size = hw->aq.asq_buf_size;
3310
3311         req_list = rte_zmalloc("macvlan_remove", ele_buff_size, 0);
3312         if (req_list == NULL) {
3313                 PMD_DRV_LOG(ERR, "Fail to allocate memory\n");
3314                 return I40E_ERR_NO_MEMORY;
3315         }
3316
3317         num = 0;
3318         do {
3319                 actual_num = (num + ele_num > total) ? (total - num) : ele_num;
3320                 memset(req_list, 0, ele_buff_size);
3321
3322                 for (i = 0; i < actual_num; i++) {
3323                         (void)rte_memcpy(req_list[i].mac_addr,
3324                                 &filter[num + i].macaddr, ETH_ADDR_LEN);
3325                         req_list[i].vlan_tag =
3326                                 rte_cpu_to_le_16(filter[num + i].vlan_id);
3327                         req_list[i].flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
3328                 }
3329
3330                 ret = i40e_aq_remove_macvlan(hw, vsi->seid, req_list,
3331                                                 actual_num, NULL);
3332                 if (ret != I40E_SUCCESS) {
3333                         PMD_DRV_LOG(ERR, "Failed to remove macvlan filter\n");
3334                         goto DONE;
3335                 }
3336                 num += actual_num;
3337         } while (num < total);
3338
3339 DONE:
3340         rte_free(req_list);
3341         return ret;
3342 }
3343
3344 /* Find out specific MAC filter */
3345 static struct i40e_mac_filter *
3346 i40e_find_mac_filter(struct i40e_vsi *vsi,
3347                          struct ether_addr *macaddr)
3348 {
3349         struct i40e_mac_filter *f;
3350
3351         TAILQ_FOREACH(f, &vsi->mac_list, next) {
3352                 if (is_same_ether_addr(macaddr, &(f->macaddr)))
3353                         return f;
3354         }
3355
3356         return NULL;
3357 }
3358
3359 static bool
3360 i40e_find_vlan_filter(struct i40e_vsi *vsi,
3361                          uint16_t vlan_id)
3362 {
3363         uint32_t vid_idx, vid_bit;
3364
3365         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
3366         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
3367
3368         if (vsi->vfta[vid_idx] & vid_bit)
3369                 return 1;
3370         else
3371                 return 0;
3372 }
3373
3374 static void
3375 i40e_set_vlan_filter(struct i40e_vsi *vsi,
3376                          uint16_t vlan_id, bool on)
3377 {
3378         uint32_t vid_idx, vid_bit;
3379
3380 #define UINT32_BIT_MASK      0x1F
3381 #define VALID_VLAN_BIT_MASK  0xFFF
3382         /* VFTA is 32-bits size array, each element contains 32 vlan bits, Find the
3383          *  element first, then find the bits it belongs to
3384          */
3385         vid_idx = (uint32_t) ((vlan_id & VALID_VLAN_BIT_MASK) >>
3386                   sizeof(uint32_t));
3387         vid_bit = (uint32_t) (1 << (vlan_id & UINT32_BIT_MASK));
3388
3389         if (on)
3390                 vsi->vfta[vid_idx] |= vid_bit;
3391         else
3392                 vsi->vfta[vid_idx] &= ~vid_bit;
3393 }
3394
3395 /**
3396  * Find all vlan options for specific mac addr,
3397  * return with actual vlan found.
3398  */
3399 static inline int
3400 i40e_find_all_vlan_for_mac(struct i40e_vsi *vsi,
3401                            struct i40e_macvlan_filter *mv_f,
3402                            int num, struct ether_addr *addr)
3403 {
3404         int i;
3405         uint32_t j, k;
3406
3407         /**
3408          * Not to use i40e_find_vlan_filter to decrease the loop time,
3409          * although the code looks complex.
3410           */
3411         if (num < vsi->vlan_num)
3412                 return I40E_ERR_PARAM;
3413
3414         i = 0;
3415         for (j = 0; j < I40E_VFTA_SIZE; j++) {
3416                 if (vsi->vfta[j]) {
3417                         for (k = 0; k < I40E_UINT32_BIT_SIZE; k++) {
3418                                 if (vsi->vfta[j] & (1 << k)) {
3419                                         if (i > num - 1) {
3420                                                 PMD_DRV_LOG(ERR, "vlan number "
3421                                                                 "not match\n");
3422                                                 return I40E_ERR_PARAM;
3423                                         }
3424                                         (void)rte_memcpy(&mv_f[i].macaddr,
3425                                                         addr, ETH_ADDR_LEN);
3426                                         mv_f[i].vlan_id =
3427                                                 j * I40E_UINT32_BIT_SIZE + k;
3428                                         i++;
3429                                 }
3430                         }
3431                 }
3432         }
3433         return I40E_SUCCESS;
3434 }
3435
3436 static inline int
3437 i40e_find_all_mac_for_vlan(struct i40e_vsi *vsi,
3438                            struct i40e_macvlan_filter *mv_f,
3439                            int num,
3440                            uint16_t vlan)
3441 {
3442         int i = 0;
3443         struct i40e_mac_filter *f;
3444
3445         if (num < vsi->mac_num)
3446                 return I40E_ERR_PARAM;
3447
3448         TAILQ_FOREACH(f, &vsi->mac_list, next) {
3449                 if (i > num - 1) {
3450                         PMD_DRV_LOG(ERR, "buffer number not match\n");
3451                         return I40E_ERR_PARAM;
3452                 }
3453                 (void)rte_memcpy(&mv_f[i].macaddr, &f->macaddr, ETH_ADDR_LEN);
3454                 mv_f[i].vlan_id = vlan;
3455                 i++;
3456         }
3457
3458         return I40E_SUCCESS;
3459 }
3460
3461 static int
3462 i40e_vsi_remove_all_macvlan_filter(struct i40e_vsi *vsi)
3463 {
3464         int i, num;
3465         struct i40e_mac_filter *f;
3466         struct i40e_macvlan_filter *mv_f;
3467         int ret = I40E_SUCCESS;
3468
3469         if (vsi == NULL || vsi->mac_num == 0)
3470                 return I40E_ERR_PARAM;
3471
3472         /* Case that no vlan is set */
3473         if (vsi->vlan_num == 0)
3474                 num = vsi->mac_num;
3475         else
3476                 num = vsi->mac_num * vsi->vlan_num;
3477
3478         mv_f = rte_zmalloc("macvlan_data", num * sizeof(*mv_f), 0);
3479         if (mv_f == NULL) {
3480                 PMD_DRV_LOG(ERR, "failed to allocate memory\n");
3481                 return I40E_ERR_NO_MEMORY;
3482         }
3483
3484         i = 0;
3485         if (vsi->vlan_num == 0) {
3486                 TAILQ_FOREACH(f, &vsi->mac_list, next) {
3487                         (void)rte_memcpy(&mv_f[i].macaddr,
3488                                 &f->macaddr, ETH_ADDR_LEN);
3489                         mv_f[i].vlan_id = 0;
3490                         i++;
3491                 }
3492         } else {
3493                 TAILQ_FOREACH(f, &vsi->mac_list, next) {
3494                         ret = i40e_find_all_vlan_for_mac(vsi,&mv_f[i],
3495                                         vsi->vlan_num, &f->macaddr);
3496                         if (ret != I40E_SUCCESS)
3497                                 goto DONE;
3498                         i += vsi->vlan_num;
3499                 }
3500         }
3501
3502         ret = i40e_remove_macvlan_filters(vsi, mv_f, num);
3503 DONE:
3504         rte_free(mv_f);
3505
3506         return ret;
3507 }
3508
3509 int
3510 i40e_vsi_add_vlan(struct i40e_vsi *vsi, uint16_t vlan)
3511 {
3512         struct i40e_macvlan_filter *mv_f;
3513         int mac_num;
3514         int ret = I40E_SUCCESS;
3515
3516         if (!vsi || vlan > ETHER_MAX_VLAN_ID)
3517                 return I40E_ERR_PARAM;
3518
3519         /* If it's already set, just return */
3520         if (i40e_find_vlan_filter(vsi,vlan))
3521                 return I40E_SUCCESS;
3522
3523         mac_num = vsi->mac_num;
3524
3525         if (mac_num == 0) {
3526                 PMD_DRV_LOG(ERR, "Error! VSI doesn't have a mac addr\n");
3527                 return I40E_ERR_PARAM;
3528         }
3529
3530         mv_f = rte_zmalloc("macvlan_data", mac_num * sizeof(*mv_f), 0);
3531
3532         if (mv_f == NULL) {
3533                 PMD_DRV_LOG(ERR, "failed to allocate memory\n");
3534                 return I40E_ERR_NO_MEMORY;
3535         }
3536
3537         ret = i40e_find_all_mac_for_vlan(vsi, mv_f, mac_num, vlan);
3538
3539         if (ret != I40E_SUCCESS)
3540                 goto DONE;
3541
3542         ret = i40e_add_macvlan_filters(vsi, mv_f, mac_num);
3543
3544         if (ret != I40E_SUCCESS)
3545                 goto DONE;
3546
3547         i40e_set_vlan_filter(vsi, vlan, 1);
3548
3549         vsi->vlan_num++;
3550         ret = I40E_SUCCESS;
3551 DONE:
3552         rte_free(mv_f);
3553         return ret;
3554 }
3555
3556 int
3557 i40e_vsi_delete_vlan(struct i40e_vsi *vsi, uint16_t vlan)
3558 {
3559         struct i40e_macvlan_filter *mv_f;
3560         int mac_num;
3561         int ret = I40E_SUCCESS;
3562
3563         /**
3564          * Vlan 0 is the generic filter for untagged packets
3565          * and can't be removed.
3566          */
3567         if (!vsi || vlan == 0 || vlan > ETHER_MAX_VLAN_ID)
3568                 return I40E_ERR_PARAM;
3569
3570         /* If can't find it, just return */
3571         if (!i40e_find_vlan_filter(vsi, vlan))
3572                 return I40E_ERR_PARAM;
3573
3574         mac_num = vsi->mac_num;
3575
3576         if (mac_num == 0) {
3577                 PMD_DRV_LOG(ERR, "Error! VSI doesn't have a mac addr\n");
3578                 return I40E_ERR_PARAM;
3579         }
3580
3581         mv_f = rte_zmalloc("macvlan_data", mac_num * sizeof(*mv_f), 0);
3582
3583         if (mv_f == NULL) {
3584                 PMD_DRV_LOG(ERR, "failed to allocate memory\n");
3585                 return I40E_ERR_NO_MEMORY;
3586         }
3587
3588         ret = i40e_find_all_mac_for_vlan(vsi, mv_f, mac_num, vlan);
3589
3590         if (ret != I40E_SUCCESS)
3591                 goto DONE;
3592
3593         ret = i40e_remove_macvlan_filters(vsi, mv_f, mac_num);
3594
3595         if (ret != I40E_SUCCESS)
3596                 goto DONE;
3597
3598         /* This is last vlan to remove, replace all mac filter with vlan 0 */
3599         if (vsi->vlan_num == 1) {
3600                 ret = i40e_find_all_mac_for_vlan(vsi, mv_f, mac_num, 0);
3601                 if (ret != I40E_SUCCESS)
3602                         goto DONE;
3603
3604                 ret = i40e_add_macvlan_filters(vsi, mv_f, mac_num);
3605                 if (ret != I40E_SUCCESS)
3606                         goto DONE;
3607         }
3608
3609         i40e_set_vlan_filter(vsi, vlan, 0);
3610
3611         vsi->vlan_num--;
3612         ret = I40E_SUCCESS;
3613 DONE:
3614         rte_free(mv_f);
3615         return ret;
3616 }
3617
3618 int
3619 i40e_vsi_add_mac(struct i40e_vsi *vsi, struct ether_addr *addr)
3620 {
3621         struct i40e_mac_filter *f;
3622         struct i40e_macvlan_filter *mv_f;
3623         int vlan_num;
3624         int ret = I40E_SUCCESS;
3625
3626         /* If it's add and we've config it, return */
3627         f = i40e_find_mac_filter(vsi, addr);
3628         if (f != NULL)
3629                 return I40E_SUCCESS;
3630
3631         /**
3632          * If vlan_num is 0, that's the first time to add mac,
3633          * set mask for vlan_id 0.
3634          */
3635         if (vsi->vlan_num == 0) {
3636                 i40e_set_vlan_filter(vsi, 0, 1);
3637                 vsi->vlan_num = 1;
3638         }
3639
3640         vlan_num = vsi->vlan_num;
3641
3642         mv_f = rte_zmalloc("macvlan_data", vlan_num * sizeof(*mv_f), 0);
3643         if (mv_f == NULL) {
3644                 PMD_DRV_LOG(ERR, "failed to allocate memory\n");
3645                 return I40E_ERR_NO_MEMORY;
3646         }
3647
3648         ret = i40e_find_all_vlan_for_mac(vsi, mv_f, vlan_num, addr);
3649         if (ret != I40E_SUCCESS)
3650                 goto DONE;
3651
3652         ret = i40e_add_macvlan_filters(vsi, mv_f, vlan_num);
3653         if (ret != I40E_SUCCESS)
3654                 goto DONE;
3655
3656         /* Add the mac addr into mac list */
3657         f = rte_zmalloc("macv_filter", sizeof(*f), 0);
3658         if (f == NULL) {
3659                 PMD_DRV_LOG(ERR, "failed to allocate memory\n");
3660                 ret = I40E_ERR_NO_MEMORY;
3661                 goto DONE;
3662         }
3663         (void)rte_memcpy(&f->macaddr, addr, ETH_ADDR_LEN);
3664         TAILQ_INSERT_TAIL(&vsi->mac_list, f, next);
3665         vsi->mac_num++;
3666
3667         ret = I40E_SUCCESS;
3668 DONE:
3669         rte_free(mv_f);
3670
3671         return ret;
3672 }
3673
3674 int
3675 i40e_vsi_delete_mac(struct i40e_vsi *vsi, struct ether_addr *addr)
3676 {
3677         struct i40e_mac_filter *f;
3678         struct i40e_macvlan_filter *mv_f;
3679         int vlan_num;
3680         int ret = I40E_SUCCESS;
3681
3682         /* Can't find it, return an error */
3683         f = i40e_find_mac_filter(vsi, addr);
3684         if (f == NULL)
3685                 return I40E_ERR_PARAM;
3686
3687         vlan_num = vsi->vlan_num;
3688         if (vlan_num == 0) {
3689                 PMD_DRV_LOG(ERR, "VLAN number shouldn't be 0\n");
3690                 return I40E_ERR_PARAM;
3691         }
3692         mv_f = rte_zmalloc("macvlan_data", vlan_num * sizeof(*mv_f), 0);
3693         if (mv_f == NULL) {
3694                 PMD_DRV_LOG(ERR, "failed to allocate memory\n");
3695                 return I40E_ERR_NO_MEMORY;
3696         }
3697
3698         ret = i40e_find_all_vlan_for_mac(vsi, mv_f, vlan_num, addr);
3699         if (ret != I40E_SUCCESS)
3700                 goto DONE;
3701
3702         ret = i40e_remove_macvlan_filters(vsi, mv_f, vlan_num);
3703         if (ret != I40E_SUCCESS)
3704                 goto DONE;
3705
3706         /* Remove the mac addr into mac list */
3707         TAILQ_REMOVE(&vsi->mac_list, f, next);
3708         rte_free(f);
3709         vsi->mac_num--;
3710
3711         ret = I40E_SUCCESS;
3712 DONE:
3713         rte_free(mv_f);
3714         return ret;
3715 }
3716
3717 /* Configure hash enable flags for RSS */
3718 static uint64_t
3719 i40e_config_hena(uint64_t flags)
3720 {
3721         uint64_t hena = 0;
3722
3723         if (!flags)
3724                 return hena;
3725
3726         if (flags & ETH_RSS_NONF_IPV4_UDP)
3727                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
3728         if (flags & ETH_RSS_NONF_IPV4_TCP)
3729                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP;
3730         if (flags & ETH_RSS_NONF_IPV4_SCTP)
3731                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP;
3732         if (flags & ETH_RSS_NONF_IPV4_OTHER)
3733                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER;
3734         if (flags & ETH_RSS_FRAG_IPV4)
3735                 hena |= 1ULL << I40E_FILTER_PCTYPE_FRAG_IPV4;
3736         if (flags & ETH_RSS_NONF_IPV6_UDP)
3737                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_UDP;
3738         if (flags & ETH_RSS_NONF_IPV6_TCP)
3739                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP;
3740         if (flags & ETH_RSS_NONF_IPV6_SCTP)
3741                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP;
3742         if (flags & ETH_RSS_NONF_IPV6_OTHER)
3743                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER;
3744         if (flags & ETH_RSS_FRAG_IPV6)
3745                 hena |= 1ULL << I40E_FILTER_PCTYPE_FRAG_IPV6;
3746         if (flags & ETH_RSS_L2_PAYLOAD)
3747                 hena |= 1ULL << I40E_FILTER_PCTYPE_L2_PAYLOAD;
3748
3749         return hena;
3750 }
3751
3752 /* Parse the hash enable flags */
3753 static uint64_t
3754 i40e_parse_hena(uint64_t flags)
3755 {
3756         uint64_t rss_hf = 0;
3757
3758         if (!flags)
3759                 return rss_hf;
3760
3761         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP))
3762                 rss_hf |= ETH_RSS_NONF_IPV4_UDP;
3763         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP))
3764                 rss_hf |= ETH_RSS_NONF_IPV4_TCP;
3765         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP))
3766                 rss_hf |= ETH_RSS_NONF_IPV4_SCTP;
3767         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER))
3768                 rss_hf |= ETH_RSS_NONF_IPV4_OTHER;
3769         if (flags & (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV4))
3770                 rss_hf |= ETH_RSS_FRAG_IPV4;
3771         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_UDP))
3772                 rss_hf |= ETH_RSS_NONF_IPV6_UDP;
3773         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP))
3774                 rss_hf |= ETH_RSS_NONF_IPV6_TCP;
3775         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP))
3776                 rss_hf |= ETH_RSS_NONF_IPV6_SCTP;
3777         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER))
3778                 rss_hf |= ETH_RSS_NONF_IPV6_OTHER;
3779         if (flags & (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV6))
3780                 rss_hf |= ETH_RSS_FRAG_IPV6;
3781         if (flags & (1ULL << I40E_FILTER_PCTYPE_L2_PAYLOAD))
3782                 rss_hf |= ETH_RSS_L2_PAYLOAD;
3783
3784         return rss_hf;
3785 }
3786
3787 /* Disable RSS */
3788 static void
3789 i40e_pf_disable_rss(struct i40e_pf *pf)
3790 {
3791         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
3792         uint64_t hena;
3793
3794         hena = (uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(0));
3795         hena |= ((uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(1))) << 32;
3796         hena &= ~I40E_RSS_HENA_ALL;
3797         I40E_WRITE_REG(hw, I40E_PFQF_HENA(0), (uint32_t)hena);
3798         I40E_WRITE_REG(hw, I40E_PFQF_HENA(1), (uint32_t)(hena >> 32));
3799         I40E_WRITE_FLUSH(hw);
3800 }
3801
3802 static int
3803 i40e_hw_rss_hash_set(struct i40e_hw *hw, struct rte_eth_rss_conf *rss_conf)
3804 {
3805         uint32_t *hash_key;
3806         uint8_t hash_key_len;
3807         uint64_t rss_hf;
3808         uint16_t i;
3809         uint64_t hena;
3810
3811         hash_key = (uint32_t *)(rss_conf->rss_key);
3812         hash_key_len = rss_conf->rss_key_len;
3813         if (hash_key != NULL && hash_key_len >=
3814                 (I40E_PFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t)) {
3815                 /* Fill in RSS hash key */
3816                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
3817                         I40E_WRITE_REG(hw, I40E_PFQF_HKEY(i), hash_key[i]);
3818         }
3819
3820         rss_hf = rss_conf->rss_hf;
3821         hena = (uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(0));
3822         hena |= ((uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(1))) << 32;
3823         hena &= ~I40E_RSS_HENA_ALL;
3824         hena |= i40e_config_hena(rss_hf);
3825         I40E_WRITE_REG(hw, I40E_PFQF_HENA(0), (uint32_t)hena);
3826         I40E_WRITE_REG(hw, I40E_PFQF_HENA(1), (uint32_t)(hena >> 32));
3827         I40E_WRITE_FLUSH(hw);
3828
3829         return 0;
3830 }
3831
3832 static int
3833 i40e_dev_rss_hash_update(struct rte_eth_dev *dev,
3834                          struct rte_eth_rss_conf *rss_conf)
3835 {
3836         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3837         uint64_t rss_hf = rss_conf->rss_hf & I40E_RSS_OFFLOAD_ALL;
3838         uint64_t hena;
3839
3840         hena = (uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(0));
3841         hena |= ((uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(1))) << 32;
3842         if (!(hena & I40E_RSS_HENA_ALL)) { /* RSS disabled */
3843                 if (rss_hf != 0) /* Enable RSS */
3844                         return -EINVAL;
3845                 return 0; /* Nothing to do */
3846         }
3847         /* RSS enabled */
3848         if (rss_hf == 0) /* Disable RSS */
3849                 return -EINVAL;
3850
3851         return i40e_hw_rss_hash_set(hw, rss_conf);
3852 }
3853
3854 static int
3855 i40e_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
3856                            struct rte_eth_rss_conf *rss_conf)
3857 {
3858         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3859         uint32_t *hash_key = (uint32_t *)(rss_conf->rss_key);
3860         uint64_t hena;
3861         uint16_t i;
3862
3863         if (hash_key != NULL) {
3864                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
3865                         hash_key[i] = I40E_READ_REG(hw, I40E_PFQF_HKEY(i));
3866                 rss_conf->rss_key_len = i * sizeof(uint32_t);
3867         }
3868         hena = (uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(0));
3869         hena |= ((uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(1))) << 32;
3870         rss_conf->rss_hf = i40e_parse_hena(hena);
3871
3872         return 0;
3873 }
3874
3875 /* Configure RSS */
3876 static int
3877 i40e_pf_config_rss(struct i40e_pf *pf)
3878 {
3879         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
3880         struct rte_eth_rss_conf rss_conf;
3881         uint32_t i, lut = 0;
3882         uint16_t j, num = i40e_prev_power_of_2(pf->dev_data->nb_rx_queues);
3883
3884         for (i = 0, j = 0; i < hw->func_caps.rss_table_size; i++, j++) {
3885                 if (j == num)
3886                         j = 0;
3887                 lut = (lut << 8) | (j & ((0x1 <<
3888                         hw->func_caps.rss_table_entry_width) - 1));
3889                 if ((i & 3) == 3)
3890                         I40E_WRITE_REG(hw, I40E_PFQF_HLUT(i >> 2), lut);
3891         }
3892
3893         rss_conf = pf->dev_data->dev_conf.rx_adv_conf.rss_conf;
3894         if ((rss_conf.rss_hf & I40E_RSS_OFFLOAD_ALL) == 0) {
3895                 i40e_pf_disable_rss(pf);
3896                 return 0;
3897         }
3898         if (rss_conf.rss_key == NULL || rss_conf.rss_key_len <
3899                 (I40E_PFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t)) {
3900                 /* Calculate the default hash key */
3901                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
3902                         rss_key_default[i] = (uint32_t)rte_rand();
3903                 rss_conf.rss_key = (uint8_t *)rss_key_default;
3904                 rss_conf.rss_key_len = (I40E_PFQF_HKEY_MAX_INDEX + 1) *
3905                                                         sizeof(uint32_t);
3906         }
3907
3908         return i40e_hw_rss_hash_set(hw, &rss_conf);
3909 }
3910
3911 static int
3912 i40e_pf_config_mq_rx(struct i40e_pf *pf)
3913 {
3914         if (!pf->dev_data->sriov.active) {
3915                 switch (pf->dev_data->dev_conf.rxmode.mq_mode) {
3916                 case ETH_MQ_RX_RSS:
3917                         i40e_pf_config_rss(pf);
3918                         break;
3919                 default:
3920                         i40e_pf_disable_rss(pf);
3921                         break;
3922                 }
3923         }
3924
3925         return 0;
3926 }
3927
3928 static int
3929 i40e_disable_queue(struct i40e_hw *hw, uint16_t q_idx)
3930 {
3931         uint16_t i;
3932         uint32_t reg;
3933
3934         /* Disable TX queue */
3935         for (i = 0; i < I40E_CHK_Q_ENA_COUNT; i++) {
3936                 reg = I40E_READ_REG(hw, I40E_QTX_ENA(q_idx));
3937                 if (!(((reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 0x1) ^
3938                         ((reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 0x1)))
3939                         break;
3940                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
3941         }
3942         if (i >= I40E_CHK_Q_ENA_COUNT) {
3943                 PMD_DRV_LOG(ERR, "Failed to disable "
3944                         "tx queue[%u]\n", q_idx);
3945                 return I40E_ERR_TIMEOUT;
3946         }
3947
3948         if (reg & I40E_QTX_ENA_QENA_STAT_MASK) {
3949                 reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
3950                 I40E_WRITE_REG(hw, I40E_QTX_ENA(q_idx), reg);
3951                 for (i = 0; i < I40E_CHK_Q_ENA_COUNT; i++) {
3952                         rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
3953                         reg = I40E_READ_REG(hw, I40E_QTX_ENA(q_idx));
3954                         if (!(reg & I40E_QTX_ENA_QENA_REQ_MASK) &&
3955                                 !(reg & I40E_QTX_ENA_QENA_STAT_MASK))
3956                                 break;
3957                 }
3958                 if (i >= I40E_CHK_Q_ENA_COUNT) {
3959                         PMD_DRV_LOG(ERR, "Failed to disable "
3960                                 "tx queue[%u]\n", q_idx);
3961                         return I40E_ERR_TIMEOUT;
3962                 }
3963         }
3964
3965         /* Disable RX queue */
3966         for (i = 0; i < I40E_CHK_Q_ENA_COUNT; i++) {
3967                 reg = I40E_READ_REG(hw, I40E_QRX_ENA(q_idx));
3968                 if (!((reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 0x1) ^
3969                         ((reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 0x1))
3970                         break;
3971                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
3972         }
3973         if (i >= I40E_CHK_Q_ENA_COUNT) {
3974                 PMD_DRV_LOG(ERR, "Failed to disable "
3975                         "rx queue[%u]\n", q_idx);
3976                 return I40E_ERR_TIMEOUT;
3977         }
3978
3979         if (reg & I40E_QRX_ENA_QENA_STAT_MASK) {
3980                 reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
3981                 I40E_WRITE_REG(hw, I40E_QRX_ENA(q_idx), reg);
3982                 for (i = 0; i < I40E_CHK_Q_ENA_COUNT; i++) {
3983                         rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
3984                         reg = I40E_READ_REG(hw, I40E_QRX_ENA(q_idx));
3985                         if (!(reg & I40E_QRX_ENA_QENA_REQ_MASK) &&
3986                                 !(reg & I40E_QRX_ENA_QENA_STAT_MASK))
3987                                 break;
3988                 }
3989                 if (i >= I40E_CHK_Q_ENA_COUNT) {
3990                         PMD_DRV_LOG(ERR, "Failed to disable "
3991                                 "rx queue[%u]\n", q_idx);
3992                         return I40E_ERR_TIMEOUT;
3993                 }
3994         }
3995
3996         return I40E_SUCCESS;
3997 }
3998
3999 static int
4000 i40e_pf_disable_all_queues(struct i40e_hw *hw)
4001 {
4002         uint32_t reg;
4003         uint16_t firstq, lastq, maxq, i;
4004         int ret;
4005         reg = I40E_READ_REG(hw, I40E_PFLAN_QALLOC);
4006         if (!(reg & I40E_PFLAN_QALLOC_VALID_MASK)) {
4007                 PMD_DRV_LOG(INFO, "PF queue allocation is invalid\n");
4008                 return I40E_ERR_PARAM;
4009         }
4010         firstq = reg & I40E_PFLAN_QALLOC_FIRSTQ_MASK;
4011         lastq = (reg & I40E_PFLAN_QALLOC_LASTQ_MASK) >>
4012                         I40E_PFLAN_QALLOC_LASTQ_SHIFT;
4013         maxq = lastq - firstq;
4014         for (i = 0; i <= maxq; i++) {
4015                 ret = i40e_disable_queue(hw, i);
4016                 if (ret != I40E_SUCCESS)
4017                         return ret;
4018         }
4019         return I40E_SUCCESS;
4020 }