lib/librte_pmd_i40e/i40e_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <inttypes.h>
  42
  43 #include <rte_string_fns.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_memzone.h>
  48 #include <rte_malloc.h>
  49 #include <rte_memcpy.h>
  50 #include <rte_dev.h>
  51
  52 #include "i40e_logs.h"
  53 #include "i40e/i40e_register_x710_int.h"
  54 #include "i40e/i40e_prototype.h"
  55 #include "i40e/i40e_adminq_cmd.h"
  56 #include "i40e/i40e_type.h"
  57 #include "i40e_ethdev.h"
  58 #include "i40e_rxtx.h"
  59 #include "i40e_pf.h"
  60
  61 #define I40E_DEFAULT_RX_FREE_THRESH  32
  62 #define I40E_DEFAULT_RX_PTHRESH      8
  63 #define I40E_DEFAULT_RX_HTHRESH      8
  64 #define I40E_DEFAULT_RX_WTHRESH      0
  65
  66 #define I40E_DEFAULT_TX_FREE_THRESH  32
  67 #define I40E_DEFAULT_TX_PTHRESH      32
  68 #define I40E_DEFAULT_TX_HTHRESH      0
  69 #define I40E_DEFAULT_TX_WTHRESH      0
  70 #define I40E_DEFAULT_TX_RSBIT_THRESH 32
  71
  72 /* Maximun number of MAC addresses */
  73 #define I40E_NUM_MACADDR_MAX       64
  74 #define I40E_CLEAR_PXE_WAIT_MS     200
  75
  76 /* Maximun number of capability elements */
  77 #define I40E_MAX_CAP_ELE_NUM       128
  78
  79 /* Wait count and inteval */
  80 #define I40E_CHK_Q_ENA_COUNT       1000
  81 #define I40E_CHK_Q_ENA_INTERVAL_US 1000
  82
  83 /* Maximun number of VSI */
  84 #define I40E_MAX_NUM_VSIS          (384UL)
  85
  86 /* Bit shift and mask */
  87 #define I40E_16_BIT_SHIFT 16
  88 #define I40E_16_BIT_MASK  0xFFFF
  89 #define I40E_32_BIT_SHIFT 32
  90 #define I40E_32_BIT_MASK  0xFFFFFFFF
  91 #define I40E_48_BIT_SHIFT 48
  92 #define I40E_48_BIT_MASK  0xFFFFFFFFFFFFULL
  93
  94 /* Default queue interrupt throttling time in microseconds*/
  95 #define I40E_ITR_INDEX_DEFAULT          0
  96 #define I40E_QUEUE_ITR_INTERVAL_DEFAULT 32 /* 32 us */
  97 #define I40E_QUEUE_ITR_INTERVAL_MAX     8160 /* 8160 us */
  98
  99 #define I40E_PRE_TX_Q_CFG_WAIT_US       10 /* 10 us */
 100
 101 static int eth_i40e_dev_init(\
 102                         __attribute__((unused)) struct eth_driver *eth_drv,
 103                         struct rte_eth_dev *eth_dev);
 104 static int i40e_dev_configure(struct rte_eth_dev *dev);
 105 static int i40e_dev_start(struct rte_eth_dev *dev);
 106 static void i40e_dev_stop(struct rte_eth_dev *dev);
 107 static void i40e_dev_close(struct rte_eth_dev *dev);
 108 static void i40e_dev_promiscuous_enable(struct rte_eth_dev *dev);
 109 static void i40e_dev_promiscuous_disable(struct rte_eth_dev *dev);
 110 static void i40e_dev_allmulticast_enable(struct rte_eth_dev *dev);
 111 static void i40e_dev_allmulticast_disable(struct rte_eth_dev *dev);
 112 static int i40e_dev_set_link_up(struct rte_eth_dev *dev);
 113 static int i40e_dev_set_link_down(struct rte_eth_dev *dev);
 114 static void i40e_dev_stats_get(struct rte_eth_dev *dev,
 115                                struct rte_eth_stats *stats);
 116 static void i40e_dev_stats_reset(struct rte_eth_dev *dev);
 117 static int i40e_dev_queue_stats_mapping_set(struct rte_eth_dev *dev,
 118                                             uint16_t queue_id,
 119                                             uint8_t stat_idx,
 120                                             uint8_t is_rx);
 121 static void i40e_dev_info_get(struct rte_eth_dev *dev,
 122                               struct rte_eth_dev_info *dev_info);
 123 static int i40e_vlan_filter_set(struct rte_eth_dev *dev,
 124                                 uint16_t vlan_id,
 125                                 int on);
 126 static void i40e_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid);
 127 static void i40e_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 128 static void i40e_vlan_strip_queue_set(struct rte_eth_dev *dev,
 129                                       uint16_t queue,
 130                                       int on);
 131 static int i40e_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid, int on);
 132 static int i40e_dev_led_on(struct rte_eth_dev *dev);
 133 static int i40e_dev_led_off(struct rte_eth_dev *dev);
 134 static int i40e_flow_ctrl_set(struct rte_eth_dev *dev,
 135                               struct rte_eth_fc_conf *fc_conf);
 136 static int i40e_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 137                                        struct rte_eth_pfc_conf *pfc_conf);
 138 static void i40e_macaddr_add(struct rte_eth_dev *dev,
 139                           struct ether_addr *mac_addr,
 140                           uint32_t index,
 141                           uint32_t pool);
 142 static void i40e_macaddr_remove(struct rte_eth_dev *dev, uint32_t index);
 143 static int i40e_dev_rss_reta_update(struct rte_eth_dev *dev,
 144                                     struct rte_eth_rss_reta *reta_conf);
 145 static int i40e_dev_rss_reta_query(struct rte_eth_dev *dev,
 146                                    struct rte_eth_rss_reta *reta_conf);
 147
 148 static int i40e_get_cap(struct i40e_hw *hw);
 149 static int i40e_pf_parameter_init(struct rte_eth_dev *dev);
 150 static int i40e_pf_setup(struct i40e_pf *pf);
 151 static int i40e_vsi_init(struct i40e_vsi *vsi);
 152 static void i40e_stat_update_32(struct i40e_hw *hw, uint32_t reg,
 153                 bool offset_loaded, uint64_t *offset, uint64_t *stat);
 154 static void i40e_stat_update_48(struct i40e_hw *hw,
 155                                uint32_t hireg,
 156                                uint32_t loreg,
 157                                bool offset_loaded,
 158                                uint64_t *offset,
 159                                uint64_t *stat);
 160 static void i40e_pf_config_irq0(struct i40e_hw *hw);
 161 static void i40e_dev_interrupt_handler(
 162                 __rte_unused struct rte_intr_handle *handle, void *param);
 163 static int i40e_res_pool_init(struct i40e_res_pool_info *pool,
 164                                 uint32_t base, uint32_t num);
 165 static void i40e_res_pool_destroy(struct i40e_res_pool_info *pool);
 166 static int i40e_res_pool_free(struct i40e_res_pool_info *pool,
 167                         uint32_t base);
 168 static int i40e_res_pool_alloc(struct i40e_res_pool_info *pool,
 169                         uint16_t num);
 170 static int i40e_dev_init_vlan(struct rte_eth_dev *dev);
 171 static int i40e_veb_release(struct i40e_veb *veb);
 172 static struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf,
 173                                                 struct i40e_vsi *vsi);
 174 static int i40e_pf_config_mq_rx(struct i40e_pf *pf);
 175 static int i40e_vsi_config_double_vlan(struct i40e_vsi *vsi, int on);
 176 static inline int i40e_find_all_vlan_for_mac(struct i40e_vsi *vsi,
 177                                              struct i40e_macvlan_filter *mv_f,
 178                                              int num,
 179                                              struct ether_addr *addr);
 180 static inline int i40e_find_all_mac_for_vlan(struct i40e_vsi *vsi,
 181                                              struct i40e_macvlan_filter *mv_f,
 182                                              int num,
 183                                              uint16_t vlan);
 184 static int i40e_vsi_remove_all_macvlan_filter(struct i40e_vsi *vsi);
 185 static int i40e_dev_rss_hash_update(struct rte_eth_dev *dev,
 186                                     struct rte_eth_rss_conf *rss_conf);
 187 static int i40e_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
 188                                       struct rte_eth_rss_conf *rss_conf);
 189
 190 /* Default hash key buffer for RSS */
 191 static uint32_t rss_key_default[I40E_PFQF_HKEY_MAX_INDEX + 1];
 192
 193 static struct rte_pci_id pci_id_i40e_map[] = {
 194 #define RTE_PCI_DEV_ID_DECL_I40E(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 195 #include "rte_pci_dev_ids.h"
 196 { .vendor_id = 0, /* sentinel */ },
 197 };
 198
 199 static struct eth_dev_ops i40e_eth_dev_ops = {
 200         .dev_configure                = i40e_dev_configure,
 201         .dev_start                    = i40e_dev_start,
 202         .dev_stop                     = i40e_dev_stop,
 203         .dev_close                    = i40e_dev_close,
 204         .promiscuous_enable           = i40e_dev_promiscuous_enable,
 205         .promiscuous_disable          = i40e_dev_promiscuous_disable,
 206         .allmulticast_enable          = i40e_dev_allmulticast_enable,
 207         .allmulticast_disable         = i40e_dev_allmulticast_disable,
 208         .dev_set_link_up              = i40e_dev_set_link_up,
 209         .dev_set_link_down            = i40e_dev_set_link_down,
 210         .link_update                  = i40e_dev_link_update,
 211         .stats_get                    = i40e_dev_stats_get,
 212         .stats_reset                  = i40e_dev_stats_reset,
 213         .queue_stats_mapping_set      = i40e_dev_queue_stats_mapping_set,
 214         .dev_infos_get                = i40e_dev_info_get,
 215         .vlan_filter_set              = i40e_vlan_filter_set,
 216         .vlan_tpid_set                = i40e_vlan_tpid_set,
 217         .vlan_offload_set             = i40e_vlan_offload_set,
 218         .vlan_strip_queue_set         = i40e_vlan_strip_queue_set,
 219         .vlan_pvid_set                = i40e_vlan_pvid_set,
 220         .rx_queue_start               = i40e_dev_rx_queue_start,
 221         .rx_queue_stop                = i40e_dev_rx_queue_stop,
 222         .tx_queue_start               = i40e_dev_tx_queue_start,
 223         .tx_queue_stop                = i40e_dev_tx_queue_stop,
 224         .rx_queue_setup               = i40e_dev_rx_queue_setup,
 225         .rx_queue_release             = i40e_dev_rx_queue_release,
 226         .rx_queue_count               = i40e_dev_rx_queue_count,
 227         .rx_descriptor_done           = i40e_dev_rx_descriptor_done,
 228         .tx_queue_setup               = i40e_dev_tx_queue_setup,
 229         .tx_queue_release             = i40e_dev_tx_queue_release,
 230         .dev_led_on                   = i40e_dev_led_on,
 231         .dev_led_off                  = i40e_dev_led_off,
 232         .flow_ctrl_set                = i40e_flow_ctrl_set,
 233         .priority_flow_ctrl_set       = i40e_priority_flow_ctrl_set,
 234         .mac_addr_add                 = i40e_macaddr_add,
 235         .mac_addr_remove              = i40e_macaddr_remove,
 236         .reta_update                  = i40e_dev_rss_reta_update,
 237         .reta_query                   = i40e_dev_rss_reta_query,
 238         .rss_hash_update              = i40e_dev_rss_hash_update,
 239         .rss_hash_conf_get            = i40e_dev_rss_hash_conf_get,
 240 };
 241
 242 static struct eth_driver rte_i40e_pmd = {
 243         {
 244                 .name = "rte_i40e_pmd",
 245                 .id_table = pci_id_i40e_map,
 246                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
 247         },
 248         .eth_dev_init = eth_i40e_dev_init,
 249         .dev_private_size = sizeof(struct i40e_adapter),
 250 };
 251
 252 static inline int
 253 i40e_prev_power_of_2(int n)
 254 {
 255        int p = n;
 256
 257        --p;
 258        p |= p >> 1;
 259        p |= p >> 2;
 260        p |= p >> 4;
 261        p |= p >> 8;
 262        p |= p >> 16;
 263        if (p == (n - 1))
 264                return n;
 265        p >>= 1;
 266
 267        return ++p;
 268 }
 269
 270 static inline int
 271 rte_i40e_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 272                                      struct rte_eth_link *link)
 273 {
 274         struct rte_eth_link *dst = link;
 275         struct rte_eth_link *src = &(dev->data->dev_link);
 276
 277         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 278                                         *(uint64_t *)src) == 0)
 279                 return -1;
 280
 281         return 0;
 282 }
 283
 284 static inline int
 285 rte_i40e_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 286                                       struct rte_eth_link *link)
 287 {
 288         struct rte_eth_link *dst = &(dev->data->dev_link);
 289         struct rte_eth_link *src = link;
 290
 291         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 292                                         *(uint64_t *)src) == 0)
 293                 return -1;
 294
 295         return 0;
 296 }
 297
 298 /*
 299  * Driver initialization routine.
 300  * Invoked once at EAL init time.
 301  * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
 302  */
 303 static int
 304 rte_i40e_pmd_init(const char *name __rte_unused,
 305                   const char *params __rte_unused)
 306 {
 307         PMD_INIT_FUNC_TRACE();
 308         rte_eth_driver_register(&rte_i40e_pmd);
 309
 310         return 0;
 311 }
 312
 313 static struct rte_driver rte_i40e_driver = {
 314         .type = PMD_PDEV,
 315         .init = rte_i40e_pmd_init,
 316 };
 317
 318 PMD_REGISTER_DRIVER(rte_i40e_driver);
 319
 320 static int
 321 eth_i40e_dev_init(__rte_unused struct eth_driver *eth_drv,
 322                   struct rte_eth_dev *dev)
 323 {
 324         struct rte_pci_device *pci_dev;
 325         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 326         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 327         struct i40e_vsi *vsi;
 328         int ret;
 329         uint32_t len;
 330         uint8_t aq_fail = 0;
 331
 332         PMD_INIT_FUNC_TRACE();
 333
 334         dev->dev_ops = &i40e_eth_dev_ops;
 335         dev->rx_pkt_burst = i40e_recv_pkts;
 336         dev->tx_pkt_burst = i40e_xmit_pkts;
 337
 338         /* for secondary processes, we don't initialise any further as primary
 339          * has already done this work. Only check we don't need a different
 340          * RX function */
 341         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 342                 if (dev->data->scattered_rx)
 343                         dev->rx_pkt_burst = i40e_recv_scattered_pkts;
 344                 return 0;
 345         }
 346         pci_dev = dev->pci_dev;
 347         pf->adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 348         pf->adapter->eth_dev = dev;
 349         pf->dev_data = dev->data;
 350
 351         hw->back = I40E_PF_TO_ADAPTER(pf);
 352         hw->hw_addr = (uint8_t *)(pci_dev->mem_resource[0].addr);
 353         if (!hw->hw_addr) {
 354                 PMD_INIT_LOG(ERR, "Hardware is not available, "
 355                              "as address is NULL");
 356                 return -ENODEV;
 357         }
 358
 359         hw->vendor_id = pci_dev->id.vendor_id;
 360         hw->device_id = pci_dev->id.device_id;
 361         hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
 362         hw->subsystem_device_id = pci_dev->id.subsystem_device_id;
 363         hw->bus.device = pci_dev->addr.devid;
 364         hw->bus.func = pci_dev->addr.function;
 365
 366         /* Make sure all is clean before doing PF reset */
 367         i40e_clear_hw(hw);
 368
 369         /* Reset here to make sure all is clean for each PF */
 370         ret = i40e_pf_reset(hw);
 371         if (ret) {
 372                 PMD_INIT_LOG(ERR, "Failed to reset pf: %d", ret);
 373                 return ret;
 374         }
 375
 376         /* Initialize the shared code (base driver) */
 377         ret = i40e_init_shared_code(hw);
 378         if (ret) {
 379                 PMD_INIT_LOG(ERR, "Failed to init shared code (base driver): %d", ret);
 380                 return ret;
 381         }
 382
 383         /* Initialize the parameters for adminq */
 384         i40e_init_adminq_parameter(hw);
 385         ret = i40e_init_adminq(hw);
 386         if (ret != I40E_SUCCESS) {
 387                 PMD_INIT_LOG(ERR, "Failed to init adminq: %d", ret);
 388                 return -EIO;
 389         }
 390         PMD_INIT_LOG(INFO, "FW %d.%d API %d.%d NVM %02d.%02d.%02d eetrack %04x",
 391                      hw->aq.fw_maj_ver, hw->aq.fw_min_ver,
 392                      hw->aq.api_maj_ver, hw->aq.api_min_ver,
 393                      ((hw->nvm.version >> 12) & 0xf),
 394                      ((hw->nvm.version >> 4) & 0xff),
 395                      (hw->nvm.version & 0xf), hw->nvm.eetrack);
 396
 397         /* Disable LLDP */
 398         ret = i40e_aq_stop_lldp(hw, true, NULL);
 399         if (ret != I40E_SUCCESS) /* Its failure can be ignored */
 400                 PMD_INIT_LOG(INFO, "Failed to stop lldp");
 401
 402         /* Clear PXE mode */
 403         i40e_clear_pxe_mode(hw);
 404
 405         /* Get hw capabilities */
 406         ret = i40e_get_cap(hw);
 407         if (ret != I40E_SUCCESS) {
 408                 PMD_INIT_LOG(ERR, "Failed to get capabilities: %d", ret);
 409                 goto err_get_capabilities;
 410         }
 411
 412         /* Initialize parameters for PF */
 413         ret = i40e_pf_parameter_init(dev);
 414         if (ret != 0) {
 415                 PMD_INIT_LOG(ERR, "Failed to do parameter init: %d", ret);
 416                 goto err_parameter_init;
 417         }
 418
 419         /* Initialize the queue management */
 420         ret = i40e_res_pool_init(&pf->qp_pool, 0, hw->func_caps.num_tx_qp);
 421         if (ret < 0) {
 422                 PMD_INIT_LOG(ERR, "Failed to init queue pool");
 423                 goto err_qp_pool_init;
 424         }
 425         ret = i40e_res_pool_init(&pf->msix_pool, 1,
 426                                 hw->func_caps.num_msix_vectors - 1);
 427         if (ret < 0) {
 428                 PMD_INIT_LOG(ERR, "Failed to init MSIX pool");
 429                 goto err_msix_pool_init;
 430         }
 431
 432         /* Initialize lan hmc */
 433         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
 434                                 hw->func_caps.num_rx_qp, 0, 0);
 435         if (ret != I40E_SUCCESS) {
 436                 PMD_INIT_LOG(ERR, "Failed to init lan hmc: %d", ret);
 437                 goto err_init_lan_hmc;
 438         }
 439
 440         /* Configure lan hmc */
 441         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
 442         if (ret != I40E_SUCCESS) {
 443                 PMD_INIT_LOG(ERR, "Failed to configure lan hmc: %d", ret);
 444                 goto err_configure_lan_hmc;
 445         }
 446
 447         /* Get and check the mac address */
 448         i40e_get_mac_addr(hw, hw->mac.addr);
 449         if (i40e_validate_mac_addr(hw->mac.addr) != I40E_SUCCESS) {
 450                 PMD_INIT_LOG(ERR, "mac address is not valid");
 451                 ret = -EIO;
 452                 goto err_get_mac_addr;
 453         }
 454         /* Copy the permanent MAC address */
 455         ether_addr_copy((struct ether_addr *) hw->mac.addr,
 456                         (struct ether_addr *) hw->mac.perm_addr);
 457
 458         /* Disable flow control */
 459         hw->fc.requested_mode = I40E_FC_NONE;
 460         i40e_set_fc(hw, &aq_fail, TRUE);
 461
 462         /* PF setup, which includes VSI setup */
 463         ret = i40e_pf_setup(pf);
 464         if (ret) {
 465                 PMD_INIT_LOG(ERR, "Failed to setup pf switch: %d", ret);
 466                 goto err_setup_pf_switch;
 467         }
 468
 469         vsi = pf->main_vsi;
 470
 471         /* Disable double vlan by default */
 472         i40e_vsi_config_double_vlan(vsi, FALSE);
 473
 474         if (!vsi->max_macaddrs)
 475                 len = ETHER_ADDR_LEN;
 476         else
 477                 len = ETHER_ADDR_LEN * vsi->max_macaddrs;
 478
 479         /* Should be after VSI initialized */
 480         dev->data->mac_addrs = rte_zmalloc("i40e", len, 0);
 481         if (!dev->data->mac_addrs) {
 482                 PMD_INIT_LOG(ERR, "Failed to allocated memory "
 483                                         "for storing mac address");
 484                 goto err_get_mac_addr;
 485         }
 486         ether_addr_copy((struct ether_addr *)hw->mac.perm_addr,
 487                                         &dev->data->mac_addrs[0]);
 488
 489         /* initialize pf host driver to setup SRIOV resource if applicable */
 490         i40e_pf_host_init(dev);
 491
 492         /* register callback func to eal lib */
 493         rte_intr_callback_register(&(pci_dev->intr_handle),
 494                 i40e_dev_interrupt_handler, (void *)dev);
 495
 496         /* configure and enable device interrupt */
 497         i40e_pf_config_irq0(hw);
 498         i40e_pf_enable_irq0(hw);
 499
 500         /* enable uio intr after callback register */
 501         rte_intr_enable(&(pci_dev->intr_handle));
 502
 503         return 0;
 504
 505 err_setup_pf_switch:
 506         rte_free(pf->main_vsi);
 507 err_get_mac_addr:
 508 err_configure_lan_hmc:
 509         (void)i40e_shutdown_lan_hmc(hw);
 510 err_init_lan_hmc:
 511         i40e_res_pool_destroy(&pf->msix_pool);
 512 err_msix_pool_init:
 513         i40e_res_pool_destroy(&pf->qp_pool);
 514 err_qp_pool_init:
 515 err_parameter_init:
 516 err_get_capabilities:
 517         (void)i40e_shutdown_adminq(hw);
 518
 519         return ret;
 520 }
 521
 522 static int
 523 i40e_dev_configure(struct rte_eth_dev *dev)
 524 {
 525         return i40e_dev_init_vlan(dev);
 526 }
 527
 528 void
 529 i40e_vsi_queues_unbind_intr(struct i40e_vsi *vsi)
 530 {
 531         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
 532         uint16_t msix_vect = vsi->msix_intr;
 533         uint16_t i;
 534
 535         for (i = 0; i < vsi->nb_qps; i++) {
 536                 I40E_WRITE_REG(hw, I40E_QINT_TQCTL(vsi->base_queue + i), 0);
 537                 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(vsi->base_queue + i), 0);
 538                 rte_wmb();
 539         }
 540
 541         if (vsi->type != I40E_VSI_SRIOV) {
 542                 I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(msix_vect - 1), 0);
 543                 I40E_WRITE_REG(hw, I40E_PFINT_ITRN(I40E_ITR_INDEX_DEFAULT,
 544                                 msix_vect - 1), 0);
 545         } else {
 546                 uint32_t reg;
 547                 reg = (hw->func_caps.num_msix_vectors_vf - 1) *
 548                         vsi->user_param + (msix_vect - 1);
 549
 550                 I40E_WRITE_REG(hw, I40E_VPINT_LNKLSTN(reg), 0);
 551         }
 552         I40E_WRITE_FLUSH(hw);
 553 }
 554
 555 static inline uint16_t
 556 i40e_calc_itr_interval(int16_t interval)
 557 {
 558         if (interval < 0 || interval > I40E_QUEUE_ITR_INTERVAL_MAX)
 559                 interval = I40E_QUEUE_ITR_INTERVAL_DEFAULT;
 560
 561         /* Convert to hardware count, as writing each 1 represents 2 us */
 562         return (interval/2);
 563 }
 564
 565 void
 566 i40e_vsi_queues_bind_intr(struct i40e_vsi *vsi)
 567 {
 568         uint32_t val;
 569         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
 570         uint16_t msix_vect = vsi->msix_intr;
 571         uint16_t interval = i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL);
 572         int i;
 573
 574         for (i = 0; i < vsi->nb_qps; i++)
 575                 I40E_WRITE_REG(hw, I40E_QINT_TQCTL(vsi->base_queue + i), 0);
 576
 577         /* Bind all RX queues to allocated MSIX interrupt */
 578         for (i = 0; i < vsi->nb_qps; i++) {
 579                 val = (msix_vect << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
 580                         (interval << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
 581                         ((vsi->base_queue + i + 1) <<
 582                         I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
 583                         (0 << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) |
 584                         I40E_QINT_RQCTL_CAUSE_ENA_MASK;
 585
 586                 if (i == vsi->nb_qps - 1)
 587                         val |= I40E_QINT_RQCTL_NEXTQ_INDX_MASK;
 588                 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(vsi->base_queue + i), val);
 589         }
 590
 591         /* Write first RX queue to Link list register as the head element */
 592         if (vsi->type != I40E_VSI_SRIOV) {
 593                 I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(msix_vect - 1),
 594                         (vsi->base_queue << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) |
 595                         (0x0 << I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT));
 596
 597                 I40E_WRITE_REG(hw, I40E_PFINT_ITRN(I40E_ITR_INDEX_DEFAULT,
 598                                 msix_vect - 1), interval);
 599
 600                 /* Disable auto-mask on enabling of all none-zero  interrupt */
 601                 I40E_WRITE_REG(hw, I40E_GLINT_CTL,
 602                                 I40E_GLINT_CTL_DIS_AUTOMASK_N_MASK);
 603         }
 604         else {
 605                 uint32_t reg;
 606                 /* num_msix_vectors_vf needs to minus irq0 */
 607                 reg = (hw->func_caps.num_msix_vectors_vf - 1) *
 608                         vsi->user_param + (msix_vect - 1);
 609
 610                 I40E_WRITE_REG(hw, I40E_VPINT_LNKLSTN(reg),
 611                         (vsi->base_queue << I40E_VPINT_LNKLSTN_FIRSTQ_INDX_SHIFT) |
 612                         (0x0 << I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_SHIFT));
 613         }
 614
 615         I40E_WRITE_FLUSH(hw);
 616 }
 617
 618 static void
 619 i40e_vsi_enable_queues_intr(struct i40e_vsi *vsi)
 620 {
 621         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
 622         uint16_t interval = i40e_calc_itr_interval(\
 623                         RTE_LIBRTE_I40E_ITR_INTERVAL);
 624
 625         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(vsi->msix_intr - 1),
 626                                         I40E_PFINT_DYN_CTLN_INTENA_MASK |
 627                                         I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
 628                                 (0 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT) |
 629                         (interval << I40E_PFINT_DYN_CTLN_INTERVAL_SHIFT));
 630 }
 631
 632 static void
 633 i40e_vsi_disable_queues_intr(struct i40e_vsi *vsi)
 634 {
 635         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
 636
 637         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(vsi->msix_intr - 1), 0);
 638 }
 639
 640 static inline uint8_t
 641 i40e_parse_link_speed(uint16_t eth_link_speed)
 642 {
 643         uint8_t link_speed = I40E_LINK_SPEED_UNKNOWN;
 644
 645         switch (eth_link_speed) {
 646         case ETH_LINK_SPEED_40G:
 647                 link_speed = I40E_LINK_SPEED_40GB;
 648                 break;
 649         case ETH_LINK_SPEED_20G:
 650                 link_speed = I40E_LINK_SPEED_20GB;
 651                 break;
 652         case ETH_LINK_SPEED_10G:
 653                 link_speed = I40E_LINK_SPEED_10GB;
 654                 break;
 655         case ETH_LINK_SPEED_1000:
 656                 link_speed = I40E_LINK_SPEED_1GB;
 657                 break;
 658         case ETH_LINK_SPEED_100:
 659                 link_speed = I40E_LINK_SPEED_100MB;
 660                 break;
 661         }
 662
 663         return link_speed;
 664 }
 665
 666 static int
 667 i40e_phy_conf_link(struct i40e_hw *hw, uint8_t abilities, uint8_t force_speed)
 668 {
 669         enum i40e_status_code status;
 670         struct i40e_aq_get_phy_abilities_resp phy_ab;
 671         struct i40e_aq_set_phy_config phy_conf;
 672         const uint8_t mask = I40E_AQ_PHY_FLAG_PAUSE_TX |
 673                         I40E_AQ_PHY_FLAG_PAUSE_RX |
 674                         I40E_AQ_PHY_FLAG_LOW_POWER;
 675         const uint8_t advt = I40E_LINK_SPEED_40GB |
 676                         I40E_LINK_SPEED_10GB |
 677                         I40E_LINK_SPEED_1GB |
 678                         I40E_LINK_SPEED_100MB;
 679         int ret = -ENOTSUP;
 680
 681         status = i40e_aq_get_phy_capabilities(hw, false, false, &phy_ab,
 682                                               NULL);
 683         if (status)
 684                 return ret;
 685
 686         memset(&phy_conf, 0, sizeof(phy_conf));
 687
 688         /* bits 0-2 use the values from get_phy_abilities_resp */
 689         abilities &= ~mask;
 690         abilities |= phy_ab.abilities & mask;
 691
 692         /* update ablities and speed */
 693         if (abilities & I40E_AQ_PHY_AN_ENABLED)
 694                 phy_conf.link_speed = advt;
 695         else
 696                 phy_conf.link_speed = force_speed;
 697
 698         phy_conf.abilities = abilities;
 699
 700         /* use get_phy_abilities_resp value for the rest */
 701         phy_conf.phy_type = phy_ab.phy_type;
 702         phy_conf.eee_capability = phy_ab.eee_capability;
 703         phy_conf.eeer = phy_ab.eeer_val;
 704         phy_conf.low_power_ctrl = phy_ab.d3_lpan;
 705
 706         PMD_DRV_LOG(DEBUG, "\tCurrent: abilities %x, link_speed %x",
 707                     phy_ab.abilities, phy_ab.link_speed);
 708         PMD_DRV_LOG(DEBUG, "\tConfig:  abilities %x, link_speed %x",
 709                     phy_conf.abilities, phy_conf.link_speed);
 710
 711         status = i40e_aq_set_phy_config(hw, &phy_conf, NULL);
 712         if (status)
 713                 return ret;
 714
 715         return I40E_SUCCESS;
 716 }
 717
 718 static int
 719 i40e_apply_link_speed(struct rte_eth_dev *dev)
 720 {
 721         uint8_t speed;
 722         uint8_t abilities = 0;
 723         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 724         struct rte_eth_conf *conf = &dev->data->dev_conf;
 725
 726         speed = i40e_parse_link_speed(conf->link_speed);
 727         abilities |= I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
 728         if (conf->link_speed == ETH_LINK_SPEED_AUTONEG)
 729                 abilities |= I40E_AQ_PHY_AN_ENABLED;
 730         else
 731                 abilities |= I40E_AQ_PHY_LINK_ENABLED;
 732
 733         return i40e_phy_conf_link(hw, abilities, speed);
 734 }
 735
 736 static int
 737 i40e_dev_start(struct rte_eth_dev *dev)
 738 {
 739         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 740         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 741         struct i40e_vsi *vsi = pf->main_vsi;
 742         int ret;
 743
 744         if ((dev->data->dev_conf.link_duplex != ETH_LINK_AUTONEG_DUPLEX) &&
 745                 (dev->data->dev_conf.link_duplex != ETH_LINK_FULL_DUPLEX)) {
 746                 PMD_INIT_LOG(ERR, "Invalid link_duplex (%hu) for port %hhu",
 747                              dev->data->dev_conf.link_duplex,
 748                              dev->data->port_id);
 749                 return -EINVAL;
 750         }
 751
 752         /* Initialize VSI */
 753         ret = i40e_vsi_init(vsi);
 754         if (ret != I40E_SUCCESS) {
 755                 PMD_DRV_LOG(ERR, "Failed to init VSI");
 756                 goto err_up;
 757         }
 758
 759         /* Map queues with MSIX interrupt */
 760         i40e_vsi_queues_bind_intr(vsi);
 761         i40e_vsi_enable_queues_intr(vsi);
 762
 763         /* Enable all queues which have been configured */
 764         ret = i40e_vsi_switch_queues(vsi, TRUE);
 765         if (ret != I40E_SUCCESS) {
 766                 PMD_DRV_LOG(ERR, "Failed to enable VSI");
 767                 goto err_up;
 768         }
 769
 770         /* Enable receiving broadcast packets */
 771         if ((vsi->type == I40E_VSI_MAIN) || (vsi->type == I40E_VSI_VMDQ2)) {
 772                 ret = i40e_aq_set_vsi_broadcast(hw, vsi->seid, true, NULL);
 773                 if (ret != I40E_SUCCESS)
 774                         PMD_DRV_LOG(INFO, "fail to set vsi broadcast");
 775         }
 776
 777         /* Apply link configure */
 778         ret = i40e_apply_link_speed(dev);
 779         if (I40E_SUCCESS != ret) {
 780                 PMD_DRV_LOG(ERR, "Fail to apply link setting");
 781                 goto err_up;
 782         }
 783
 784         return I40E_SUCCESS;
 785
 786 err_up:
 787         i40e_vsi_switch_queues(vsi, FALSE);
 788
 789         return ret;
 790 }
 791
 792 static void
 793 i40e_dev_stop(struct rte_eth_dev *dev)
 794 {
 795         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 796         struct i40e_vsi *vsi = pf->main_vsi;
 797
 798         /* Disable all queues */
 799         i40e_vsi_switch_queues(vsi, FALSE);
 800
 801         /* Set link down */
 802         i40e_dev_set_link_down(dev);
 803
 804         /* un-map queues with interrupt registers */
 805         i40e_vsi_disable_queues_intr(vsi);
 806         i40e_vsi_queues_unbind_intr(vsi);
 807 }
 808
 809 static void
 810 i40e_dev_close(struct rte_eth_dev *dev)
 811 {
 812         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 813         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 814         uint32_t reg;
 815
 816         PMD_INIT_FUNC_TRACE();
 817
 818         i40e_dev_stop(dev);
 819
 820         /* Disable interrupt */
 821         i40e_pf_disable_irq0(hw);
 822         rte_intr_disable(&(dev->pci_dev->intr_handle));
 823
 824         /* shutdown and destroy the HMC */
 825         i40e_shutdown_lan_hmc(hw);
 826
 827         /* release all the existing VSIs and VEBs */
 828         i40e_vsi_release(pf->main_vsi);
 829
 830         /* shutdown the adminq */
 831         i40e_aq_queue_shutdown(hw, true);
 832         i40e_shutdown_adminq(hw);
 833
 834         i40e_res_pool_destroy(&pf->qp_pool);
 835         i40e_res_pool_destroy(&pf->msix_pool);
 836
 837         /* force a PF reset to clean anything leftover */
 838         reg = I40E_READ_REG(hw, I40E_PFGEN_CTRL);
 839         I40E_WRITE_REG(hw, I40E_PFGEN_CTRL,
 840                         (reg | I40E_PFGEN_CTRL_PFSWR_MASK));
 841         I40E_WRITE_FLUSH(hw);
 842 }
 843
 844 static void
 845 i40e_dev_promiscuous_enable(struct rte_eth_dev *dev)
 846 {
 847         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 848         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 849         struct i40e_vsi *vsi = pf->main_vsi;
 850         int status;
 851
 852         status = i40e_aq_set_vsi_unicast_promiscuous(hw, vsi->seid,
 853                                                         true, NULL);
 854         if (status != I40E_SUCCESS)
 855                 PMD_DRV_LOG(ERR, "Failed to enable unicast promiscuous");
 856
 857         status = i40e_aq_set_vsi_multicast_promiscuous(hw, vsi->seid,
 858                                                         TRUE, NULL);
 859         if (status != I40E_SUCCESS)
 860                 PMD_DRV_LOG(ERR, "Failed to enable multicast promiscuous");
 861
 862 }
 863
 864 static void
 865 i40e_dev_promiscuous_disable(struct rte_eth_dev *dev)
 866 {
 867         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 868         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 869         struct i40e_vsi *vsi = pf->main_vsi;
 870         int status;
 871
 872         status = i40e_aq_set_vsi_unicast_promiscuous(hw, vsi->seid,
 873                                                         false, NULL);
 874         if (status != I40E_SUCCESS)
 875                 PMD_DRV_LOG(ERR, "Failed to disable unicast promiscuous");
 876
 877         status = i40e_aq_set_vsi_multicast_promiscuous(hw, vsi->seid,
 878                                                         false, NULL);
 879         if (status != I40E_SUCCESS)
 880                 PMD_DRV_LOG(ERR, "Failed to disable multicast promiscuous");
 881 }
 882
 883 static void
 884 i40e_dev_allmulticast_enable(struct rte_eth_dev *dev)
 885 {
 886         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 887         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 888         struct i40e_vsi *vsi = pf->main_vsi;
 889         int ret;
 890
 891         ret = i40e_aq_set_vsi_multicast_promiscuous(hw, vsi->seid, TRUE, NULL);
 892         if (ret != I40E_SUCCESS)
 893                 PMD_DRV_LOG(ERR, "Failed to enable multicast promiscuous");
 894 }
 895
 896 static void
 897 i40e_dev_allmulticast_disable(struct rte_eth_dev *dev)
 898 {
 899         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 900         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 901         struct i40e_vsi *vsi = pf->main_vsi;
 902         int ret;
 903
 904         if (dev->data->promiscuous == 1)
 905                 return; /* must remain in all_multicast mode */
 906
 907         ret = i40e_aq_set_vsi_multicast_promiscuous(hw,
 908                                 vsi->seid, FALSE, NULL);
 909         if (ret != I40E_SUCCESS)
 910                 PMD_DRV_LOG(ERR, "Failed to disable multicast promiscuous");
 911 }
 912
 913 /*
 914  * Set device link up.
 915  */
 916 static int
 917 i40e_dev_set_link_up(struct rte_eth_dev *dev)
 918 {
 919         /* re-apply link speed setting */
 920         return i40e_apply_link_speed(dev);
 921 }
 922
 923 /*
 924  * Set device link down.
 925  */
 926 static int
 927 i40e_dev_set_link_down(__rte_unused struct rte_eth_dev *dev)
 928 {
 929         uint8_t speed = I40E_LINK_SPEED_UNKNOWN;
 930         uint8_t abilities = I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
 931         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 932
 933         return i40e_phy_conf_link(hw, abilities, speed);
 934 }
 935
 936 int
 937 i40e_dev_link_update(struct rte_eth_dev *dev,
 938                      __rte_unused int wait_to_complete)
 939 {
 940         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 941         struct i40e_link_status link_status;
 942         struct rte_eth_link link, old;
 943         int status;
 944
 945         memset(&link, 0, sizeof(link));
 946         memset(&old, 0, sizeof(old));
 947         memset(&link_status, 0, sizeof(link_status));
 948         rte_i40e_dev_atomic_read_link_status(dev, &old);
 949
 950         /* Get link status information from hardware */
 951         status = i40e_aq_get_link_info(hw, false, &link_status, NULL);
 952         if (status != I40E_SUCCESS) {
 953                 link.link_speed = ETH_LINK_SPEED_100;
 954                 link.link_duplex = ETH_LINK_FULL_DUPLEX;
 955                 PMD_DRV_LOG(ERR, "Failed to get link info");
 956                 goto out;
 957         }
 958
 959         link.link_status = link_status.link_info & I40E_AQ_LINK_UP;
 960
 961         if (!link.link_status)
 962                 goto out;
 963
 964         /* i40e uses full duplex only */
 965         link.link_duplex = ETH_LINK_FULL_DUPLEX;
 966
 967         /* Parse the link status */
 968         switch (link_status.link_speed) {
 969         case I40E_LINK_SPEED_100MB:
 970                 link.link_speed = ETH_LINK_SPEED_100;
 971                 break;
 972         case I40E_LINK_SPEED_1GB:
 973                 link.link_speed = ETH_LINK_SPEED_1000;
 974                 break;
 975         case I40E_LINK_SPEED_10GB:
 976                 link.link_speed = ETH_LINK_SPEED_10G;
 977                 break;
 978         case I40E_LINK_SPEED_20GB:
 979                 link.link_speed = ETH_LINK_SPEED_20G;
 980                 break;
 981         case I40E_LINK_SPEED_40GB:
 982                 link.link_speed = ETH_LINK_SPEED_40G;
 983                 break;
 984         default:
 985                 link.link_speed = ETH_LINK_SPEED_100;
 986                 break;
 987         }
 988
 989 out:
 990         rte_i40e_dev_atomic_write_link_status(dev, &link);
 991         if (link.link_status == old.link_status)
 992                 return -1;
 993
 994         return 0;
 995 }
 996
 997 /* Get all the statistics of a VSI */
 998 void
 999 i40e_update_vsi_stats(struct i40e_vsi *vsi)
1000 {
1001         struct i40e_eth_stats *oes = &vsi->eth_stats_offset;
1002         struct i40e_eth_stats *nes = &vsi->eth_stats;
1003         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1004         int idx = rte_le_to_cpu_16(vsi->info.stat_counter_idx);
1005
1006         i40e_stat_update_48(hw, I40E_GLV_GORCH(idx), I40E_GLV_GORCL(idx),
1007                             vsi->offset_loaded, &oes->rx_bytes,
1008                             &nes->rx_bytes);
1009         i40e_stat_update_48(hw, I40E_GLV_UPRCH(idx), I40E_GLV_UPRCL(idx),
1010                             vsi->offset_loaded, &oes->rx_unicast,
1011                             &nes->rx_unicast);
1012         i40e_stat_update_48(hw, I40E_GLV_MPRCH(idx), I40E_GLV_MPRCL(idx),
1013                             vsi->offset_loaded, &oes->rx_multicast,
1014                             &nes->rx_multicast);
1015         i40e_stat_update_48(hw, I40E_GLV_BPRCH(idx), I40E_GLV_BPRCL(idx),
1016                             vsi->offset_loaded, &oes->rx_broadcast,
1017                             &nes->rx_broadcast);
1018         i40e_stat_update_32(hw, I40E_GLV_RDPC(idx), vsi->offset_loaded,
1019                             &oes->rx_discards, &nes->rx_discards);
1020         /* GLV_REPC not supported */
1021         /* GLV_RMPC not supported */
1022         i40e_stat_update_32(hw, I40E_GLV_RUPP(idx), vsi->offset_loaded,
1023                             &oes->rx_unknown_protocol,
1024                             &nes->rx_unknown_protocol);
1025         i40e_stat_update_48(hw, I40E_GLV_GOTCH(idx), I40E_GLV_GOTCL(idx),
1026                             vsi->offset_loaded, &oes->tx_bytes,
1027                             &nes->tx_bytes);
1028         i40e_stat_update_48(hw, I40E_GLV_UPTCH(idx), I40E_GLV_UPTCL(idx),
1029                             vsi->offset_loaded, &oes->tx_unicast,
1030                             &nes->tx_unicast);
1031         i40e_stat_update_48(hw, I40E_GLV_MPTCH(idx), I40E_GLV_MPTCL(idx),
1032                             vsi->offset_loaded, &oes->tx_multicast,
1033                             &nes->tx_multicast);
1034         i40e_stat_update_48(hw, I40E_GLV_BPTCH(idx), I40E_GLV_BPTCL(idx),
1035                             vsi->offset_loaded,  &oes->tx_broadcast,
1036                             &nes->tx_broadcast);
1037         /* GLV_TDPC not supported */
1038         i40e_stat_update_32(hw, I40E_GLV_TEPC(idx), vsi->offset_loaded,
1039                             &oes->tx_errors, &nes->tx_errors);
1040         vsi->offset_loaded = true;
1041
1042         PMD_DRV_LOG(DEBUG, "***************** VSI[%u] stats start *******************",
1043                     vsi->vsi_id);
1044         PMD_DRV_LOG(DEBUG, "rx_bytes:            %lu", nes->rx_bytes);
1045         PMD_DRV_LOG(DEBUG, "rx_unicast:          %lu", nes->rx_unicast);
1046         PMD_DRV_LOG(DEBUG, "rx_multicast:        %lu", nes->rx_multicast);
1047         PMD_DRV_LOG(DEBUG, "rx_broadcast:        %lu", nes->rx_broadcast);
1048         PMD_DRV_LOG(DEBUG, "rx_discards:         %lu", nes->rx_discards);
1049         PMD_DRV_LOG(DEBUG, "rx_unknown_protocol: %lu",
1050                     nes->rx_unknown_protocol);
1051         PMD_DRV_LOG(DEBUG, "tx_bytes:            %lu", nes->tx_bytes);
1052         PMD_DRV_LOG(DEBUG, "tx_unicast:          %lu", nes->tx_unicast);
1053         PMD_DRV_LOG(DEBUG, "tx_multicast:        %lu", nes->tx_multicast);
1054         PMD_DRV_LOG(DEBUG, "tx_broadcast:        %lu", nes->tx_broadcast);
1055         PMD_DRV_LOG(DEBUG, "tx_discards:         %lu", nes->tx_discards);
1056         PMD_DRV_LOG(DEBUG, "tx_errors:           %lu", nes->tx_errors);
1057         PMD_DRV_LOG(DEBUG, "***************** VSI[%u] stats end *******************",
1058                     vsi->vsi_id);
1059 }
1060
1061 /* Get all statistics of a port */
1062 static void
1063 i40e_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1064 {
1065         uint32_t i;
1066         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1067         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1068         struct i40e_hw_port_stats *ns = &pf->stats; /* new stats */
1069         struct i40e_hw_port_stats *os = &pf->stats_offset; /* old stats */
1070
1071         /* Get statistics of struct i40e_eth_stats */
1072         i40e_stat_update_48(hw, I40E_GLPRT_GORCH(hw->port),
1073                             I40E_GLPRT_GORCL(hw->port),
1074                             pf->offset_loaded, &os->eth.rx_bytes,
1075                             &ns->eth.rx_bytes);
1076         i40e_stat_update_48(hw, I40E_GLPRT_UPRCH(hw->port),
1077                             I40E_GLPRT_UPRCL(hw->port),
1078                             pf->offset_loaded, &os->eth.rx_unicast,
1079                             &ns->eth.rx_unicast);
1080         i40e_stat_update_48(hw, I40E_GLPRT_MPRCH(hw->port),
1081                             I40E_GLPRT_MPRCL(hw->port),
1082                             pf->offset_loaded, &os->eth.rx_multicast,
1083                             &ns->eth.rx_multicast);
1084         i40e_stat_update_48(hw, I40E_GLPRT_BPRCH(hw->port),
1085                             I40E_GLPRT_BPRCL(hw->port),
1086                             pf->offset_loaded, &os->eth.rx_broadcast,
1087                             &ns->eth.rx_broadcast);
1088         i40e_stat_update_32(hw, I40E_GLPRT_RDPC(hw->port),
1089                             pf->offset_loaded, &os->eth.rx_discards,
1090                             &ns->eth.rx_discards);
1091         /* GLPRT_REPC not supported */
1092         /* GLPRT_RMPC not supported */
1093         i40e_stat_update_32(hw, I40E_GLPRT_RUPP(hw->port),
1094                             pf->offset_loaded,
1095                             &os->eth.rx_unknown_protocol,
1096                             &ns->eth.rx_unknown_protocol);
1097         i40e_stat_update_48(hw, I40E_GLPRT_GOTCH(hw->port),
1098                             I40E_GLPRT_GOTCL(hw->port),
1099                             pf->offset_loaded, &os->eth.tx_bytes,
1100                             &ns->eth.tx_bytes);
1101         i40e_stat_update_48(hw, I40E_GLPRT_UPTCH(hw->port),
1102                             I40E_GLPRT_UPTCL(hw->port),
1103                             pf->offset_loaded, &os->eth.tx_unicast,
1104                             &ns->eth.tx_unicast);
1105         i40e_stat_update_48(hw, I40E_GLPRT_MPTCH(hw->port),
1106                             I40E_GLPRT_MPTCL(hw->port),
1107                             pf->offset_loaded, &os->eth.tx_multicast,
1108                             &ns->eth.tx_multicast);
1109         i40e_stat_update_48(hw, I40E_GLPRT_BPTCH(hw->port),
1110                             I40E_GLPRT_BPTCL(hw->port),
1111                             pf->offset_loaded, &os->eth.tx_broadcast,
1112                             &ns->eth.tx_broadcast);
1113         i40e_stat_update_32(hw, I40E_GLPRT_TDPC(hw->port),
1114                             pf->offset_loaded, &os->eth.tx_discards,
1115                             &ns->eth.tx_discards);
1116         /* GLPRT_TEPC not supported */
1117
1118         /* additional port specific stats */
1119         i40e_stat_update_32(hw, I40E_GLPRT_TDOLD(hw->port),
1120                             pf->offset_loaded, &os->tx_dropped_link_down,
1121                             &ns->tx_dropped_link_down);
1122         i40e_stat_update_32(hw, I40E_GLPRT_CRCERRS(hw->port),
1123                             pf->offset_loaded, &os->crc_errors,
1124                             &ns->crc_errors);
1125         i40e_stat_update_32(hw, I40E_GLPRT_ILLERRC(hw->port),
1126                             pf->offset_loaded, &os->illegal_bytes,
1127                             &ns->illegal_bytes);
1128         /* GLPRT_ERRBC not supported */
1129         i40e_stat_update_32(hw, I40E_GLPRT_MLFC(hw->port),
1130                             pf->offset_loaded, &os->mac_local_faults,
1131                             &ns->mac_local_faults);
1132         i40e_stat_update_32(hw, I40E_GLPRT_MRFC(hw->port),
1133                             pf->offset_loaded, &os->mac_remote_faults,
1134                             &ns->mac_remote_faults);
1135         i40e_stat_update_32(hw, I40E_GLPRT_RLEC(hw->port),
1136                             pf->offset_loaded, &os->rx_length_errors,
1137                             &ns->rx_length_errors);
1138         i40e_stat_update_32(hw, I40E_GLPRT_LXONRXC(hw->port),
1139                             pf->offset_loaded, &os->link_xon_rx,
1140                             &ns->link_xon_rx);
1141         i40e_stat_update_32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
1142                             pf->offset_loaded, &os->link_xoff_rx,
1143                             &ns->link_xoff_rx);
1144         for (i = 0; i < 8; i++) {
1145                 i40e_stat_update_32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
1146                                     pf->offset_loaded,
1147                                     &os->priority_xon_rx[i],
1148                                     &ns->priority_xon_rx[i]);
1149                 i40e_stat_update_32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
1150                                     pf->offset_loaded,
1151                                     &os->priority_xoff_rx[i],
1152                                     &ns->priority_xoff_rx[i]);
1153         }
1154         i40e_stat_update_32(hw, I40E_GLPRT_LXONTXC(hw->port),
1155                             pf->offset_loaded, &os->link_xon_tx,
1156                             &ns->link_xon_tx);
1157         i40e_stat_update_32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
1158                             pf->offset_loaded, &os->link_xoff_tx,
1159                             &ns->link_xoff_tx);
1160         for (i = 0; i < 8; i++) {
1161                 i40e_stat_update_32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
1162                                     pf->offset_loaded,
1163                                     &os->priority_xon_tx[i],
1164                                     &ns->priority_xon_tx[i]);
1165                 i40e_stat_update_32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
1166                                     pf->offset_loaded,
1167                                     &os->priority_xoff_tx[i],
1168                                     &ns->priority_xoff_tx[i]);
1169                 i40e_stat_update_32(hw, I40E_GLPRT_RXON2OFFCNT(hw->port, i),
1170                                     pf->offset_loaded,
1171                                     &os->priority_xon_2_xoff[i],
1172                                     &ns->priority_xon_2_xoff[i]);
1173         }
1174         i40e_stat_update_48(hw, I40E_GLPRT_PRC64H(hw->port),
1175                             I40E_GLPRT_PRC64L(hw->port),
1176                             pf->offset_loaded, &os->rx_size_64,
1177                             &ns->rx_size_64);
1178         i40e_stat_update_48(hw, I40E_GLPRT_PRC127H(hw->port),
1179                             I40E_GLPRT_PRC127L(hw->port),
1180                             pf->offset_loaded, &os->rx_size_127,
1181                             &ns->rx_size_127);
1182         i40e_stat_update_48(hw, I40E_GLPRT_PRC255H(hw->port),
1183                             I40E_GLPRT_PRC255L(hw->port),
1184                             pf->offset_loaded, &os->rx_size_255,
1185                             &ns->rx_size_255);
1186         i40e_stat_update_48(hw, I40E_GLPRT_PRC511H(hw->port),
1187                             I40E_GLPRT_PRC511L(hw->port),
1188                             pf->offset_loaded, &os->rx_size_511,
1189                             &ns->rx_size_511);
1190         i40e_stat_update_48(hw, I40E_GLPRT_PRC1023H(hw->port),
1191                             I40E_GLPRT_PRC1023L(hw->port),
1192                             pf->offset_loaded, &os->rx_size_1023,
1193                             &ns->rx_size_1023);
1194         i40e_stat_update_48(hw, I40E_GLPRT_PRC1522H(hw->port),
1195                             I40E_GLPRT_PRC1522L(hw->port),
1196                             pf->offset_loaded, &os->rx_size_1522,
1197                             &ns->rx_size_1522);
1198         i40e_stat_update_48(hw, I40E_GLPRT_PRC9522H(hw->port),
1199                             I40E_GLPRT_PRC9522L(hw->port),
1200                             pf->offset_loaded, &os->rx_size_big,
1201                             &ns->rx_size_big);
1202         i40e_stat_update_32(hw, I40E_GLPRT_RUC(hw->port),
1203                             pf->offset_loaded, &os->rx_undersize,
1204                             &ns->rx_undersize);
1205         i40e_stat_update_32(hw, I40E_GLPRT_RFC(hw->port),
1206                             pf->offset_loaded, &os->rx_fragments,
1207                             &ns->rx_fragments);
1208         i40e_stat_update_32(hw, I40E_GLPRT_ROC(hw->port),
1209                             pf->offset_loaded, &os->rx_oversize,
1210                             &ns->rx_oversize);
1211         i40e_stat_update_32(hw, I40E_GLPRT_RJC(hw->port),
1212                             pf->offset_loaded, &os->rx_jabber,
1213                             &ns->rx_jabber);
1214         i40e_stat_update_48(hw, I40E_GLPRT_PTC64H(hw->port),
1215                             I40E_GLPRT_PTC64L(hw->port),
1216                             pf->offset_loaded, &os->tx_size_64,
1217                             &ns->tx_size_64);
1218         i40e_stat_update_48(hw, I40E_GLPRT_PTC127H(hw->port),
1219                             I40E_GLPRT_PTC127L(hw->port),
1220                             pf->offset_loaded, &os->tx_size_127,
1221                             &ns->tx_size_127);
1222         i40e_stat_update_48(hw, I40E_GLPRT_PTC255H(hw->port),
1223                             I40E_GLPRT_PTC255L(hw->port),
1224                             pf->offset_loaded, &os->tx_size_255,
1225                             &ns->tx_size_255);
1226         i40e_stat_update_48(hw, I40E_GLPRT_PTC511H(hw->port),
1227                             I40E_GLPRT_PTC511L(hw->port),
1228                             pf->offset_loaded, &os->tx_size_511,
1229                             &ns->tx_size_511);
1230         i40e_stat_update_48(hw, I40E_GLPRT_PTC1023H(hw->port),
1231                             I40E_GLPRT_PTC1023L(hw->port),
1232                             pf->offset_loaded, &os->tx_size_1023,
1233                             &ns->tx_size_1023);
1234         i40e_stat_update_48(hw, I40E_GLPRT_PTC1522H(hw->port),
1235                             I40E_GLPRT_PTC1522L(hw->port),
1236                             pf->offset_loaded, &os->tx_size_1522,
1237                             &ns->tx_size_1522);
1238         i40e_stat_update_48(hw, I40E_GLPRT_PTC9522H(hw->port),
1239                             I40E_GLPRT_PTC9522L(hw->port),
1240                             pf->offset_loaded, &os->tx_size_big,
1241                             &ns->tx_size_big);
1242         /* GLPRT_MSPDC not supported */
1243         /* GLPRT_XEC not supported */
1244
1245         pf->offset_loaded = true;
1246
1247         if (pf->main_vsi)
1248                 i40e_update_vsi_stats(pf->main_vsi);
1249
1250         stats->ipackets = ns->eth.rx_unicast + ns->eth.rx_multicast +
1251                                                 ns->eth.rx_broadcast;
1252         stats->opackets = ns->eth.tx_unicast + ns->eth.tx_multicast +
1253                                                 ns->eth.tx_broadcast;
1254         stats->ibytes   = ns->eth.rx_bytes;
1255         stats->obytes   = ns->eth.tx_bytes;
1256         stats->oerrors  = ns->eth.tx_errors;
1257         stats->imcasts  = ns->eth.rx_multicast;
1258
1259         /* Rx Errors */
1260         stats->ibadcrc  = ns->crc_errors;
1261         stats->ibadlen  = ns->rx_length_errors + ns->rx_undersize +
1262                         ns->rx_oversize + ns->rx_fragments + ns->rx_jabber;
1263         stats->imissed  = ns->eth.rx_discards;
1264         stats->ierrors  = stats->ibadcrc + stats->ibadlen + stats->imissed;
1265
1266         PMD_DRV_LOG(DEBUG, "***************** PF stats start *******************");
1267         PMD_DRV_LOG(DEBUG, "rx_bytes:            %lu", ns->eth.rx_bytes);
1268         PMD_DRV_LOG(DEBUG, "rx_unicast:          %lu", ns->eth.rx_unicast);
1269         PMD_DRV_LOG(DEBUG, "rx_multicast:        %lu", ns->eth.rx_multicast);
1270         PMD_DRV_LOG(DEBUG, "rx_broadcast:        %lu", ns->eth.rx_broadcast);
1271         PMD_DRV_LOG(DEBUG, "rx_discards:         %lu", ns->eth.rx_discards);
1272         PMD_DRV_LOG(DEBUG, "rx_unknown_protocol: %lu",
1273                     ns->eth.rx_unknown_protocol);
1274         PMD_DRV_LOG(DEBUG, "tx_bytes:            %lu", ns->eth.tx_bytes);
1275         PMD_DRV_LOG(DEBUG, "tx_unicast:          %lu", ns->eth.tx_unicast);
1276         PMD_DRV_LOG(DEBUG, "tx_multicast:        %lu", ns->eth.tx_multicast);
1277         PMD_DRV_LOG(DEBUG, "tx_broadcast:        %lu", ns->eth.tx_broadcast);
1278         PMD_DRV_LOG(DEBUG, "tx_discards:         %lu", ns->eth.tx_discards);
1279         PMD_DRV_LOG(DEBUG, "tx_errors:           %lu", ns->eth.tx_errors);
1280
1281         PMD_DRV_LOG(DEBUG, "tx_dropped_link_down:     %lu",
1282                     ns->tx_dropped_link_down);
1283         PMD_DRV_LOG(DEBUG, "crc_errors:               %lu", ns->crc_errors);
1284         PMD_DRV_LOG(DEBUG, "illegal_bytes:            %lu",
1285                     ns->illegal_bytes);
1286         PMD_DRV_LOG(DEBUG, "error_bytes:              %lu", ns->error_bytes);
1287         PMD_DRV_LOG(DEBUG, "mac_local_faults:         %lu",
1288                     ns->mac_local_faults);
1289         PMD_DRV_LOG(DEBUG, "mac_remote_faults:        %lu",
1290                     ns->mac_remote_faults);
1291         PMD_DRV_LOG(DEBUG, "rx_length_errors:         %lu",
1292                     ns->rx_length_errors);
1293         PMD_DRV_LOG(DEBUG, "link_xon_rx:              %lu", ns->link_xon_rx);
1294         PMD_DRV_LOG(DEBUG, "link_xoff_rx:             %lu", ns->link_xoff_rx);
1295         for (i = 0; i < 8; i++) {
1296                 PMD_DRV_LOG(DEBUG, "priority_xon_rx[%d]:      %lu",
1297                                 i, ns->priority_xon_rx[i]);
1298                 PMD_DRV_LOG(DEBUG, "priority_xoff_rx[%d]:     %lu",
1299                                 i, ns->priority_xoff_rx[i]);
1300         }
1301         PMD_DRV_LOG(DEBUG, "link_xon_tx:              %lu", ns->link_xon_tx);
1302         PMD_DRV_LOG(DEBUG, "link_xoff_tx:             %lu", ns->link_xoff_tx);
1303         for (i = 0; i < 8; i++) {
1304                 PMD_DRV_LOG(DEBUG, "priority_xon_tx[%d]:      %lu",
1305                                 i, ns->priority_xon_tx[i]);
1306                 PMD_DRV_LOG(DEBUG, "priority_xoff_tx[%d]:     %lu",
1307                                 i, ns->priority_xoff_tx[i]);
1308                 PMD_DRV_LOG(DEBUG, "priority_xon_2_xoff[%d]:  %lu",
1309                                 i, ns->priority_xon_2_xoff[i]);
1310         }
1311         PMD_DRV_LOG(DEBUG, "rx_size_64:               %lu", ns->rx_size_64);
1312         PMD_DRV_LOG(DEBUG, "rx_size_127:              %lu", ns->rx_size_127);
1313         PMD_DRV_LOG(DEBUG, "rx_size_255:              %lu", ns->rx_size_255);
1314         PMD_DRV_LOG(DEBUG, "rx_size_511:              %lu", ns->rx_size_511);
1315         PMD_DRV_LOG(DEBUG, "rx_size_1023:             %lu", ns->rx_size_1023);
1316         PMD_DRV_LOG(DEBUG, "rx_size_1522:             %lu", ns->rx_size_1522);
1317         PMD_DRV_LOG(DEBUG, "rx_size_big:              %lu", ns->rx_size_big);
1318         PMD_DRV_LOG(DEBUG, "rx_undersize:             %lu", ns->rx_undersize);
1319         PMD_DRV_LOG(DEBUG, "rx_fragments:             %lu", ns->rx_fragments);
1320         PMD_DRV_LOG(DEBUG, "rx_oversize:              %lu", ns->rx_oversize);
1321         PMD_DRV_LOG(DEBUG, "rx_jabber:                %lu", ns->rx_jabber);
1322         PMD_DRV_LOG(DEBUG, "tx_size_64:               %lu", ns->tx_size_64);
1323         PMD_DRV_LOG(DEBUG, "tx_size_127:              %lu", ns->tx_size_127);
1324         PMD_DRV_LOG(DEBUG, "tx_size_255:              %lu", ns->tx_size_255);
1325         PMD_DRV_LOG(DEBUG, "tx_size_511:              %lu", ns->tx_size_511);
1326         PMD_DRV_LOG(DEBUG, "tx_size_1023:             %lu", ns->tx_size_1023);
1327         PMD_DRV_LOG(DEBUG, "tx_size_1522:             %lu", ns->tx_size_1522);
1328         PMD_DRV_LOG(DEBUG, "tx_size_big:              %lu", ns->tx_size_big);
1329         PMD_DRV_LOG(DEBUG, "mac_short_packet_dropped: %lu",
1330                         ns->mac_short_packet_dropped);
1331         PMD_DRV_LOG(DEBUG, "checksum_error:           %lu",
1332                     ns->checksum_error);
1333         PMD_DRV_LOG(DEBUG, "***************** PF stats end ********************");
1334 }
1335
1336 /* Reset the statistics */
1337 static void
1338 i40e_dev_stats_reset(struct rte_eth_dev *dev)
1339 {
1340         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1341
1342         /* It results in reloading the start point of each counter */
1343         pf->offset_loaded = false;
1344 }
1345
1346 static int
1347 i40e_dev_queue_stats_mapping_set(__rte_unused struct rte_eth_dev *dev,
1348                                  __rte_unused uint16_t queue_id,
1349                                  __rte_unused uint8_t stat_idx,
1350                                  __rte_unused uint8_t is_rx)
1351 {
1352         PMD_INIT_FUNC_TRACE();
1353
1354         return -ENOSYS;
1355 }
1356
1357 static void
1358 i40e_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1359 {
1360         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1361         struct i40e_vsi *vsi = pf->main_vsi;
1362
1363         dev_info->max_rx_queues = vsi->nb_qps;
1364         dev_info->max_tx_queues = vsi->nb_qps;
1365         dev_info->min_rx_bufsize = I40E_BUF_SIZE_MIN;
1366         dev_info->max_rx_pktlen = I40E_FRAME_SIZE_MAX;
1367         dev_info->max_mac_addrs = vsi->max_macaddrs;
1368         dev_info->max_vfs = dev->pci_dev->max_vfs;
1369         dev_info->rx_offload_capa =
1370                 DEV_RX_OFFLOAD_VLAN_STRIP |
1371                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1372                 DEV_RX_OFFLOAD_UDP_CKSUM |
1373                 DEV_RX_OFFLOAD_TCP_CKSUM;
1374         dev_info->tx_offload_capa =
1375                 DEV_TX_OFFLOAD_VLAN_INSERT |
1376                 DEV_TX_OFFLOAD_IPV4_CKSUM |
1377                 DEV_TX_OFFLOAD_UDP_CKSUM |
1378                 DEV_TX_OFFLOAD_TCP_CKSUM |
1379                 DEV_TX_OFFLOAD_SCTP_CKSUM;
1380
1381         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1382                 .rx_thresh = {
1383                         .pthresh = I40E_DEFAULT_RX_PTHRESH,
1384                         .hthresh = I40E_DEFAULT_RX_HTHRESH,
1385                         .wthresh = I40E_DEFAULT_RX_WTHRESH,
1386                 },
1387                 .rx_free_thresh = I40E_DEFAULT_RX_FREE_THRESH,
1388                 .rx_drop_en = 0,
1389         };
1390
1391         dev_info->default_txconf = (struct rte_eth_txconf) {
1392                 .tx_thresh = {
1393                         .pthresh = I40E_DEFAULT_TX_PTHRESH,
1394                         .hthresh = I40E_DEFAULT_TX_HTHRESH,
1395                         .wthresh = I40E_DEFAULT_TX_WTHRESH,
1396                 },
1397                 .tx_free_thresh = I40E_DEFAULT_TX_FREE_THRESH,
1398                 .tx_rs_thresh = I40E_DEFAULT_TX_RSBIT_THRESH,
1399                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS | ETH_TXQ_FLAGS_NOOFFLOADS,
1400         };
1401
1402 }
1403
1404 static int
1405 i40e_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1406 {
1407         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1408         struct i40e_vsi *vsi = pf->main_vsi;
1409         PMD_INIT_FUNC_TRACE();
1410
1411         if (on)
1412                 return i40e_vsi_add_vlan(vsi, vlan_id);
1413         else
1414                 return i40e_vsi_delete_vlan(vsi, vlan_id);
1415 }
1416
1417 static void
1418 i40e_vlan_tpid_set(__rte_unused struct rte_eth_dev *dev,
1419                    __rte_unused uint16_t tpid)
1420 {
1421         PMD_INIT_FUNC_TRACE();
1422 }
1423
1424 static void
1425 i40e_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1426 {
1427         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1428         struct i40e_vsi *vsi = pf->main_vsi;
1429
1430         if (mask & ETH_VLAN_STRIP_MASK) {
1431                 /* Enable or disable VLAN stripping */
1432                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1433                         i40e_vsi_config_vlan_stripping(vsi, TRUE);
1434                 else
1435                         i40e_vsi_config_vlan_stripping(vsi, FALSE);
1436         }
1437
1438         if (mask & ETH_VLAN_EXTEND_MASK) {
1439                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1440                         i40e_vsi_config_double_vlan(vsi, TRUE);
1441                 else
1442                         i40e_vsi_config_double_vlan(vsi, FALSE);
1443         }
1444 }
1445
1446 static void
1447 i40e_vlan_strip_queue_set(__rte_unused struct rte_eth_dev *dev,
1448                           __rte_unused uint16_t queue,
1449                           __rte_unused int on)
1450 {
1451         PMD_INIT_FUNC_TRACE();
1452 }
1453
1454 static int
1455 i40e_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid, int on)
1456 {
1457         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1458         struct i40e_vsi *vsi = pf->main_vsi;
1459         struct rte_eth_dev_data *data = I40E_VSI_TO_DEV_DATA(vsi);
1460         struct i40e_vsi_vlan_pvid_info info;
1461
1462         memset(&info, 0, sizeof(info));
1463         info.on = on;
1464         if (info.on)
1465                 info.config.pvid = pvid;
1466         else {
1467                 info.config.reject.tagged =
1468                                 data->dev_conf.txmode.hw_vlan_reject_tagged;
1469                 info.config.reject.untagged =
1470                                 data->dev_conf.txmode.hw_vlan_reject_untagged;
1471         }
1472
1473         return i40e_vsi_vlan_pvid_set(vsi, &info);
1474 }
1475
1476 static int
1477 i40e_dev_led_on(struct rte_eth_dev *dev)
1478 {
1479         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1480         uint32_t mode = i40e_led_get(hw);
1481
1482         if (mode == 0)
1483                 i40e_led_set(hw, 0xf, true); /* 0xf means led always true */
1484
1485         return 0;
1486 }
1487
1488 static int
1489 i40e_dev_led_off(struct rte_eth_dev *dev)
1490 {
1491         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1492         uint32_t mode = i40e_led_get(hw);
1493
1494         if (mode != 0)
1495                 i40e_led_set(hw, 0, false);
1496
1497         return 0;
1498 }
1499
1500 static int
1501 i40e_flow_ctrl_set(__rte_unused struct rte_eth_dev *dev,
1502                    __rte_unused struct rte_eth_fc_conf *fc_conf)
1503 {
1504         PMD_INIT_FUNC_TRACE();
1505
1506         return -ENOSYS;
1507 }
1508
1509 static int
1510 i40e_priority_flow_ctrl_set(__rte_unused struct rte_eth_dev *dev,
1511                             __rte_unused struct rte_eth_pfc_conf *pfc_conf)
1512 {
1513         PMD_INIT_FUNC_TRACE();
1514
1515         return -ENOSYS;
1516 }
1517
1518 /* Add a MAC address, and update filters */
1519 static void
1520 i40e_macaddr_add(struct rte_eth_dev *dev,
1521                  struct ether_addr *mac_addr,
1522                  __attribute__((unused)) uint32_t index,
1523                  __attribute__((unused)) uint32_t pool)
1524 {
1525         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1526         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1527         struct i40e_vsi *vsi = pf->main_vsi;
1528         struct ether_addr old_mac;
1529         int ret;
1530
1531         if (!is_valid_assigned_ether_addr(mac_addr)) {
1532                 PMD_DRV_LOG(ERR, "Invalid ethernet address");
1533                 return;
1534         }
1535
1536         if (is_same_ether_addr(mac_addr, &(pf->dev_addr))) {
1537                 PMD_DRV_LOG(INFO, "Ignore adding permanent mac address");
1538                 return;
1539         }
1540
1541         /* Write mac address */
1542         ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_ONLY,
1543                                         mac_addr->addr_bytes, NULL);
1544         if (ret != I40E_SUCCESS) {
1545                 PMD_DRV_LOG(ERR, "Failed to write mac address");
1546                 return;
1547         }
1548
1549         (void)rte_memcpy(&old_mac, hw->mac.addr, ETHER_ADDR_LEN);
1550         (void)rte_memcpy(hw->mac.addr, mac_addr->addr_bytes,
1551                         ETHER_ADDR_LEN);
1552
1553         ret = i40e_vsi_add_mac(vsi, mac_addr);
1554         if (ret != I40E_SUCCESS) {
1555                 PMD_DRV_LOG(ERR, "Failed to add MACVLAN filter");
1556                 return;
1557         }
1558
1559         ether_addr_copy(mac_addr, &pf->dev_addr);
1560         i40e_vsi_delete_mac(vsi, &old_mac);
1561 }
1562
1563 /* Remove a MAC address, and update filters */
1564 static void
1565 i40e_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1566 {
1567         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1568         struct i40e_vsi *vsi = pf->main_vsi;
1569         struct rte_eth_dev_data *data = I40E_VSI_TO_DEV_DATA(vsi);
1570         struct ether_addr *macaddr;
1571         int ret;
1572         struct i40e_hw *hw =
1573                 I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1574
1575         if (index >= vsi->max_macaddrs)
1576                 return;
1577
1578         macaddr = &(data->mac_addrs[index]);
1579         if (!is_valid_assigned_ether_addr(macaddr))
1580                 return;
1581
1582         ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_ONLY,
1583                                         hw->mac.perm_addr, NULL);
1584         if (ret != I40E_SUCCESS) {
1585                 PMD_DRV_LOG(ERR, "Failed to write mac address");
1586                 return;
1587         }
1588
1589         (void)rte_memcpy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN);
1590
1591         ret = i40e_vsi_delete_mac(vsi, macaddr);
1592         if (ret != I40E_SUCCESS)
1593                 return;
1594
1595         /* Clear device address as it has been removed */
1596         if (is_same_ether_addr(&(pf->dev_addr), macaddr))
1597                 memset(&pf->dev_addr, 0, sizeof(struct ether_addr));
1598 }
1599
1600 static int
1601 i40e_dev_rss_reta_update(struct rte_eth_dev *dev,
1602                          struct rte_eth_rss_reta *reta_conf)
1603 {
1604         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1605         uint32_t lut, l;
1606         uint8_t i, j, mask, max = ETH_RSS_RETA_NUM_ENTRIES / 2;
1607
1608         for (i = 0; i < ETH_RSS_RETA_NUM_ENTRIES; i += 4) {
1609                 if (i < max)
1610                         mask = (uint8_t)((reta_conf->mask_lo >> i) & 0xF);
1611                 else
1612                         mask = (uint8_t)((reta_conf->mask_hi >>
1613                                                 (i - max)) & 0xF);
1614
1615                 if (!mask)
1616                         continue;
1617
1618                 if (mask == 0xF)
1619                         l = 0;
1620                 else
1621                         l = I40E_READ_REG(hw, I40E_PFQF_HLUT(i >> 2));
1622
1623                 for (j = 0, lut = 0; j < 4; j++) {
1624                         if (mask & (0x1 << j))
1625                                 lut |= reta_conf->reta[i + j] << (8 * j);
1626                         else
1627                                 lut |= l & (0xFF << (8 * j));
1628                 }
1629                 I40E_WRITE_REG(hw, I40E_PFQF_HLUT(i >> 2), lut);
1630         }
1631
1632         return 0;
1633 }
1634
1635 static int
1636 i40e_dev_rss_reta_query(struct rte_eth_dev *dev,
1637                         struct rte_eth_rss_reta *reta_conf)
1638 {
1639         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1640         uint32_t lut;
1641         uint8_t i, j, mask, max = ETH_RSS_RETA_NUM_ENTRIES / 2;
1642
1643         for (i = 0; i < ETH_RSS_RETA_NUM_ENTRIES; i += 4) {
1644                 if (i < max)
1645                         mask = (uint8_t)((reta_conf->mask_lo >> i) & 0xF);
1646                 else
1647                         mask = (uint8_t)((reta_conf->mask_hi >>
1648                                                 (i - max)) & 0xF);
1649
1650                 if (!mask)
1651                         continue;
1652
1653                 lut = I40E_READ_REG(hw, I40E_PFQF_HLUT(i >> 2));
1654                 for (j = 0; j < 4; j++) {
1655                         if (mask & (0x1 << j))
1656                                 reta_conf->reta[i + j] =
1657                                         (uint8_t)((lut >> (8 * j)) & 0xFF);
1658                 }
1659         }
1660
1661         return 0;
1662 }
1663
1664 /**
1665  * i40e_allocate_dma_mem_d - specific memory alloc for shared code (base driver)
1666  * @hw:   pointer to the HW structure
1667  * @mem:  pointer to mem struct to fill out
1668  * @size: size of memory requested
1669  * @alignment: what to align the allocation to
1670  **/
1671 enum i40e_status_code
1672 i40e_allocate_dma_mem_d(__attribute__((unused)) struct i40e_hw *hw,
1673                         struct i40e_dma_mem *mem,
1674                         u64 size,
1675                         u32 alignment)
1676 {
1677         static uint64_t id = 0;
1678         const struct rte_memzone *mz = NULL;
1679         char z_name[RTE_MEMZONE_NAMESIZE];
1680
1681         if (!mem)
1682                 return I40E_ERR_PARAM;
1683
1684         id++;
1685         snprintf(z_name, sizeof(z_name), "i40e_dma_%"PRIu64, id);
1686 #ifdef RTE_LIBRTE_XEN_DOM0
1687         mz = rte_memzone_reserve_bounded(z_name, size, 0, 0, alignment,
1688                                                         RTE_PGSIZE_2M);
1689 #else
1690         mz = rte_memzone_reserve_aligned(z_name, size, 0, 0, alignment);
1691 #endif
1692         if (!mz)
1693                 return I40E_ERR_NO_MEMORY;
1694
1695         mem->id = id;
1696         mem->size = size;
1697         mem->va = mz->addr;
1698 #ifdef RTE_LIBRTE_XEN_DOM0
1699         mem->pa = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1700 #else
1701         mem->pa = mz->phys_addr;
1702 #endif
1703
1704         return I40E_SUCCESS;
1705 }
1706
1707 /**
1708  * i40e_free_dma_mem_d - specific memory free for shared code (base driver)
1709  * @hw:   pointer to the HW structure
1710  * @mem:  ptr to mem struct to free
1711  **/
1712 enum i40e_status_code
1713 i40e_free_dma_mem_d(__attribute__((unused)) struct i40e_hw *hw,
1714                     struct i40e_dma_mem *mem)
1715 {
1716         if (!mem || !mem->va)
1717                 return I40E_ERR_PARAM;
1718
1719         mem->va = NULL;
1720         mem->pa = (u64)0;
1721
1722         return I40E_SUCCESS;
1723 }
1724
1725 /**
1726  * i40e_allocate_virt_mem_d - specific memory alloc for shared code (base driver)
1727  * @hw:   pointer to the HW structure
1728  * @mem:  pointer to mem struct to fill out
1729  * @size: size of memory requested
1730  **/
1731 enum i40e_status_code
1732 i40e_allocate_virt_mem_d(__attribute__((unused)) struct i40e_hw *hw,
1733                          struct i40e_virt_mem *mem,
1734                          u32 size)
1735 {
1736         if (!mem)
1737                 return I40E_ERR_PARAM;
1738
1739         mem->size = size;
1740         mem->va = rte_zmalloc("i40e", size, 0);
1741
1742         if (mem->va)
1743                 return I40E_SUCCESS;
1744         else
1745                 return I40E_ERR_NO_MEMORY;
1746 }
1747
1748 /**
1749  * i40e_free_virt_mem_d - specific memory free for shared code (base driver)
1750  * @hw:   pointer to the HW structure
1751  * @mem:  pointer to mem struct to free
1752  **/
1753 enum i40e_status_code
1754 i40e_free_virt_mem_d(__attribute__((unused)) struct i40e_hw *hw,
1755                      struct i40e_virt_mem *mem)
1756 {
1757         if (!mem)
1758                 return I40E_ERR_PARAM;
1759
1760         rte_free(mem->va);
1761         mem->va = NULL;
1762
1763         return I40E_SUCCESS;
1764 }
1765
1766 void
1767 i40e_init_spinlock_d(struct i40e_spinlock *sp)
1768 {
1769         rte_spinlock_init(&sp->spinlock);
1770 }
1771
1772 void
1773 i40e_acquire_spinlock_d(struct i40e_spinlock *sp)
1774 {
1775         rte_spinlock_lock(&sp->spinlock);
1776 }
1777
1778 void
1779 i40e_release_spinlock_d(struct i40e_spinlock *sp)
1780 {
1781         rte_spinlock_unlock(&sp->spinlock);
1782 }
1783
1784 void
1785 i40e_destroy_spinlock_d(__attribute__((unused)) struct i40e_spinlock *sp)
1786 {
1787         return;
1788 }
1789
1790 /**
1791  * Get the hardware capabilities, which will be parsed
1792  * and saved into struct i40e_hw.
1793  */
1794 static int
1795 i40e_get_cap(struct i40e_hw *hw)
1796 {
1797         struct i40e_aqc_list_capabilities_element_resp *buf;
1798         uint16_t len, size = 0;
1799         int ret;
1800
1801         /* Calculate a huge enough buff for saving response data temporarily */
1802         len = sizeof(struct i40e_aqc_list_capabilities_element_resp) *
1803                                                 I40E_MAX_CAP_ELE_NUM;
1804         buf = rte_zmalloc("i40e", len, 0);
1805         if (!buf) {
1806                 PMD_DRV_LOG(ERR, "Failed to allocate memory");
1807                 return I40E_ERR_NO_MEMORY;
1808         }
1809
1810         /* Get, parse the capabilities and save it to hw */
1811         ret = i40e_aq_discover_capabilities(hw, buf, len, &size,
1812                         i40e_aqc_opc_list_func_capabilities, NULL);
1813         if (ret != I40E_SUCCESS)
1814                 PMD_DRV_LOG(ERR, "Failed to discover capabilities");
1815
1816         /* Free the temporary buffer after being used */
1817         rte_free(buf);
1818
1819         return ret;
1820 }
1821
1822 static int
1823 i40e_pf_parameter_init(struct rte_eth_dev *dev)
1824 {
1825         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1826         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
1827         uint16_t sum_queues = 0, sum_vsis;
1828
1829         /* First check if FW support SRIOV */
1830         if (dev->pci_dev->max_vfs && !hw->func_caps.sr_iov_1_1) {
1831                 PMD_INIT_LOG(ERR, "HW configuration doesn't support SRIOV");
1832                 return -EINVAL;
1833         }
1834
1835         pf->flags = I40E_FLAG_HEADER_SPLIT_DISABLED;
1836         pf->max_num_vsi = RTE_MIN(hw->func_caps.num_vsis, I40E_MAX_NUM_VSIS);
1837         PMD_INIT_LOG(INFO, "Max supported VSIs:%u", pf->max_num_vsi);
1838         /* Allocate queues for pf */
1839         if (hw->func_caps.rss) {
1840                 pf->flags |= I40E_FLAG_RSS;
1841                 pf->lan_nb_qps = RTE_MIN(hw->func_caps.num_tx_qp,
1842                         (uint32_t)(1 << hw->func_caps.rss_table_entry_width));
1843                 pf->lan_nb_qps = i40e_prev_power_of_2(pf->lan_nb_qps);
1844         } else
1845                 pf->lan_nb_qps = 1;
1846         sum_queues = pf->lan_nb_qps;
1847         /* Default VSI is not counted in */
1848         sum_vsis = 0;
1849         PMD_INIT_LOG(INFO, "PF queue pairs:%u", pf->lan_nb_qps);
1850
1851         if (hw->func_caps.sr_iov_1_1 && dev->pci_dev->max_vfs) {
1852                 pf->flags |= I40E_FLAG_SRIOV;
1853                 pf->vf_nb_qps = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VF;
1854                 if (dev->pci_dev->max_vfs > hw->func_caps.num_vfs) {
1855                         PMD_INIT_LOG(ERR, "Config VF number %u, "
1856                                      "max supported %u.",
1857                                      dev->pci_dev->max_vfs,
1858                                      hw->func_caps.num_vfs);
1859                         return -EINVAL;
1860                 }
1861                 if (pf->vf_nb_qps > I40E_MAX_QP_NUM_PER_VF) {
1862                         PMD_INIT_LOG(ERR, "FVL VF queue %u, "
1863                                      "max support %u queues.",
1864                                      pf->vf_nb_qps, I40E_MAX_QP_NUM_PER_VF);
1865                         return -EINVAL;
1866                 }
1867                 pf->vf_num = dev->pci_dev->max_vfs;
1868                 sum_queues += pf->vf_nb_qps * pf->vf_num;
1869                 sum_vsis   += pf->vf_num;
1870                 PMD_INIT_LOG(INFO, "Max VF num:%u each has queue pairs:%u",
1871                              pf->vf_num, pf->vf_nb_qps);
1872         } else
1873                 pf->vf_num = 0;
1874
1875         if (hw->func_caps.vmdq) {
1876                 pf->flags |= I40E_FLAG_VMDQ;
1877                 pf->vmdq_nb_qps = I40E_DEFAULT_QP_NUM_VMDQ;
1878                 sum_queues += pf->vmdq_nb_qps;
1879                 sum_vsis += 1;
1880                 PMD_INIT_LOG(INFO, "VMDQ queue pairs:%u", pf->vmdq_nb_qps);
1881         }
1882
1883         if (hw->func_caps.fd) {
1884                 pf->flags |= I40E_FLAG_FDIR;
1885                 pf->fdir_nb_qps = I40E_DEFAULT_QP_NUM_FDIR;
1886                 /**
1887                  * Each flow director consumes one VSI and one queue,
1888                  * but can't calculate out predictably here.
1889                  */
1890         }
1891
1892         if (sum_vsis > pf->max_num_vsi ||
1893                 sum_queues > hw->func_caps.num_rx_qp) {
1894                 PMD_INIT_LOG(ERR, "VSI/QUEUE setting can't be satisfied");
1895                 PMD_INIT_LOG(ERR, "Max VSIs: %u, asked:%u",
1896                              pf->max_num_vsi, sum_vsis);
1897                 PMD_INIT_LOG(ERR, "Total queue pairs:%u, asked:%u",
1898                              hw->func_caps.num_rx_qp, sum_queues);
1899                 return -EINVAL;
1900         }
1901
1902         /* Each VSI occupy 1 MSIX interrupt at least, plus IRQ0 for misc intr
1903          * cause */
1904         if (sum_vsis > hw->func_caps.num_msix_vectors - 1) {
1905                 PMD_INIT_LOG(ERR, "Too many VSIs(%u), MSIX intr(%u) not enough",
1906                              sum_vsis, hw->func_caps.num_msix_vectors);
1907                 return -EINVAL;
1908         }
1909         return I40E_SUCCESS;
1910 }
1911
1912 static int
1913 i40e_pf_get_switch_config(struct i40e_pf *pf)
1914 {
1915         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
1916         struct i40e_aqc_get_switch_config_resp *switch_config;
1917         struct i40e_aqc_switch_config_element_resp *element;
1918         uint16_t start_seid = 0, num_reported;
1919         int ret;
1920
1921         switch_config = (struct i40e_aqc_get_switch_config_resp *)\
1922                         rte_zmalloc("i40e", I40E_AQ_LARGE_BUF, 0);
1923         if (!switch_config) {
1924                 PMD_DRV_LOG(ERR, "Failed to allocated memory");
1925                 return -ENOMEM;
1926         }
1927
1928         /* Get the switch configurations */
1929         ret = i40e_aq_get_switch_config(hw, switch_config,
1930                 I40E_AQ_LARGE_BUF, &start_seid, NULL);
1931         if (ret != I40E_SUCCESS) {
1932                 PMD_DRV_LOG(ERR, "Failed to get switch configurations");
1933                 goto fail;
1934         }
1935         num_reported = rte_le_to_cpu_16(switch_config->header.num_reported);
1936         if (num_reported != 1) { /* The number should be 1 */
1937                 PMD_DRV_LOG(ERR, "Wrong number of switch config reported");
1938                 goto fail;
1939         }
1940
1941         /* Parse the switch configuration elements */
1942         element = &(switch_config->element[0]);
1943         if (element->element_type == I40E_SWITCH_ELEMENT_TYPE_VSI) {
1944                 pf->mac_seid = rte_le_to_cpu_16(element->uplink_seid);
1945                 pf->main_vsi_seid = rte_le_to_cpu_16(element->seid);
1946         } else
1947                 PMD_DRV_LOG(INFO, "Unknown element type");
1948
1949 fail:
1950         rte_free(switch_config);
1951
1952         return ret;
1953 }
1954
1955 static int
1956 i40e_res_pool_init (struct i40e_res_pool_info *pool, uint32_t base,
1957                         uint32_t num)
1958 {
1959         struct pool_entry *entry;
1960
1961         if (pool == NULL || num == 0)
1962                 return -EINVAL;
1963
1964         entry = rte_zmalloc("i40e", sizeof(*entry), 0);
1965         if (entry == NULL) {
1966                 PMD_DRV_LOG(ERR, "Failed to allocate memory for resource pool");
1967                 return -ENOMEM;
1968         }
1969
1970         /* queue heap initialize */
1971         pool->num_free = num;
1972         pool->num_alloc = 0;
1973         pool->base = base;
1974         LIST_INIT(&pool->alloc_list);
1975         LIST_INIT(&pool->free_list);
1976
1977         /* Initialize element  */
1978         entry->base = 0;
1979         entry->len = num;
1980
1981         LIST_INSERT_HEAD(&pool->free_list, entry, next);
1982         return 0;
1983 }
1984
1985 static void
1986 i40e_res_pool_destroy(struct i40e_res_pool_info *pool)
1987 {
1988         struct pool_entry *entry;
1989
1990         if (pool == NULL)
1991                 return;
1992
1993         LIST_FOREACH(entry, &pool->alloc_list, next) {
1994                 LIST_REMOVE(entry, next);
1995                 rte_free(entry);
1996         }
1997
1998         LIST_FOREACH(entry, &pool->free_list, next) {
1999                 LIST_REMOVE(entry, next);
2000                 rte_free(entry);
2001         }
2002
2003         pool->num_free = 0;
2004         pool->num_alloc = 0;
2005         pool->base = 0;
2006         LIST_INIT(&pool->alloc_list);
2007         LIST_INIT(&pool->free_list);
2008 }
2009
2010 static int
2011 i40e_res_pool_free(struct i40e_res_pool_info *pool,
2012                        uint32_t base)
2013 {
2014         struct pool_entry *entry, *next, *prev, *valid_entry = NULL;
2015         uint32_t pool_offset;
2016         int insert;
2017
2018         if (pool == NULL) {
2019                 PMD_DRV_LOG(ERR, "Invalid parameter");
2020                 return -EINVAL;
2021         }
2022
2023         pool_offset = base - pool->base;
2024         /* Lookup in alloc list */
2025         LIST_FOREACH(entry, &pool->alloc_list, next) {
2026                 if (entry->base == pool_offset) {
2027                         valid_entry = entry;
2028                         LIST_REMOVE(entry, next);
2029                         break;
2030                 }
2031         }
2032
2033         /* Not find, return */
2034         if (valid_entry == NULL) {
2035                 PMD_DRV_LOG(ERR, "Failed to find entry");
2036                 return -EINVAL;
2037         }
2038
2039         /**
2040          * Found it, move it to free list  and try to merge.
2041          * In order to make merge easier, always sort it by qbase.
2042          * Find adjacent prev and last entries.
2043          */
2044         prev = next = NULL;
2045         LIST_FOREACH(entry, &pool->free_list, next) {
2046                 if (entry->base > valid_entry->base) {
2047                         next = entry;
2048                         break;
2049                 }
2050                 prev = entry;
2051         }
2052
2053         insert = 0;
2054         /* Try to merge with next one*/
2055         if (next != NULL) {
2056                 /* Merge with next one */
2057                 if (valid_entry->base + valid_entry->len == next->base) {
2058                         next->base = valid_entry->base;
2059                         next->len += valid_entry->len;
2060                         rte_free(valid_entry);
2061                         valid_entry = next;
2062                         insert = 1;
2063                 }
2064         }
2065
2066         if (prev != NULL) {
2067                 /* Merge with previous one */
2068                 if (prev->base + prev->len == valid_entry->base) {
2069                         prev->len += valid_entry->len;
2070                         /* If it merge with next one, remove next node */
2071                         if (insert == 1) {
2072                                 LIST_REMOVE(valid_entry, next);
2073                                 rte_free(valid_entry);
2074                         } else {
2075                                 rte_free(valid_entry);
2076                                 insert = 1;
2077                         }
2078                 }
2079         }
2080
2081         /* Not find any entry to merge, insert */
2082         if (insert == 0) {
2083                 if (prev != NULL)
2084                         LIST_INSERT_AFTER(prev, valid_entry, next);
2085                 else if (next != NULL)
2086                         LIST_INSERT_BEFORE(next, valid_entry, next);
2087                 else /* It's empty list, insert to head */
2088                         LIST_INSERT_HEAD(&pool->free_list, valid_entry, next);
2089         }
2090
2091         pool->num_free += valid_entry->len;
2092         pool->num_alloc -= valid_entry->len;
2093
2094         return 0;
2095 }
2096
2097 static int
2098 i40e_res_pool_alloc(struct i40e_res_pool_info *pool,
2099                        uint16_t num)
2100 {
2101         struct pool_entry *entry, *valid_entry;
2102
2103         if (pool == NULL || num == 0) {
2104                 PMD_DRV_LOG(ERR, "Invalid parameter");
2105                 return -EINVAL;
2106         }
2107
2108         if (pool->num_free < num) {
2109                 PMD_DRV_LOG(ERR, "No resource. ask:%u, available:%u",
2110                             num, pool->num_free);
2111                 return -ENOMEM;
2112         }
2113
2114         valid_entry = NULL;
2115         /* Lookup  in free list and find most fit one */
2116         LIST_FOREACH(entry, &pool->free_list, next) {
2117                 if (entry->len >= num) {
2118                         /* Find best one */
2119                         if (entry->len == num) {
2120                                 valid_entry = entry;
2121                                 break;
2122                         }
2123                         if (valid_entry == NULL || valid_entry->len > entry->len)
2124                                 valid_entry = entry;
2125                 }
2126         }
2127
2128         /* Not find one to satisfy the request, return */
2129         if (valid_entry == NULL) {
2130                 PMD_DRV_LOG(ERR, "No valid entry found");
2131                 return -ENOMEM;
2132         }
2133         /**
2134          * The entry have equal queue number as requested,
2135          * remove it from alloc_list.
2136          */
2137         if (valid_entry->len == num) {
2138                 LIST_REMOVE(valid_entry, next);
2139         } else {
2140                 /**
2141                  * The entry have more numbers than requested,
2142                  * create a new entry for alloc_list and minus its
2143                  * queue base and number in free_list.
2144                  */
2145                 entry = rte_zmalloc("res_pool", sizeof(*entry), 0);
2146                 if (entry == NULL) {
2147                         PMD_DRV_LOG(ERR, "Failed to allocate memory for "
2148                                     "resource pool");
2149                         return -ENOMEM;
2150                 }
2151                 entry->base = valid_entry->base;
2152                 entry->len = num;
2153                 valid_entry->base += num;
2154                 valid_entry->len -= num;
2155                 valid_entry = entry;
2156         }
2157
2158         /* Insert it into alloc list, not sorted */
2159         LIST_INSERT_HEAD(&pool->alloc_list, valid_entry, next);
2160
2161         pool->num_free -= valid_entry->len;
2162         pool->num_alloc += valid_entry->len;
2163
2164         return (valid_entry->base + pool->base);
2165 }
2166
2167 /**
2168  * bitmap_is_subset - Check whether src2 is subset of src1
2169  **/
2170 static inline int
2171 bitmap_is_subset(uint8_t src1, uint8_t src2)
2172 {
2173         return !((src1 ^ src2) & src2);
2174 }
2175
2176 static int
2177 validate_tcmap_parameter(struct i40e_vsi *vsi, uint8_t enabled_tcmap)
2178 {
2179         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2180
2181         /* If DCB is not supported, only default TC is supported */
2182         if (!hw->func_caps.dcb && enabled_tcmap != I40E_DEFAULT_TCMAP) {
2183                 PMD_DRV_LOG(ERR, "DCB is not enabled, only TC0 is supported");
2184                 return -EINVAL;
2185         }
2186
2187         if (!bitmap_is_subset(hw->func_caps.enabled_tcmap, enabled_tcmap)) {
2188                 PMD_DRV_LOG(ERR, "Enabled TC map 0x%x not applicable to "
2189                             "HW support 0x%x", hw->func_caps.enabled_tcmap,
2190                             enabled_tcmap);
2191                 return -EINVAL;
2192         }
2193         return I40E_SUCCESS;
2194 }
2195
2196 int
2197 i40e_vsi_vlan_pvid_set(struct i40e_vsi *vsi,
2198                                 struct i40e_vsi_vlan_pvid_info *info)
2199 {
2200         struct i40e_hw *hw;
2201         struct i40e_vsi_context ctxt;
2202         uint8_t vlan_flags = 0;
2203         int ret;
2204
2205         if (vsi == NULL || info == NULL) {
2206                 PMD_DRV_LOG(ERR, "invalid parameters");
2207                 return I40E_ERR_PARAM;
2208         }
2209
2210         if (info->on) {
2211                 vsi->info.pvid = info->config.pvid;
2212                 /**
2213                  * If insert pvid is enabled, only tagged pkts are
2214                  * allowed to be sent out.
2215                  */
2216                 vlan_flags |= I40E_AQ_VSI_PVLAN_INSERT_PVID |
2217                                 I40E_AQ_VSI_PVLAN_MODE_TAGGED;
2218         } else {
2219                 vsi->info.pvid = 0;
2220                 if (info->config.reject.tagged == 0)
2221                         vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_TAGGED;
2222
2223                 if (info->config.reject.untagged == 0)
2224                         vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_UNTAGGED;
2225         }
2226         vsi->info.port_vlan_flags &= ~(I40E_AQ_VSI_PVLAN_INSERT_PVID |
2227                                         I40E_AQ_VSI_PVLAN_MODE_MASK);
2228         vsi->info.port_vlan_flags |= vlan_flags;
2229         vsi->info.valid_sections =
2230                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
2231         memset(&ctxt, 0, sizeof(ctxt));
2232         (void)rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
2233         ctxt.seid = vsi->seid;
2234
2235         hw = I40E_VSI_TO_HW(vsi);
2236         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
2237         if (ret != I40E_SUCCESS)
2238                 PMD_DRV_LOG(ERR, "Failed to update VSI params");
2239
2240         return ret;
2241 }
2242
2243 static int
2244 i40e_vsi_update_tc_bandwidth(struct i40e_vsi *vsi, uint8_t enabled_tcmap)
2245 {
2246         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2247         int i, ret;
2248         struct i40e_aqc_configure_vsi_tc_bw_data tc_bw_data;
2249
2250         ret = validate_tcmap_parameter(vsi, enabled_tcmap);
2251         if (ret != I40E_SUCCESS)
2252                 return ret;
2253
2254         if (!vsi->seid) {
2255                 PMD_DRV_LOG(ERR, "seid not valid");
2256                 return -EINVAL;
2257         }
2258
2259         memset(&tc_bw_data, 0, sizeof(tc_bw_data));
2260         tc_bw_data.tc_valid_bits = enabled_tcmap;
2261         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
2262                 tc_bw_data.tc_bw_credits[i] =
2263                         (enabled_tcmap & (1 << i)) ? 1 : 0;
2264
2265         ret = i40e_aq_config_vsi_tc_bw(hw, vsi->seid, &tc_bw_data, NULL);
2266         if (ret != I40E_SUCCESS) {
2267                 PMD_DRV_LOG(ERR, "Failed to configure TC BW");
2268                 return ret;
2269         }
2270
2271         (void)rte_memcpy(vsi->info.qs_handle, tc_bw_data.qs_handles,
2272                                         sizeof(vsi->info.qs_handle));
2273         return I40E_SUCCESS;
2274 }
2275
2276 static int
2277 i40e_vsi_config_tc_queue_mapping(struct i40e_vsi *vsi,
2278                                  struct i40e_aqc_vsi_properties_data *info,
2279                                  uint8_t enabled_tcmap)
2280 {
2281         int ret, total_tc = 0, i;
2282         uint16_t qpnum_per_tc, bsf, qp_idx;
2283
2284         ret = validate_tcmap_parameter(vsi, enabled_tcmap);
2285         if (ret != I40E_SUCCESS)
2286                 return ret;
2287
2288         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
2289                 if (enabled_tcmap & (1 << i))
2290                         total_tc++;
2291         vsi->enabled_tc = enabled_tcmap;
2292
2293         /* Number of queues per enabled TC */
2294         qpnum_per_tc = i40e_prev_power_of_2(vsi->nb_qps / total_tc);
2295         qpnum_per_tc = RTE_MIN(qpnum_per_tc, I40E_MAX_Q_PER_TC);
2296         bsf = rte_bsf32(qpnum_per_tc);
2297
2298         /* Adjust the queue number to actual queues that can be applied */
2299         vsi->nb_qps = qpnum_per_tc * total_tc;
2300
2301         /**
2302          * Configure TC and queue mapping parameters, for enabled TC,
2303          * allocate qpnum_per_tc queues to this traffic. For disabled TC,
2304          * default queue will serve it.
2305          */
2306         qp_idx = 0;
2307         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
2308                 if (vsi->enabled_tc & (1 << i)) {
2309                         info->tc_mapping[i] = rte_cpu_to_le_16((qp_idx <<
2310                                         I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
2311                                 (bsf << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT));
2312                         qp_idx += qpnum_per_tc;
2313                 } else
2314                         info->tc_mapping[i] = 0;
2315         }
2316
2317         /* Associate queue number with VSI */
2318         if (vsi->type == I40E_VSI_SRIOV) {
2319                 info->mapping_flags |=
2320                         rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
2321                 for (i = 0; i < vsi->nb_qps; i++)
2322                         info->queue_mapping[i] =
2323                                 rte_cpu_to_le_16(vsi->base_queue + i);
2324         } else {
2325                 info->mapping_flags |=
2326                         rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_CONTIG);
2327                 info->queue_mapping[0] = rte_cpu_to_le_16(vsi->base_queue);
2328         }
2329         info->valid_sections =
2330                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID);
2331
2332         return I40E_SUCCESS;
2333 }
2334
2335 static int
2336 i40e_veb_release(struct i40e_veb *veb)
2337 {
2338         struct i40e_vsi *vsi;
2339         struct i40e_hw *hw;
2340
2341         if (veb == NULL || veb->associate_vsi == NULL)
2342                 return -EINVAL;
2343
2344         if (!TAILQ_EMPTY(&veb->head)) {
2345                 PMD_DRV_LOG(ERR, "VEB still has VSI attached, can't remove");
2346                 return -EACCES;
2347         }
2348
2349         vsi = veb->associate_vsi;
2350         hw = I40E_VSI_TO_HW(vsi);
2351
2352         vsi->uplink_seid = veb->uplink_seid;
2353         i40e_aq_delete_element(hw, veb->seid, NULL);
2354         rte_free(veb);
2355         vsi->veb = NULL;
2356         return I40E_SUCCESS;
2357 }
2358
2359 /* Setup a veb */
2360 static struct i40e_veb *
2361 i40e_veb_setup(struct i40e_pf *pf, struct i40e_vsi *vsi)
2362 {
2363         struct i40e_veb *veb;
2364         int ret;
2365         struct i40e_hw *hw;
2366
2367         if (NULL == pf || vsi == NULL) {
2368                 PMD_DRV_LOG(ERR, "veb setup failed, "
2369                             "associated VSI shouldn't null");
2370                 return NULL;
2371         }
2372         hw = I40E_PF_TO_HW(pf);
2373
2374         veb = rte_zmalloc("i40e_veb", sizeof(struct i40e_veb), 0);
2375         if (!veb) {
2376                 PMD_DRV_LOG(ERR, "Failed to allocate memory for veb");
2377                 goto fail;
2378         }
2379
2380         veb->associate_vsi = vsi;
2381         TAILQ_INIT(&veb->head);
2382         veb->uplink_seid = vsi->uplink_seid;
2383
2384         ret = i40e_aq_add_veb(hw, veb->uplink_seid, vsi->seid,
2385                 I40E_DEFAULT_TCMAP, false, false, &veb->seid, NULL);
2386
2387         if (ret != I40E_SUCCESS) {
2388                 PMD_DRV_LOG(ERR, "Add veb failed, aq_err: %d",
2389                             hw->aq.asq_last_status);
2390                 goto fail;
2391         }
2392
2393         /* get statistics index */
2394         ret = i40e_aq_get_veb_parameters(hw, veb->seid, NULL, NULL,
2395                                 &veb->stats_idx, NULL, NULL, NULL);
2396         if (ret != I40E_SUCCESS) {
2397                 PMD_DRV_LOG(ERR, "Get veb statics index failed, aq_err: %d",
2398                             hw->aq.asq_last_status);
2399                 goto fail;
2400         }
2401
2402         /* Get VEB bandwidth, to be implemented */
2403         /* Now associated vsi binding to the VEB, set uplink to this VEB */
2404         vsi->uplink_seid = veb->seid;
2405
2406         return veb;
2407 fail:
2408         rte_free(veb);
2409         return NULL;
2410 }
2411
2412 int
2413 i40e_vsi_release(struct i40e_vsi *vsi)
2414 {
2415         struct i40e_pf *pf;
2416         struct i40e_hw *hw;
2417         struct i40e_vsi_list *vsi_list;
2418         int ret;
2419         struct i40e_mac_filter *f;
2420
2421         if (!vsi)
2422                 return I40E_SUCCESS;
2423
2424         pf = I40E_VSI_TO_PF(vsi);
2425         hw = I40E_VSI_TO_HW(vsi);
2426
2427         /* VSI has child to attach, release child first */
2428         if (vsi->veb) {
2429                 TAILQ_FOREACH(vsi_list, &vsi->veb->head, list) {
2430                         if (i40e_vsi_release(vsi_list->vsi) != I40E_SUCCESS)
2431                                 return -1;
2432                         TAILQ_REMOVE(&vsi->veb->head, vsi_list, list);
2433                 }
2434                 i40e_veb_release(vsi->veb);
2435         }
2436
2437         /* Remove all macvlan filters of the VSI */
2438         i40e_vsi_remove_all_macvlan_filter(vsi);
2439         TAILQ_FOREACH(f, &vsi->mac_list, next)
2440                 rte_free(f);
2441
2442         if (vsi->type != I40E_VSI_MAIN) {
2443                 /* Remove vsi from parent's sibling list */
2444                 if (vsi->parent_vsi == NULL || vsi->parent_vsi->veb == NULL) {
2445                         PMD_DRV_LOG(ERR, "VSI's parent VSI is NULL");
2446                         return I40E_ERR_PARAM;
2447                 }
2448                 TAILQ_REMOVE(&vsi->parent_vsi->veb->head,
2449                                 &vsi->sib_vsi_list, list);
2450
2451                 /* Remove all switch element of the VSI */
2452                 ret = i40e_aq_delete_element(hw, vsi->seid, NULL);
2453                 if (ret != I40E_SUCCESS)
2454                         PMD_DRV_LOG(ERR, "Failed to delete element");
2455         }
2456         i40e_res_pool_free(&pf->qp_pool, vsi->base_queue);
2457
2458         if (vsi->type != I40E_VSI_SRIOV)
2459                 i40e_res_pool_free(&pf->msix_pool, vsi->msix_intr);
2460         rte_free(vsi);
2461
2462         return I40E_SUCCESS;
2463 }
2464
2465 static int
2466 i40e_update_default_filter_setting(struct i40e_vsi *vsi)
2467 {
2468         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2469         struct i40e_aqc_remove_macvlan_element_data def_filter;
2470         int ret;
2471
2472         if (vsi->type != I40E_VSI_MAIN)
2473                 return I40E_ERR_CONFIG;
2474         memset(&def_filter, 0, sizeof(def_filter));
2475         (void)rte_memcpy(def_filter.mac_addr, hw->mac.perm_addr,
2476                                         ETH_ADDR_LEN);
2477         def_filter.vlan_tag = 0;
2478         def_filter.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
2479                                 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
2480         ret = i40e_aq_remove_macvlan(hw, vsi->seid, &def_filter, 1, NULL);
2481         if (ret != I40E_SUCCESS) {
2482                 struct i40e_mac_filter *f;
2483
2484                 PMD_DRV_LOG(WARNING, "Cannot remove the default "
2485                             "macvlan filter");
2486                 /* It needs to add the permanent mac into mac list */
2487                 f = rte_zmalloc("macv_filter", sizeof(*f), 0);
2488                 if (f == NULL) {
2489                         PMD_DRV_LOG(ERR, "failed to allocate memory");
2490                         return I40E_ERR_NO_MEMORY;
2491                 }
2492                 (void)rte_memcpy(&f->macaddr.addr_bytes, hw->mac.perm_addr,
2493                                 ETH_ADDR_LEN);
2494                 TAILQ_INSERT_TAIL(&vsi->mac_list, f, next);
2495                 vsi->mac_num++;
2496
2497                 return ret;
2498         }
2499
2500         return i40e_vsi_add_mac(vsi, (struct ether_addr *)(hw->mac.perm_addr));
2501 }
2502
2503 static int
2504 i40e_vsi_dump_bw_config(struct i40e_vsi *vsi)
2505 {
2506         struct i40e_aqc_query_vsi_bw_config_resp bw_config;
2507         struct i40e_aqc_query_vsi_ets_sla_config_resp ets_sla_config;
2508         struct i40e_hw *hw = &vsi->adapter->hw;
2509         i40e_status ret;
2510         int i;
2511
2512         memset(&bw_config, 0, sizeof(bw_config));
2513         ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
2514         if (ret != I40E_SUCCESS) {
2515                 PMD_DRV_LOG(ERR, "VSI failed to get bandwidth configuration %u",
2516                             hw->aq.asq_last_status);
2517                 return ret;
2518         }
2519
2520         memset(&ets_sla_config, 0, sizeof(ets_sla_config));
2521         ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid,
2522                                         &ets_sla_config, NULL);
2523         if (ret != I40E_SUCCESS) {
2524                 PMD_DRV_LOG(ERR, "VSI failed to get TC bandwdith "
2525                             "configuration %u", hw->aq.asq_last_status);
2526                 return ret;
2527         }
2528
2529         /* Not store the info yet, just print out */
2530         PMD_DRV_LOG(INFO, "VSI bw limit:%u", bw_config.port_bw_limit);
2531         PMD_DRV_LOG(INFO, "VSI max_bw:%u", bw_config.max_bw);
2532         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
2533                 PMD_DRV_LOG(INFO, "\tVSI TC%u:share credits %u", i,
2534                             ets_sla_config.share_credits[i]);
2535                 PMD_DRV_LOG(INFO, "\tVSI TC%u:credits %u", i,
2536                             rte_le_to_cpu_16(ets_sla_config.credits[i]));
2537                 PMD_DRV_LOG(INFO, "\tVSI TC%u: max credits: %u", i,
2538                             rte_le_to_cpu_16(ets_sla_config.credits[i / 4]) >>
2539                             (i * 4));
2540         }
2541
2542         return 0;
2543 }
2544
2545 /* Setup a VSI */
2546 struct i40e_vsi *
2547 i40e_vsi_setup(struct i40e_pf *pf,
2548                enum i40e_vsi_type type,
2549                struct i40e_vsi *uplink_vsi,
2550                uint16_t user_param)
2551 {
2552         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
2553         struct i40e_vsi *vsi;
2554         int ret;
2555         struct i40e_vsi_context ctxt;
2556         struct ether_addr broadcast =
2557                 {.addr_bytes = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}};
2558
2559         if (type != I40E_VSI_MAIN && uplink_vsi == NULL) {
2560                 PMD_DRV_LOG(ERR, "VSI setup failed, "
2561                             "VSI link shouldn't be NULL");
2562                 return NULL;
2563         }
2564
2565         if (type == I40E_VSI_MAIN && uplink_vsi != NULL) {
2566                 PMD_DRV_LOG(ERR, "VSI setup failed, MAIN VSI "
2567                             "uplink VSI should be NULL");
2568                 return NULL;
2569         }
2570
2571         /* If uplink vsi didn't setup VEB, create one first */
2572         if (type != I40E_VSI_MAIN && uplink_vsi->veb == NULL) {
2573                 uplink_vsi->veb = i40e_veb_setup(pf, uplink_vsi);
2574
2575                 if (NULL == uplink_vsi->veb) {
2576                         PMD_DRV_LOG(ERR, "VEB setup failed");
2577                         return NULL;
2578                 }
2579         }
2580
2581         vsi = rte_zmalloc("i40e_vsi", sizeof(struct i40e_vsi), 0);
2582         if (!vsi) {
2583                 PMD_DRV_LOG(ERR, "Failed to allocate memory for vsi");
2584                 return NULL;
2585         }
2586         TAILQ_INIT(&vsi->mac_list);
2587         vsi->type = type;
2588         vsi->adapter = I40E_PF_TO_ADAPTER(pf);
2589         vsi->max_macaddrs = I40E_NUM_MACADDR_MAX;
2590         vsi->parent_vsi = uplink_vsi;
2591         vsi->user_param = user_param;
2592         /* Allocate queues */
2593         switch (vsi->type) {
2594         case I40E_VSI_MAIN  :
2595                 vsi->nb_qps = pf->lan_nb_qps;
2596                 break;
2597         case I40E_VSI_SRIOV :
2598                 vsi->nb_qps = pf->vf_nb_qps;
2599                 break;
2600         default:
2601                 goto fail_mem;
2602         }
2603         ret = i40e_res_pool_alloc(&pf->qp_pool, vsi->nb_qps);
2604         if (ret < 0) {
2605                 PMD_DRV_LOG(ERR, "VSI %d allocate queue failed %d",
2606                                 vsi->seid, ret);
2607                 goto fail_mem;
2608         }
2609         vsi->base_queue = ret;
2610
2611         /* VF has MSIX interrupt in VF range, don't allocate here */
2612         if (type != I40E_VSI_SRIOV) {
2613                 ret = i40e_res_pool_alloc(&pf->msix_pool, 1);
2614                 if (ret < 0) {
2615                         PMD_DRV_LOG(ERR, "VSI %d get heap failed %d", vsi->seid, ret);
2616                         goto fail_queue_alloc;
2617                 }
2618                 vsi->msix_intr = ret;
2619         } else
2620                 vsi->msix_intr = 0;
2621         /* Add VSI */
2622         if (type == I40E_VSI_MAIN) {
2623                 /* For main VSI, no need to add since it's default one */
2624                 vsi->uplink_seid = pf->mac_seid;
2625                 vsi->seid = pf->main_vsi_seid;
2626                 /* Bind queues with specific MSIX interrupt */
2627                 /**
2628                  * Needs 2 interrupt at least, one for misc cause which will
2629                  * enabled from OS side, Another for queues binding the
2630                  * interrupt from device side only.
2631                  */
2632
2633                 /* Get default VSI parameters from hardware */
2634                 memset(&ctxt, 0, sizeof(ctxt));
2635                 ctxt.seid = vsi->seid;
2636                 ctxt.pf_num = hw->pf_id;
2637                 ctxt.uplink_seid = vsi->uplink_seid;
2638                 ctxt.vf_num = 0;
2639                 ret = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
2640                 if (ret != I40E_SUCCESS) {
2641                         PMD_DRV_LOG(ERR, "Failed to get VSI params");
2642                         goto fail_msix_alloc;
2643                 }
2644                 (void)rte_memcpy(&vsi->info, &ctxt.info,
2645                         sizeof(struct i40e_aqc_vsi_properties_data));
2646                 vsi->vsi_id = ctxt.vsi_number;
2647                 vsi->info.valid_sections = 0;
2648
2649                 /* Configure tc, enabled TC0 only */
2650                 if (i40e_vsi_update_tc_bandwidth(vsi, I40E_DEFAULT_TCMAP) !=
2651                         I40E_SUCCESS) {
2652                         PMD_DRV_LOG(ERR, "Failed to update TC bandwidth");
2653                         goto fail_msix_alloc;
2654                 }
2655
2656                 /* TC, queue mapping */
2657                 memset(&ctxt, 0, sizeof(ctxt));
2658                 vsi->info.valid_sections |=
2659                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
2660                 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2661                                         I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2662                 (void)rte_memcpy(&ctxt.info, &vsi->info,
2663                         sizeof(struct i40e_aqc_vsi_properties_data));
2664                 ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
2665                                                 I40E_DEFAULT_TCMAP);
2666                 if (ret != I40E_SUCCESS) {
2667                         PMD_DRV_LOG(ERR, "Failed to configure "
2668                                     "TC queue mapping");
2669                         goto fail_msix_alloc;
2670                 }
2671                 ctxt.seid = vsi->seid;
2672                 ctxt.pf_num = hw->pf_id;
2673                 ctxt.uplink_seid = vsi->uplink_seid;
2674                 ctxt.vf_num = 0;
2675
2676                 /* Update VSI parameters */
2677                 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
2678                 if (ret != I40E_SUCCESS) {
2679                         PMD_DRV_LOG(ERR, "Failed to update VSI params");
2680                         goto fail_msix_alloc;
2681                 }
2682
2683                 (void)rte_memcpy(&vsi->info.tc_mapping, &ctxt.info.tc_mapping,
2684                                                 sizeof(vsi->info.tc_mapping));
2685                 (void)rte_memcpy(&vsi->info.queue_mapping,
2686                                 &ctxt.info.queue_mapping,
2687                         sizeof(vsi->info.queue_mapping));
2688                 vsi->info.mapping_flags = ctxt.info.mapping_flags;
2689                 vsi->info.valid_sections = 0;
2690
2691                 (void)rte_memcpy(pf->dev_addr.addr_bytes, hw->mac.perm_addr,
2692                                 ETH_ADDR_LEN);
2693
2694                 /**
2695                  * Updating default filter settings are necessary to prevent
2696                  * reception of tagged packets.
2697                  * Some old firmware configurations load a default macvlan
2698                  * filter which accepts both tagged and untagged packets.
2699                  * The updating is to use a normal filter instead if needed.
2700                  * For NVM 4.2.2 or after, the updating is not needed anymore.
2701                  * The firmware with correct configurations load the default
2702                  * macvlan filter which is expected and cannot be removed.
2703                  */
2704                 i40e_update_default_filter_setting(vsi);
2705         } else if (type == I40E_VSI_SRIOV) {
2706                 memset(&ctxt, 0, sizeof(ctxt));
2707                 /**
2708                  * For other VSI, the uplink_seid equals to uplink VSI's
2709                  * uplink_seid since they share same VEB
2710                  */
2711                 vsi->uplink_seid = uplink_vsi->uplink_seid;
2712                 ctxt.pf_num = hw->pf_id;
2713                 ctxt.vf_num = hw->func_caps.vf_base_id + user_param;
2714                 ctxt.uplink_seid = vsi->uplink_seid;
2715                 ctxt.connection_type = 0x1;
2716                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
2717
2718                 /* Configure switch ID */
2719                 ctxt.info.valid_sections |=
2720                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SWITCH_VALID);
2721                 ctxt.info.switch_id =
2722                         rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
2723                 /* Configure port/vlan */
2724                 ctxt.info.valid_sections |=
2725                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
2726                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
2727                 ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
2728                                                 I40E_DEFAULT_TCMAP);
2729                 if (ret != I40E_SUCCESS) {
2730                         PMD_DRV_LOG(ERR, "Failed to configure "
2731                                     "TC queue mapping");
2732                         goto fail_msix_alloc;
2733                 }
2734                 ctxt.info.up_enable_bits = I40E_DEFAULT_TCMAP;
2735                 ctxt.info.valid_sections |=
2736                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SCHED_VALID);
2737                 /**
2738                  * Since VSI is not created yet, only configure parameter,
2739                  * will add vsi below.
2740                  */
2741         }
2742         else {
2743                 PMD_DRV_LOG(ERR, "VSI: Not support other type VSI yet");
2744                 goto fail_msix_alloc;
2745         }
2746
2747         if (vsi->type != I40E_VSI_MAIN) {
2748                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
2749                 if (ret) {
2750                         PMD_DRV_LOG(ERR, "add vsi failed, aq_err=%d",
2751                                     hw->aq.asq_last_status);
2752                         goto fail_msix_alloc;
2753                 }
2754                 memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
2755                 vsi->info.valid_sections = 0;
2756                 vsi->seid = ctxt.seid;
2757                 vsi->vsi_id = ctxt.vsi_number;
2758                 vsi->sib_vsi_list.vsi = vsi;
2759                 TAILQ_INSERT_TAIL(&uplink_vsi->veb->head,
2760                                 &vsi->sib_vsi_list, list);
2761         }
2762
2763         /* MAC/VLAN configuration */
2764         ret = i40e_vsi_add_mac(vsi, &broadcast);
2765         if (ret != I40E_SUCCESS) {
2766                 PMD_DRV_LOG(ERR, "Failed to add MACVLAN filter");
2767                 goto fail_msix_alloc;
2768         }
2769
2770         /* Get VSI BW information */
2771         i40e_vsi_dump_bw_config(vsi);
2772         return vsi;
2773 fail_msix_alloc:
2774         i40e_res_pool_free(&pf->msix_pool,vsi->msix_intr);
2775 fail_queue_alloc:
2776         i40e_res_pool_free(&pf->qp_pool,vsi->base_queue);
2777 fail_mem:
2778         rte_free(vsi);
2779         return NULL;
2780 }
2781
2782 /* Configure vlan stripping on or off */
2783 int
2784 i40e_vsi_config_vlan_stripping(struct i40e_vsi *vsi, bool on)
2785 {
2786         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2787         struct i40e_vsi_context ctxt;
2788         uint8_t vlan_flags;
2789         int ret = I40E_SUCCESS;
2790
2791         /* Check if it has been already on or off */
2792         if (vsi->info.valid_sections &
2793                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID)) {
2794                 if (on) {
2795                         if ((vsi->info.port_vlan_flags &
2796                                 I40E_AQ_VSI_PVLAN_EMOD_MASK) == 0)
2797                                 return 0; /* already on */
2798                 } else {
2799                         if ((vsi->info.port_vlan_flags &
2800                                 I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
2801                                 I40E_AQ_VSI_PVLAN_EMOD_MASK)
2802                                 return 0; /* already off */
2803                 }
2804         }
2805
2806         if (on)
2807                 vlan_flags = I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2808         else
2809                 vlan_flags = I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
2810         vsi->info.valid_sections =
2811                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
2812         vsi->info.port_vlan_flags &= ~(I40E_AQ_VSI_PVLAN_EMOD_MASK);
2813         vsi->info.port_vlan_flags |= vlan_flags;
2814         ctxt.seid = vsi->seid;
2815         (void)rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
2816         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
2817         if (ret)
2818                 PMD_DRV_LOG(INFO, "Update VSI failed to %s vlan stripping",
2819                             on ? "enable" : "disable");
2820
2821         return ret;
2822 }
2823
2824 static int
2825 i40e_dev_init_vlan(struct rte_eth_dev *dev)
2826 {
2827         struct rte_eth_dev_data *data = dev->data;
2828         int ret;
2829
2830         /* Apply vlan offload setting */
2831         i40e_vlan_offload_set(dev, ETH_VLAN_STRIP_MASK);
2832
2833         /* Apply double-vlan setting, not implemented yet */
2834
2835         /* Apply pvid setting */
2836         ret = i40e_vlan_pvid_set(dev, data->dev_conf.txmode.pvid,
2837                                 data->dev_conf.txmode.hw_vlan_insert_pvid);
2838         if (ret)
2839                 PMD_DRV_LOG(INFO, "Failed to update VSI params");
2840
2841         return ret;
2842 }
2843
2844 static int
2845 i40e_vsi_config_double_vlan(struct i40e_vsi *vsi, int on)
2846 {
2847         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2848
2849         return i40e_aq_set_port_parameters(hw, vsi->seid, 0, 1, on, NULL);
2850 }
2851
2852 static int
2853 i40e_update_flow_control(struct i40e_hw *hw)
2854 {
2855 #define I40E_LINK_PAUSE_RXTX (I40E_AQ_LINK_PAUSE_RX | I40E_AQ_LINK_PAUSE_TX)
2856         struct i40e_link_status link_status;
2857         uint32_t rxfc = 0, txfc = 0, reg;
2858         uint8_t an_info;
2859         int ret;
2860
2861         memset(&link_status, 0, sizeof(link_status));
2862         ret = i40e_aq_get_link_info(hw, FALSE, &link_status, NULL);
2863         if (ret != I40E_SUCCESS) {
2864                 PMD_DRV_LOG(ERR, "Failed to get link status information");
2865                 goto write_reg; /* Disable flow control */
2866         }
2867
2868         an_info = hw->phy.link_info.an_info;
2869         if (!(an_info & I40E_AQ_AN_COMPLETED)) {
2870                 PMD_DRV_LOG(INFO, "Link auto negotiation not completed");
2871                 ret = I40E_ERR_NOT_READY;
2872                 goto write_reg; /* Disable flow control */
2873         }
2874         /**
2875          * If link auto negotiation is enabled, flow control needs to
2876          * be configured according to it
2877          */
2878         switch (an_info & I40E_LINK_PAUSE_RXTX) {
2879         case I40E_LINK_PAUSE_RXTX:
2880                 rxfc = 1;
2881                 txfc = 1;
2882                 hw->fc.current_mode = I40E_FC_FULL;
2883                 break;
2884         case I40E_AQ_LINK_PAUSE_RX:
2885                 rxfc = 1;
2886                 hw->fc.current_mode = I40E_FC_RX_PAUSE;
2887                 break;
2888         case I40E_AQ_LINK_PAUSE_TX:
2889                 txfc = 1;
2890                 hw->fc.current_mode = I40E_FC_TX_PAUSE;
2891                 break;
2892         default:
2893                 hw->fc.current_mode = I40E_FC_NONE;
2894                 break;
2895         }
2896
2897 write_reg:
2898         I40E_WRITE_REG(hw, I40E_PRTDCB_FCCFG,
2899                 txfc << I40E_PRTDCB_FCCFG_TFCE_SHIFT);
2900         reg = I40E_READ_REG(hw, I40E_PRTDCB_MFLCN);
2901         reg &= ~I40E_PRTDCB_MFLCN_RFCE_MASK;
2902         reg |= rxfc << I40E_PRTDCB_MFLCN_RFCE_SHIFT;
2903         I40E_WRITE_REG(hw, I40E_PRTDCB_MFLCN, reg);
2904
2905         return ret;
2906 }
2907
2908 /* PF setup */
2909 static int
2910 i40e_pf_setup(struct i40e_pf *pf)
2911 {
2912         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
2913         struct i40e_filter_control_settings settings;
2914         struct rte_eth_dev_data *dev_data = pf->dev_data;
2915         struct i40e_vsi *vsi;
2916         int ret;
2917
2918         /* Clear all stats counters */
2919         pf->offset_loaded = FALSE;
2920         memset(&pf->stats, 0, sizeof(struct i40e_hw_port_stats));
2921         memset(&pf->stats_offset, 0, sizeof(struct i40e_hw_port_stats));
2922
2923         ret = i40e_pf_get_switch_config(pf);
2924         if (ret != I40E_SUCCESS) {
2925                 PMD_DRV_LOG(ERR, "Could not get switch config, err %d", ret);
2926                 return ret;
2927         }
2928
2929         /* VSI setup */
2930         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, NULL, 0);
2931         if (!vsi) {
2932                 PMD_DRV_LOG(ERR, "Setup of main vsi failed");
2933                 return I40E_ERR_NOT_READY;
2934         }
2935         pf->main_vsi = vsi;
2936         dev_data->nb_rx_queues = vsi->nb_qps;
2937         dev_data->nb_tx_queues = vsi->nb_qps;
2938
2939         /* Configure filter control */
2940         memset(&settings, 0, sizeof(settings));
2941         settings.hash_lut_size = I40E_HASH_LUT_SIZE_128;
2942         /* Enable ethtype and macvlan filters */
2943         settings.enable_ethtype = TRUE;
2944         settings.enable_macvlan = TRUE;
2945         ret = i40e_set_filter_control(hw, &settings);
2946         if (ret)
2947                 PMD_INIT_LOG(WARNING, "setup_pf_filter_control failed: %d",
2948                                                                 ret);
2949
2950         /* Update flow control according to the auto negotiation */
2951         i40e_update_flow_control(hw);
2952
2953         return I40E_SUCCESS;
2954 }
2955
2956 int
2957 i40e_switch_tx_queue(struct i40e_hw *hw, uint16_t q_idx, bool on)
2958 {
2959         uint32_t reg;
2960         uint16_t j;
2961
2962         /**
2963          * Set or clear TX Queue Disable flags,
2964          * which is required by hardware.
2965          */
2966         i40e_pre_tx_queue_cfg(hw, q_idx, on);
2967         rte_delay_us(I40E_PRE_TX_Q_CFG_WAIT_US);
2968
2969         /* Wait until the request is finished */
2970         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
2971                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
2972                 reg = I40E_READ_REG(hw, I40E_QTX_ENA(q_idx));
2973                 if (!(((reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 0x1) ^
2974                         ((reg >> I40E_QTX_ENA_QENA_STAT_SHIFT)
2975                                                         & 0x1))) {
2976                         break;
2977                 }
2978         }
2979         if (on) {
2980                 if (reg & I40E_QTX_ENA_QENA_STAT_MASK)
2981                         return I40E_SUCCESS; /* already on, skip next steps */
2982
2983                 I40E_WRITE_REG(hw, I40E_QTX_HEAD(q_idx), 0);
2984                 reg |= I40E_QTX_ENA_QENA_REQ_MASK;
2985         } else {
2986                 if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK))
2987                         return I40E_SUCCESS; /* already off, skip next steps */
2988                 reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
2989         }
2990         /* Write the register */
2991         I40E_WRITE_REG(hw, I40E_QTX_ENA(q_idx), reg);
2992         /* Check the result */
2993         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
2994                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
2995                 reg = I40E_READ_REG(hw, I40E_QTX_ENA(q_idx));
2996                 if (on) {
2997                         if ((reg & I40E_QTX_ENA_QENA_REQ_MASK) &&
2998                                 (reg & I40E_QTX_ENA_QENA_STAT_MASK))
2999                                 break;
3000                 } else {
3001                         if (!(reg & I40E_QTX_ENA_QENA_REQ_MASK) &&
3002                                 !(reg & I40E_QTX_ENA_QENA_STAT_MASK))
3003                                 break;
3004                 }
3005         }
3006         /* Check if it is timeout */
3007         if (j >= I40E_CHK_Q_ENA_COUNT) {
3008                 PMD_DRV_LOG(ERR, "Failed to %s tx queue[%u]",
3009                             (on ? "enable" : "disable"), q_idx);
3010                 return I40E_ERR_TIMEOUT;
3011         }
3012
3013         return I40E_SUCCESS;
3014 }
3015
3016 /* Swith on or off the tx queues */
3017 static int
3018 i40e_vsi_switch_tx_queues(struct i40e_vsi *vsi, bool on)
3019 {
3020         struct rte_eth_dev_data *dev_data = I40E_VSI_TO_DEV_DATA(vsi);
3021         struct i40e_tx_queue *txq;
3022         struct rte_eth_dev *dev = I40E_VSI_TO_ETH_DEV(vsi);
3023         uint16_t i;
3024         int ret;
3025
3026         for (i = 0; i < dev_data->nb_tx_queues; i++) {
3027                 txq = dev_data->tx_queues[i];
3028                 /* Don't operate the queue if not configured or
3029                  * if starting only per queue */
3030                 if (!txq->q_set || (on && txq->tx_deferred_start))
3031                         continue;
3032                 if (on)
3033                         ret = i40e_dev_tx_queue_start(dev, i);
3034                 else
3035                         ret = i40e_dev_tx_queue_stop(dev, i);
3036                 if ( ret != I40E_SUCCESS)
3037                         return ret;
3038         }
3039
3040         return I40E_SUCCESS;
3041 }
3042
3043 int
3044 i40e_switch_rx_queue(struct i40e_hw *hw, uint16_t q_idx, bool on)
3045 {
3046         uint32_t reg;
3047         uint16_t j;
3048
3049         /* Wait until the request is finished */
3050         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
3051                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
3052                 reg = I40E_READ_REG(hw, I40E_QRX_ENA(q_idx));
3053                 if (!((reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 0x1) ^
3054                         ((reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 0x1))
3055                         break;
3056         }
3057
3058         if (on) {
3059                 if (reg & I40E_QRX_ENA_QENA_STAT_MASK)
3060                         return I40E_SUCCESS; /* Already on, skip next steps */
3061                 reg |= I40E_QRX_ENA_QENA_REQ_MASK;
3062         } else {
3063                 if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK))
3064                         return I40E_SUCCESS; /* Already off, skip next steps */
3065                 reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
3066         }
3067
3068         /* Write the register */
3069         I40E_WRITE_REG(hw, I40E_QRX_ENA(q_idx), reg);
3070         /* Check the result */
3071         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
3072                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
3073                 reg = I40E_READ_REG(hw, I40E_QRX_ENA(q_idx));
3074                 if (on) {
3075                         if ((reg & I40E_QRX_ENA_QENA_REQ_MASK) &&
3076                                 (reg & I40E_QRX_ENA_QENA_STAT_MASK))
3077                                 break;
3078                 } else {
3079                         if (!(reg & I40E_QRX_ENA_QENA_REQ_MASK) &&
3080                                 !(reg & I40E_QRX_ENA_QENA_STAT_MASK))
3081                                 break;
3082                 }
3083         }
3084
3085         /* Check if it is timeout */
3086         if (j >= I40E_CHK_Q_ENA_COUNT) {
3087                 PMD_DRV_LOG(ERR, "Failed to %s rx queue[%u]",
3088                             (on ? "enable" : "disable"), q_idx);
3089                 return I40E_ERR_TIMEOUT;
3090         }
3091
3092         return I40E_SUCCESS;
3093 }
3094 /* Switch on or off the rx queues */
3095 static int
3096 i40e_vsi_switch_rx_queues(struct i40e_vsi *vsi, bool on)
3097 {
3098         struct rte_eth_dev_data *dev_data = I40E_VSI_TO_DEV_DATA(vsi);
3099         struct i40e_rx_queue *rxq;
3100         struct rte_eth_dev *dev = I40E_VSI_TO_ETH_DEV(vsi);
3101         uint16_t i;
3102         int ret;
3103
3104         for (i = 0; i < dev_data->nb_rx_queues; i++) {
3105                 rxq = dev_data->rx_queues[i];
3106                 /* Don't operate the queue if not configured or
3107                  * if starting only per queue */
3108                 if (!rxq->q_set || (on && rxq->rx_deferred_start))
3109                         continue;
3110                 if (on)
3111                         ret = i40e_dev_rx_queue_start(dev, i);
3112                 else
3113                         ret = i40e_dev_rx_queue_stop(dev, i);
3114                 if (ret != I40E_SUCCESS)
3115                         return ret;
3116         }
3117
3118         return I40E_SUCCESS;
3119 }
3120
3121 /* Switch on or off all the rx/tx queues */
3122 int
3123 i40e_vsi_switch_queues(struct i40e_vsi *vsi, bool on)
3124 {
3125         int ret;
3126
3127         if (on) {
3128                 /* enable rx queues before enabling tx queues */
3129                 ret = i40e_vsi_switch_rx_queues(vsi, on);
3130                 if (ret) {
3131                         PMD_DRV_LOG(ERR, "Failed to switch rx queues");
3132                         return ret;
3133                 }
3134                 ret = i40e_vsi_switch_tx_queues(vsi, on);
3135         } else {
3136                 /* Stop tx queues before stopping rx queues */
3137                 ret = i40e_vsi_switch_tx_queues(vsi, on);
3138                 if (ret) {
3139                         PMD_DRV_LOG(ERR, "Failed to switch tx queues");
3140                         return ret;
3141                 }
3142                 ret = i40e_vsi_switch_rx_queues(vsi, on);
3143         }
3144
3145         return ret;
3146 }
3147
3148 /* Initialize VSI for TX */
3149 static int
3150 i40e_vsi_tx_init(struct i40e_vsi *vsi)
3151 {
3152         struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
3153         struct rte_eth_dev_data *data = pf->dev_data;
3154         uint16_t i;
3155         uint32_t ret = I40E_SUCCESS;
3156
3157         for (i = 0; i < data->nb_tx_queues; i++) {
3158                 ret = i40e_tx_queue_init(data->tx_queues[i]);
3159                 if (ret != I40E_SUCCESS)
3160                         break;
3161         }
3162
3163         return ret;
3164 }
3165
3166 /* Initialize VSI for RX */
3167 static int
3168 i40e_vsi_rx_init(struct i40e_vsi *vsi)
3169 {
3170         struct i40e_pf *pf = I40E_VSI_TO_PF(vsi);
3171         struct rte_eth_dev_data *data = pf->dev_data;
3172         int ret = I40E_SUCCESS;
3173         uint16_t i;
3174
3175         i40e_pf_config_mq_rx(pf);
3176         for (i = 0; i < data->nb_rx_queues; i++) {
3177                 ret = i40e_rx_queue_init(data->rx_queues[i]);
3178                 if (ret != I40E_SUCCESS) {
3179                         PMD_DRV_LOG(ERR, "Failed to do RX queue "
3180                                     "initialization");
3181                         break;
3182                 }
3183         }
3184
3185         return ret;
3186 }
3187
3188 /* Initialize VSI */
3189 static int
3190 i40e_vsi_init(struct i40e_vsi *vsi)
3191 {
3192         int err;
3193
3194         err = i40e_vsi_tx_init(vsi);
3195         if (err) {
3196                 PMD_DRV_LOG(ERR, "Failed to do vsi TX initialization");
3197                 return err;
3198         }
3199         err = i40e_vsi_rx_init(vsi);
3200         if (err) {
3201                 PMD_DRV_LOG(ERR, "Failed to do vsi RX initialization");
3202                 return err;
3203         }
3204
3205         return err;
3206 }
3207
3208 static void
3209 i40e_stat_update_32(struct i40e_hw *hw,
3210                    uint32_t reg,
3211                    bool offset_loaded,
3212                    uint64_t *offset,
3213                    uint64_t *stat)
3214 {
3215         uint64_t new_data;
3216
3217         new_data = (uint64_t)I40E_READ_REG(hw, reg);
3218         if (!offset_loaded)
3219                 *offset = new_data;
3220
3221         if (new_data >= *offset)
3222                 *stat = (uint64_t)(new_data - *offset);
3223         else
3224                 *stat = (uint64_t)((new_data +
3225                         ((uint64_t)1 << I40E_32_BIT_SHIFT)) - *offset);
3226 }
3227
3228 static void
3229 i40e_stat_update_48(struct i40e_hw *hw,
3230                    uint32_t hireg,
3231                    uint32_t loreg,
3232                    bool offset_loaded,
3233                    uint64_t *offset,
3234                    uint64_t *stat)
3235 {
3236         uint64_t new_data;
3237
3238         new_data = (uint64_t)I40E_READ_REG(hw, loreg);
3239         new_data |= ((uint64_t)(I40E_READ_REG(hw, hireg) &
3240                         I40E_16_BIT_MASK)) << I40E_32_BIT_SHIFT;
3241
3242         if (!offset_loaded)
3243                 *offset = new_data;
3244
3245         if (new_data >= *offset)
3246                 *stat = new_data - *offset;
3247         else
3248                 *stat = (uint64_t)((new_data +
3249                         ((uint64_t)1 << I40E_48_BIT_SHIFT)) - *offset);
3250
3251         *stat &= I40E_48_BIT_MASK;
3252 }
3253
3254 /* Disable IRQ0 */
3255 void
3256 i40e_pf_disable_irq0(struct i40e_hw *hw)
3257 {
3258         /* Disable all interrupt types */
3259         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, 0);
3260         I40E_WRITE_FLUSH(hw);
3261 }
3262
3263 /* Enable IRQ0 */
3264 void
3265 i40e_pf_enable_irq0(struct i40e_hw *hw)
3266 {
3267         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0,
3268                 I40E_PFINT_DYN_CTL0_INTENA_MASK |
3269                 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
3270                 I40E_PFINT_DYN_CTL0_ITR_INDX_MASK);
3271         I40E_WRITE_FLUSH(hw);
3272 }
3273
3274 static void
3275 i40e_pf_config_irq0(struct i40e_hw *hw)
3276 {
3277         uint32_t enable;
3278
3279         /* read pending request and disable first */
3280         i40e_pf_disable_irq0(hw);
3281         /**
3282          * Enable all interrupt error options to detect possible errors,
3283          * other informative int are ignored
3284          */
3285         enable = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK |
3286                  I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK |
3287                  I40E_PFINT_ICR0_ENA_GRST_MASK |
3288                  I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
3289                  I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK |
3290                  I40E_PFINT_ICR0_ENA_HMC_ERR_MASK |
3291                  I40E_PFINT_ICR0_ENA_VFLR_MASK |
3292                  I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3293
3294         I40E_WRITE_REG(hw, I40E_PFINT_ICR0_ENA, enable);
3295         I40E_WRITE_REG(hw, I40E_PFINT_STAT_CTL0,
3296                 I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_MASK);
3297
3298         /* Link no queues with irq0 */
3299         I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0,
3300                 I40E_PFINT_LNKLST0_FIRSTQ_INDX_MASK);
3301 }
3302
3303 static void
3304 i40e_dev_handle_vfr_event(struct rte_eth_dev *dev)
3305 {
3306         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3307         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3308         int i;
3309         uint16_t abs_vf_id;
3310         uint32_t index, offset, val;
3311
3312         if (!pf->vfs)
3313                 return;
3314         /**
3315          * Try to find which VF trigger a reset, use absolute VF id to access
3316          * since the reg is global register.
3317          */
3318         for (i = 0; i < pf->vf_num; i++) {
3319                 abs_vf_id = hw->func_caps.vf_base_id + i;
3320                 index = abs_vf_id / I40E_UINT32_BIT_SIZE;
3321                 offset = abs_vf_id % I40E_UINT32_BIT_SIZE;
3322                 val = I40E_READ_REG(hw, I40E_GLGEN_VFLRSTAT(index));
3323                 /* VFR event occured */
3324                 if (val & (0x1 << offset)) {
3325                         int ret;
3326
3327                         /* Clear the event first */
3328                         I40E_WRITE_REG(hw, I40E_GLGEN_VFLRSTAT(index),
3329                                                         (0x1 << offset));
3330                         PMD_DRV_LOG(INFO, "VF %u reset occured", abs_vf_id);
3331                         /**
3332                          * Only notify a VF reset event occured,
3333                          * don't trigger another SW reset
3334                          */
3335                         ret = i40e_pf_host_vf_reset(&pf->vfs[i], 0);
3336                         if (ret != I40E_SUCCESS)
3337                                 PMD_DRV_LOG(ERR, "Failed to do VF reset");
3338                 }
3339         }
3340 }
3341
3342 static void
3343 i40e_dev_handle_aq_msg(struct rte_eth_dev *dev)
3344 {
3345         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3346         struct i40e_arq_event_info info;
3347         uint16_t pending, opcode;
3348         int ret;
3349
3350         info.buf_len = I40E_AQ_BUF_SZ;
3351         info.msg_buf = rte_zmalloc("msg_buffer", info.buf_len, 0);
3352         if (!info.msg_buf) {
3353                 PMD_DRV_LOG(ERR, "Failed to allocate mem");
3354                 return;
3355         }
3356
3357         pending = 1;
3358         while (pending) {
3359                 ret = i40e_clean_arq_element(hw, &info, &pending);
3360
3361                 if (ret != I40E_SUCCESS) {
3362                         PMD_DRV_LOG(INFO, "Failed to read msg from AdminQ, "
3363                                     "aq_err: %u", hw->aq.asq_last_status);
3364                         break;
3365                 }
3366                 opcode = rte_le_to_cpu_16(info.desc.opcode);
3367
3368                 switch (opcode) {
3369                 case i40e_aqc_opc_send_msg_to_pf:
3370                         /* Refer to i40e_aq_send_msg_to_pf() for argument layout*/
3371                         i40e_pf_host_handle_vf_msg(dev,
3372                                         rte_le_to_cpu_16(info.desc.retval),
3373                                         rte_le_to_cpu_32(info.desc.cookie_high),
3374                                         rte_le_to_cpu_32(info.desc.cookie_low),
3375                                         info.msg_buf,
3376                                         info.msg_len);
3377                         break;
3378                 default:
3379                         PMD_DRV_LOG(ERR, "Request %u is not supported yet",
3380                                     opcode);
3381                         break;
3382                 }
3383         }
3384         rte_free(info.msg_buf);
3385 }
3386
3387 /**
3388  * Interrupt handler triggered by NIC  for handling
3389  * specific interrupt.
3390  *
3391  * @param handle
3392  *  Pointer to interrupt handle.
3393  * @param param
3394  *  The address of parameter (struct rte_eth_dev *) regsitered before.
3395  *
3396  * @return
3397  *  void
3398  */
3399 static void
3400 i40e_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
3401                            void *param)
3402 {
3403         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
3404         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3405         uint32_t cause, enable;
3406
3407         i40e_pf_disable_irq0(hw);
3408
3409         cause = I40E_READ_REG(hw, I40E_PFINT_ICR0);
3410         enable = I40E_READ_REG(hw, I40E_PFINT_ICR0_ENA);
3411
3412         /* Shared IRQ case, return */
3413         if (!(cause & I40E_PFINT_ICR0_INTEVENT_MASK)) {
3414                 PMD_DRV_LOG(INFO, "Port%d INT0:share IRQ case, "
3415                             "no INT event to process", hw->pf_id);
3416                 goto done;
3417         }
3418
3419         if (cause & I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK) {
3420                 PMD_DRV_LOG(INFO, "INT:Link status changed");
3421                 i40e_dev_link_update(dev, 0);
3422         }
3423
3424         if (cause & I40E_PFINT_ICR0_ECC_ERR_MASK)
3425                 PMD_DRV_LOG(INFO, "INT:Unrecoverable ECC Error");
3426
3427         if (cause & I40E_PFINT_ICR0_MAL_DETECT_MASK)
3428                 PMD_DRV_LOG(INFO, "INT:Malicious programming detected");
3429
3430         if (cause & I40E_PFINT_ICR0_GRST_MASK)
3431                 PMD_DRV_LOG(INFO, "INT:Global Resets Requested");
3432
3433         if (cause & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)
3434                 PMD_DRV_LOG(INFO, "INT:PCI EXCEPTION occured");
3435
3436         if (cause & I40E_PFINT_ICR0_HMC_ERR_MASK)
3437                 PMD_DRV_LOG(INFO, "INT:HMC error occured");
3438
3439         /* Add processing func to deal with VF reset vent */
3440         if (cause & I40E_PFINT_ICR0_VFLR_MASK) {
3441                 PMD_DRV_LOG(INFO, "INT:VF reset detected");
3442                 i40e_dev_handle_vfr_event(dev);
3443         }
3444         /* Find admin queue event */
3445         if (cause & I40E_PFINT_ICR0_ADMINQ_MASK) {
3446                 PMD_DRV_LOG(INFO, "INT:ADMINQ event");
3447                 i40e_dev_handle_aq_msg(dev);
3448         }
3449
3450 done:
3451         I40E_WRITE_REG(hw, I40E_PFINT_ICR0_ENA, enable);
3452         /* Re-enable interrupt from device side */
3453         i40e_pf_enable_irq0(hw);
3454         /* Re-enable interrupt from host side */
3455         rte_intr_enable(&(dev->pci_dev->intr_handle));
3456 }
3457
3458 static int
3459 i40e_add_macvlan_filters(struct i40e_vsi *vsi,
3460                          struct i40e_macvlan_filter *filter,
3461                          int total)
3462 {
3463         int ele_num, ele_buff_size;
3464         int num, actual_num, i;
3465         int ret = I40E_SUCCESS;
3466         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
3467         struct i40e_aqc_add_macvlan_element_data *req_list;
3468
3469         if (filter == NULL  || total == 0)
3470                 return I40E_ERR_PARAM;
3471         ele_num = hw->aq.asq_buf_size / sizeof(*req_list);
3472         ele_buff_size = hw->aq.asq_buf_size;
3473
3474         req_list = rte_zmalloc("macvlan_add", ele_buff_size, 0);
3475         if (req_list == NULL) {
3476                 PMD_DRV_LOG(ERR, "Fail to allocate memory");
3477                 return I40E_ERR_NO_MEMORY;
3478         }
3479
3480         num = 0;
3481         do {
3482                 actual_num = (num + ele_num > total) ? (total - num) : ele_num;
3483                 memset(req_list, 0, ele_buff_size);
3484
3485                 for (i = 0; i < actual_num; i++) {
3486                         (void)rte_memcpy(req_list[i].mac_addr,
3487                                 &filter[num + i].macaddr, ETH_ADDR_LEN);
3488                         req_list[i].vlan_tag =
3489                                 rte_cpu_to_le_16(filter[num + i].vlan_id);
3490                         req_list[i].flags = rte_cpu_to_le_16(\
3491                                 I40E_AQC_MACVLAN_ADD_PERFECT_MATCH);
3492                         req_list[i].queue_number = 0;
3493                 }
3494
3495                 ret = i40e_aq_add_macvlan(hw, vsi->seid, req_list,
3496                                                 actual_num, NULL);
3497                 if (ret != I40E_SUCCESS) {
3498                         PMD_DRV_LOG(ERR, "Failed to add macvlan filter");
3499                         goto DONE;
3500                 }
3501                 num += actual_num;
3502         } while (num < total);
3503
3504 DONE:
3505         rte_free(req_list);
3506         return ret;
3507 }
3508
3509 static int
3510 i40e_remove_macvlan_filters(struct i40e_vsi *vsi,
3511                             struct i40e_macvlan_filter *filter,
3512                             int total)
3513 {
3514         int ele_num, ele_buff_size;
3515         int num, actual_num, i;
3516         int ret = I40E_SUCCESS;
3517         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
3518         struct i40e_aqc_remove_macvlan_element_data *req_list;
3519
3520         if (filter == NULL  || total == 0)
3521                 return I40E_ERR_PARAM;
3522
3523         ele_num = hw->aq.asq_buf_size / sizeof(*req_list);
3524         ele_buff_size = hw->aq.asq_buf_size;
3525
3526         req_list = rte_zmalloc("macvlan_remove", ele_buff_size, 0);
3527         if (req_list == NULL) {
3528                 PMD_DRV_LOG(ERR, "Fail to allocate memory");
3529                 return I40E_ERR_NO_MEMORY;
3530         }
3531
3532         num = 0;
3533         do {
3534                 actual_num = (num + ele_num > total) ? (total - num) : ele_num;
3535                 memset(req_list, 0, ele_buff_size);
3536
3537                 for (i = 0; i < actual_num; i++) {
3538                         (void)rte_memcpy(req_list[i].mac_addr,
3539                                 &filter[num + i].macaddr, ETH_ADDR_LEN);
3540                         req_list[i].vlan_tag =
3541                                 rte_cpu_to_le_16(filter[num + i].vlan_id);
3542                         req_list[i].flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
3543                 }
3544
3545                 ret = i40e_aq_remove_macvlan(hw, vsi->seid, req_list,
3546                                                 actual_num, NULL);
3547                 if (ret != I40E_SUCCESS) {
3548                         PMD_DRV_LOG(ERR, "Failed to remove macvlan filter");
3549                         goto DONE;
3550                 }
3551                 num += actual_num;
3552         } while (num < total);
3553
3554 DONE:
3555         rte_free(req_list);
3556         return ret;
3557 }
3558
3559 /* Find out specific MAC filter */
3560 static struct i40e_mac_filter *
3561 i40e_find_mac_filter(struct i40e_vsi *vsi,
3562                          struct ether_addr *macaddr)
3563 {
3564         struct i40e_mac_filter *f;
3565
3566         TAILQ_FOREACH(f, &vsi->mac_list, next) {
3567                 if (is_same_ether_addr(macaddr, &(f->macaddr)))
3568                         return f;
3569         }
3570
3571         return NULL;
3572 }
3573
3574 static bool
3575 i40e_find_vlan_filter(struct i40e_vsi *vsi,
3576                          uint16_t vlan_id)
3577 {
3578         uint32_t vid_idx, vid_bit;
3579
3580         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
3581         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
3582
3583         if (vsi->vfta[vid_idx] & vid_bit)
3584                 return 1;
3585         else
3586                 return 0;
3587 }
3588
3589 static void
3590 i40e_set_vlan_filter(struct i40e_vsi *vsi,
3591                          uint16_t vlan_id, bool on)
3592 {
3593         uint32_t vid_idx, vid_bit;
3594
3595 #define UINT32_BIT_MASK      0x1F
3596 #define VALID_VLAN_BIT_MASK  0xFFF
3597         /* VFTA is 32-bits size array, each element contains 32 vlan bits, Find the
3598          *  element first, then find the bits it belongs to
3599          */
3600         vid_idx = (uint32_t) ((vlan_id & VALID_VLAN_BIT_MASK) >>
3601                   sizeof(uint32_t));
3602         vid_bit = (uint32_t) (1 << (vlan_id & UINT32_BIT_MASK));
3603
3604         if (on)
3605                 vsi->vfta[vid_idx] |= vid_bit;
3606         else
3607                 vsi->vfta[vid_idx] &= ~vid_bit;
3608 }
3609
3610 /**
3611  * Find all vlan options for specific mac addr,
3612  * return with actual vlan found.
3613  */
3614 static inline int
3615 i40e_find_all_vlan_for_mac(struct i40e_vsi *vsi,
3616                            struct i40e_macvlan_filter *mv_f,
3617                            int num, struct ether_addr *addr)
3618 {
3619         int i;
3620         uint32_t j, k;
3621
3622         /**
3623          * Not to use i40e_find_vlan_filter to decrease the loop time,
3624          * although the code looks complex.
3625           */
3626         if (num < vsi->vlan_num)
3627                 return I40E_ERR_PARAM;
3628
3629         i = 0;
3630         for (j = 0; j < I40E_VFTA_SIZE; j++) {
3631                 if (vsi->vfta[j]) {
3632                         for (k = 0; k < I40E_UINT32_BIT_SIZE; k++) {
3633                                 if (vsi->vfta[j] & (1 << k)) {
3634                                         if (i > num - 1) {
3635                                                 PMD_DRV_LOG(ERR, "vlan number "
3636                                                             "not match");
3637                                                 return I40E_ERR_PARAM;
3638                                         }
3639                                         (void)rte_memcpy(&mv_f[i].macaddr,
3640                                                         addr, ETH_ADDR_LEN);
3641                                         mv_f[i].vlan_id =
3642                                                 j * I40E_UINT32_BIT_SIZE + k;
3643                                         i++;
3644                                 }
3645                         }
3646                 }
3647         }
3648         return I40E_SUCCESS;
3649 }
3650
3651 static inline int
3652 i40e_find_all_mac_for_vlan(struct i40e_vsi *vsi,
3653                            struct i40e_macvlan_filter *mv_f,
3654                            int num,
3655                            uint16_t vlan)
3656 {
3657         int i = 0;
3658         struct i40e_mac_filter *f;
3659
3660         if (num < vsi->mac_num)
3661                 return I40E_ERR_PARAM;
3662
3663         TAILQ_FOREACH(f, &vsi->mac_list, next) {
3664                 if (i > num - 1) {
3665                         PMD_DRV_LOG(ERR, "buffer number not match");
3666                         return I40E_ERR_PARAM;
3667                 }
3668                 (void)rte_memcpy(&mv_f[i].macaddr, &f->macaddr, ETH_ADDR_LEN);
3669                 mv_f[i].vlan_id = vlan;
3670                 i++;
3671         }
3672
3673         return I40E_SUCCESS;
3674 }
3675
3676 static int
3677 i40e_vsi_remove_all_macvlan_filter(struct i40e_vsi *vsi)
3678 {
3679         int i, num;
3680         struct i40e_mac_filter *f;
3681         struct i40e_macvlan_filter *mv_f;
3682         int ret = I40E_SUCCESS;
3683
3684         if (vsi == NULL || vsi->mac_num == 0)
3685                 return I40E_ERR_PARAM;
3686
3687         /* Case that no vlan is set */
3688         if (vsi->vlan_num == 0)
3689                 num = vsi->mac_num;
3690         else
3691                 num = vsi->mac_num * vsi->vlan_num;
3692
3693         mv_f = rte_zmalloc("macvlan_data", num * sizeof(*mv_f), 0);
3694         if (mv_f == NULL) {
3695                 PMD_DRV_LOG(ERR, "failed to allocate memory");
3696                 return I40E_ERR_NO_MEMORY;
3697         }
3698
3699         i = 0;
3700         if (vsi->vlan_num == 0) {
3701                 TAILQ_FOREACH(f, &vsi->mac_list, next) {
3702                         (void)rte_memcpy(&mv_f[i].macaddr,
3703                                 &f->macaddr, ETH_ADDR_LEN);
3704                         mv_f[i].vlan_id = 0;
3705                         i++;
3706                 }
3707         } else {
3708                 TAILQ_FOREACH(f, &vsi->mac_list, next) {
3709                         ret = i40e_find_all_vlan_for_mac(vsi,&mv_f[i],
3710                                         vsi->vlan_num, &f->macaddr);
3711                         if (ret != I40E_SUCCESS)
3712                                 goto DONE;
3713                         i += vsi->vlan_num;
3714                 }
3715         }
3716
3717         ret = i40e_remove_macvlan_filters(vsi, mv_f, num);
3718 DONE:
3719         rte_free(mv_f);
3720
3721         return ret;
3722 }
3723
3724 int
3725 i40e_vsi_add_vlan(struct i40e_vsi *vsi, uint16_t vlan)
3726 {
3727         struct i40e_macvlan_filter *mv_f;
3728         int mac_num;
3729         int ret = I40E_SUCCESS;
3730
3731         if (!vsi || vlan > ETHER_MAX_VLAN_ID)
3732                 return I40E_ERR_PARAM;
3733
3734         /* If it's already set, just return */
3735         if (i40e_find_vlan_filter(vsi,vlan))
3736                 return I40E_SUCCESS;
3737
3738         mac_num = vsi->mac_num;
3739
3740         if (mac_num == 0) {
3741                 PMD_DRV_LOG(ERR, "Error! VSI doesn't have a mac addr");
3742                 return I40E_ERR_PARAM;
3743         }
3744
3745         mv_f = rte_zmalloc("macvlan_data", mac_num * sizeof(*mv_f), 0);
3746
3747         if (mv_f == NULL) {
3748                 PMD_DRV_LOG(ERR, "failed to allocate memory");
3749                 return I40E_ERR_NO_MEMORY;
3750         }
3751
3752         ret = i40e_find_all_mac_for_vlan(vsi, mv_f, mac_num, vlan);
3753
3754         if (ret != I40E_SUCCESS)
3755                 goto DONE;
3756
3757         ret = i40e_add_macvlan_filters(vsi, mv_f, mac_num);
3758
3759         if (ret != I40E_SUCCESS)
3760                 goto DONE;
3761
3762         i40e_set_vlan_filter(vsi, vlan, 1);
3763
3764         vsi->vlan_num++;
3765         ret = I40E_SUCCESS;
3766 DONE:
3767         rte_free(mv_f);
3768         return ret;
3769 }
3770
3771 int
3772 i40e_vsi_delete_vlan(struct i40e_vsi *vsi, uint16_t vlan)
3773 {
3774         struct i40e_macvlan_filter *mv_f;
3775         int mac_num;
3776         int ret = I40E_SUCCESS;
3777
3778         /**
3779          * Vlan 0 is the generic filter for untagged packets
3780          * and can't be removed.
3781          */
3782         if (!vsi || vlan == 0 || vlan > ETHER_MAX_VLAN_ID)
3783                 return I40E_ERR_PARAM;
3784
3785         /* If can't find it, just return */
3786         if (!i40e_find_vlan_filter(vsi, vlan))
3787                 return I40E_ERR_PARAM;
3788
3789         mac_num = vsi->mac_num;
3790
3791         if (mac_num == 0) {
3792                 PMD_DRV_LOG(ERR, "Error! VSI doesn't have a mac addr");
3793                 return I40E_ERR_PARAM;
3794         }
3795
3796         mv_f = rte_zmalloc("macvlan_data", mac_num * sizeof(*mv_f), 0);
3797
3798         if (mv_f == NULL) {
3799                 PMD_DRV_LOG(ERR, "failed to allocate memory");
3800                 return I40E_ERR_NO_MEMORY;
3801         }
3802
3803         ret = i40e_find_all_mac_for_vlan(vsi, mv_f, mac_num, vlan);
3804
3805         if (ret != I40E_SUCCESS)
3806                 goto DONE;
3807
3808         ret = i40e_remove_macvlan_filters(vsi, mv_f, mac_num);
3809
3810         if (ret != I40E_SUCCESS)
3811                 goto DONE;
3812
3813         /* This is last vlan to remove, replace all mac filter with vlan 0 */
3814         if (vsi->vlan_num == 1) {
3815                 ret = i40e_find_all_mac_for_vlan(vsi, mv_f, mac_num, 0);
3816                 if (ret != I40E_SUCCESS)
3817                         goto DONE;
3818
3819                 ret = i40e_add_macvlan_filters(vsi, mv_f, mac_num);
3820                 if (ret != I40E_SUCCESS)
3821                         goto DONE;
3822         }
3823
3824         i40e_set_vlan_filter(vsi, vlan, 0);
3825
3826         vsi->vlan_num--;
3827         ret = I40E_SUCCESS;
3828 DONE:
3829         rte_free(mv_f);
3830         return ret;
3831 }
3832
3833 int
3834 i40e_vsi_add_mac(struct i40e_vsi *vsi, struct ether_addr *addr)
3835 {
3836         struct i40e_mac_filter *f;
3837         struct i40e_macvlan_filter *mv_f;
3838         int vlan_num;
3839         int ret = I40E_SUCCESS;
3840
3841         /* If it's add and we've config it, return */
3842         f = i40e_find_mac_filter(vsi, addr);
3843         if (f != NULL)
3844                 return I40E_SUCCESS;
3845
3846         /**
3847          * If vlan_num is 0, that's the first time to add mac,
3848          * set mask for vlan_id 0.
3849          */
3850         if (vsi->vlan_num == 0) {
3851                 i40e_set_vlan_filter(vsi, 0, 1);
3852                 vsi->vlan_num = 1;
3853         }
3854
3855         vlan_num = vsi->vlan_num;
3856
3857         mv_f = rte_zmalloc("macvlan_data", vlan_num * sizeof(*mv_f), 0);
3858         if (mv_f == NULL) {
3859                 PMD_DRV_LOG(ERR, "failed to allocate memory");
3860                 return I40E_ERR_NO_MEMORY;
3861         }
3862
3863         ret = i40e_find_all_vlan_for_mac(vsi, mv_f, vlan_num, addr);
3864         if (ret != I40E_SUCCESS)
3865                 goto DONE;
3866
3867         ret = i40e_add_macvlan_filters(vsi, mv_f, vlan_num);
3868         if (ret != I40E_SUCCESS)
3869                 goto DONE;
3870
3871         /* Add the mac addr into mac list */
3872         f = rte_zmalloc("macv_filter", sizeof(*f), 0);
3873         if (f == NULL) {
3874                 PMD_DRV_LOG(ERR, "failed to allocate memory");
3875                 ret = I40E_ERR_NO_MEMORY;
3876                 goto DONE;
3877         }
3878         (void)rte_memcpy(&f->macaddr, addr, ETH_ADDR_LEN);
3879         TAILQ_INSERT_TAIL(&vsi->mac_list, f, next);
3880         vsi->mac_num++;
3881
3882         ret = I40E_SUCCESS;
3883 DONE:
3884         rte_free(mv_f);
3885
3886         return ret;
3887 }
3888
3889 int
3890 i40e_vsi_delete_mac(struct i40e_vsi *vsi, struct ether_addr *addr)
3891 {
3892         struct i40e_mac_filter *f;
3893         struct i40e_macvlan_filter *mv_f;
3894         int vlan_num;
3895         int ret = I40E_SUCCESS;
3896
3897         /* Can't find it, return an error */
3898         f = i40e_find_mac_filter(vsi, addr);
3899         if (f == NULL)
3900                 return I40E_ERR_PARAM;
3901
3902         vlan_num = vsi->vlan_num;
3903         if (vlan_num == 0) {
3904                 PMD_DRV_LOG(ERR, "VLAN number shouldn't be 0");
3905                 return I40E_ERR_PARAM;
3906         }
3907         mv_f = rte_zmalloc("macvlan_data", vlan_num * sizeof(*mv_f), 0);
3908         if (mv_f == NULL) {
3909                 PMD_DRV_LOG(ERR, "failed to allocate memory");
3910                 return I40E_ERR_NO_MEMORY;
3911         }
3912
3913         ret = i40e_find_all_vlan_for_mac(vsi, mv_f, vlan_num, addr);
3914         if (ret != I40E_SUCCESS)
3915                 goto DONE;
3916
3917         ret = i40e_remove_macvlan_filters(vsi, mv_f, vlan_num);
3918         if (ret != I40E_SUCCESS)
3919                 goto DONE;
3920
3921         /* Remove the mac addr into mac list */
3922         TAILQ_REMOVE(&vsi->mac_list, f, next);
3923         rte_free(f);
3924         vsi->mac_num--;
3925
3926         ret = I40E_SUCCESS;
3927 DONE:
3928         rte_free(mv_f);
3929         return ret;
3930 }
3931
3932 /* Configure hash enable flags for RSS */
3933 uint64_t
3934 i40e_config_hena(uint64_t flags)
3935 {
3936         uint64_t hena = 0;
3937
3938         if (!flags)
3939                 return hena;
3940
3941         if (flags & ETH_RSS_NONF_IPV4_UDP)
3942                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
3943         if (flags & ETH_RSS_NONF_IPV4_TCP)
3944                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP;
3945         if (flags & ETH_RSS_NONF_IPV4_SCTP)
3946                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP;
3947         if (flags & ETH_RSS_NONF_IPV4_OTHER)
3948                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER;
3949         if (flags & ETH_RSS_FRAG_IPV4)
3950                 hena |= 1ULL << I40E_FILTER_PCTYPE_FRAG_IPV4;
3951         if (flags & ETH_RSS_NONF_IPV6_UDP)
3952                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_UDP;
3953         if (flags & ETH_RSS_NONF_IPV6_TCP)
3954                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP;
3955         if (flags & ETH_RSS_NONF_IPV6_SCTP)
3956                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP;
3957         if (flags & ETH_RSS_NONF_IPV6_OTHER)
3958                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER;
3959         if (flags & ETH_RSS_FRAG_IPV6)
3960                 hena |= 1ULL << I40E_FILTER_PCTYPE_FRAG_IPV6;
3961         if (flags & ETH_RSS_L2_PAYLOAD)
3962                 hena |= 1ULL << I40E_FILTER_PCTYPE_L2_PAYLOAD;
3963
3964         return hena;
3965 }
3966
3967 /* Parse the hash enable flags */
3968 uint64_t
3969 i40e_parse_hena(uint64_t flags)
3970 {
3971         uint64_t rss_hf = 0;
3972
3973         if (!flags)
3974                 return rss_hf;
3975
3976         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP))
3977                 rss_hf |= ETH_RSS_NONF_IPV4_UDP;
3978         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP))
3979                 rss_hf |= ETH_RSS_NONF_IPV4_TCP;
3980         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP))
3981                 rss_hf |= ETH_RSS_NONF_IPV4_SCTP;
3982         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER))
3983                 rss_hf |= ETH_RSS_NONF_IPV4_OTHER;
3984         if (flags & (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV4))
3985                 rss_hf |= ETH_RSS_FRAG_IPV4;
3986         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_UDP))
3987                 rss_hf |= ETH_RSS_NONF_IPV6_UDP;
3988         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP))
3989                 rss_hf |= ETH_RSS_NONF_IPV6_TCP;
3990         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP))
3991                 rss_hf |= ETH_RSS_NONF_IPV6_SCTP;
3992         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER))
3993                 rss_hf |= ETH_RSS_NONF_IPV6_OTHER;
3994         if (flags & (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV6))
3995                 rss_hf |= ETH_RSS_FRAG_IPV6;
3996         if (flags & (1ULL << I40E_FILTER_PCTYPE_L2_PAYLOAD))
3997                 rss_hf |= ETH_RSS_L2_PAYLOAD;
3998
3999         return rss_hf;
4000 }
4001
4002 /* Disable RSS */
4003 static void
4004 i40e_pf_disable_rss(struct i40e_pf *pf)
4005 {
4006         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
4007         uint64_t hena;
4008
4009         hena = (uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(0));
4010         hena |= ((uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(1))) << 32;
4011         hena &= ~I40E_RSS_HENA_ALL;
4012         I40E_WRITE_REG(hw, I40E_PFQF_HENA(0), (uint32_t)hena);
4013         I40E_WRITE_REG(hw, I40E_PFQF_HENA(1), (uint32_t)(hena >> 32));
4014         I40E_WRITE_FLUSH(hw);
4015 }
4016
4017 static int
4018 i40e_hw_rss_hash_set(struct i40e_hw *hw, struct rte_eth_rss_conf *rss_conf)
4019 {
4020         uint32_t *hash_key;
4021         uint8_t hash_key_len;
4022         uint64_t rss_hf;
4023         uint16_t i;
4024         uint64_t hena;
4025
4026         hash_key = (uint32_t *)(rss_conf->rss_key);
4027         hash_key_len = rss_conf->rss_key_len;
4028         if (hash_key != NULL && hash_key_len >=
4029                 (I40E_PFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t)) {
4030                 /* Fill in RSS hash key */
4031                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
4032                         I40E_WRITE_REG(hw, I40E_PFQF_HKEY(i), hash_key[i]);
4033         }
4034
4035         rss_hf = rss_conf->rss_hf;
4036         hena = (uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(0));
4037         hena |= ((uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(1))) << 32;
4038         hena &= ~I40E_RSS_HENA_ALL;
4039         hena |= i40e_config_hena(rss_hf);
4040         I40E_WRITE_REG(hw, I40E_PFQF_HENA(0), (uint32_t)hena);
4041         I40E_WRITE_REG(hw, I40E_PFQF_HENA(1), (uint32_t)(hena >> 32));
4042         I40E_WRITE_FLUSH(hw);
4043
4044         return 0;
4045 }
4046
4047 static int
4048 i40e_dev_rss_hash_update(struct rte_eth_dev *dev,
4049                          struct rte_eth_rss_conf *rss_conf)
4050 {
4051         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4052         uint64_t rss_hf = rss_conf->rss_hf & I40E_RSS_OFFLOAD_ALL;
4053         uint64_t hena;
4054
4055         hena = (uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(0));
4056         hena |= ((uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(1))) << 32;
4057         if (!(hena & I40E_RSS_HENA_ALL)) { /* RSS disabled */
4058                 if (rss_hf != 0) /* Enable RSS */
4059                         return -EINVAL;
4060                 return 0; /* Nothing to do */
4061         }
4062         /* RSS enabled */
4063         if (rss_hf == 0) /* Disable RSS */
4064                 return -EINVAL;
4065
4066         return i40e_hw_rss_hash_set(hw, rss_conf);
4067 }
4068
4069 static int
4070 i40e_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
4071                            struct rte_eth_rss_conf *rss_conf)
4072 {
4073         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4074         uint32_t *hash_key = (uint32_t *)(rss_conf->rss_key);
4075         uint64_t hena;
4076         uint16_t i;
4077
4078         if (hash_key != NULL) {
4079                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
4080                         hash_key[i] = I40E_READ_REG(hw, I40E_PFQF_HKEY(i));
4081                 rss_conf->rss_key_len = i * sizeof(uint32_t);
4082         }
4083         hena = (uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(0));
4084         hena |= ((uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(1))) << 32;
4085         rss_conf->rss_hf = i40e_parse_hena(hena);
4086
4087         return 0;
4088 }
4089
4090 /* Configure RSS */
4091 static int
4092 i40e_pf_config_rss(struct i40e_pf *pf)
4093 {
4094         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
4095         struct rte_eth_rss_conf rss_conf;
4096         uint32_t i, lut = 0;
4097         uint16_t j, num = i40e_prev_power_of_2(pf->dev_data->nb_rx_queues);
4098
4099         for (i = 0, j = 0; i < hw->func_caps.rss_table_size; i++, j++) {
4100                 if (j == num)
4101                         j = 0;
4102                 lut = (lut << 8) | (j & ((0x1 <<
4103                         hw->func_caps.rss_table_entry_width) - 1));
4104                 if ((i & 3) == 3)
4105                         I40E_WRITE_REG(hw, I40E_PFQF_HLUT(i >> 2), lut);
4106         }
4107
4108         rss_conf = pf->dev_data->dev_conf.rx_adv_conf.rss_conf;
4109         if ((rss_conf.rss_hf & I40E_RSS_OFFLOAD_ALL) == 0) {
4110                 i40e_pf_disable_rss(pf);
4111                 return 0;
4112         }
4113         if (rss_conf.rss_key == NULL || rss_conf.rss_key_len <
4114                 (I40E_PFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t)) {
4115                 /* Calculate the default hash key */
4116                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
4117                         rss_key_default[i] = (uint32_t)rte_rand();
4118                 rss_conf.rss_key = (uint8_t *)rss_key_default;
4119                 rss_conf.rss_key_len = (I40E_PFQF_HKEY_MAX_INDEX + 1) *
4120                                                         sizeof(uint32_t);
4121         }
4122
4123         return i40e_hw_rss_hash_set(hw, &rss_conf);
4124 }
4125
4126 static int
4127 i40e_pf_config_mq_rx(struct i40e_pf *pf)
4128 {
4129         if (!pf->dev_data->sriov.active) {
4130                 switch (pf->dev_data->dev_conf.rxmode.mq_mode) {
4131                 case ETH_MQ_RX_RSS:
4132                         i40e_pf_config_rss(pf);
4133                         break;
4134                 default:
4135                         i40e_pf_disable_rss(pf);
4136                         break;
4137                 }
4138         }
4139
4140         return 0;
4141 }