lib/librte_pmd_i40e/i40e_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <inttypes.h>
  42
  43 #include <rte_string_fns.h>
  44 #include <rte_pci.h>
  45 #include <rte_ether.h>
  46 #include <rte_ethdev.h>
  47 #include <rte_memzone.h>
  48 #include <rte_malloc.h>
  49 #include <rte_memcpy.h>
  50 #include <rte_alarm.h>
  51 #include <rte_dev.h>
  52 #include <rte_eth_ctrl.h>
  53
  54 #include "i40e_logs.h"
  55 #include "i40e/i40e_prototype.h"
  56 #include "i40e/i40e_adminq_cmd.h"
  57 #include "i40e/i40e_type.h"
  58 #include "i40e_ethdev.h"
  59 #include "i40e_rxtx.h"
  60 #include "i40e_pf.h"
  61
  62 /* Maximun number of MAC addresses */
  63 #define I40E_NUM_MACADDR_MAX       64
  64 #define I40E_CLEAR_PXE_WAIT_MS     200
  65
  66 /* Maximun number of capability elements */
  67 #define I40E_MAX_CAP_ELE_NUM       128
  68
  69 /* Wait count and inteval */
  70 #define I40E_CHK_Q_ENA_COUNT       1000
  71 #define I40E_CHK_Q_ENA_INTERVAL_US 1000
  72
  73 /* Maximun number of VSI */
  74 #define I40E_MAX_NUM_VSIS          (384UL)
  75
  76 /* Default queue interrupt throttling time in microseconds*/
  77 #define I40E_ITR_INDEX_DEFAULT          0
  78 #define I40E_QUEUE_ITR_INTERVAL_DEFAULT 32 /* 32 us */
  79 #define I40E_QUEUE_ITR_INTERVAL_MAX     8160 /* 8160 us */
  80
  81 #define I40E_PRE_TX_Q_CFG_WAIT_US       10 /* 10 us */
  82
  83 /* Mask of PF interrupt causes */
  84 #define I40E_PFINT_ICR0_ENA_MASK ( \
  85                 I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | \
  86                 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | \
  87                 I40E_PFINT_ICR0_ENA_GRST_MASK | \
  88                 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | \
  89                 I40E_PFINT_ICR0_ENA_STORM_DETECT_MASK | \
  90                 I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK | \
  91                 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | \
  92                 I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK | \
  93                 I40E_PFINT_ICR0_ENA_VFLR_MASK | \
  94                 I40E_PFINT_ICR0_ENA_ADMINQ_MASK)
  95
  96 static int eth_i40e_dev_init(\
  97                         __attribute__((unused)) struct eth_driver *eth_drv,
  98                         struct rte_eth_dev *eth_dev);
  99 static int i40e_dev_configure(struct rte_eth_dev *dev);
 100 static int i40e_dev_start(struct rte_eth_dev *dev);
 101 static void i40e_dev_stop(struct rte_eth_dev *dev);
 102 static void i40e_dev_close(struct rte_eth_dev *dev);
 103 static void i40e_dev_promiscuous_enable(struct rte_eth_dev *dev);
 104 static void i40e_dev_promiscuous_disable(struct rte_eth_dev *dev);
 105 static void i40e_dev_allmulticast_enable(struct rte_eth_dev *dev);
 106 static void i40e_dev_allmulticast_disable(struct rte_eth_dev *dev);
 107 static int i40e_dev_set_link_up(struct rte_eth_dev *dev);
 108 static int i40e_dev_set_link_down(struct rte_eth_dev *dev);
 109 static void i40e_dev_stats_get(struct rte_eth_dev *dev,
 110                                struct rte_eth_stats *stats);
 111 static void i40e_dev_stats_reset(struct rte_eth_dev *dev);
 112 static int i40e_dev_queue_stats_mapping_set(struct rte_eth_dev *dev,
 113                                             uint16_t queue_id,
 114                                             uint8_t stat_idx,
 115                                             uint8_t is_rx);
 116 static void i40e_dev_info_get(struct rte_eth_dev *dev,
 117                               struct rte_eth_dev_info *dev_info);
 118 static int i40e_vlan_filter_set(struct rte_eth_dev *dev,
 119                                 uint16_t vlan_id,
 120                                 int on);
 121 static void i40e_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid);
 122 static void i40e_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 123 static void i40e_vlan_strip_queue_set(struct rte_eth_dev *dev,
 124                                       uint16_t queue,
 125                                       int on);
 126 static int i40e_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid, int on);
 127 static int i40e_dev_led_on(struct rte_eth_dev *dev);
 128 static int i40e_dev_led_off(struct rte_eth_dev *dev);
 129 static int i40e_flow_ctrl_set(struct rte_eth_dev *dev,
 130                               struct rte_eth_fc_conf *fc_conf);
 131 static int i40e_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 132                                        struct rte_eth_pfc_conf *pfc_conf);
 133 static void i40e_macaddr_add(struct rte_eth_dev *dev,
 134                           struct ether_addr *mac_addr,
 135                           uint32_t index,
 136                           uint32_t pool);
 137 static void i40e_macaddr_remove(struct rte_eth_dev *dev, uint32_t index);
 138 static int i40e_dev_rss_reta_update(struct rte_eth_dev *dev,
 139                                     struct rte_eth_rss_reta_entry64 *reta_conf,
 140                                     uint16_t reta_size);
 141 static int i40e_dev_rss_reta_query(struct rte_eth_dev *dev,
 142                                    struct rte_eth_rss_reta_entry64 *reta_conf,
 143                                    uint16_t reta_size);
 144
 145 static int i40e_get_cap(struct i40e_hw *hw);
 146 static int i40e_pf_parameter_init(struct rte_eth_dev *dev);
 147 static int i40e_pf_setup(struct i40e_pf *pf);
 148 static int i40e_dev_rxtx_init(struct i40e_pf *pf);
 149 static int i40e_vmdq_setup(struct rte_eth_dev *dev);
 150 static void i40e_stat_update_32(struct i40e_hw *hw, uint32_t reg,
 151                 bool offset_loaded, uint64_t *offset, uint64_t *stat);
 152 static void i40e_stat_update_48(struct i40e_hw *hw,
 153                                uint32_t hireg,
 154                                uint32_t loreg,
 155                                bool offset_loaded,
 156                                uint64_t *offset,
 157                                uint64_t *stat);
 158 static void i40e_pf_config_irq0(struct i40e_hw *hw);
 159 static void i40e_dev_interrupt_handler(
 160                 __rte_unused struct rte_intr_handle *handle, void *param);
 161 static int i40e_res_pool_init(struct i40e_res_pool_info *pool,
 162                                 uint32_t base, uint32_t num);
 163 static void i40e_res_pool_destroy(struct i40e_res_pool_info *pool);
 164 static int i40e_res_pool_free(struct i40e_res_pool_info *pool,
 165                         uint32_t base);
 166 static int i40e_res_pool_alloc(struct i40e_res_pool_info *pool,
 167                         uint16_t num);
 168 static int i40e_dev_init_vlan(struct rte_eth_dev *dev);
 169 static int i40e_veb_release(struct i40e_veb *veb);
 170 static struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf,
 171                                                 struct i40e_vsi *vsi);
 172 static int i40e_pf_config_mq_rx(struct i40e_pf *pf);
 173 static int i40e_vsi_config_double_vlan(struct i40e_vsi *vsi, int on);
 174 static inline int i40e_find_all_vlan_for_mac(struct i40e_vsi *vsi,
 175                                              struct i40e_macvlan_filter *mv_f,
 176                                              int num,
 177                                              struct ether_addr *addr);
 178 static inline int i40e_find_all_mac_for_vlan(struct i40e_vsi *vsi,
 179                                              struct i40e_macvlan_filter *mv_f,
 180                                              int num,
 181                                              uint16_t vlan);
 182 static int i40e_vsi_remove_all_macvlan_filter(struct i40e_vsi *vsi);
 183 static int i40e_dev_rss_hash_update(struct rte_eth_dev *dev,
 184                                     struct rte_eth_rss_conf *rss_conf);
 185 static int i40e_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
 186                                       struct rte_eth_rss_conf *rss_conf);
 187 static int i40e_dev_udp_tunnel_add(struct rte_eth_dev *dev,
 188                                 struct rte_eth_udp_tunnel *udp_tunnel);
 189 static int i40e_dev_udp_tunnel_del(struct rte_eth_dev *dev,
 190                                 struct rte_eth_udp_tunnel *udp_tunnel);
 191 static int i40e_ethertype_filter_set(struct i40e_pf *pf,
 192                         struct rte_eth_ethertype_filter *filter,
 193                         bool add);
 194 static int i40e_ethertype_filter_handle(struct rte_eth_dev *dev,
 195                                 enum rte_filter_op filter_op,
 196                                 void *arg);
 197 static int i40e_dev_filter_ctrl(struct rte_eth_dev *dev,
 198                                 enum rte_filter_type filter_type,
 199                                 enum rte_filter_op filter_op,
 200                                 void *arg);
 201
 202 /* Default hash key buffer for RSS */
 203 static uint32_t rss_key_default[I40E_PFQF_HKEY_MAX_INDEX + 1];
 204
 205 static struct rte_pci_id pci_id_i40e_map[] = {
 206 #define RTE_PCI_DEV_ID_DECL_I40E(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 207 #include "rte_pci_dev_ids.h"
 208 { .vendor_id = 0, /* sentinel */ },
 209 };
 210
 211 static struct eth_dev_ops i40e_eth_dev_ops = {
 212         .dev_configure                = i40e_dev_configure,
 213         .dev_start                    = i40e_dev_start,
 214         .dev_stop                     = i40e_dev_stop,
 215         .dev_close                    = i40e_dev_close,
 216         .promiscuous_enable           = i40e_dev_promiscuous_enable,
 217         .promiscuous_disable          = i40e_dev_promiscuous_disable,
 218         .allmulticast_enable          = i40e_dev_allmulticast_enable,
 219         .allmulticast_disable         = i40e_dev_allmulticast_disable,
 220         .dev_set_link_up              = i40e_dev_set_link_up,
 221         .dev_set_link_down            = i40e_dev_set_link_down,
 222         .link_update                  = i40e_dev_link_update,
 223         .stats_get                    = i40e_dev_stats_get,
 224         .stats_reset                  = i40e_dev_stats_reset,
 225         .queue_stats_mapping_set      = i40e_dev_queue_stats_mapping_set,
 226         .dev_infos_get                = i40e_dev_info_get,
 227         .vlan_filter_set              = i40e_vlan_filter_set,
 228         .vlan_tpid_set                = i40e_vlan_tpid_set,
 229         .vlan_offload_set             = i40e_vlan_offload_set,
 230         .vlan_strip_queue_set         = i40e_vlan_strip_queue_set,
 231         .vlan_pvid_set                = i40e_vlan_pvid_set,
 232         .rx_queue_start               = i40e_dev_rx_queue_start,
 233         .rx_queue_stop                = i40e_dev_rx_queue_stop,
 234         .tx_queue_start               = i40e_dev_tx_queue_start,
 235         .tx_queue_stop                = i40e_dev_tx_queue_stop,
 236         .rx_queue_setup               = i40e_dev_rx_queue_setup,
 237         .rx_queue_release             = i40e_dev_rx_queue_release,
 238         .rx_queue_count               = i40e_dev_rx_queue_count,
 239         .rx_descriptor_done           = i40e_dev_rx_descriptor_done,
 240         .tx_queue_setup               = i40e_dev_tx_queue_setup,
 241         .tx_queue_release             = i40e_dev_tx_queue_release,
 242         .dev_led_on                   = i40e_dev_led_on,
 243         .dev_led_off                  = i40e_dev_led_off,
 244         .flow_ctrl_set                = i40e_flow_ctrl_set,
 245         .priority_flow_ctrl_set       = i40e_priority_flow_ctrl_set,
 246         .mac_addr_add                 = i40e_macaddr_add,
 247         .mac_addr_remove              = i40e_macaddr_remove,
 248         .reta_update                  = i40e_dev_rss_reta_update,
 249         .reta_query                   = i40e_dev_rss_reta_query,
 250         .rss_hash_update              = i40e_dev_rss_hash_update,
 251         .rss_hash_conf_get            = i40e_dev_rss_hash_conf_get,
 252         .udp_tunnel_add               = i40e_dev_udp_tunnel_add,
 253         .udp_tunnel_del               = i40e_dev_udp_tunnel_del,
 254         .filter_ctrl                  = i40e_dev_filter_ctrl,
 255 };
 256
 257 static struct eth_driver rte_i40e_pmd = {
 258         {
 259                 .name = "rte_i40e_pmd",
 260                 .id_table = pci_id_i40e_map,
 261                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
 262         },
 263         .eth_dev_init = eth_i40e_dev_init,
 264         .dev_private_size = sizeof(struct i40e_adapter),
 265 };
 266
 267 static inline int
 268 i40e_align_floor(int n)
 269 {
 270         if (n == 0)
 271                 return 0;
 272         return (1 << (sizeof(n) * CHAR_BIT - 1 - __builtin_clz(n)));
 273 }
 274
 275 static inline int
 276 rte_i40e_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 277                                      struct rte_eth_link *link)
 278 {
 279         struct rte_eth_link *dst = link;
 280         struct rte_eth_link *src = &(dev->data->dev_link);
 281
 282         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 283                                         *(uint64_t *)src) == 0)
 284                 return -1;
 285
 286         return 0;
 287 }
 288
 289 static inline int
 290 rte_i40e_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 291                                       struct rte_eth_link *link)
 292 {
 293         struct rte_eth_link *dst = &(dev->data->dev_link);
 294         struct rte_eth_link *src = link;
 295
 296         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 297                                         *(uint64_t *)src) == 0)
 298                 return -1;
 299
 300         return 0;
 301 }
 302
 303 /*
 304  * Driver initialization routine.
 305  * Invoked once at EAL init time.
 306  * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
 307  */
 308 static int
 309 rte_i40e_pmd_init(const char *name __rte_unused,
 310                   const char *params __rte_unused)
 311 {
 312         PMD_INIT_FUNC_TRACE();
 313         rte_eth_driver_register(&rte_i40e_pmd);
 314
 315         return 0;
 316 }
 317
 318 static struct rte_driver rte_i40e_driver = {
 319         .type = PMD_PDEV,
 320         .init = rte_i40e_pmd_init,
 321 };
 322
 323 PMD_REGISTER_DRIVER(rte_i40e_driver);
 324
 325 /*
 326  * Initialize registers for flexible payload, which should be set by NVM.
 327  * This should be removed from code once it is fixed in NVM.
 328  */
 329 #ifndef I40E_GLQF_ORT
 330 #define I40E_GLQF_ORT(_i)    (0x00268900 + ((_i) * 4))
 331 #endif
 332 #ifndef I40E_GLQF_PIT
 333 #define I40E_GLQF_PIT(_i)    (0x00268C80 + ((_i) * 4))
 334 #endif
 335
 336 static inline void i40e_flex_payload_reg_init(struct i40e_hw *hw)
 337 {
 338         I40E_WRITE_REG(hw, I40E_GLQF_ORT(18), 0x00000030);
 339         I40E_WRITE_REG(hw, I40E_GLQF_ORT(19), 0x00000030);
 340         I40E_WRITE_REG(hw, I40E_GLQF_ORT(26), 0x0000002B);
 341         I40E_WRITE_REG(hw, I40E_GLQF_ORT(30), 0x0000002B);
 342         I40E_WRITE_REG(hw, I40E_GLQF_ORT(33), 0x000000E0);
 343         I40E_WRITE_REG(hw, I40E_GLQF_ORT(34), 0x000000E3);
 344         I40E_WRITE_REG(hw, I40E_GLQF_ORT(35), 0x000000E6);
 345         I40E_WRITE_REG(hw, I40E_GLQF_ORT(20), 0x00000031);
 346         I40E_WRITE_REG(hw, I40E_GLQF_ORT(23), 0x00000031);
 347         I40E_WRITE_REG(hw, I40E_GLQF_ORT(63), 0x0000002D);
 348
 349         /* GLQF_PIT Registers */
 350         I40E_WRITE_REG(hw, I40E_GLQF_PIT(16), 0x00007480);
 351         I40E_WRITE_REG(hw, I40E_GLQF_PIT(17), 0x00007440);
 352 }
 353
 354 static int
 355 eth_i40e_dev_init(__rte_unused struct eth_driver *eth_drv,
 356                   struct rte_eth_dev *dev)
 357 {
 358         struct rte_pci_device *pci_dev;
 359         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 360         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 361         struct i40e_vsi *vsi;
 362         int ret;
 363         uint32_t len;
 364         uint8_t aq_fail = 0;
 365
 366         PMD_INIT_FUNC_TRACE();
 367
 368         dev->dev_ops = &i40e_eth_dev_ops;
 369         dev->rx_pkt_burst = i40e_recv_pkts;
 370         dev->tx_pkt_burst = i40e_xmit_pkts;
 371
 372         /* for secondary processes, we don't initialise any further as primary
 373          * has already done this work. Only check we don't need a different
 374          * RX function */
 375         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 376                 if (dev->data->scattered_rx)
 377                         dev->rx_pkt_burst = i40e_recv_scattered_pkts;
 378                 return 0;
 379         }
 380         pci_dev = dev->pci_dev;
 381         pf->adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 382         pf->adapter->eth_dev = dev;
 383         pf->dev_data = dev->data;
 384
 385         hw->back = I40E_PF_TO_ADAPTER(pf);
 386         hw->hw_addr = (uint8_t *)(pci_dev->mem_resource[0].addr);
 387         if (!hw->hw_addr) {
 388                 PMD_INIT_LOG(ERR, "Hardware is not available, "
 389                              "as address is NULL");
 390                 return -ENODEV;
 391         }
 392
 393         hw->vendor_id = pci_dev->id.vendor_id;
 394         hw->device_id = pci_dev->id.device_id;
 395         hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
 396         hw->subsystem_device_id = pci_dev->id.subsystem_device_id;
 397         hw->bus.device = pci_dev->addr.devid;
 398         hw->bus.func = pci_dev->addr.function;
 399
 400         /* Make sure all is clean before doing PF reset */
 401         i40e_clear_hw(hw);
 402
 403         /* Reset here to make sure all is clean for each PF */
 404         ret = i40e_pf_reset(hw);
 405         if (ret) {
 406                 PMD_INIT_LOG(ERR, "Failed to reset pf: %d", ret);
 407                 return ret;
 408         }
 409
 410         /* Initialize the shared code (base driver) */
 411         ret = i40e_init_shared_code(hw);
 412         if (ret) {
 413                 PMD_INIT_LOG(ERR, "Failed to init shared code (base driver): %d", ret);
 414                 return ret;
 415         }
 416
 417         /*
 418          * To work around the NVM issue,initialize registers
 419          * for flexible payload by software.
 420          * It should be removed once issues are fixed in NVM.
 421          */
 422         i40e_flex_payload_reg_init(hw);
 423
 424         /* Initialize the parameters for adminq */
 425         i40e_init_adminq_parameter(hw);
 426         ret = i40e_init_adminq(hw);
 427         if (ret != I40E_SUCCESS) {
 428                 PMD_INIT_LOG(ERR, "Failed to init adminq: %d", ret);
 429                 return -EIO;
 430         }
 431         PMD_INIT_LOG(INFO, "FW %d.%d API %d.%d NVM %02d.%02d.%02d eetrack %04x",
 432                      hw->aq.fw_maj_ver, hw->aq.fw_min_ver,
 433                      hw->aq.api_maj_ver, hw->aq.api_min_ver,
 434                      ((hw->nvm.version >> 12) & 0xf),
 435                      ((hw->nvm.version >> 4) & 0xff),
 436                      (hw->nvm.version & 0xf), hw->nvm.eetrack);
 437
 438         /* Disable LLDP */
 439         ret = i40e_aq_stop_lldp(hw, true, NULL);
 440         if (ret != I40E_SUCCESS) /* Its failure can be ignored */
 441                 PMD_INIT_LOG(INFO, "Failed to stop lldp");
 442
 443         /* Clear PXE mode */
 444         i40e_clear_pxe_mode(hw);
 445
 446         /* Get hw capabilities */
 447         ret = i40e_get_cap(hw);
 448         if (ret != I40E_SUCCESS) {
 449                 PMD_INIT_LOG(ERR, "Failed to get capabilities: %d", ret);
 450                 goto err_get_capabilities;
 451         }
 452
 453         /* Initialize parameters for PF */
 454         ret = i40e_pf_parameter_init(dev);
 455         if (ret != 0) {
 456                 PMD_INIT_LOG(ERR, "Failed to do parameter init: %d", ret);
 457                 goto err_parameter_init;
 458         }
 459
 460         /* Initialize the queue management */
 461         ret = i40e_res_pool_init(&pf->qp_pool, 0, hw->func_caps.num_tx_qp);
 462         if (ret < 0) {
 463                 PMD_INIT_LOG(ERR, "Failed to init queue pool");
 464                 goto err_qp_pool_init;
 465         }
 466         ret = i40e_res_pool_init(&pf->msix_pool, 1,
 467                                 hw->func_caps.num_msix_vectors - 1);
 468         if (ret < 0) {
 469                 PMD_INIT_LOG(ERR, "Failed to init MSIX pool");
 470                 goto err_msix_pool_init;
 471         }
 472
 473         /* Initialize lan hmc */
 474         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
 475                                 hw->func_caps.num_rx_qp, 0, 0);
 476         if (ret != I40E_SUCCESS) {
 477                 PMD_INIT_LOG(ERR, "Failed to init lan hmc: %d", ret);
 478                 goto err_init_lan_hmc;
 479         }
 480
 481         /* Configure lan hmc */
 482         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
 483         if (ret != I40E_SUCCESS) {
 484                 PMD_INIT_LOG(ERR, "Failed to configure lan hmc: %d", ret);
 485                 goto err_configure_lan_hmc;
 486         }
 487
 488         /* Get and check the mac address */
 489         i40e_get_mac_addr(hw, hw->mac.addr);
 490         if (i40e_validate_mac_addr(hw->mac.addr) != I40E_SUCCESS) {
 491                 PMD_INIT_LOG(ERR, "mac address is not valid");
 492                 ret = -EIO;
 493                 goto err_get_mac_addr;
 494         }
 495         /* Copy the permanent MAC address */
 496         ether_addr_copy((struct ether_addr *) hw->mac.addr,
 497                         (struct ether_addr *) hw->mac.perm_addr);
 498
 499         /* Disable flow control */
 500         hw->fc.requested_mode = I40E_FC_NONE;
 501         i40e_set_fc(hw, &aq_fail, TRUE);
 502
 503         /* PF setup, which includes VSI setup */
 504         ret = i40e_pf_setup(pf);
 505         if (ret) {
 506                 PMD_INIT_LOG(ERR, "Failed to setup pf switch: %d", ret);
 507                 goto err_setup_pf_switch;
 508         }
 509
 510         vsi = pf->main_vsi;
 511
 512         /* Disable double vlan by default */
 513         i40e_vsi_config_double_vlan(vsi, FALSE);
 514
 515         if (!vsi->max_macaddrs)
 516                 len = ETHER_ADDR_LEN;
 517         else
 518                 len = ETHER_ADDR_LEN * vsi->max_macaddrs;
 519
 520         /* Should be after VSI initialized */
 521         dev->data->mac_addrs = rte_zmalloc("i40e", len, 0);
 522         if (!dev->data->mac_addrs) {
 523                 PMD_INIT_LOG(ERR, "Failed to allocated memory "
 524                                         "for storing mac address");
 525                 goto err_mac_alloc;
 526         }
 527         ether_addr_copy((struct ether_addr *)hw->mac.perm_addr,
 528                                         &dev->data->mac_addrs[0]);
 529
 530         /* initialize pf host driver to setup SRIOV resource if applicable */
 531         i40e_pf_host_init(dev);
 532
 533         /* register callback func to eal lib */
 534         rte_intr_callback_register(&(pci_dev->intr_handle),
 535                 i40e_dev_interrupt_handler, (void *)dev);
 536
 537         /* configure and enable device interrupt */
 538         i40e_pf_config_irq0(hw);
 539         i40e_pf_enable_irq0(hw);
 540
 541         /* enable uio intr after callback register */
 542         rte_intr_enable(&(pci_dev->intr_handle));
 543
 544         return 0;
 545
 546 err_mac_alloc:
 547         i40e_vsi_release(pf->main_vsi);
 548 err_setup_pf_switch:
 549 err_get_mac_addr:
 550 err_configure_lan_hmc:
 551         (void)i40e_shutdown_lan_hmc(hw);
 552 err_init_lan_hmc:
 553         i40e_res_pool_destroy(&pf->msix_pool);
 554 err_msix_pool_init:
 555         i40e_res_pool_destroy(&pf->qp_pool);
 556 err_qp_pool_init:
 557 err_parameter_init:
 558 err_get_capabilities:
 559         (void)i40e_shutdown_adminq(hw);
 560
 561         return ret;
 562 }
 563
 564 static int
 565 i40e_dev_configure(struct rte_eth_dev *dev)
 566 {
 567         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 568         enum rte_eth_rx_mq_mode mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 569         int ret;
 570
 571         if (dev->data->dev_conf.fdir_conf.mode == RTE_FDIR_MODE_PERFECT) {
 572                 ret = i40e_fdir_setup(pf);
 573                 if (ret != I40E_SUCCESS) {
 574                         PMD_DRV_LOG(ERR, "Failed to setup flow director.");
 575                         return -ENOTSUP;
 576                 }
 577                 ret = i40e_fdir_configure(dev);
 578                 if (ret < 0) {
 579                         PMD_DRV_LOG(ERR, "failed to configure fdir.");
 580                         goto err;
 581                 }
 582         } else
 583                 i40e_fdir_teardown(pf);
 584
 585         ret = i40e_dev_init_vlan(dev);
 586         if (ret < 0)
 587                 goto err;
 588
 589         /* VMDQ setup.
 590          *  Needs to move VMDQ setting out of i40e_pf_config_mq_rx() as VMDQ and
 591          *  RSS setting have different requirements.
 592          *  General PMD driver call sequence are NIC init, configure,
 593          *  rx/tx_queue_setup and dev_start. In rx/tx_queue_setup() function, it
 594          *  will try to lookup the VSI that specific queue belongs to if VMDQ
 595          *  applicable. So, VMDQ setting has to be done before
 596          *  rx/tx_queue_setup(). This function is good  to place vmdq_setup.
 597          *  For RSS setting, it will try to calculate actual configured RX queue
 598          *  number, which will be available after rx_queue_setup(). dev_start()
 599          *  function is good to place RSS setup.
 600          */
 601         if (mq_mode & ETH_MQ_RX_VMDQ_FLAG) {
 602                 ret = i40e_vmdq_setup(dev);
 603                 if (ret)
 604                         goto err;
 605         }
 606         return 0;
 607 err:
 608         i40e_fdir_teardown(pf);
 609         return ret;
 610 }
 611
 612 void
 613 i40e_vsi_queues_unbind_intr(struct i40e_vsi *vsi)
 614 {
 615         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
 616         uint16_t msix_vect = vsi->msix_intr;
 617         uint16_t i;
 618
 619         for (i = 0; i < vsi->nb_qps; i++) {
 620                 I40E_WRITE_REG(hw, I40E_QINT_TQCTL(vsi->base_queue + i), 0);
 621                 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(vsi->base_queue + i), 0);
 622                 rte_wmb();
 623         }
 624
 625         if (vsi->type != I40E_VSI_SRIOV) {
 626                 I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(msix_vect - 1), 0);
 627                 I40E_WRITE_REG(hw, I40E_PFINT_ITRN(I40E_ITR_INDEX_DEFAULT,
 628                                 msix_vect - 1), 0);
 629         } else {
 630                 uint32_t reg;
 631                 reg = (hw->func_caps.num_msix_vectors_vf - 1) *
 632                         vsi->user_param + (msix_vect - 1);
 633
 634                 I40E_WRITE_REG(hw, I40E_VPINT_LNKLSTN(reg), 0);
 635         }
 636         I40E_WRITE_FLUSH(hw);
 637 }
 638
 639 static inline uint16_t
 640 i40e_calc_itr_interval(int16_t interval)
 641 {
 642         if (interval < 0 || interval > I40E_QUEUE_ITR_INTERVAL_MAX)
 643                 interval = I40E_QUEUE_ITR_INTERVAL_DEFAULT;
 644
 645         /* Convert to hardware count, as writing each 1 represents 2 us */
 646         return (interval/2);
 647 }
 648
 649 void
 650 i40e_vsi_queues_bind_intr(struct i40e_vsi *vsi)
 651 {
 652         uint32_t val;
 653         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
 654         uint16_t msix_vect = vsi->msix_intr;
 655         int i;
 656
 657         for (i = 0; i < vsi->nb_qps; i++)
 658                 I40E_WRITE_REG(hw, I40E_QINT_TQCTL(vsi->base_queue + i), 0);
 659
 660         /* Bind all RX queues to allocated MSIX interrupt */
 661         for (i = 0; i < vsi->nb_qps; i++) {
 662                 val = (msix_vect << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
 663                         I40E_QINT_RQCTL_ITR_INDX_MASK |
 664                         ((vsi->base_queue + i + 1) <<
 665                         I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
 666                         (0 << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) |
 667                         I40E_QINT_RQCTL_CAUSE_ENA_MASK;
 668
 669                 if (i == vsi->nb_qps - 1)
 670                         val |= I40E_QINT_RQCTL_NEXTQ_INDX_MASK;
 671                 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(vsi->base_queue + i), val);
 672         }
 673
 674         /* Write first RX queue to Link list register as the head element */
 675         if (vsi->type != I40E_VSI_SRIOV) {
 676                 uint16_t interval =
 677                         i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL);
 678
 679                 I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(msix_vect - 1),
 680                                                 (vsi->base_queue <<
 681                                 I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) |
 682                         (0x0 << I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT));
 683
 684                 I40E_WRITE_REG(hw, I40E_PFINT_ITRN(I40E_ITR_INDEX_DEFAULT,
 685                                                 msix_vect - 1), interval);
 686
 687 #ifndef I40E_GLINT_CTL
 688 #define I40E_GLINT_CTL                     0x0003F800
 689 #define I40E_GLINT_CTL_DIS_AUTOMASK_N_MASK 0x4
 690 #endif
 691                 /* Disable auto-mask on enabling of all none-zero  interrupt */
 692                 I40E_WRITE_REG(hw, I40E_GLINT_CTL,
 693                         I40E_GLINT_CTL_DIS_AUTOMASK_N_MASK);
 694         } else {
 695                 uint32_t reg;
 696
 697                 /* num_msix_vectors_vf needs to minus irq0 */
 698                 reg = (hw->func_caps.num_msix_vectors_vf - 1) *
 699                         vsi->user_param + (msix_vect - 1);
 700
 701                 I40E_WRITE_REG(hw, I40E_VPINT_LNKLSTN(reg), (vsi->base_queue <<
 702                                         I40E_VPINT_LNKLSTN_FIRSTQ_INDX_SHIFT) |
 703                                 (0x0 << I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_SHIFT));
 704         }
 705
 706         I40E_WRITE_FLUSH(hw);
 707 }
 708
 709 static void
 710 i40e_vsi_enable_queues_intr(struct i40e_vsi *vsi)
 711 {
 712         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
 713         uint16_t interval = i40e_calc_itr_interval(\
 714                         RTE_LIBRTE_I40E_ITR_INTERVAL);
 715
 716         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(vsi->msix_intr - 1),
 717                                         I40E_PFINT_DYN_CTLN_INTENA_MASK |
 718                                         I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
 719                                 (0 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT) |
 720                         (interval << I40E_PFINT_DYN_CTLN_INTERVAL_SHIFT));
 721 }
 722
 723 static void
 724 i40e_vsi_disable_queues_intr(struct i40e_vsi *vsi)
 725 {
 726         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
 727
 728         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(vsi->msix_intr - 1), 0);
 729 }
 730
 731 static inline uint8_t
 732 i40e_parse_link_speed(uint16_t eth_link_speed)
 733 {
 734         uint8_t link_speed = I40E_LINK_SPEED_UNKNOWN;
 735
 736         switch (eth_link_speed) {
 737         case ETH_LINK_SPEED_40G:
 738                 link_speed = I40E_LINK_SPEED_40GB;
 739                 break;
 740         case ETH_LINK_SPEED_20G:
 741                 link_speed = I40E_LINK_SPEED_20GB;
 742                 break;
 743         case ETH_LINK_SPEED_10G:
 744                 link_speed = I40E_LINK_SPEED_10GB;
 745                 break;
 746         case ETH_LINK_SPEED_1000:
 747                 link_speed = I40E_LINK_SPEED_1GB;
 748                 break;
 749         case ETH_LINK_SPEED_100:
 750                 link_speed = I40E_LINK_SPEED_100MB;
 751                 break;
 752         }
 753
 754         return link_speed;
 755 }
 756
 757 static int
 758 i40e_phy_conf_link(struct i40e_hw *hw, uint8_t abilities, uint8_t force_speed)
 759 {
 760         enum i40e_status_code status;
 761         struct i40e_aq_get_phy_abilities_resp phy_ab;
 762         struct i40e_aq_set_phy_config phy_conf;
 763         const uint8_t mask = I40E_AQ_PHY_FLAG_PAUSE_TX |
 764                         I40E_AQ_PHY_FLAG_PAUSE_RX |
 765                         I40E_AQ_PHY_FLAG_LOW_POWER;
 766         const uint8_t advt = I40E_LINK_SPEED_40GB |
 767                         I40E_LINK_SPEED_10GB |
 768                         I40E_LINK_SPEED_1GB |
 769                         I40E_LINK_SPEED_100MB;
 770         int ret = -ENOTSUP;
 771
 772         status = i40e_aq_get_phy_capabilities(hw, false, false, &phy_ab,
 773                                               NULL);
 774         if (status)
 775                 return ret;
 776
 777         memset(&phy_conf, 0, sizeof(phy_conf));
 778
 779         /* bits 0-2 use the values from get_phy_abilities_resp */
 780         abilities &= ~mask;
 781         abilities |= phy_ab.abilities & mask;
 782
 783         /* update ablities and speed */
 784         if (abilities & I40E_AQ_PHY_AN_ENABLED)
 785                 phy_conf.link_speed = advt;
 786         else
 787                 phy_conf.link_speed = force_speed;
 788
 789         phy_conf.abilities = abilities;
 790
 791         /* use get_phy_abilities_resp value for the rest */
 792         phy_conf.phy_type = phy_ab.phy_type;
 793         phy_conf.eee_capability = phy_ab.eee_capability;
 794         phy_conf.eeer = phy_ab.eeer_val;
 795         phy_conf.low_power_ctrl = phy_ab.d3_lpan;
 796
 797         PMD_DRV_LOG(DEBUG, "\tCurrent: abilities %x, link_speed %x",
 798                     phy_ab.abilities, phy_ab.link_speed);
 799         PMD_DRV_LOG(DEBUG, "\tConfig:  abilities %x, link_speed %x",
 800                     phy_conf.abilities, phy_conf.link_speed);
 801
 802         status = i40e_aq_set_phy_config(hw, &phy_conf, NULL);
 803         if (status)
 804                 return ret;
 805
 806         return I40E_SUCCESS;
 807 }
 808
 809 static int
 810 i40e_apply_link_speed(struct rte_eth_dev *dev)
 811 {
 812         uint8_t speed;
 813         uint8_t abilities = 0;
 814         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 815         struct rte_eth_conf *conf = &dev->data->dev_conf;
 816
 817         speed = i40e_parse_link_speed(conf->link_speed);
 818         abilities |= I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
 819         if (conf->link_speed == ETH_LINK_SPEED_AUTONEG)
 820                 abilities |= I40E_AQ_PHY_AN_ENABLED;
 821         else
 822                 abilities |= I40E_AQ_PHY_LINK_ENABLED;
 823
 824         return i40e_phy_conf_link(hw, abilities, speed);
 825 }
 826
 827 static int
 828 i40e_dev_start(struct rte_eth_dev *dev)
 829 {
 830         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 831         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 832         struct i40e_vsi *main_vsi = pf->main_vsi;
 833         int ret, i;
 834
 835         if ((dev->data->dev_conf.link_duplex != ETH_LINK_AUTONEG_DUPLEX) &&
 836                 (dev->data->dev_conf.link_duplex != ETH_LINK_FULL_DUPLEX)) {
 837                 PMD_INIT_LOG(ERR, "Invalid link_duplex (%hu) for port %hhu",
 838                              dev->data->dev_conf.link_duplex,
 839                              dev->data->port_id);
 840                 return -EINVAL;
 841         }
 842
 843         /* Initialize VSI */
 844         ret = i40e_dev_rxtx_init(pf);
 845         if (ret != I40E_SUCCESS) {
 846                 PMD_DRV_LOG(ERR, "Failed to init rx/tx queues");
 847                 goto err_up;
 848         }
 849
 850         /* Map queues with MSIX interrupt */
 851         i40e_vsi_queues_bind_intr(main_vsi);
 852         i40e_vsi_enable_queues_intr(main_vsi);
 853
 854         /* Map VMDQ VSI queues with MSIX interrupt */
 855         for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
 856                 i40e_vsi_queues_bind_intr(pf->vmdq[i].vsi);
 857                 i40e_vsi_enable_queues_intr(pf->vmdq[i].vsi);
 858         }
 859
 860         /* enable FDIR MSIX interrupt */
 861         if (pf->fdir.fdir_vsi) {
 862                 i40e_vsi_queues_bind_intr(pf->fdir.fdir_vsi);
 863                 i40e_vsi_enable_queues_intr(pf->fdir.fdir_vsi);
 864         }
 865
 866         /* Enable all queues which have been configured */
 867         ret = i40e_dev_switch_queues(pf, TRUE);
 868         if (ret != I40E_SUCCESS) {
 869                 PMD_DRV_LOG(ERR, "Failed to enable VSI");
 870                 goto err_up;
 871         }
 872
 873         /* Enable receiving broadcast packets */
 874         ret = i40e_aq_set_vsi_broadcast(hw, main_vsi->seid, true, NULL);
 875         if (ret != I40E_SUCCESS)
 876                 PMD_DRV_LOG(INFO, "fail to set vsi broadcast");
 877
 878         for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
 879                 ret = i40e_aq_set_vsi_broadcast(hw, pf->vmdq[i].vsi->seid,
 880                                                 true, NULL);
 881                 if (ret != I40E_SUCCESS)
 882                         PMD_DRV_LOG(INFO, "fail to set vsi broadcast");
 883         }
 884
 885         /* Apply link configure */
 886         ret = i40e_apply_link_speed(dev);
 887         if (I40E_SUCCESS != ret) {
 888                 PMD_DRV_LOG(ERR, "Fail to apply link setting");
 889                 goto err_up;
 890         }
 891
 892         return I40E_SUCCESS;
 893
 894 err_up:
 895         i40e_dev_switch_queues(pf, FALSE);
 896         i40e_dev_clear_queues(dev);
 897
 898         return ret;
 899 }
 900
 901 static void
 902 i40e_dev_stop(struct rte_eth_dev *dev)
 903 {
 904         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 905         struct i40e_vsi *main_vsi = pf->main_vsi;
 906         int i;
 907
 908         /* Disable all queues */
 909         i40e_dev_switch_queues(pf, FALSE);
 910
 911         /* un-map queues with interrupt registers */
 912         i40e_vsi_disable_queues_intr(main_vsi);
 913         i40e_vsi_queues_unbind_intr(main_vsi);
 914
 915         for (i = 0; i < pf->nb_cfg_vmdq_vsi; i++) {
 916                 i40e_vsi_disable_queues_intr(pf->vmdq[i].vsi);
 917                 i40e_vsi_queues_unbind_intr(pf->vmdq[i].vsi);
 918         }
 919
 920         if (pf->fdir.fdir_vsi) {
 921                 i40e_vsi_queues_bind_intr(pf->fdir.fdir_vsi);
 922                 i40e_vsi_enable_queues_intr(pf->fdir.fdir_vsi);
 923         }
 924         /* Clear all queues and release memory */
 925         i40e_dev_clear_queues(dev);
 926
 927         /* Set link down */
 928         i40e_dev_set_link_down(dev);
 929
 930 }
 931
 932 static void
 933 i40e_dev_close(struct rte_eth_dev *dev)
 934 {
 935         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 936         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 937         uint32_t reg;
 938
 939         PMD_INIT_FUNC_TRACE();
 940
 941         i40e_dev_stop(dev);
 942
 943         /* Disable interrupt */
 944         i40e_pf_disable_irq0(hw);
 945         rte_intr_disable(&(dev->pci_dev->intr_handle));
 946
 947         /* shutdown and destroy the HMC */
 948         i40e_shutdown_lan_hmc(hw);
 949
 950         /* release all the existing VSIs and VEBs */
 951         i40e_fdir_teardown(pf);
 952         i40e_vsi_release(pf->main_vsi);
 953
 954         /* shutdown the adminq */
 955         i40e_aq_queue_shutdown(hw, true);
 956         i40e_shutdown_adminq(hw);
 957
 958         i40e_res_pool_destroy(&pf->qp_pool);
 959         i40e_res_pool_destroy(&pf->msix_pool);
 960
 961         /* force a PF reset to clean anything leftover */
 962         reg = I40E_READ_REG(hw, I40E_PFGEN_CTRL);
 963         I40E_WRITE_REG(hw, I40E_PFGEN_CTRL,
 964                         (reg | I40E_PFGEN_CTRL_PFSWR_MASK));
 965         I40E_WRITE_FLUSH(hw);
 966 }
 967
 968 static void
 969 i40e_dev_promiscuous_enable(struct rte_eth_dev *dev)
 970 {
 971         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 972         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 973         struct i40e_vsi *vsi = pf->main_vsi;
 974         int status;
 975
 976         status = i40e_aq_set_vsi_unicast_promiscuous(hw, vsi->seid,
 977                                                         true, NULL);
 978         if (status != I40E_SUCCESS)
 979                 PMD_DRV_LOG(ERR, "Failed to enable unicast promiscuous");
 980
 981         status = i40e_aq_set_vsi_multicast_promiscuous(hw, vsi->seid,
 982                                                         TRUE, NULL);
 983         if (status != I40E_SUCCESS)
 984                 PMD_DRV_LOG(ERR, "Failed to enable multicast promiscuous");
 985
 986 }
 987
 988 static void
 989 i40e_dev_promiscuous_disable(struct rte_eth_dev *dev)
 990 {
 991         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 992         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 993         struct i40e_vsi *vsi = pf->main_vsi;
 994         int status;
 995
 996         status = i40e_aq_set_vsi_unicast_promiscuous(hw, vsi->seid,
 997                                                         false, NULL);
 998         if (status != I40E_SUCCESS)
 999                 PMD_DRV_LOG(ERR, "Failed to disable unicast promiscuous");
1000
1001         status = i40e_aq_set_vsi_multicast_promiscuous(hw, vsi->seid,
1002                                                         false, NULL);
1003         if (status != I40E_SUCCESS)
1004                 PMD_DRV_LOG(ERR, "Failed to disable multicast promiscuous");
1005 }
1006
1007 static void
1008 i40e_dev_allmulticast_enable(struct rte_eth_dev *dev)
1009 {
1010         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1011         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1012         struct i40e_vsi *vsi = pf->main_vsi;
1013         int ret;
1014
1015         ret = i40e_aq_set_vsi_multicast_promiscuous(hw, vsi->seid, TRUE, NULL);
1016         if (ret != I40E_SUCCESS)
1017                 PMD_DRV_LOG(ERR, "Failed to enable multicast promiscuous");
1018 }
1019
1020 static void
1021 i40e_dev_allmulticast_disable(struct rte_eth_dev *dev)
1022 {
1023         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1024         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1025         struct i40e_vsi *vsi = pf->main_vsi;
1026         int ret;
1027
1028         if (dev->data->promiscuous == 1)
1029                 return; /* must remain in all_multicast mode */
1030
1031         ret = i40e_aq_set_vsi_multicast_promiscuous(hw,
1032                                 vsi->seid, FALSE, NULL);
1033         if (ret != I40E_SUCCESS)
1034                 PMD_DRV_LOG(ERR, "Failed to disable multicast promiscuous");
1035 }
1036
1037 /*
1038  * Set device link up.
1039  */
1040 static int
1041 i40e_dev_set_link_up(struct rte_eth_dev *dev)
1042 {
1043         /* re-apply link speed setting */
1044         return i40e_apply_link_speed(dev);
1045 }
1046
1047 /*
1048  * Set device link down.
1049  */
1050 static int
1051 i40e_dev_set_link_down(__rte_unused struct rte_eth_dev *dev)
1052 {
1053         uint8_t speed = I40E_LINK_SPEED_UNKNOWN;
1054         uint8_t abilities = I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
1055         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1056
1057         return i40e_phy_conf_link(hw, abilities, speed);
1058 }
1059
1060 int
1061 i40e_dev_link_update(struct rte_eth_dev *dev,
1062                      __rte_unused int wait_to_complete)
1063 {
1064         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1065         struct i40e_link_status link_status;
1066         struct rte_eth_link link, old;
1067         int status;
1068
1069         memset(&link, 0, sizeof(link));
1070         memset(&old, 0, sizeof(old));
1071         memset(&link_status, 0, sizeof(link_status));
1072         rte_i40e_dev_atomic_read_link_status(dev, &old);
1073
1074         /* Get link status information from hardware */
1075         status = i40e_aq_get_link_info(hw, false, &link_status, NULL);
1076         if (status != I40E_SUCCESS) {
1077                 link.link_speed = ETH_LINK_SPEED_100;
1078                 link.link_duplex = ETH_LINK_FULL_DUPLEX;
1079                 PMD_DRV_LOG(ERR, "Failed to get link info");
1080                 goto out;
1081         }
1082
1083         link.link_status = link_status.link_info & I40E_AQ_LINK_UP;
1084
1085         if (!link.link_status)
1086                 goto out;
1087
1088         /* i40e uses full duplex only */
1089         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1090
1091         /* Parse the link status */
1092         switch (link_status.link_speed) {
1093         case I40E_LINK_SPEED_100MB:
1094                 link.link_speed = ETH_LINK_SPEED_100;
1095                 break;
1096         case I40E_LINK_SPEED_1GB:
1097                 link.link_speed = ETH_LINK_SPEED_1000;
1098                 break;
1099         case I40E_LINK_SPEED_10GB:
1100                 link.link_speed = ETH_LINK_SPEED_10G;
1101                 break;
1102         case I40E_LINK_SPEED_20GB:
1103                 link.link_speed = ETH_LINK_SPEED_20G;
1104                 break;
1105         case I40E_LINK_SPEED_40GB:
1106                 link.link_speed = ETH_LINK_SPEED_40G;
1107                 break;
1108         default:
1109                 link.link_speed = ETH_LINK_SPEED_100;
1110                 break;
1111         }
1112
1113 out:
1114         rte_i40e_dev_atomic_write_link_status(dev, &link);
1115         if (link.link_status == old.link_status)
1116                 return -1;
1117
1118         return 0;
1119 }
1120
1121 /* Get all the statistics of a VSI */
1122 void
1123 i40e_update_vsi_stats(struct i40e_vsi *vsi)
1124 {
1125         struct i40e_eth_stats *oes = &vsi->eth_stats_offset;
1126         struct i40e_eth_stats *nes = &vsi->eth_stats;
1127         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
1128         int idx = rte_le_to_cpu_16(vsi->info.stat_counter_idx);
1129
1130         i40e_stat_update_48(hw, I40E_GLV_GORCH(idx), I40E_GLV_GORCL(idx),
1131                             vsi->offset_loaded, &oes->rx_bytes,
1132                             &nes->rx_bytes);
1133         i40e_stat_update_48(hw, I40E_GLV_UPRCH(idx), I40E_GLV_UPRCL(idx),
1134                             vsi->offset_loaded, &oes->rx_unicast,
1135                             &nes->rx_unicast);
1136         i40e_stat_update_48(hw, I40E_GLV_MPRCH(idx), I40E_GLV_MPRCL(idx),
1137                             vsi->offset_loaded, &oes->rx_multicast,
1138                             &nes->rx_multicast);
1139         i40e_stat_update_48(hw, I40E_GLV_BPRCH(idx), I40E_GLV_BPRCL(idx),
1140                             vsi->offset_loaded, &oes->rx_broadcast,
1141                             &nes->rx_broadcast);
1142         i40e_stat_update_32(hw, I40E_GLV_RDPC(idx), vsi->offset_loaded,
1143                             &oes->rx_discards, &nes->rx_discards);
1144         /* GLV_REPC not supported */
1145         /* GLV_RMPC not supported */
1146         i40e_stat_update_32(hw, I40E_GLV_RUPP(idx), vsi->offset_loaded,
1147                             &oes->rx_unknown_protocol,
1148                             &nes->rx_unknown_protocol);
1149         i40e_stat_update_48(hw, I40E_GLV_GOTCH(idx), I40E_GLV_GOTCL(idx),
1150                             vsi->offset_loaded, &oes->tx_bytes,
1151                             &nes->tx_bytes);
1152         i40e_stat_update_48(hw, I40E_GLV_UPTCH(idx), I40E_GLV_UPTCL(idx),
1153                             vsi->offset_loaded, &oes->tx_unicast,
1154                             &nes->tx_unicast);
1155         i40e_stat_update_48(hw, I40E_GLV_MPTCH(idx), I40E_GLV_MPTCL(idx),
1156                             vsi->offset_loaded, &oes->tx_multicast,
1157                             &nes->tx_multicast);
1158         i40e_stat_update_48(hw, I40E_GLV_BPTCH(idx), I40E_GLV_BPTCL(idx),
1159                             vsi->offset_loaded,  &oes->tx_broadcast,
1160                             &nes->tx_broadcast);
1161         /* GLV_TDPC not supported */
1162         i40e_stat_update_32(hw, I40E_GLV_TEPC(idx), vsi->offset_loaded,
1163                             &oes->tx_errors, &nes->tx_errors);
1164         vsi->offset_loaded = true;
1165
1166         PMD_DRV_LOG(DEBUG, "***************** VSI[%u] stats start *******************",
1167                     vsi->vsi_id);
1168         PMD_DRV_LOG(DEBUG, "rx_bytes:            %lu", nes->rx_bytes);
1169         PMD_DRV_LOG(DEBUG, "rx_unicast:          %lu", nes->rx_unicast);
1170         PMD_DRV_LOG(DEBUG, "rx_multicast:        %lu", nes->rx_multicast);
1171         PMD_DRV_LOG(DEBUG, "rx_broadcast:        %lu", nes->rx_broadcast);
1172         PMD_DRV_LOG(DEBUG, "rx_discards:         %lu", nes->rx_discards);
1173         PMD_DRV_LOG(DEBUG, "rx_unknown_protocol: %lu",
1174                     nes->rx_unknown_protocol);
1175         PMD_DRV_LOG(DEBUG, "tx_bytes:            %lu", nes->tx_bytes);
1176         PMD_DRV_LOG(DEBUG, "tx_unicast:          %lu", nes->tx_unicast);
1177         PMD_DRV_LOG(DEBUG, "tx_multicast:        %lu", nes->tx_multicast);
1178         PMD_DRV_LOG(DEBUG, "tx_broadcast:        %lu", nes->tx_broadcast);
1179         PMD_DRV_LOG(DEBUG, "tx_discards:         %lu", nes->tx_discards);
1180         PMD_DRV_LOG(DEBUG, "tx_errors:           %lu", nes->tx_errors);
1181         PMD_DRV_LOG(DEBUG, "***************** VSI[%u] stats end *******************",
1182                     vsi->vsi_id);
1183 }
1184
1185 /* Get all statistics of a port */
1186 static void
1187 i40e_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1188 {
1189         uint32_t i;
1190         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1191         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1192         struct i40e_hw_port_stats *ns = &pf->stats; /* new stats */
1193         struct i40e_hw_port_stats *os = &pf->stats_offset; /* old stats */
1194
1195         /* Get statistics of struct i40e_eth_stats */
1196         i40e_stat_update_48(hw, I40E_GLPRT_GORCH(hw->port),
1197                             I40E_GLPRT_GORCL(hw->port),
1198                             pf->offset_loaded, &os->eth.rx_bytes,
1199                             &ns->eth.rx_bytes);
1200         i40e_stat_update_48(hw, I40E_GLPRT_UPRCH(hw->port),
1201                             I40E_GLPRT_UPRCL(hw->port),
1202                             pf->offset_loaded, &os->eth.rx_unicast,
1203                             &ns->eth.rx_unicast);
1204         i40e_stat_update_48(hw, I40E_GLPRT_MPRCH(hw->port),
1205                             I40E_GLPRT_MPRCL(hw->port),
1206                             pf->offset_loaded, &os->eth.rx_multicast,
1207                             &ns->eth.rx_multicast);
1208         i40e_stat_update_48(hw, I40E_GLPRT_BPRCH(hw->port),
1209                             I40E_GLPRT_BPRCL(hw->port),
1210                             pf->offset_loaded, &os->eth.rx_broadcast,
1211                             &ns->eth.rx_broadcast);
1212         i40e_stat_update_32(hw, I40E_GLPRT_RDPC(hw->port),
1213                             pf->offset_loaded, &os->eth.rx_discards,
1214                             &ns->eth.rx_discards);
1215         /* GLPRT_REPC not supported */
1216         /* GLPRT_RMPC not supported */
1217         i40e_stat_update_32(hw, I40E_GLPRT_RUPP(hw->port),
1218                             pf->offset_loaded,
1219                             &os->eth.rx_unknown_protocol,
1220                             &ns->eth.rx_unknown_protocol);
1221         i40e_stat_update_48(hw, I40E_GLPRT_GOTCH(hw->port),
1222                             I40E_GLPRT_GOTCL(hw->port),
1223                             pf->offset_loaded, &os->eth.tx_bytes,
1224                             &ns->eth.tx_bytes);
1225         i40e_stat_update_48(hw, I40E_GLPRT_UPTCH(hw->port),
1226                             I40E_GLPRT_UPTCL(hw->port),
1227                             pf->offset_loaded, &os->eth.tx_unicast,
1228                             &ns->eth.tx_unicast);
1229         i40e_stat_update_48(hw, I40E_GLPRT_MPTCH(hw->port),
1230                             I40E_GLPRT_MPTCL(hw->port),
1231                             pf->offset_loaded, &os->eth.tx_multicast,
1232                             &ns->eth.tx_multicast);
1233         i40e_stat_update_48(hw, I40E_GLPRT_BPTCH(hw->port),
1234                             I40E_GLPRT_BPTCL(hw->port),
1235                             pf->offset_loaded, &os->eth.tx_broadcast,
1236                             &ns->eth.tx_broadcast);
1237         i40e_stat_update_32(hw, I40E_GLPRT_TDPC(hw->port),
1238                             pf->offset_loaded, &os->eth.tx_discards,
1239                             &ns->eth.tx_discards);
1240         /* GLPRT_TEPC not supported */
1241
1242         /* additional port specific stats */
1243         i40e_stat_update_32(hw, I40E_GLPRT_TDOLD(hw->port),
1244                             pf->offset_loaded, &os->tx_dropped_link_down,
1245                             &ns->tx_dropped_link_down);
1246         i40e_stat_update_32(hw, I40E_GLPRT_CRCERRS(hw->port),
1247                             pf->offset_loaded, &os->crc_errors,
1248                             &ns->crc_errors);
1249         i40e_stat_update_32(hw, I40E_GLPRT_ILLERRC(hw->port),
1250                             pf->offset_loaded, &os->illegal_bytes,
1251                             &ns->illegal_bytes);
1252         /* GLPRT_ERRBC not supported */
1253         i40e_stat_update_32(hw, I40E_GLPRT_MLFC(hw->port),
1254                             pf->offset_loaded, &os->mac_local_faults,
1255                             &ns->mac_local_faults);
1256         i40e_stat_update_32(hw, I40E_GLPRT_MRFC(hw->port),
1257                             pf->offset_loaded, &os->mac_remote_faults,
1258                             &ns->mac_remote_faults);
1259         i40e_stat_update_32(hw, I40E_GLPRT_RLEC(hw->port),
1260                             pf->offset_loaded, &os->rx_length_errors,
1261                             &ns->rx_length_errors);
1262         i40e_stat_update_32(hw, I40E_GLPRT_LXONRXC(hw->port),
1263                             pf->offset_loaded, &os->link_xon_rx,
1264                             &ns->link_xon_rx);
1265         i40e_stat_update_32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
1266                             pf->offset_loaded, &os->link_xoff_rx,
1267                             &ns->link_xoff_rx);
1268         for (i = 0; i < 8; i++) {
1269                 i40e_stat_update_32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
1270                                     pf->offset_loaded,
1271                                     &os->priority_xon_rx[i],
1272                                     &ns->priority_xon_rx[i]);
1273                 i40e_stat_update_32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
1274                                     pf->offset_loaded,
1275                                     &os->priority_xoff_rx[i],
1276                                     &ns->priority_xoff_rx[i]);
1277         }
1278         i40e_stat_update_32(hw, I40E_GLPRT_LXONTXC(hw->port),
1279                             pf->offset_loaded, &os->link_xon_tx,
1280                             &ns->link_xon_tx);
1281         i40e_stat_update_32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
1282                             pf->offset_loaded, &os->link_xoff_tx,
1283                             &ns->link_xoff_tx);
1284         for (i = 0; i < 8; i++) {
1285                 i40e_stat_update_32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
1286                                     pf->offset_loaded,
1287                                     &os->priority_xon_tx[i],
1288                                     &ns->priority_xon_tx[i]);
1289                 i40e_stat_update_32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
1290                                     pf->offset_loaded,
1291                                     &os->priority_xoff_tx[i],
1292                                     &ns->priority_xoff_tx[i]);
1293                 i40e_stat_update_32(hw, I40E_GLPRT_RXON2OFFCNT(hw->port, i),
1294                                     pf->offset_loaded,
1295                                     &os->priority_xon_2_xoff[i],
1296                                     &ns->priority_xon_2_xoff[i]);
1297         }
1298         i40e_stat_update_48(hw, I40E_GLPRT_PRC64H(hw->port),
1299                             I40E_GLPRT_PRC64L(hw->port),
1300                             pf->offset_loaded, &os->rx_size_64,
1301                             &ns->rx_size_64);
1302         i40e_stat_update_48(hw, I40E_GLPRT_PRC127H(hw->port),
1303                             I40E_GLPRT_PRC127L(hw->port),
1304                             pf->offset_loaded, &os->rx_size_127,
1305                             &ns->rx_size_127);
1306         i40e_stat_update_48(hw, I40E_GLPRT_PRC255H(hw->port),
1307                             I40E_GLPRT_PRC255L(hw->port),
1308                             pf->offset_loaded, &os->rx_size_255,
1309                             &ns->rx_size_255);
1310         i40e_stat_update_48(hw, I40E_GLPRT_PRC511H(hw->port),
1311                             I40E_GLPRT_PRC511L(hw->port),
1312                             pf->offset_loaded, &os->rx_size_511,
1313                             &ns->rx_size_511);
1314         i40e_stat_update_48(hw, I40E_GLPRT_PRC1023H(hw->port),
1315                             I40E_GLPRT_PRC1023L(hw->port),
1316                             pf->offset_loaded, &os->rx_size_1023,
1317                             &ns->rx_size_1023);
1318         i40e_stat_update_48(hw, I40E_GLPRT_PRC1522H(hw->port),
1319                             I40E_GLPRT_PRC1522L(hw->port),
1320                             pf->offset_loaded, &os->rx_size_1522,
1321                             &ns->rx_size_1522);
1322         i40e_stat_update_48(hw, I40E_GLPRT_PRC9522H(hw->port),
1323                             I40E_GLPRT_PRC9522L(hw->port),
1324                             pf->offset_loaded, &os->rx_size_big,
1325                             &ns->rx_size_big);
1326         i40e_stat_update_32(hw, I40E_GLPRT_RUC(hw->port),
1327                             pf->offset_loaded, &os->rx_undersize,
1328                             &ns->rx_undersize);
1329         i40e_stat_update_32(hw, I40E_GLPRT_RFC(hw->port),
1330                             pf->offset_loaded, &os->rx_fragments,
1331                             &ns->rx_fragments);
1332         i40e_stat_update_32(hw, I40E_GLPRT_ROC(hw->port),
1333                             pf->offset_loaded, &os->rx_oversize,
1334                             &ns->rx_oversize);
1335         i40e_stat_update_32(hw, I40E_GLPRT_RJC(hw->port),
1336                             pf->offset_loaded, &os->rx_jabber,
1337                             &ns->rx_jabber);
1338         i40e_stat_update_48(hw, I40E_GLPRT_PTC64H(hw->port),
1339                             I40E_GLPRT_PTC64L(hw->port),
1340                             pf->offset_loaded, &os->tx_size_64,
1341                             &ns->tx_size_64);
1342         i40e_stat_update_48(hw, I40E_GLPRT_PTC127H(hw->port),
1343                             I40E_GLPRT_PTC127L(hw->port),
1344                             pf->offset_loaded, &os->tx_size_127,
1345                             &ns->tx_size_127);
1346         i40e_stat_update_48(hw, I40E_GLPRT_PTC255H(hw->port),
1347                             I40E_GLPRT_PTC255L(hw->port),
1348                             pf->offset_loaded, &os->tx_size_255,
1349                             &ns->tx_size_255);
1350         i40e_stat_update_48(hw, I40E_GLPRT_PTC511H(hw->port),
1351                             I40E_GLPRT_PTC511L(hw->port),
1352                             pf->offset_loaded, &os->tx_size_511,
1353                             &ns->tx_size_511);
1354         i40e_stat_update_48(hw, I40E_GLPRT_PTC1023H(hw->port),
1355                             I40E_GLPRT_PTC1023L(hw->port),
1356                             pf->offset_loaded, &os->tx_size_1023,
1357                             &ns->tx_size_1023);
1358         i40e_stat_update_48(hw, I40E_GLPRT_PTC1522H(hw->port),
1359                             I40E_GLPRT_PTC1522L(hw->port),
1360                             pf->offset_loaded, &os->tx_size_1522,
1361                             &ns->tx_size_1522);
1362         i40e_stat_update_48(hw, I40E_GLPRT_PTC9522H(hw->port),
1363                             I40E_GLPRT_PTC9522L(hw->port),
1364                             pf->offset_loaded, &os->tx_size_big,
1365                             &ns->tx_size_big);
1366         i40e_stat_update_32(hw, I40E_GLQF_PCNT(pf->fdir.match_counter_index),
1367                            pf->offset_loaded,
1368                            &os->fd_sb_match, &ns->fd_sb_match);
1369         /* GLPRT_MSPDC not supported */
1370         /* GLPRT_XEC not supported */
1371
1372         pf->offset_loaded = true;
1373
1374         if (pf->main_vsi)
1375                 i40e_update_vsi_stats(pf->main_vsi);
1376
1377         stats->ipackets = ns->eth.rx_unicast + ns->eth.rx_multicast +
1378                                                 ns->eth.rx_broadcast;
1379         stats->opackets = ns->eth.tx_unicast + ns->eth.tx_multicast +
1380                                                 ns->eth.tx_broadcast;
1381         stats->ibytes   = ns->eth.rx_bytes;
1382         stats->obytes   = ns->eth.tx_bytes;
1383         stats->oerrors  = ns->eth.tx_errors;
1384         stats->imcasts  = ns->eth.rx_multicast;
1385         stats->fdirmatch = ns->fd_sb_match;
1386
1387         /* Rx Errors */
1388         stats->ibadcrc  = ns->crc_errors;
1389         stats->ibadlen  = ns->rx_length_errors + ns->rx_undersize +
1390                         ns->rx_oversize + ns->rx_fragments + ns->rx_jabber;
1391         stats->imissed  = ns->eth.rx_discards;
1392         stats->ierrors  = stats->ibadcrc + stats->ibadlen + stats->imissed;
1393
1394         PMD_DRV_LOG(DEBUG, "***************** PF stats start *******************");
1395         PMD_DRV_LOG(DEBUG, "rx_bytes:            %lu", ns->eth.rx_bytes);
1396         PMD_DRV_LOG(DEBUG, "rx_unicast:          %lu", ns->eth.rx_unicast);
1397         PMD_DRV_LOG(DEBUG, "rx_multicast:        %lu", ns->eth.rx_multicast);
1398         PMD_DRV_LOG(DEBUG, "rx_broadcast:        %lu", ns->eth.rx_broadcast);
1399         PMD_DRV_LOG(DEBUG, "rx_discards:         %lu", ns->eth.rx_discards);
1400         PMD_DRV_LOG(DEBUG, "rx_unknown_protocol: %lu",
1401                     ns->eth.rx_unknown_protocol);
1402         PMD_DRV_LOG(DEBUG, "tx_bytes:            %lu", ns->eth.tx_bytes);
1403         PMD_DRV_LOG(DEBUG, "tx_unicast:          %lu", ns->eth.tx_unicast);
1404         PMD_DRV_LOG(DEBUG, "tx_multicast:        %lu", ns->eth.tx_multicast);
1405         PMD_DRV_LOG(DEBUG, "tx_broadcast:        %lu", ns->eth.tx_broadcast);
1406         PMD_DRV_LOG(DEBUG, "tx_discards:         %lu", ns->eth.tx_discards);
1407         PMD_DRV_LOG(DEBUG, "tx_errors:           %lu", ns->eth.tx_errors);
1408
1409         PMD_DRV_LOG(DEBUG, "tx_dropped_link_down:     %lu",
1410                     ns->tx_dropped_link_down);
1411         PMD_DRV_LOG(DEBUG, "crc_errors:               %lu", ns->crc_errors);
1412         PMD_DRV_LOG(DEBUG, "illegal_bytes:            %lu",
1413                     ns->illegal_bytes);
1414         PMD_DRV_LOG(DEBUG, "error_bytes:              %lu", ns->error_bytes);
1415         PMD_DRV_LOG(DEBUG, "mac_local_faults:         %lu",
1416                     ns->mac_local_faults);
1417         PMD_DRV_LOG(DEBUG, "mac_remote_faults:        %lu",
1418                     ns->mac_remote_faults);
1419         PMD_DRV_LOG(DEBUG, "rx_length_errors:         %lu",
1420                     ns->rx_length_errors);
1421         PMD_DRV_LOG(DEBUG, "link_xon_rx:              %lu", ns->link_xon_rx);
1422         PMD_DRV_LOG(DEBUG, "link_xoff_rx:             %lu", ns->link_xoff_rx);
1423         for (i = 0; i < 8; i++) {
1424                 PMD_DRV_LOG(DEBUG, "priority_xon_rx[%d]:      %lu",
1425                                 i, ns->priority_xon_rx[i]);
1426                 PMD_DRV_LOG(DEBUG, "priority_xoff_rx[%d]:     %lu",
1427                                 i, ns->priority_xoff_rx[i]);
1428         }
1429         PMD_DRV_LOG(DEBUG, "link_xon_tx:              %lu", ns->link_xon_tx);
1430         PMD_DRV_LOG(DEBUG, "link_xoff_tx:             %lu", ns->link_xoff_tx);
1431         for (i = 0; i < 8; i++) {
1432                 PMD_DRV_LOG(DEBUG, "priority_xon_tx[%d]:      %lu",
1433                                 i, ns->priority_xon_tx[i]);
1434                 PMD_DRV_LOG(DEBUG, "priority_xoff_tx[%d]:     %lu",
1435                                 i, ns->priority_xoff_tx[i]);
1436                 PMD_DRV_LOG(DEBUG, "priority_xon_2_xoff[%d]:  %lu",
1437                                 i, ns->priority_xon_2_xoff[i]);
1438         }
1439         PMD_DRV_LOG(DEBUG, "rx_size_64:               %lu", ns->rx_size_64);
1440         PMD_DRV_LOG(DEBUG, "rx_size_127:              %lu", ns->rx_size_127);
1441         PMD_DRV_LOG(DEBUG, "rx_size_255:              %lu", ns->rx_size_255);
1442         PMD_DRV_LOG(DEBUG, "rx_size_511:              %lu", ns->rx_size_511);
1443         PMD_DRV_LOG(DEBUG, "rx_size_1023:             %lu", ns->rx_size_1023);
1444         PMD_DRV_LOG(DEBUG, "rx_size_1522:             %lu", ns->rx_size_1522);
1445         PMD_DRV_LOG(DEBUG, "rx_size_big:              %lu", ns->rx_size_big);
1446         PMD_DRV_LOG(DEBUG, "rx_undersize:             %lu", ns->rx_undersize);
1447         PMD_DRV_LOG(DEBUG, "rx_fragments:             %lu", ns->rx_fragments);
1448         PMD_DRV_LOG(DEBUG, "rx_oversize:              %lu", ns->rx_oversize);
1449         PMD_DRV_LOG(DEBUG, "rx_jabber:                %lu", ns->rx_jabber);
1450         PMD_DRV_LOG(DEBUG, "tx_size_64:               %lu", ns->tx_size_64);
1451         PMD_DRV_LOG(DEBUG, "tx_size_127:              %lu", ns->tx_size_127);
1452         PMD_DRV_LOG(DEBUG, "tx_size_255:              %lu", ns->tx_size_255);
1453         PMD_DRV_LOG(DEBUG, "tx_size_511:              %lu", ns->tx_size_511);
1454         PMD_DRV_LOG(DEBUG, "tx_size_1023:             %lu", ns->tx_size_1023);
1455         PMD_DRV_LOG(DEBUG, "tx_size_1522:             %lu", ns->tx_size_1522);
1456         PMD_DRV_LOG(DEBUG, "tx_size_big:              %lu", ns->tx_size_big);
1457         PMD_DRV_LOG(DEBUG, "mac_short_packet_dropped: %lu",
1458                         ns->mac_short_packet_dropped);
1459         PMD_DRV_LOG(DEBUG, "checksum_error:           %lu",
1460                     ns->checksum_error);
1461         PMD_DRV_LOG(DEBUG, "fdir_match:               %lu", ns->fd_sb_match);
1462         PMD_DRV_LOG(DEBUG, "***************** PF stats end ********************");
1463 }
1464
1465 /* Reset the statistics */
1466 static void
1467 i40e_dev_stats_reset(struct rte_eth_dev *dev)
1468 {
1469         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1470
1471         /* It results in reloading the start point of each counter */
1472         pf->offset_loaded = false;
1473 }
1474
1475 static int
1476 i40e_dev_queue_stats_mapping_set(__rte_unused struct rte_eth_dev *dev,
1477                                  __rte_unused uint16_t queue_id,
1478                                  __rte_unused uint8_t stat_idx,
1479                                  __rte_unused uint8_t is_rx)
1480 {
1481         PMD_INIT_FUNC_TRACE();
1482
1483         return -ENOSYS;
1484 }
1485
1486 static void
1487 i40e_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1488 {
1489         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1490         struct i40e_vsi *vsi = pf->main_vsi;
1491
1492         dev_info->max_rx_queues = vsi->nb_qps;
1493         dev_info->max_tx_queues = vsi->nb_qps;
1494         dev_info->min_rx_bufsize = I40E_BUF_SIZE_MIN;
1495         dev_info->max_rx_pktlen = I40E_FRAME_SIZE_MAX;
1496         dev_info->max_mac_addrs = vsi->max_macaddrs;
1497         dev_info->max_vfs = dev->pci_dev->max_vfs;
1498         dev_info->rx_offload_capa =
1499                 DEV_RX_OFFLOAD_VLAN_STRIP |
1500                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1501                 DEV_RX_OFFLOAD_UDP_CKSUM |
1502                 DEV_RX_OFFLOAD_TCP_CKSUM;
1503         dev_info->tx_offload_capa =
1504                 DEV_TX_OFFLOAD_VLAN_INSERT |
1505                 DEV_TX_OFFLOAD_IPV4_CKSUM |
1506                 DEV_TX_OFFLOAD_UDP_CKSUM |
1507                 DEV_TX_OFFLOAD_TCP_CKSUM |
1508                 DEV_TX_OFFLOAD_SCTP_CKSUM;
1509         dev_info->reta_size = pf->hash_lut_size;
1510
1511         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1512                 .rx_thresh = {
1513                         .pthresh = I40E_DEFAULT_RX_PTHRESH,
1514                         .hthresh = I40E_DEFAULT_RX_HTHRESH,
1515                         .wthresh = I40E_DEFAULT_RX_WTHRESH,
1516                 },
1517                 .rx_free_thresh = I40E_DEFAULT_RX_FREE_THRESH,
1518                 .rx_drop_en = 0,
1519         };
1520
1521         dev_info->default_txconf = (struct rte_eth_txconf) {
1522                 .tx_thresh = {
1523                         .pthresh = I40E_DEFAULT_TX_PTHRESH,
1524                         .hthresh = I40E_DEFAULT_TX_HTHRESH,
1525                         .wthresh = I40E_DEFAULT_TX_WTHRESH,
1526                 },
1527                 .tx_free_thresh = I40E_DEFAULT_TX_FREE_THRESH,
1528                 .tx_rs_thresh = I40E_DEFAULT_TX_RSBIT_THRESH,
1529                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
1530                                 ETH_TXQ_FLAGS_NOOFFLOADS,
1531         };
1532
1533         if (pf->flags | I40E_FLAG_VMDQ) {
1534                 dev_info->max_vmdq_pools = pf->max_nb_vmdq_vsi;
1535                 dev_info->vmdq_queue_base = dev_info->max_rx_queues;
1536                 dev_info->vmdq_queue_num = pf->vmdq_nb_qps *
1537                                                 pf->max_nb_vmdq_vsi;
1538                 dev_info->vmdq_pool_base = I40E_VMDQ_POOL_BASE;
1539                 dev_info->max_rx_queues += dev_info->vmdq_queue_num;
1540                 dev_info->max_tx_queues += dev_info->vmdq_queue_num;
1541         }
1542 }
1543
1544 static int
1545 i40e_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1546 {
1547         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1548         struct i40e_vsi *vsi = pf->main_vsi;
1549         PMD_INIT_FUNC_TRACE();
1550
1551         if (on)
1552                 return i40e_vsi_add_vlan(vsi, vlan_id);
1553         else
1554                 return i40e_vsi_delete_vlan(vsi, vlan_id);
1555 }
1556
1557 static void
1558 i40e_vlan_tpid_set(__rte_unused struct rte_eth_dev *dev,
1559                    __rte_unused uint16_t tpid)
1560 {
1561         PMD_INIT_FUNC_TRACE();
1562 }
1563
1564 static void
1565 i40e_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1566 {
1567         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1568         struct i40e_vsi *vsi = pf->main_vsi;
1569
1570         if (mask & ETH_VLAN_STRIP_MASK) {
1571                 /* Enable or disable VLAN stripping */
1572                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1573                         i40e_vsi_config_vlan_stripping(vsi, TRUE);
1574                 else
1575                         i40e_vsi_config_vlan_stripping(vsi, FALSE);
1576         }
1577
1578         if (mask & ETH_VLAN_EXTEND_MASK) {
1579                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1580                         i40e_vsi_config_double_vlan(vsi, TRUE);
1581                 else
1582                         i40e_vsi_config_double_vlan(vsi, FALSE);
1583         }
1584 }
1585
1586 static void
1587 i40e_vlan_strip_queue_set(__rte_unused struct rte_eth_dev *dev,
1588                           __rte_unused uint16_t queue,
1589                           __rte_unused int on)
1590 {
1591         PMD_INIT_FUNC_TRACE();
1592 }
1593
1594 static int
1595 i40e_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid, int on)
1596 {
1597         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1598         struct i40e_vsi *vsi = pf->main_vsi;
1599         struct rte_eth_dev_data *data = I40E_VSI_TO_DEV_DATA(vsi);
1600         struct i40e_vsi_vlan_pvid_info info;
1601
1602         memset(&info, 0, sizeof(info));
1603         info.on = on;
1604         if (info.on)
1605                 info.config.pvid = pvid;
1606         else {
1607                 info.config.reject.tagged =
1608                                 data->dev_conf.txmode.hw_vlan_reject_tagged;
1609                 info.config.reject.untagged =
1610                                 data->dev_conf.txmode.hw_vlan_reject_untagged;
1611         }
1612
1613         return i40e_vsi_vlan_pvid_set(vsi, &info);
1614 }
1615
1616 static int
1617 i40e_dev_led_on(struct rte_eth_dev *dev)
1618 {
1619         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1620         uint32_t mode = i40e_led_get(hw);
1621
1622         if (mode == 0)
1623                 i40e_led_set(hw, 0xf, true); /* 0xf means led always true */
1624
1625         return 0;
1626 }
1627
1628 static int
1629 i40e_dev_led_off(struct rte_eth_dev *dev)
1630 {
1631         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1632         uint32_t mode = i40e_led_get(hw);
1633
1634         if (mode != 0)
1635                 i40e_led_set(hw, 0, false);
1636
1637         return 0;
1638 }
1639
1640 static int
1641 i40e_flow_ctrl_set(__rte_unused struct rte_eth_dev *dev,
1642                    __rte_unused struct rte_eth_fc_conf *fc_conf)
1643 {
1644         PMD_INIT_FUNC_TRACE();
1645
1646         return -ENOSYS;
1647 }
1648
1649 static int
1650 i40e_priority_flow_ctrl_set(__rte_unused struct rte_eth_dev *dev,
1651                             __rte_unused struct rte_eth_pfc_conf *pfc_conf)
1652 {
1653         PMD_INIT_FUNC_TRACE();
1654
1655         return -ENOSYS;
1656 }
1657
1658 /* Add a MAC address, and update filters */
1659 static void
1660 i40e_macaddr_add(struct rte_eth_dev *dev,
1661                  struct ether_addr *mac_addr,
1662                  __rte_unused uint32_t index,
1663                  uint32_t pool)
1664 {
1665         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1666         struct i40e_mac_filter_info mac_filter;
1667         struct i40e_vsi *vsi;
1668         int ret;
1669
1670         /* If VMDQ not enabled or configured, return */
1671         if (pool != 0 && (!(pf->flags | I40E_FLAG_VMDQ) || !pf->nb_cfg_vmdq_vsi)) {
1672                 PMD_DRV_LOG(ERR, "VMDQ not %s, can't set mac to pool %u",
1673                         pf->flags | I40E_FLAG_VMDQ ? "configured" : "enabled",
1674                         pool);
1675                 return;
1676         }
1677
1678         if (pool > pf->nb_cfg_vmdq_vsi) {
1679                 PMD_DRV_LOG(ERR, "Pool number %u invalid. Max pool is %u",
1680                                 pool, pf->nb_cfg_vmdq_vsi);
1681                 return;
1682         }
1683
1684         (void)rte_memcpy(&mac_filter.mac_addr, mac_addr, ETHER_ADDR_LEN);
1685         mac_filter.filter_type = RTE_MACVLAN_PERFECT_MATCH;
1686
1687         if (pool == 0)
1688                 vsi = pf->main_vsi;
1689         else
1690                 vsi = pf->vmdq[pool - 1].vsi;
1691
1692         ret = i40e_vsi_add_mac(vsi, &mac_filter);
1693         if (ret != I40E_SUCCESS) {
1694                 PMD_DRV_LOG(ERR, "Failed to add MACVLAN filter");
1695                 return;
1696         }
1697 }
1698
1699 /* Remove a MAC address, and update filters */
1700 static void
1701 i40e_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1702 {
1703         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1704         struct i40e_vsi *vsi;
1705         struct rte_eth_dev_data *data = dev->data;
1706         struct ether_addr *macaddr;
1707         int ret;
1708         uint32_t i;
1709         uint64_t pool_sel;
1710
1711         macaddr = &(data->mac_addrs[index]);
1712
1713         pool_sel = dev->data->mac_pool_sel[index];
1714
1715         for (i = 0; i < sizeof(pool_sel) * CHAR_BIT; i++) {
1716                 if (pool_sel & (1ULL << i)) {
1717                         if (i == 0)
1718                                 vsi = pf->main_vsi;
1719                         else {
1720                                 /* No VMDQ pool enabled or configured */
1721                                 if (!(pf->flags | I40E_FLAG_VMDQ) ||
1722                                         (i > pf->nb_cfg_vmdq_vsi)) {
1723                                         PMD_DRV_LOG(ERR, "No VMDQ pool enabled"
1724                                                         "/configured");
1725                                         return;
1726                                 }
1727                                 vsi = pf->vmdq[i - 1].vsi;
1728                         }
1729                         ret = i40e_vsi_delete_mac(vsi, macaddr);
1730
1731                         if (ret) {
1732                                 PMD_DRV_LOG(ERR, "Failed to remove MACVLAN filter");
1733                                 return;
1734                         }
1735                 }
1736         }
1737 }
1738
1739 /* Set perfect match or hash match of MAC and VLAN for a VF */
1740 static int
1741 i40e_vf_mac_filter_set(struct i40e_pf *pf,
1742                  struct rte_eth_mac_filter *filter,
1743                  bool add)
1744 {
1745         struct i40e_hw *hw;
1746         struct i40e_mac_filter_info mac_filter;
1747         struct ether_addr old_mac;
1748         struct ether_addr *new_mac;
1749         struct i40e_pf_vf *vf = NULL;
1750         uint16_t vf_id;
1751         int ret;
1752
1753         if (pf == NULL) {
1754                 PMD_DRV_LOG(ERR, "Invalid PF argument.");
1755                 return -EINVAL;
1756         }
1757         hw = I40E_PF_TO_HW(pf);
1758
1759         if (filter == NULL) {
1760                 PMD_DRV_LOG(ERR, "Invalid mac filter argument.");
1761                 return -EINVAL;
1762         }
1763
1764         new_mac = &filter->mac_addr;
1765
1766         if (is_zero_ether_addr(new_mac)) {
1767                 PMD_DRV_LOG(ERR, "Invalid ethernet address.");
1768                 return -EINVAL;
1769         }
1770
1771         vf_id = filter->dst_id;
1772
1773         if (vf_id > pf->vf_num - 1 || !pf->vfs) {
1774                 PMD_DRV_LOG(ERR, "Invalid argument.");
1775                 return -EINVAL;
1776         }
1777         vf = &pf->vfs[vf_id];
1778
1779         if (add && is_same_ether_addr(new_mac, &(pf->dev_addr))) {
1780                 PMD_DRV_LOG(INFO, "Ignore adding permanent MAC address.");
1781                 return -EINVAL;
1782         }
1783
1784         if (add) {
1785                 (void)rte_memcpy(&old_mac, hw->mac.addr, ETHER_ADDR_LEN);
1786                 (void)rte_memcpy(hw->mac.addr, new_mac->addr_bytes,
1787                                 ETHER_ADDR_LEN);
1788                 (void)rte_memcpy(&mac_filter.mac_addr, &filter->mac_addr,
1789                                  ETHER_ADDR_LEN);
1790
1791                 mac_filter.filter_type = filter->filter_type;
1792                 ret = i40e_vsi_add_mac(vf->vsi, &mac_filter);
1793                 if (ret != I40E_SUCCESS) {
1794                         PMD_DRV_LOG(ERR, "Failed to add MAC filter.");
1795                         return -1;
1796                 }
1797                 ether_addr_copy(new_mac, &pf->dev_addr);
1798         } else {
1799                 (void)rte_memcpy(hw->mac.addr, hw->mac.perm_addr,
1800                                 ETHER_ADDR_LEN);
1801                 ret = i40e_vsi_delete_mac(vf->vsi, &filter->mac_addr);
1802                 if (ret != I40E_SUCCESS) {
1803                         PMD_DRV_LOG(ERR, "Failed to delete MAC filter.");
1804                         return -1;
1805                 }
1806
1807                 /* Clear device address as it has been removed */
1808                 if (is_same_ether_addr(&(pf->dev_addr), new_mac))
1809                         memset(&pf->dev_addr, 0, sizeof(struct ether_addr));
1810         }
1811
1812         return 0;
1813 }
1814
1815 /* MAC filter handle */
1816 static int
1817 i40e_mac_filter_handle(struct rte_eth_dev *dev, enum rte_filter_op filter_op,
1818                 void *arg)
1819 {
1820         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1821         struct rte_eth_mac_filter *filter;
1822         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
1823         int ret = I40E_NOT_SUPPORTED;
1824
1825         filter = (struct rte_eth_mac_filter *)(arg);
1826
1827         switch (filter_op) {
1828         case RTE_ETH_FILTER_NOP:
1829                 ret = I40E_SUCCESS;
1830                 break;
1831         case RTE_ETH_FILTER_ADD:
1832                 i40e_pf_disable_irq0(hw);
1833                 if (filter->is_vf)
1834                         ret = i40e_vf_mac_filter_set(pf, filter, 1);
1835                 i40e_pf_enable_irq0(hw);
1836                 break;
1837         case RTE_ETH_FILTER_DELETE:
1838                 i40e_pf_disable_irq0(hw);
1839                 if (filter->is_vf)
1840                         ret = i40e_vf_mac_filter_set(pf, filter, 0);
1841                 i40e_pf_enable_irq0(hw);
1842                 break;
1843         default:
1844                 PMD_DRV_LOG(ERR, "unknown operation %u", filter_op);
1845                 ret = I40E_ERR_PARAM;
1846                 break;
1847         }
1848
1849         return ret;
1850 }
1851
1852 static int
1853 i40e_dev_rss_reta_update(struct rte_eth_dev *dev,
1854                          struct rte_eth_rss_reta_entry64 *reta_conf,
1855                          uint16_t reta_size)
1856 {
1857         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1858         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1859         uint32_t lut, l;
1860         uint16_t i, j, lut_size = pf->hash_lut_size;
1861         uint16_t idx, shift;
1862         uint8_t mask;
1863
1864         if (reta_size != lut_size ||
1865                 reta_size > ETH_RSS_RETA_SIZE_512) {
1866                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
1867                         "(%d) doesn't match the number hardware can supported "
1868                                         "(%d)\n", reta_size, lut_size);
1869                 return -EINVAL;
1870         }
1871
1872         for (i = 0; i < reta_size; i += I40E_4_BIT_WIDTH) {
1873                 idx = i / RTE_RETA_GROUP_SIZE;
1874                 shift = i % RTE_RETA_GROUP_SIZE;
1875                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1876                                                 I40E_4_BIT_MASK);
1877                 if (!mask)
1878                         continue;
1879                 if (mask == I40E_4_BIT_MASK)
1880                         l = 0;
1881                 else
1882                         l = I40E_READ_REG(hw, I40E_PFQF_HLUT(i >> 2));
1883                 for (j = 0, lut = 0; j < I40E_4_BIT_WIDTH; j++) {
1884                         if (mask & (0x1 << j))
1885                                 lut |= reta_conf[idx].reta[shift + j] <<
1886                                                         (CHAR_BIT * j);
1887                         else
1888                                 lut |= l & (I40E_8_BIT_MASK << (CHAR_BIT * j));
1889                 }
1890                 I40E_WRITE_REG(hw, I40E_PFQF_HLUT(i >> 2), lut);
1891         }
1892
1893         return 0;
1894 }
1895
1896 static int
1897 i40e_dev_rss_reta_query(struct rte_eth_dev *dev,
1898                         struct rte_eth_rss_reta_entry64 *reta_conf,
1899                         uint16_t reta_size)
1900 {
1901         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1902         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1903         uint32_t lut;
1904         uint16_t i, j, lut_size = pf->hash_lut_size;
1905         uint16_t idx, shift;
1906         uint8_t mask;
1907
1908         if (reta_size != lut_size ||
1909                 reta_size > ETH_RSS_RETA_SIZE_512) {
1910                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
1911                         "(%d) doesn't match the number hardware can supported "
1912                                         "(%d)\n", reta_size, lut_size);
1913                 return -EINVAL;
1914         }
1915
1916         for (i = 0; i < reta_size; i += I40E_4_BIT_WIDTH) {
1917                 idx = i / RTE_RETA_GROUP_SIZE;
1918                 shift = i % RTE_RETA_GROUP_SIZE;
1919                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1920                                                 I40E_4_BIT_MASK);
1921                 if (!mask)
1922                         continue;
1923
1924                 lut = I40E_READ_REG(hw, I40E_PFQF_HLUT(i >> 2));
1925                 for (j = 0; j < I40E_4_BIT_WIDTH; j++) {
1926                         if (mask & (0x1 << j))
1927                                 reta_conf[idx].reta[shift] = ((lut >>
1928                                         (CHAR_BIT * j)) & I40E_8_BIT_MASK);
1929                 }
1930         }
1931
1932         return 0;
1933 }
1934
1935 /**
1936  * i40e_allocate_dma_mem_d - specific memory alloc for shared code (base driver)
1937  * @hw:   pointer to the HW structure
1938  * @mem:  pointer to mem struct to fill out
1939  * @size: size of memory requested
1940  * @alignment: what to align the allocation to
1941  **/
1942 enum i40e_status_code
1943 i40e_allocate_dma_mem_d(__attribute__((unused)) struct i40e_hw *hw,
1944                         struct i40e_dma_mem *mem,
1945                         u64 size,
1946                         u32 alignment)
1947 {
1948         static uint64_t id = 0;
1949         const struct rte_memzone *mz = NULL;
1950         char z_name[RTE_MEMZONE_NAMESIZE];
1951
1952         if (!mem)
1953                 return I40E_ERR_PARAM;
1954
1955         id++;
1956         snprintf(z_name, sizeof(z_name), "i40e_dma_%"PRIu64, id);
1957 #ifdef RTE_LIBRTE_XEN_DOM0
1958         mz = rte_memzone_reserve_bounded(z_name, size, 0, 0, alignment,
1959                                                         RTE_PGSIZE_2M);
1960 #else
1961         mz = rte_memzone_reserve_aligned(z_name, size, 0, 0, alignment);
1962 #endif
1963         if (!mz)
1964                 return I40E_ERR_NO_MEMORY;
1965
1966         mem->id = id;
1967         mem->size = size;
1968         mem->va = mz->addr;
1969 #ifdef RTE_LIBRTE_XEN_DOM0
1970         mem->pa = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1971 #else
1972         mem->pa = mz->phys_addr;
1973 #endif
1974
1975         return I40E_SUCCESS;
1976 }
1977
1978 /**
1979  * i40e_free_dma_mem_d - specific memory free for shared code (base driver)
1980  * @hw:   pointer to the HW structure
1981  * @mem:  ptr to mem struct to free
1982  **/
1983 enum i40e_status_code
1984 i40e_free_dma_mem_d(__attribute__((unused)) struct i40e_hw *hw,
1985                     struct i40e_dma_mem *mem)
1986 {
1987         if (!mem || !mem->va)
1988                 return I40E_ERR_PARAM;
1989
1990         mem->va = NULL;
1991         mem->pa = (u64)0;
1992
1993         return I40E_SUCCESS;
1994 }
1995
1996 /**
1997  * i40e_allocate_virt_mem_d - specific memory alloc for shared code (base driver)
1998  * @hw:   pointer to the HW structure
1999  * @mem:  pointer to mem struct to fill out
2000  * @size: size of memory requested
2001  **/
2002 enum i40e_status_code
2003 i40e_allocate_virt_mem_d(__attribute__((unused)) struct i40e_hw *hw,
2004                          struct i40e_virt_mem *mem,
2005                          u32 size)
2006 {
2007         if (!mem)
2008                 return I40E_ERR_PARAM;
2009
2010         mem->size = size;
2011         mem->va = rte_zmalloc("i40e", size, 0);
2012
2013         if (mem->va)
2014                 return I40E_SUCCESS;
2015         else
2016                 return I40E_ERR_NO_MEMORY;
2017 }
2018
2019 /**
2020  * i40e_free_virt_mem_d - specific memory free for shared code (base driver)
2021  * @hw:   pointer to the HW structure
2022  * @mem:  pointer to mem struct to free
2023  **/
2024 enum i40e_status_code
2025 i40e_free_virt_mem_d(__attribute__((unused)) struct i40e_hw *hw,
2026                      struct i40e_virt_mem *mem)
2027 {
2028         if (!mem)
2029                 return I40E_ERR_PARAM;
2030
2031         rte_free(mem->va);
2032         mem->va = NULL;
2033
2034         return I40E_SUCCESS;
2035 }
2036
2037 void
2038 i40e_init_spinlock_d(struct i40e_spinlock *sp)
2039 {
2040         rte_spinlock_init(&sp->spinlock);
2041 }
2042
2043 void
2044 i40e_acquire_spinlock_d(struct i40e_spinlock *sp)
2045 {
2046         rte_spinlock_lock(&sp->spinlock);
2047 }
2048
2049 void
2050 i40e_release_spinlock_d(struct i40e_spinlock *sp)
2051 {
2052         rte_spinlock_unlock(&sp->spinlock);
2053 }
2054
2055 void
2056 i40e_destroy_spinlock_d(__attribute__((unused)) struct i40e_spinlock *sp)
2057 {
2058         return;
2059 }
2060
2061 /**
2062  * Get the hardware capabilities, which will be parsed
2063  * and saved into struct i40e_hw.
2064  */
2065 static int
2066 i40e_get_cap(struct i40e_hw *hw)
2067 {
2068         struct i40e_aqc_list_capabilities_element_resp *buf;
2069         uint16_t len, size = 0;
2070         int ret;
2071
2072         /* Calculate a huge enough buff for saving response data temporarily */
2073         len = sizeof(struct i40e_aqc_list_capabilities_element_resp) *
2074                                                 I40E_MAX_CAP_ELE_NUM;
2075         buf = rte_zmalloc("i40e", len, 0);
2076         if (!buf) {
2077                 PMD_DRV_LOG(ERR, "Failed to allocate memory");
2078                 return I40E_ERR_NO_MEMORY;
2079         }
2080
2081         /* Get, parse the capabilities and save it to hw */
2082         ret = i40e_aq_discover_capabilities(hw, buf, len, &size,
2083                         i40e_aqc_opc_list_func_capabilities, NULL);
2084         if (ret != I40E_SUCCESS)
2085                 PMD_DRV_LOG(ERR, "Failed to discover capabilities");
2086
2087         /* Free the temporary buffer after being used */
2088         rte_free(buf);
2089
2090         return ret;
2091 }
2092
2093 static int
2094 i40e_pf_parameter_init(struct rte_eth_dev *dev)
2095 {
2096         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2097         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
2098         uint16_t sum_queues = 0, sum_vsis, left_queues;
2099
2100         /* First check if FW support SRIOV */
2101         if (dev->pci_dev->max_vfs && !hw->func_caps.sr_iov_1_1) {
2102                 PMD_INIT_LOG(ERR, "HW configuration doesn't support SRIOV");
2103                 return -EINVAL;
2104         }
2105
2106         pf->flags = I40E_FLAG_HEADER_SPLIT_DISABLED;
2107         pf->max_num_vsi = RTE_MIN(hw->func_caps.num_vsis, I40E_MAX_NUM_VSIS);
2108         PMD_INIT_LOG(INFO, "Max supported VSIs:%u", pf->max_num_vsi);
2109         /* Allocate queues for pf */
2110         if (hw->func_caps.rss) {
2111                 pf->flags |= I40E_FLAG_RSS;
2112                 pf->lan_nb_qps = RTE_MIN(hw->func_caps.num_tx_qp,
2113                         (uint32_t)(1 << hw->func_caps.rss_table_entry_width));
2114                 pf->lan_nb_qps = i40e_align_floor(pf->lan_nb_qps);
2115         } else
2116                 pf->lan_nb_qps = 1;
2117         sum_queues = pf->lan_nb_qps;
2118         /* Default VSI is not counted in */
2119         sum_vsis = 0;
2120         PMD_INIT_LOG(INFO, "PF queue pairs:%u", pf->lan_nb_qps);
2121
2122         if (hw->func_caps.sr_iov_1_1 && dev->pci_dev->max_vfs) {
2123                 pf->flags |= I40E_FLAG_SRIOV;
2124                 pf->vf_nb_qps = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VF;
2125                 if (dev->pci_dev->max_vfs > hw->func_caps.num_vfs) {
2126                         PMD_INIT_LOG(ERR, "Config VF number %u, "
2127                                      "max supported %u.",
2128                                      dev->pci_dev->max_vfs,
2129                                      hw->func_caps.num_vfs);
2130                         return -EINVAL;
2131                 }
2132                 if (pf->vf_nb_qps > I40E_MAX_QP_NUM_PER_VF) {
2133                         PMD_INIT_LOG(ERR, "FVL VF queue %u, "
2134                                      "max support %u queues.",
2135                                      pf->vf_nb_qps, I40E_MAX_QP_NUM_PER_VF);
2136                         return -EINVAL;
2137                 }
2138                 pf->vf_num = dev->pci_dev->max_vfs;
2139                 sum_queues += pf->vf_nb_qps * pf->vf_num;
2140                 sum_vsis   += pf->vf_num;
2141                 PMD_INIT_LOG(INFO, "Max VF num:%u each has queue pairs:%u",
2142                              pf->vf_num, pf->vf_nb_qps);
2143         } else
2144                 pf->vf_num = 0;
2145
2146         if (hw->func_caps.vmdq) {
2147                 pf->flags |= I40E_FLAG_VMDQ;
2148                 pf->vmdq_nb_qps = RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM;
2149                 pf->max_nb_vmdq_vsi = 1;
2150                 /*
2151                  * If VMDQ available, assume a single VSI can be created.  Will adjust
2152                  * later.
2153                  */
2154                 sum_queues += pf->vmdq_nb_qps * pf->max_nb_vmdq_vsi;
2155                 sum_vsis += pf->max_nb_vmdq_vsi;
2156         } else {
2157                 pf->vmdq_nb_qps = 0;
2158                 pf->max_nb_vmdq_vsi = 0;
2159         }
2160         pf->nb_cfg_vmdq_vsi = 0;
2161
2162         if (hw->func_caps.fd) {
2163                 pf->flags |= I40E_FLAG_FDIR;
2164                 pf->fdir_nb_qps = I40E_DEFAULT_QP_NUM_FDIR;
2165                 /**
2166                  * Each flow director consumes one VSI and one queue,
2167                  * but can't calculate out predictably here.
2168                  */
2169         }
2170
2171         if (sum_vsis > pf->max_num_vsi ||
2172                 sum_queues > hw->func_caps.num_rx_qp) {
2173                 PMD_INIT_LOG(ERR, "VSI/QUEUE setting can't be satisfied");
2174                 PMD_INIT_LOG(ERR, "Max VSIs: %u, asked:%u",
2175                              pf->max_num_vsi, sum_vsis);
2176                 PMD_INIT_LOG(ERR, "Total queue pairs:%u, asked:%u",
2177                              hw->func_caps.num_rx_qp, sum_queues);
2178                 return -EINVAL;
2179         }
2180
2181         /* Adjust VMDQ setting to support as many VMs as possible */
2182         if (pf->flags & I40E_FLAG_VMDQ) {
2183                 left_queues = hw->func_caps.num_rx_qp - sum_queues;
2184
2185                 pf->max_nb_vmdq_vsi += RTE_MIN(left_queues / pf->vmdq_nb_qps,
2186                                         pf->max_num_vsi - sum_vsis);
2187
2188                 /* Limit the max VMDQ number that rte_ether that can support  */
2189                 pf->max_nb_vmdq_vsi = RTE_MIN(pf->max_nb_vmdq_vsi,
2190                                         ETH_64_POOLS - 1);
2191
2192                 PMD_INIT_LOG(INFO, "Max VMDQ VSI num:%u",
2193                                 pf->max_nb_vmdq_vsi);
2194                 PMD_INIT_LOG(INFO, "VMDQ queue pairs:%u", pf->vmdq_nb_qps);
2195         }
2196
2197         /* Each VSI occupy 1 MSIX interrupt at least, plus IRQ0 for misc intr
2198          * cause */
2199         if (sum_vsis > hw->func_caps.num_msix_vectors - 1) {
2200                 PMD_INIT_LOG(ERR, "Too many VSIs(%u), MSIX intr(%u) not enough",
2201                              sum_vsis, hw->func_caps.num_msix_vectors);
2202                 return -EINVAL;
2203         }
2204         return I40E_SUCCESS;
2205 }
2206
2207 static int
2208 i40e_pf_get_switch_config(struct i40e_pf *pf)
2209 {
2210         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
2211         struct i40e_aqc_get_switch_config_resp *switch_config;
2212         struct i40e_aqc_switch_config_element_resp *element;
2213         uint16_t start_seid = 0, num_reported;
2214         int ret;
2215
2216         switch_config = (struct i40e_aqc_get_switch_config_resp *)\
2217                         rte_zmalloc("i40e", I40E_AQ_LARGE_BUF, 0);
2218         if (!switch_config) {
2219                 PMD_DRV_LOG(ERR, "Failed to allocated memory");
2220                 return -ENOMEM;
2221         }
2222
2223         /* Get the switch configurations */
2224         ret = i40e_aq_get_switch_config(hw, switch_config,
2225                 I40E_AQ_LARGE_BUF, &start_seid, NULL);
2226         if (ret != I40E_SUCCESS) {
2227                 PMD_DRV_LOG(ERR, "Failed to get switch configurations");
2228                 goto fail;
2229         }
2230         num_reported = rte_le_to_cpu_16(switch_config->header.num_reported);
2231         if (num_reported != 1) { /* The number should be 1 */
2232                 PMD_DRV_LOG(ERR, "Wrong number of switch config reported");
2233                 goto fail;
2234         }
2235
2236         /* Parse the switch configuration elements */
2237         element = &(switch_config->element[0]);
2238         if (element->element_type == I40E_SWITCH_ELEMENT_TYPE_VSI) {
2239                 pf->mac_seid = rte_le_to_cpu_16(element->uplink_seid);
2240                 pf->main_vsi_seid = rte_le_to_cpu_16(element->seid);
2241         } else
2242                 PMD_DRV_LOG(INFO, "Unknown element type");
2243
2244 fail:
2245         rte_free(switch_config);
2246
2247         return ret;
2248 }
2249
2250 static int
2251 i40e_res_pool_init (struct i40e_res_pool_info *pool, uint32_t base,
2252                         uint32_t num)
2253 {
2254         struct pool_entry *entry;
2255
2256         if (pool == NULL || num == 0)
2257                 return -EINVAL;
2258
2259         entry = rte_zmalloc("i40e", sizeof(*entry), 0);
2260         if (entry == NULL) {
2261                 PMD_DRV_LOG(ERR, "Failed to allocate memory for resource pool");
2262                 return -ENOMEM;
2263         }
2264
2265         /* queue heap initialize */
2266         pool->num_free = num;
2267         pool->num_alloc = 0;
2268         pool->base = base;
2269         LIST_INIT(&pool->alloc_list);
2270         LIST_INIT(&pool->free_list);
2271
2272         /* Initialize element  */
2273         entry->base = 0;
2274         entry->len = num;
2275
2276         LIST_INSERT_HEAD(&pool->free_list, entry, next);
2277         return 0;
2278 }
2279
2280 static void
2281 i40e_res_pool_destroy(struct i40e_res_pool_info *pool)
2282 {
2283         struct pool_entry *entry;
2284
2285         if (pool == NULL)
2286                 return;
2287
2288         LIST_FOREACH(entry, &pool->alloc_list, next) {
2289                 LIST_REMOVE(entry, next);
2290                 rte_free(entry);
2291         }
2292
2293         LIST_FOREACH(entry, &pool->free_list, next) {
2294                 LIST_REMOVE(entry, next);
2295                 rte_free(entry);
2296         }
2297
2298         pool->num_free = 0;
2299         pool->num_alloc = 0;
2300         pool->base = 0;
2301         LIST_INIT(&pool->alloc_list);
2302         LIST_INIT(&pool->free_list);
2303 }
2304
2305 static int
2306 i40e_res_pool_free(struct i40e_res_pool_info *pool,
2307                        uint32_t base)
2308 {
2309         struct pool_entry *entry, *next, *prev, *valid_entry = NULL;
2310         uint32_t pool_offset;
2311         int insert;
2312
2313         if (pool == NULL) {
2314                 PMD_DRV_LOG(ERR, "Invalid parameter");
2315                 return -EINVAL;
2316         }
2317
2318         pool_offset = base - pool->base;
2319         /* Lookup in alloc list */
2320         LIST_FOREACH(entry, &pool->alloc_list, next) {
2321                 if (entry->base == pool_offset) {
2322                         valid_entry = entry;
2323                         LIST_REMOVE(entry, next);
2324                         break;
2325                 }
2326         }
2327
2328         /* Not find, return */
2329         if (valid_entry == NULL) {
2330                 PMD_DRV_LOG(ERR, "Failed to find entry");
2331                 return -EINVAL;
2332         }
2333
2334         /**
2335          * Found it, move it to free list  and try to merge.
2336          * In order to make merge easier, always sort it by qbase.
2337          * Find adjacent prev and last entries.
2338          */
2339         prev = next = NULL;
2340         LIST_FOREACH(entry, &pool->free_list, next) {
2341                 if (entry->base > valid_entry->base) {
2342                         next = entry;
2343                         break;
2344                 }
2345                 prev = entry;
2346         }
2347
2348         insert = 0;
2349         /* Try to merge with next one*/
2350         if (next != NULL) {
2351                 /* Merge with next one */
2352                 if (valid_entry->base + valid_entry->len == next->base) {
2353                         next->base = valid_entry->base;
2354                         next->len += valid_entry->len;
2355                         rte_free(valid_entry);
2356                         valid_entry = next;
2357                         insert = 1;
2358                 }
2359         }
2360
2361         if (prev != NULL) {
2362                 /* Merge with previous one */
2363                 if (prev->base + prev->len == valid_entry->base) {
2364                         prev->len += valid_entry->len;
2365                         /* If it merge with next one, remove next node */
2366                         if (insert == 1) {
2367                                 LIST_REMOVE(valid_entry, next);
2368                                 rte_free(valid_entry);
2369                         } else {
2370                                 rte_free(valid_entry);
2371                                 insert = 1;
2372                         }
2373                 }
2374         }
2375
2376         /* Not find any entry to merge, insert */
2377         if (insert == 0) {
2378                 if (prev != NULL)
2379                         LIST_INSERT_AFTER(prev, valid_entry, next);
2380                 else if (next != NULL)
2381                         LIST_INSERT_BEFORE(next, valid_entry, next);
2382                 else /* It's empty list, insert to head */
2383                         LIST_INSERT_HEAD(&pool->free_list, valid_entry, next);
2384         }
2385
2386         pool->num_free += valid_entry->len;
2387         pool->num_alloc -= valid_entry->len;
2388
2389         return 0;
2390 }
2391
2392 static int
2393 i40e_res_pool_alloc(struct i40e_res_pool_info *pool,
2394                        uint16_t num)
2395 {
2396         struct pool_entry *entry, *valid_entry;
2397
2398         if (pool == NULL || num == 0) {
2399                 PMD_DRV_LOG(ERR, "Invalid parameter");
2400                 return -EINVAL;
2401         }
2402
2403         if (pool->num_free < num) {
2404                 PMD_DRV_LOG(ERR, "No resource. ask:%u, available:%u",
2405                             num, pool->num_free);
2406                 return -ENOMEM;
2407         }
2408
2409         valid_entry = NULL;
2410         /* Lookup  in free list and find most fit one */
2411         LIST_FOREACH(entry, &pool->free_list, next) {
2412                 if (entry->len >= num) {
2413                         /* Find best one */
2414                         if (entry->len == num) {
2415                                 valid_entry = entry;
2416                                 break;
2417                         }
2418                         if (valid_entry == NULL || valid_entry->len > entry->len)
2419                                 valid_entry = entry;
2420                 }
2421         }
2422
2423         /* Not find one to satisfy the request, return */
2424         if (valid_entry == NULL) {
2425                 PMD_DRV_LOG(ERR, "No valid entry found");
2426                 return -ENOMEM;
2427         }
2428         /**
2429          * The entry have equal queue number as requested,
2430          * remove it from alloc_list.
2431          */
2432         if (valid_entry->len == num) {
2433                 LIST_REMOVE(valid_entry, next);
2434         } else {
2435                 /**
2436                  * The entry have more numbers than requested,
2437                  * create a new entry for alloc_list and minus its
2438                  * queue base and number in free_list.
2439                  */
2440                 entry = rte_zmalloc("res_pool", sizeof(*entry), 0);
2441                 if (entry == NULL) {
2442                         PMD_DRV_LOG(ERR, "Failed to allocate memory for "
2443                                     "resource pool");
2444                         return -ENOMEM;
2445                 }
2446                 entry->base = valid_entry->base;
2447                 entry->len = num;
2448                 valid_entry->base += num;
2449                 valid_entry->len -= num;
2450                 valid_entry = entry;
2451         }
2452
2453         /* Insert it into alloc list, not sorted */
2454         LIST_INSERT_HEAD(&pool->alloc_list, valid_entry, next);
2455
2456         pool->num_free -= valid_entry->len;
2457         pool->num_alloc += valid_entry->len;
2458
2459         return (valid_entry->base + pool->base);
2460 }
2461
2462 /**
2463  * bitmap_is_subset - Check whether src2 is subset of src1
2464  **/
2465 static inline int
2466 bitmap_is_subset(uint8_t src1, uint8_t src2)
2467 {
2468         return !((src1 ^ src2) & src2);
2469 }
2470
2471 static int
2472 validate_tcmap_parameter(struct i40e_vsi *vsi, uint8_t enabled_tcmap)
2473 {
2474         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2475
2476         /* If DCB is not supported, only default TC is supported */
2477         if (!hw->func_caps.dcb && enabled_tcmap != I40E_DEFAULT_TCMAP) {
2478                 PMD_DRV_LOG(ERR, "DCB is not enabled, only TC0 is supported");
2479                 return -EINVAL;
2480         }
2481
2482         if (!bitmap_is_subset(hw->func_caps.enabled_tcmap, enabled_tcmap)) {
2483                 PMD_DRV_LOG(ERR, "Enabled TC map 0x%x not applicable to "
2484                             "HW support 0x%x", hw->func_caps.enabled_tcmap,
2485                             enabled_tcmap);
2486                 return -EINVAL;
2487         }
2488         return I40E_SUCCESS;
2489 }
2490
2491 int
2492 i40e_vsi_vlan_pvid_set(struct i40e_vsi *vsi,
2493                                 struct i40e_vsi_vlan_pvid_info *info)
2494 {
2495         struct i40e_hw *hw;
2496         struct i40e_vsi_context ctxt;
2497         uint8_t vlan_flags = 0;
2498         int ret;
2499
2500         if (vsi == NULL || info == NULL) {
2501                 PMD_DRV_LOG(ERR, "invalid parameters");
2502                 return I40E_ERR_PARAM;
2503         }
2504
2505         if (info->on) {
2506                 vsi->info.pvid = info->config.pvid;
2507                 /**
2508                  * If insert pvid is enabled, only tagged pkts are
2509                  * allowed to be sent out.
2510                  */
2511                 vlan_flags |= I40E_AQ_VSI_PVLAN_INSERT_PVID |
2512                                 I40E_AQ_VSI_PVLAN_MODE_TAGGED;
2513         } else {
2514                 vsi->info.pvid = 0;
2515                 if (info->config.reject.tagged == 0)
2516                         vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_TAGGED;
2517
2518                 if (info->config.reject.untagged == 0)
2519                         vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_UNTAGGED;
2520         }
2521         vsi->info.port_vlan_flags &= ~(I40E_AQ_VSI_PVLAN_INSERT_PVID |
2522                                         I40E_AQ_VSI_PVLAN_MODE_MASK);
2523         vsi->info.port_vlan_flags |= vlan_flags;
2524         vsi->info.valid_sections =
2525                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
2526         memset(&ctxt, 0, sizeof(ctxt));
2527         (void)rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
2528         ctxt.seid = vsi->seid;
2529
2530         hw = I40E_VSI_TO_HW(vsi);
2531         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
2532         if (ret != I40E_SUCCESS)
2533                 PMD_DRV_LOG(ERR, "Failed to update VSI params");
2534
2535         return ret;
2536 }
2537
2538 static int
2539 i40e_vsi_update_tc_bandwidth(struct i40e_vsi *vsi, uint8_t enabled_tcmap)
2540 {
2541         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2542         int i, ret;
2543         struct i40e_aqc_configure_vsi_tc_bw_data tc_bw_data;
2544
2545         ret = validate_tcmap_parameter(vsi, enabled_tcmap);
2546         if (ret != I40E_SUCCESS)
2547                 return ret;
2548
2549         if (!vsi->seid) {
2550                 PMD_DRV_LOG(ERR, "seid not valid");
2551                 return -EINVAL;
2552         }
2553
2554         memset(&tc_bw_data, 0, sizeof(tc_bw_data));
2555         tc_bw_data.tc_valid_bits = enabled_tcmap;
2556         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
2557                 tc_bw_data.tc_bw_credits[i] =
2558                         (enabled_tcmap & (1 << i)) ? 1 : 0;
2559
2560         ret = i40e_aq_config_vsi_tc_bw(hw, vsi->seid, &tc_bw_data, NULL);
2561         if (ret != I40E_SUCCESS) {
2562                 PMD_DRV_LOG(ERR, "Failed to configure TC BW");
2563                 return ret;
2564         }
2565
2566         (void)rte_memcpy(vsi->info.qs_handle, tc_bw_data.qs_handles,
2567                                         sizeof(vsi->info.qs_handle));
2568         return I40E_SUCCESS;
2569 }
2570
2571 static int
2572 i40e_vsi_config_tc_queue_mapping(struct i40e_vsi *vsi,
2573                                  struct i40e_aqc_vsi_properties_data *info,
2574                                  uint8_t enabled_tcmap)
2575 {
2576         int ret, total_tc = 0, i;
2577         uint16_t qpnum_per_tc, bsf, qp_idx;
2578
2579         ret = validate_tcmap_parameter(vsi, enabled_tcmap);
2580         if (ret != I40E_SUCCESS)
2581                 return ret;
2582
2583         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
2584                 if (enabled_tcmap & (1 << i))
2585                         total_tc++;
2586         vsi->enabled_tc = enabled_tcmap;
2587
2588         /* Number of queues per enabled TC */
2589         qpnum_per_tc = i40e_align_floor(vsi->nb_qps / total_tc);
2590         qpnum_per_tc = RTE_MIN(qpnum_per_tc, I40E_MAX_Q_PER_TC);
2591         bsf = rte_bsf32(qpnum_per_tc);
2592
2593         /* Adjust the queue number to actual queues that can be applied */
2594         vsi->nb_qps = qpnum_per_tc * total_tc;
2595
2596         /**
2597          * Configure TC and queue mapping parameters, for enabled TC,
2598          * allocate qpnum_per_tc queues to this traffic. For disabled TC,
2599          * default queue will serve it.
2600          */
2601         qp_idx = 0;
2602         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
2603                 if (vsi->enabled_tc & (1 << i)) {
2604                         info->tc_mapping[i] = rte_cpu_to_le_16((qp_idx <<
2605                                         I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
2606                                 (bsf << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT));
2607                         qp_idx += qpnum_per_tc;
2608                 } else
2609                         info->tc_mapping[i] = 0;
2610         }
2611
2612         /* Associate queue number with VSI */
2613         if (vsi->type == I40E_VSI_SRIOV) {
2614                 info->mapping_flags |=
2615                         rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
2616                 for (i = 0; i < vsi->nb_qps; i++)
2617                         info->queue_mapping[i] =
2618                                 rte_cpu_to_le_16(vsi->base_queue + i);
2619         } else {
2620                 info->mapping_flags |=
2621                         rte_cpu_to_le_16(I40E_AQ_VSI_QUE_MAP_CONTIG);
2622                 info->queue_mapping[0] = rte_cpu_to_le_16(vsi->base_queue);
2623         }
2624         info->valid_sections =
2625                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID);
2626
2627         return I40E_SUCCESS;
2628 }
2629
2630 static int
2631 i40e_veb_release(struct i40e_veb *veb)
2632 {
2633         struct i40e_vsi *vsi;
2634         struct i40e_hw *hw;
2635
2636         if (veb == NULL || veb->associate_vsi == NULL)
2637                 return -EINVAL;
2638
2639         if (!TAILQ_EMPTY(&veb->head)) {
2640                 PMD_DRV_LOG(ERR, "VEB still has VSI attached, can't remove");
2641                 return -EACCES;
2642         }
2643
2644         vsi = veb->associate_vsi;
2645         hw = I40E_VSI_TO_HW(vsi);
2646
2647         vsi->uplink_seid = veb->uplink_seid;
2648         i40e_aq_delete_element(hw, veb->seid, NULL);
2649         rte_free(veb);
2650         vsi->veb = NULL;
2651         return I40E_SUCCESS;
2652 }
2653
2654 /* Setup a veb */
2655 static struct i40e_veb *
2656 i40e_veb_setup(struct i40e_pf *pf, struct i40e_vsi *vsi)
2657 {
2658         struct i40e_veb *veb;
2659         int ret;
2660         struct i40e_hw *hw;
2661
2662         if (NULL == pf || vsi == NULL) {
2663                 PMD_DRV_LOG(ERR, "veb setup failed, "
2664                             "associated VSI shouldn't null");
2665                 return NULL;
2666         }
2667         hw = I40E_PF_TO_HW(pf);
2668
2669         veb = rte_zmalloc("i40e_veb", sizeof(struct i40e_veb), 0);
2670         if (!veb) {
2671                 PMD_DRV_LOG(ERR, "Failed to allocate memory for veb");
2672                 goto fail;
2673         }
2674
2675         veb->associate_vsi = vsi;
2676         TAILQ_INIT(&veb->head);
2677         veb->uplink_seid = vsi->uplink_seid;
2678
2679         ret = i40e_aq_add_veb(hw, veb->uplink_seid, vsi->seid,
2680                 I40E_DEFAULT_TCMAP, false, false, &veb->seid, NULL);
2681
2682         if (ret != I40E_SUCCESS) {
2683                 PMD_DRV_LOG(ERR, "Add veb failed, aq_err: %d",
2684                             hw->aq.asq_last_status);
2685                 goto fail;
2686         }
2687
2688         /* get statistics index */
2689         ret = i40e_aq_get_veb_parameters(hw, veb->seid, NULL, NULL,
2690                                 &veb->stats_idx, NULL, NULL, NULL);
2691         if (ret != I40E_SUCCESS) {
2692                 PMD_DRV_LOG(ERR, "Get veb statics index failed, aq_err: %d",
2693                             hw->aq.asq_last_status);
2694                 goto fail;
2695         }
2696
2697         /* Get VEB bandwidth, to be implemented */
2698         /* Now associated vsi binding to the VEB, set uplink to this VEB */
2699         vsi->uplink_seid = veb->seid;
2700
2701         return veb;
2702 fail:
2703         rte_free(veb);
2704         return NULL;
2705 }
2706
2707 int
2708 i40e_vsi_release(struct i40e_vsi *vsi)
2709 {
2710         struct i40e_pf *pf;
2711         struct i40e_hw *hw;
2712         struct i40e_vsi_list *vsi_list;
2713         int ret;
2714         struct i40e_mac_filter *f;
2715
2716         if (!vsi)
2717                 return I40E_SUCCESS;
2718
2719         pf = I40E_VSI_TO_PF(vsi);
2720         hw = I40E_VSI_TO_HW(vsi);
2721
2722         /* VSI has child to attach, release child first */
2723         if (vsi->veb) {
2724                 TAILQ_FOREACH(vsi_list, &vsi->veb->head, list) {
2725                         if (i40e_vsi_release(vsi_list->vsi) != I40E_SUCCESS)
2726                                 return -1;
2727                         TAILQ_REMOVE(&vsi->veb->head, vsi_list, list);
2728                 }
2729                 i40e_veb_release(vsi->veb);
2730         }
2731
2732         /* Remove all macvlan filters of the VSI */
2733         i40e_vsi_remove_all_macvlan_filter(vsi);
2734         TAILQ_FOREACH(f, &vsi->mac_list, next)
2735                 rte_free(f);
2736
2737         if (vsi->type != I40E_VSI_MAIN) {
2738                 /* Remove vsi from parent's sibling list */
2739                 if (vsi->parent_vsi == NULL || vsi->parent_vsi->veb == NULL) {
2740                         PMD_DRV_LOG(ERR, "VSI's parent VSI is NULL");
2741                         return I40E_ERR_PARAM;
2742                 }
2743                 TAILQ_REMOVE(&vsi->parent_vsi->veb->head,
2744                                 &vsi->sib_vsi_list, list);
2745
2746                 /* Remove all switch element of the VSI */
2747                 ret = i40e_aq_delete_element(hw, vsi->seid, NULL);
2748                 if (ret != I40E_SUCCESS)
2749                         PMD_DRV_LOG(ERR, "Failed to delete element");
2750         }
2751         i40e_res_pool_free(&pf->qp_pool, vsi->base_queue);
2752
2753         if (vsi->type != I40E_VSI_SRIOV)
2754                 i40e_res_pool_free(&pf->msix_pool, vsi->msix_intr);
2755         rte_free(vsi);
2756
2757         return I40E_SUCCESS;
2758 }
2759
2760 static int
2761 i40e_update_default_filter_setting(struct i40e_vsi *vsi)
2762 {
2763         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2764         struct i40e_aqc_remove_macvlan_element_data def_filter;
2765         struct i40e_mac_filter_info filter;
2766         int ret;
2767
2768         if (vsi->type != I40E_VSI_MAIN)
2769                 return I40E_ERR_CONFIG;
2770         memset(&def_filter, 0, sizeof(def_filter));
2771         (void)rte_memcpy(def_filter.mac_addr, hw->mac.perm_addr,
2772                                         ETH_ADDR_LEN);
2773         def_filter.vlan_tag = 0;
2774         def_filter.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
2775                                 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
2776         ret = i40e_aq_remove_macvlan(hw, vsi->seid, &def_filter, 1, NULL);
2777         if (ret != I40E_SUCCESS) {
2778                 struct i40e_mac_filter *f;
2779                 struct ether_addr *mac;
2780
2781                 PMD_DRV_LOG(WARNING, "Cannot remove the default "
2782                             "macvlan filter");
2783                 /* It needs to add the permanent mac into mac list */
2784                 f = rte_zmalloc("macv_filter", sizeof(*f), 0);
2785                 if (f == NULL) {
2786                         PMD_DRV_LOG(ERR, "failed to allocate memory");
2787                         return I40E_ERR_NO_MEMORY;
2788                 }
2789                 mac = &f->mac_info.mac_addr;
2790                 (void)rte_memcpy(&mac->addr_bytes, hw->mac.perm_addr,
2791                                 ETH_ADDR_LEN);
2792                 f->mac_info.filter_type = RTE_MACVLAN_PERFECT_MATCH;
2793                 TAILQ_INSERT_TAIL(&vsi->mac_list, f, next);
2794                 vsi->mac_num++;
2795
2796                 return ret;
2797         }
2798         (void)rte_memcpy(&filter.mac_addr,
2799                 (struct ether_addr *)(hw->mac.perm_addr), ETH_ADDR_LEN);
2800         filter.filter_type = RTE_MACVLAN_PERFECT_MATCH;
2801         return i40e_vsi_add_mac(vsi, &filter);
2802 }
2803
2804 static int
2805 i40e_vsi_dump_bw_config(struct i40e_vsi *vsi)
2806 {
2807         struct i40e_aqc_query_vsi_bw_config_resp bw_config;
2808         struct i40e_aqc_query_vsi_ets_sla_config_resp ets_sla_config;
2809         struct i40e_hw *hw = &vsi->adapter->hw;
2810         i40e_status ret;
2811         int i;
2812
2813         memset(&bw_config, 0, sizeof(bw_config));
2814         ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
2815         if (ret != I40E_SUCCESS) {
2816                 PMD_DRV_LOG(ERR, "VSI failed to get bandwidth configuration %u",
2817                             hw->aq.asq_last_status);
2818                 return ret;
2819         }
2820
2821         memset(&ets_sla_config, 0, sizeof(ets_sla_config));
2822         ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid,
2823                                         &ets_sla_config, NULL);
2824         if (ret != I40E_SUCCESS) {
2825                 PMD_DRV_LOG(ERR, "VSI failed to get TC bandwdith "
2826                             "configuration %u", hw->aq.asq_last_status);
2827                 return ret;
2828         }
2829
2830         /* Not store the info yet, just print out */
2831         PMD_DRV_LOG(INFO, "VSI bw limit:%u", bw_config.port_bw_limit);
2832         PMD_DRV_LOG(INFO, "VSI max_bw:%u", bw_config.max_bw);
2833         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
2834                 PMD_DRV_LOG(INFO, "\tVSI TC%u:share credits %u", i,
2835                             ets_sla_config.share_credits[i]);
2836                 PMD_DRV_LOG(INFO, "\tVSI TC%u:credits %u", i,
2837                             rte_le_to_cpu_16(ets_sla_config.credits[i]));
2838                 PMD_DRV_LOG(INFO, "\tVSI TC%u: max credits: %u", i,
2839                             rte_le_to_cpu_16(ets_sla_config.credits[i / 4]) >>
2840                             (i * 4));
2841         }
2842
2843         return 0;
2844 }
2845
2846 /* Setup a VSI */
2847 struct i40e_vsi *
2848 i40e_vsi_setup(struct i40e_pf *pf,
2849                enum i40e_vsi_type type,
2850                struct i40e_vsi *uplink_vsi,
2851                uint16_t user_param)
2852 {
2853         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
2854         struct i40e_vsi *vsi;
2855         struct i40e_mac_filter_info filter;
2856         int ret;
2857         struct i40e_vsi_context ctxt;
2858         struct ether_addr broadcast =
2859                 {.addr_bytes = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}};
2860
2861         if (type != I40E_VSI_MAIN && uplink_vsi == NULL) {
2862                 PMD_DRV_LOG(ERR, "VSI setup failed, "
2863                             "VSI link shouldn't be NULL");
2864                 return NULL;
2865         }
2866
2867         if (type == I40E_VSI_MAIN && uplink_vsi != NULL) {
2868                 PMD_DRV_LOG(ERR, "VSI setup failed, MAIN VSI "
2869                             "uplink VSI should be NULL");
2870                 return NULL;
2871         }
2872
2873         /* If uplink vsi didn't setup VEB, create one first */
2874         if (type != I40E_VSI_MAIN && uplink_vsi->veb == NULL) {
2875                 uplink_vsi->veb = i40e_veb_setup(pf, uplink_vsi);
2876
2877                 if (NULL == uplink_vsi->veb) {
2878                         PMD_DRV_LOG(ERR, "VEB setup failed");
2879                         return NULL;
2880                 }
2881         }
2882
2883         vsi = rte_zmalloc("i40e_vsi", sizeof(struct i40e_vsi), 0);
2884         if (!vsi) {
2885                 PMD_DRV_LOG(ERR, "Failed to allocate memory for vsi");
2886                 return NULL;
2887         }
2888         TAILQ_INIT(&vsi->mac_list);
2889         vsi->type = type;
2890         vsi->adapter = I40E_PF_TO_ADAPTER(pf);
2891         vsi->max_macaddrs = I40E_NUM_MACADDR_MAX;
2892         vsi->parent_vsi = uplink_vsi;
2893         vsi->user_param = user_param;
2894         /* Allocate queues */
2895         switch (vsi->type) {
2896         case I40E_VSI_MAIN  :
2897                 vsi->nb_qps = pf->lan_nb_qps;
2898                 break;
2899         case I40E_VSI_SRIOV :
2900                 vsi->nb_qps = pf->vf_nb_qps;
2901                 break;
2902         case I40E_VSI_VMDQ2:
2903                 vsi->nb_qps = pf->vmdq_nb_qps;
2904                 break;
2905         case I40E_VSI_FDIR:
2906                 vsi->nb_qps = pf->fdir_nb_qps;
2907                 break;
2908         default:
2909                 goto fail_mem;
2910         }
2911         /*
2912          * The filter status descriptor is reported in rx queue 0,
2913          * while the tx queue for fdir filter programming has no
2914          * such constraints, can be non-zero queues.
2915          * To simplify it, choose FDIR vsi use queue 0 pair.
2916          * To make sure it will use queue 0 pair, queue allocation
2917          * need be done before this function is called
2918          */
2919         if (type != I40E_VSI_FDIR) {
2920                 ret = i40e_res_pool_alloc(&pf->qp_pool, vsi->nb_qps);
2921                         if (ret < 0) {
2922                                 PMD_DRV_LOG(ERR, "VSI %d allocate queue failed %d",
2923                                                 vsi->seid, ret);
2924                                 goto fail_mem;
2925                         }
2926                         vsi->base_queue = ret;
2927         } else
2928                 vsi->base_queue = I40E_FDIR_QUEUE_ID;
2929
2930         /* VF has MSIX interrupt in VF range, don't allocate here */
2931         if (type != I40E_VSI_SRIOV) {
2932                 ret = i40e_res_pool_alloc(&pf->msix_pool, 1);
2933                 if (ret < 0) {
2934                         PMD_DRV_LOG(ERR, "VSI %d get heap failed %d", vsi->seid, ret);
2935                         goto fail_queue_alloc;
2936                 }
2937                 vsi->msix_intr = ret;
2938         } else
2939                 vsi->msix_intr = 0;
2940         /* Add VSI */
2941         if (type == I40E_VSI_MAIN) {
2942                 /* For main VSI, no need to add since it's default one */
2943                 vsi->uplink_seid = pf->mac_seid;
2944                 vsi->seid = pf->main_vsi_seid;
2945                 /* Bind queues with specific MSIX interrupt */
2946                 /**
2947                  * Needs 2 interrupt at least, one for misc cause which will
2948                  * enabled from OS side, Another for queues binding the
2949                  * interrupt from device side only.
2950                  */
2951
2952                 /* Get default VSI parameters from hardware */
2953                 memset(&ctxt, 0, sizeof(ctxt));
2954                 ctxt.seid = vsi->seid;
2955                 ctxt.pf_num = hw->pf_id;
2956                 ctxt.uplink_seid = vsi->uplink_seid;
2957                 ctxt.vf_num = 0;
2958                 ret = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
2959                 if (ret != I40E_SUCCESS) {
2960                         PMD_DRV_LOG(ERR, "Failed to get VSI params");
2961                         goto fail_msix_alloc;
2962                 }
2963                 (void)rte_memcpy(&vsi->info, &ctxt.info,
2964                         sizeof(struct i40e_aqc_vsi_properties_data));
2965                 vsi->vsi_id = ctxt.vsi_number;
2966                 vsi->info.valid_sections = 0;
2967
2968                 /* Configure tc, enabled TC0 only */
2969                 if (i40e_vsi_update_tc_bandwidth(vsi, I40E_DEFAULT_TCMAP) !=
2970                         I40E_SUCCESS) {
2971                         PMD_DRV_LOG(ERR, "Failed to update TC bandwidth");
2972                         goto fail_msix_alloc;
2973                 }
2974
2975                 /* TC, queue mapping */
2976                 memset(&ctxt, 0, sizeof(ctxt));
2977                 vsi->info.valid_sections |=
2978                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
2979                 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2980                                         I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2981                 (void)rte_memcpy(&ctxt.info, &vsi->info,
2982                         sizeof(struct i40e_aqc_vsi_properties_data));
2983                 ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
2984                                                 I40E_DEFAULT_TCMAP);
2985                 if (ret != I40E_SUCCESS) {
2986                         PMD_DRV_LOG(ERR, "Failed to configure "
2987                                     "TC queue mapping");
2988                         goto fail_msix_alloc;
2989                 }
2990                 ctxt.seid = vsi->seid;
2991                 ctxt.pf_num = hw->pf_id;
2992                 ctxt.uplink_seid = vsi->uplink_seid;
2993                 ctxt.vf_num = 0;
2994
2995                 /* Update VSI parameters */
2996                 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
2997                 if (ret != I40E_SUCCESS) {
2998                         PMD_DRV_LOG(ERR, "Failed to update VSI params");
2999                         goto fail_msix_alloc;
3000                 }
3001
3002                 (void)rte_memcpy(&vsi->info.tc_mapping, &ctxt.info.tc_mapping,
3003                                                 sizeof(vsi->info.tc_mapping));
3004                 (void)rte_memcpy(&vsi->info.queue_mapping,
3005                                 &ctxt.info.queue_mapping,
3006                         sizeof(vsi->info.queue_mapping));
3007                 vsi->info.mapping_flags = ctxt.info.mapping_flags;
3008                 vsi->info.valid_sections = 0;
3009
3010                 (void)rte_memcpy(pf->dev_addr.addr_bytes, hw->mac.perm_addr,
3011                                 ETH_ADDR_LEN);
3012
3013                 /**
3014                  * Updating default filter settings are necessary to prevent
3015                  * reception of tagged packets.
3016                  * Some old firmware configurations load a default macvlan
3017                  * filter which accepts both tagged and untagged packets.
3018                  * The updating is to use a normal filter instead if needed.
3019                  * For NVM 4.2.2 or after, the updating is not needed anymore.
3020                  * The firmware with correct configurations load the default
3021                  * macvlan filter which is expected and cannot be removed.
3022                  */
3023                 i40e_update_default_filter_setting(vsi);
3024         } else if (type == I40E_VSI_SRIOV) {
3025                 memset(&ctxt, 0, sizeof(ctxt));
3026                 /**
3027                  * For other VSI, the uplink_seid equals to uplink VSI's
3028                  * uplink_seid since they share same VEB
3029                  */
3030                 vsi->uplink_seid = uplink_vsi->uplink_seid;
3031                 ctxt.pf_num = hw->pf_id;
3032                 ctxt.vf_num = hw->func_caps.vf_base_id + user_param;
3033                 ctxt.uplink_seid = vsi->uplink_seid;
3034                 ctxt.connection_type = 0x1;
3035                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
3036
3037                 /* Configure switch ID */
3038                 ctxt.info.valid_sections |=
3039                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SWITCH_VALID);
3040                 ctxt.info.switch_id =
3041                         rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
3042                 /* Configure port/vlan */
3043                 ctxt.info.valid_sections |=
3044                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
3045                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
3046                 ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
3047                                                 I40E_DEFAULT_TCMAP);
3048                 if (ret != I40E_SUCCESS) {
3049                         PMD_DRV_LOG(ERR, "Failed to configure "
3050                                     "TC queue mapping");
3051                         goto fail_msix_alloc;
3052                 }
3053                 ctxt.info.up_enable_bits = I40E_DEFAULT_TCMAP;
3054                 ctxt.info.valid_sections |=
3055                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SCHED_VALID);
3056                 /**
3057                  * Since VSI is not created yet, only configure parameter,
3058                  * will add vsi below.
3059                  */
3060         } else if (type == I40E_VSI_VMDQ2) {
3061                 memset(&ctxt, 0, sizeof(ctxt));
3062                 /*
3063                  * For other VSI, the uplink_seid equals to uplink VSI's
3064                  * uplink_seid since they share same VEB
3065                  */
3066                 vsi->uplink_seid = uplink_vsi->uplink_seid;
3067                 ctxt.pf_num = hw->pf_id;
3068                 ctxt.vf_num = 0;
3069                 ctxt.uplink_seid = vsi->uplink_seid;
3070                 ctxt.connection_type = 0x1;
3071                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
3072
3073                 ctxt.info.valid_sections |=
3074                                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SWITCH_VALID);
3075                 /* user_param carries flag to enable loop back */
3076                 if (user_param) {
3077                         ctxt.info.switch_id =
3078                         rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
3079                         ctxt.info.switch_id |=
3080                         rte_cpu_to_le_16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
3081                 }
3082
3083                 /* Configure port/vlan */
3084                 ctxt.info.valid_sections |=
3085                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
3086                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
3087                 ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
3088                                                 I40E_DEFAULT_TCMAP);
3089                 if (ret != I40E_SUCCESS) {
3090                         PMD_DRV_LOG(ERR, "Failed to configure "
3091                                         "TC queue mapping");
3092                         goto fail_msix_alloc;
3093                 }
3094                 ctxt.info.up_enable_bits = I40E_DEFAULT_TCMAP;
3095                 ctxt.info.valid_sections |=
3096                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SCHED_VALID);
3097         } else if (type == I40E_VSI_FDIR) {
3098                 vsi->uplink_seid = uplink_vsi->uplink_seid;
3099                 ctxt.pf_num = hw->pf_id;
3100                 ctxt.vf_num = 0;
3101                 ctxt.uplink_seid = vsi->uplink_seid;
3102                 ctxt.connection_type = 0x1;     /* regular data port */
3103                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
3104                 ret = i40e_vsi_config_tc_queue_mapping(vsi, &ctxt.info,
3105                                                 I40E_DEFAULT_TCMAP);
3106                 if (ret != I40E_SUCCESS) {
3107                         PMD_DRV_LOG(ERR, "Failed to configure "
3108                                         "TC queue mapping.");
3109                         goto fail_msix_alloc;
3110                 }
3111                 ctxt.info.up_enable_bits = I40E_DEFAULT_TCMAP;
3112                 ctxt.info.valid_sections |=
3113                         rte_cpu_to_le_16(I40E_AQ_VSI_PROP_SCHED_VALID);
3114         } else {
3115                 PMD_DRV_LOG(ERR, "VSI: Not support other type VSI yet");
3116                 goto fail_msix_alloc;
3117         }
3118
3119         if (vsi->type != I40E_VSI_MAIN) {
3120                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
3121                 if (ret) {
3122                         PMD_DRV_LOG(ERR, "add vsi failed, aq_err=%d",
3123                                     hw->aq.asq_last_status);
3124                         goto fail_msix_alloc;
3125                 }
3126                 memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
3127                 vsi->info.valid_sections = 0;
3128                 vsi->seid = ctxt.seid;
3129                 vsi->vsi_id = ctxt.vsi_number;
3130                 vsi->sib_vsi_list.vsi = vsi;
3131                 TAILQ_INSERT_TAIL(&uplink_vsi->veb->head,
3132                                 &vsi->sib_vsi_list, list);
3133         }
3134
3135         /* MAC/VLAN configuration */
3136         (void)rte_memcpy(&filter.mac_addr, &broadcast, ETHER_ADDR_LEN);
3137         filter.filter_type = RTE_MACVLAN_PERFECT_MATCH;
3138
3139         ret = i40e_vsi_add_mac(vsi, &filter);
3140         if (ret != I40E_SUCCESS) {
3141                 PMD_DRV_LOG(ERR, "Failed to add MACVLAN filter");
3142                 goto fail_msix_alloc;
3143         }
3144
3145         /* Get VSI BW information */
3146         i40e_vsi_dump_bw_config(vsi);
3147         return vsi;
3148 fail_msix_alloc:
3149         i40e_res_pool_free(&pf->msix_pool,vsi->msix_intr);
3150 fail_queue_alloc:
3151         i40e_res_pool_free(&pf->qp_pool,vsi->base_queue);
3152 fail_mem:
3153         rte_free(vsi);
3154         return NULL;
3155 }
3156
3157 /* Configure vlan stripping on or off */
3158 int
3159 i40e_vsi_config_vlan_stripping(struct i40e_vsi *vsi, bool on)
3160 {
3161         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
3162         struct i40e_vsi_context ctxt;
3163         uint8_t vlan_flags;
3164         int ret = I40E_SUCCESS;
3165
3166         /* Check if it has been already on or off */
3167         if (vsi->info.valid_sections &
3168                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID)) {
3169                 if (on) {
3170                         if ((vsi->info.port_vlan_flags &
3171                                 I40E_AQ_VSI_PVLAN_EMOD_MASK) == 0)
3172                                 return 0; /* already on */
3173                 } else {
3174                         if ((vsi->info.port_vlan_flags &
3175                                 I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
3176                                 I40E_AQ_VSI_PVLAN_EMOD_MASK)
3177                                 return 0; /* already off */
3178                 }
3179         }
3180
3181         if (on)
3182                 vlan_flags = I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
3183         else
3184                 vlan_flags = I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
3185         vsi->info.valid_sections =
3186                 rte_cpu_to_le_16(I40E_AQ_VSI_PROP_VLAN_VALID);
3187         vsi->info.port_vlan_flags &= ~(I40E_AQ_VSI_PVLAN_EMOD_MASK);
3188         vsi->info.port_vlan_flags |= vlan_flags;
3189         ctxt.seid = vsi->seid;
3190         (void)rte_memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
3191         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
3192         if (ret)
3193                 PMD_DRV_LOG(INFO, "Update VSI failed to %s vlan stripping",
3194                             on ? "enable" : "disable");
3195
3196         return ret;
3197 }
3198
3199 static int
3200 i40e_dev_init_vlan(struct rte_eth_dev *dev)
3201 {
3202         struct rte_eth_dev_data *data = dev->data;
3203         int ret;
3204
3205         /* Apply vlan offload setting */
3206         i40e_vlan_offload_set(dev, ETH_VLAN_STRIP_MASK);
3207
3208         /* Apply double-vlan setting, not implemented yet */
3209
3210         /* Apply pvid setting */
3211         ret = i40e_vlan_pvid_set(dev, data->dev_conf.txmode.pvid,
3212                                 data->dev_conf.txmode.hw_vlan_insert_pvid);
3213         if (ret)
3214                 PMD_DRV_LOG(INFO, "Failed to update VSI params");
3215
3216         return ret;
3217 }
3218
3219 static int
3220 i40e_vsi_config_double_vlan(struct i40e_vsi *vsi, int on)
3221 {
3222         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
3223
3224         return i40e_aq_set_port_parameters(hw, vsi->seid, 0, 1, on, NULL);
3225 }
3226
3227 static int
3228 i40e_update_flow_control(struct i40e_hw *hw)
3229 {
3230 #define I40E_LINK_PAUSE_RXTX (I40E_AQ_LINK_PAUSE_RX | I40E_AQ_LINK_PAUSE_TX)
3231         struct i40e_link_status link_status;
3232         uint32_t rxfc = 0, txfc = 0, reg;
3233         uint8_t an_info;
3234         int ret;
3235
3236         memset(&link_status, 0, sizeof(link_status));
3237         ret = i40e_aq_get_link_info(hw, FALSE, &link_status, NULL);
3238         if (ret != I40E_SUCCESS) {
3239                 PMD_DRV_LOG(ERR, "Failed to get link status information");
3240                 goto write_reg; /* Disable flow control */
3241         }
3242
3243         an_info = hw->phy.link_info.an_info;
3244         if (!(an_info & I40E_AQ_AN_COMPLETED)) {
3245                 PMD_DRV_LOG(INFO, "Link auto negotiation not completed");
3246                 ret = I40E_ERR_NOT_READY;
3247                 goto write_reg; /* Disable flow control */
3248         }
3249         /**
3250          * If link auto negotiation is enabled, flow control needs to
3251          * be configured according to it
3252          */
3253         switch (an_info & I40E_LINK_PAUSE_RXTX) {
3254         case I40E_LINK_PAUSE_RXTX:
3255                 rxfc = 1;
3256                 txfc = 1;
3257                 hw->fc.current_mode = I40E_FC_FULL;
3258                 break;
3259         case I40E_AQ_LINK_PAUSE_RX:
3260                 rxfc = 1;
3261                 hw->fc.current_mode = I40E_FC_RX_PAUSE;
3262                 break;
3263         case I40E_AQ_LINK_PAUSE_TX:
3264                 txfc = 1;
3265                 hw->fc.current_mode = I40E_FC_TX_PAUSE;
3266                 break;
3267         default:
3268                 hw->fc.current_mode = I40E_FC_NONE;
3269                 break;
3270         }
3271
3272 write_reg:
3273         I40E_WRITE_REG(hw, I40E_PRTDCB_FCCFG,
3274                 txfc << I40E_PRTDCB_FCCFG_TFCE_SHIFT);
3275         reg = I40E_READ_REG(hw, I40E_PRTDCB_MFLCN);
3276         reg &= ~I40E_PRTDCB_MFLCN_RFCE_MASK;
3277         reg |= rxfc << I40E_PRTDCB_MFLCN_RFCE_SHIFT;
3278         I40E_WRITE_REG(hw, I40E_PRTDCB_MFLCN, reg);
3279
3280         return ret;
3281 }
3282
3283 /* PF setup */
3284 static int
3285 i40e_pf_setup(struct i40e_pf *pf)
3286 {
3287         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
3288         struct i40e_filter_control_settings settings;
3289         struct i40e_vsi *vsi;
3290         int ret;
3291
3292         /* Clear all stats counters */
3293         pf->offset_loaded = FALSE;
3294         memset(&pf->stats, 0, sizeof(struct i40e_hw_port_stats));
3295         memset(&pf->stats_offset, 0, sizeof(struct i40e_hw_port_stats));
3296
3297         ret = i40e_pf_get_switch_config(pf);
3298         if (ret != I40E_SUCCESS) {
3299                 PMD_DRV_LOG(ERR, "Could not get switch config, err %d", ret);
3300                 return ret;
3301         }
3302         if (pf->flags & I40E_FLAG_FDIR) {
3303                 /* make queue allocated first, let FDIR use queue pair 0*/
3304                 ret = i40e_res_pool_alloc(&pf->qp_pool, I40E_DEFAULT_QP_NUM_FDIR);
3305                 if (ret != I40E_FDIR_QUEUE_ID) {
3306                         PMD_DRV_LOG(ERR, "queue allocation fails for FDIR :"
3307                                     " ret =%d", ret);
3308                         pf->flags &= ~I40E_FLAG_FDIR;
3309                 }
3310         }
3311         /*  main VSI setup */
3312         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, NULL, 0);
3313         if (!vsi) {
3314                 PMD_DRV_LOG(ERR, "Setup of main vsi failed");
3315                 return I40E_ERR_NOT_READY;
3316         }
3317         pf->main_vsi = vsi;
3318
3319         /* Configure filter control */
3320         memset(&settings, 0, sizeof(settings));
3321         if (hw->func_caps.rss_table_size == ETH_RSS_RETA_SIZE_128)
3322                 settings.hash_lut_size = I40E_HASH_LUT_SIZE_128;
3323         else if (hw->func_caps.rss_table_size == ETH_RSS_RETA_SIZE_512)
3324                 settings.hash_lut_size = I40E_HASH_LUT_SIZE_512;
3325         else {
3326                 PMD_DRV_LOG(ERR, "Hash lookup table size (%u) not supported\n",
3327                                                 hw->func_caps.rss_table_size);
3328                 return I40E_ERR_PARAM;
3329         }
3330         PMD_DRV_LOG(INFO, "Hardware capability of hash lookup table "
3331                         "size: %u\n", hw->func_caps.rss_table_size);
3332         pf->hash_lut_size = hw->func_caps.rss_table_size;
3333
3334         /* Enable ethtype and macvlan filters */
3335         settings.enable_ethtype = TRUE;
3336         settings.enable_macvlan = TRUE;
3337         ret = i40e_set_filter_control(hw, &settings);
3338         if (ret)
3339                 PMD_INIT_LOG(WARNING, "setup_pf_filter_control failed: %d",
3340                                                                 ret);
3341
3342         /* Update flow control according to the auto negotiation */
3343         i40e_update_flow_control(hw);
3344
3345         return I40E_SUCCESS;
3346 }
3347
3348 int
3349 i40e_switch_tx_queue(struct i40e_hw *hw, uint16_t q_idx, bool on)
3350 {
3351         uint32_t reg;
3352         uint16_t j;
3353
3354         /**
3355          * Set or clear TX Queue Disable flags,
3356          * which is required by hardware.
3357          */
3358         i40e_pre_tx_queue_cfg(hw, q_idx, on);
3359         rte_delay_us(I40E_PRE_TX_Q_CFG_WAIT_US);
3360
3361         /* Wait until the request is finished */
3362         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
3363                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
3364                 reg = I40E_READ_REG(hw, I40E_QTX_ENA(q_idx));
3365                 if (!(((reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 0x1) ^
3366                         ((reg >> I40E_QTX_ENA_QENA_STAT_SHIFT)
3367                                                         & 0x1))) {
3368                         break;
3369                 }
3370         }
3371         if (on) {
3372                 if (reg & I40E_QTX_ENA_QENA_STAT_MASK)
3373                         return I40E_SUCCESS; /* already on, skip next steps */
3374
3375                 I40E_WRITE_REG(hw, I40E_QTX_HEAD(q_idx), 0);
3376                 reg |= I40E_QTX_ENA_QENA_REQ_MASK;
3377         } else {
3378                 if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK))
3379                         return I40E_SUCCESS; /* already off, skip next steps */
3380                 reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
3381         }
3382         /* Write the register */
3383         I40E_WRITE_REG(hw, I40E_QTX_ENA(q_idx), reg);
3384         /* Check the result */
3385         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
3386                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
3387                 reg = I40E_READ_REG(hw, I40E_QTX_ENA(q_idx));
3388                 if (on) {
3389                         if ((reg & I40E_QTX_ENA_QENA_REQ_MASK) &&
3390                                 (reg & I40E_QTX_ENA_QENA_STAT_MASK))
3391                                 break;
3392                 } else {
3393                         if (!(reg & I40E_QTX_ENA_QENA_REQ_MASK) &&
3394                                 !(reg & I40E_QTX_ENA_QENA_STAT_MASK))
3395                                 break;
3396                 }
3397         }
3398         /* Check if it is timeout */
3399         if (j >= I40E_CHK_Q_ENA_COUNT) {
3400                 PMD_DRV_LOG(ERR, "Failed to %s tx queue[%u]",
3401                             (on ? "enable" : "disable"), q_idx);
3402                 return I40E_ERR_TIMEOUT;
3403         }
3404
3405         return I40E_SUCCESS;
3406 }
3407
3408 /* Swith on or off the tx queues */
3409 static int
3410 i40e_dev_switch_tx_queues(struct i40e_pf *pf, bool on)
3411 {
3412         struct rte_eth_dev_data *dev_data = pf->dev_data;
3413         struct i40e_tx_queue *txq;
3414         struct rte_eth_dev *dev = pf->adapter->eth_dev;
3415         uint16_t i;
3416         int ret;
3417
3418         for (i = 0; i < dev_data->nb_tx_queues; i++) {
3419                 txq = dev_data->tx_queues[i];
3420                 /* Don't operate the queue if not configured or
3421                  * if starting only per queue */
3422                 if (!txq || !txq->q_set || (on && txq->tx_deferred_start))
3423                         continue;
3424                 if (on)
3425                         ret = i40e_dev_tx_queue_start(dev, i);
3426                 else
3427                         ret = i40e_dev_tx_queue_stop(dev, i);
3428                 if ( ret != I40E_SUCCESS)
3429                         return ret;
3430         }
3431
3432         return I40E_SUCCESS;
3433 }
3434
3435 int
3436 i40e_switch_rx_queue(struct i40e_hw *hw, uint16_t q_idx, bool on)
3437 {
3438         uint32_t reg;
3439         uint16_t j;
3440
3441         /* Wait until the request is finished */
3442         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
3443                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
3444                 reg = I40E_READ_REG(hw, I40E_QRX_ENA(q_idx));
3445                 if (!((reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 0x1) ^
3446                         ((reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 0x1))
3447                         break;
3448         }
3449
3450         if (on) {
3451                 if (reg & I40E_QRX_ENA_QENA_STAT_MASK)
3452                         return I40E_SUCCESS; /* Already on, skip next steps */
3453                 reg |= I40E_QRX_ENA_QENA_REQ_MASK;
3454         } else {
3455                 if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK))
3456                         return I40E_SUCCESS; /* Already off, skip next steps */
3457                 reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
3458         }
3459
3460         /* Write the register */
3461         I40E_WRITE_REG(hw, I40E_QRX_ENA(q_idx), reg);
3462         /* Check the result */
3463         for (j = 0; j < I40E_CHK_Q_ENA_COUNT; j++) {
3464                 rte_delay_us(I40E_CHK_Q_ENA_INTERVAL_US);
3465                 reg = I40E_READ_REG(hw, I40E_QRX_ENA(q_idx));
3466                 if (on) {
3467                         if ((reg & I40E_QRX_ENA_QENA_REQ_MASK) &&
3468                                 (reg & I40E_QRX_ENA_QENA_STAT_MASK))
3469                                 break;
3470                 } else {
3471                         if (!(reg & I40E_QRX_ENA_QENA_REQ_MASK) &&
3472                                 !(reg & I40E_QRX_ENA_QENA_STAT_MASK))
3473                                 break;
3474                 }
3475         }
3476
3477         /* Check if it is timeout */
3478         if (j >= I40E_CHK_Q_ENA_COUNT) {
3479                 PMD_DRV_LOG(ERR, "Failed to %s rx queue[%u]",
3480                             (on ? "enable" : "disable"), q_idx);
3481                 return I40E_ERR_TIMEOUT;
3482         }
3483
3484         return I40E_SUCCESS;
3485 }
3486 /* Switch on or off the rx queues */
3487 static int
3488 i40e_dev_switch_rx_queues(struct i40e_pf *pf, bool on)
3489 {
3490         struct rte_eth_dev_data *dev_data = pf->dev_data;
3491         struct i40e_rx_queue *rxq;
3492         struct rte_eth_dev *dev = pf->adapter->eth_dev;
3493         uint16_t i;
3494         int ret;
3495
3496         for (i = 0; i < dev_data->nb_rx_queues; i++) {
3497                 rxq = dev_data->rx_queues[i];
3498                 /* Don't operate the queue if not configured or
3499                  * if starting only per queue */
3500                 if (!rxq || !rxq->q_set || (on && rxq->rx_deferred_start))
3501                         continue;
3502                 if (on)
3503                         ret = i40e_dev_rx_queue_start(dev, i);
3504                 else
3505                         ret = i40e_dev_rx_queue_stop(dev, i);
3506                 if (ret != I40E_SUCCESS)
3507                         return ret;
3508         }
3509
3510         return I40E_SUCCESS;
3511 }
3512
3513 /* Switch on or off all the rx/tx queues */
3514 int
3515 i40e_dev_switch_queues(struct i40e_pf *pf, bool on)
3516 {
3517         int ret;
3518
3519         if (on) {
3520                 /* enable rx queues before enabling tx queues */
3521                 ret = i40e_dev_switch_rx_queues(pf, on);
3522                 if (ret) {
3523                         PMD_DRV_LOG(ERR, "Failed to switch rx queues");
3524                         return ret;
3525                 }
3526                 ret = i40e_dev_switch_tx_queues(pf, on);
3527         } else {
3528                 /* Stop tx queues before stopping rx queues */
3529                 ret = i40e_dev_switch_tx_queues(pf, on);
3530                 if (ret) {
3531                         PMD_DRV_LOG(ERR, "Failed to switch tx queues");
3532                         return ret;
3533                 }
3534                 ret = i40e_dev_switch_rx_queues(pf, on);
3535         }
3536
3537         return ret;
3538 }
3539
3540 /* Initialize VSI for TX */
3541 static int
3542 i40e_dev_tx_init(struct i40e_pf *pf)
3543 {
3544         struct rte_eth_dev_data *data = pf->dev_data;
3545         uint16_t i;
3546         uint32_t ret = I40E_SUCCESS;
3547         struct i40e_tx_queue *txq;
3548
3549         for (i = 0; i < data->nb_tx_queues; i++) {
3550                 txq = data->tx_queues[i];
3551                 if (!txq || !txq->q_set)
3552                         continue;
3553                 ret = i40e_tx_queue_init(txq);
3554                 if (ret != I40E_SUCCESS)
3555                         break;
3556         }
3557
3558         return ret;
3559 }
3560
3561 /* Initialize VSI for RX */
3562 static int
3563 i40e_dev_rx_init(struct i40e_pf *pf)
3564 {
3565         struct rte_eth_dev_data *data = pf->dev_data;
3566         int ret = I40E_SUCCESS;
3567         uint16_t i;
3568         struct i40e_rx_queue *rxq;
3569
3570         i40e_pf_config_mq_rx(pf);
3571         for (i = 0; i < data->nb_rx_queues; i++) {
3572                 rxq = data->rx_queues[i];
3573                 if (!rxq || !rxq->q_set)
3574                         continue;
3575
3576                 ret = i40e_rx_queue_init(rxq);
3577                 if (ret != I40E_SUCCESS) {
3578                         PMD_DRV_LOG(ERR, "Failed to do RX queue "
3579                                     "initialization");
3580                         break;
3581                 }
3582         }
3583
3584         return ret;
3585 }
3586
3587 static int
3588 i40e_dev_rxtx_init(struct i40e_pf *pf)
3589 {
3590         int err;
3591
3592         err = i40e_dev_tx_init(pf);
3593         if (err) {
3594                 PMD_DRV_LOG(ERR, "Failed to do TX initialization");
3595                 return err;
3596         }
3597         err = i40e_dev_rx_init(pf);
3598         if (err) {
3599                 PMD_DRV_LOG(ERR, "Failed to do RX initialization");
3600                 return err;
3601         }
3602
3603         return err;
3604 }
3605
3606 static int
3607 i40e_vmdq_setup(struct rte_eth_dev *dev)
3608 {
3609         struct rte_eth_conf *conf = &dev->data->dev_conf;
3610         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3611         int i, err, conf_vsis, j, loop;
3612         struct i40e_vsi *vsi;
3613         struct i40e_vmdq_info *vmdq_info;
3614         struct rte_eth_vmdq_rx_conf *vmdq_conf;
3615         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
3616
3617         /*
3618          * Disable interrupt to avoid message from VF. Furthermore, it will
3619          * avoid race condition in VSI creation/destroy.
3620          */
3621         i40e_pf_disable_irq0(hw);
3622
3623         if ((pf->flags & I40E_FLAG_VMDQ) == 0) {
3624                 PMD_INIT_LOG(ERR, "FW doesn't support VMDQ");
3625                 return -ENOTSUP;
3626         }
3627
3628         conf_vsis = conf->rx_adv_conf.vmdq_rx_conf.nb_queue_pools;
3629         if (conf_vsis > pf->max_nb_vmdq_vsi) {
3630                 PMD_INIT_LOG(ERR, "VMDQ config: %u, max support:%u",
3631                         conf->rx_adv_conf.vmdq_rx_conf.nb_queue_pools,
3632                         pf->max_nb_vmdq_vsi);
3633                 return -ENOTSUP;
3634         }
3635
3636         if (pf->vmdq != NULL) {
3637                 PMD_INIT_LOG(INFO, "VMDQ already configured");
3638                 return 0;
3639         }
3640
3641         pf->vmdq = rte_zmalloc("vmdq_info_struct",
3642                                 sizeof(*vmdq_info) * conf_vsis, 0);
3643
3644         if (pf->vmdq == NULL) {
3645                 PMD_INIT_LOG(ERR, "Failed to allocate memory");
3646                 return -ENOMEM;
3647         }
3648
3649         vmdq_conf = &conf->rx_adv_conf.vmdq_rx_conf;
3650
3651         /* Create VMDQ VSI */
3652         for (i = 0; i < conf_vsis; i++) {
3653                 vsi = i40e_vsi_setup(pf, I40E_VSI_VMDQ2, pf->main_vsi,
3654                                 vmdq_conf->enable_loop_back);
3655                 if (vsi == NULL) {
3656                         PMD_INIT_LOG(ERR, "Failed to create VMDQ VSI");
3657                         err = -1;
3658                         goto err_vsi_setup;
3659                 }
3660                 vmdq_info = &pf->vmdq[i];
3661                 vmdq_info->pf = pf;
3662                 vmdq_info->vsi = vsi;
3663         }
3664         pf->nb_cfg_vmdq_vsi = conf_vsis;
3665
3666         /* Configure Vlan */
3667         loop = sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT;
3668         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
3669                 for (j = 0; j < loop && j < pf->nb_cfg_vmdq_vsi; j++) {
3670                         if (vmdq_conf->pool_map[i].pools & (1UL << j)) {
3671                                 PMD_INIT_LOG(INFO, "Add vlan %u to vmdq pool %u",
3672                                         vmdq_conf->pool_map[i].vlan_id, j);
3673
3674                                 err = i40e_vsi_add_vlan(pf->vmdq[j].vsi,
3675                                                 vmdq_conf->pool_map[i].vlan_id);
3676                                 if (err) {
3677                                         PMD_INIT_LOG(ERR, "Failed to add vlan");
3678                                         err = -1;
3679                                         goto err_vsi_setup;
3680                                 }
3681                         }
3682                 }
3683         }
3684
3685         i40e_pf_enable_irq0(hw);
3686
3687         return 0;
3688
3689 err_vsi_setup:
3690         for (i = 0; i < conf_vsis; i++)
3691                 if (pf->vmdq[i].vsi == NULL)
3692                         break;
3693                 else
3694                         i40e_vsi_release(pf->vmdq[i].vsi);
3695
3696         rte_free(pf->vmdq);
3697         pf->vmdq = NULL;
3698         i40e_pf_enable_irq0(hw);
3699         return err;
3700 }
3701
3702 static void
3703 i40e_stat_update_32(struct i40e_hw *hw,
3704                    uint32_t reg,
3705                    bool offset_loaded,
3706                    uint64_t *offset,
3707                    uint64_t *stat)
3708 {
3709         uint64_t new_data;
3710
3711         new_data = (uint64_t)I40E_READ_REG(hw, reg);
3712         if (!offset_loaded)
3713                 *offset = new_data;
3714
3715         if (new_data >= *offset)
3716                 *stat = (uint64_t)(new_data - *offset);
3717         else
3718                 *stat = (uint64_t)((new_data +
3719                         ((uint64_t)1 << I40E_32_BIT_WIDTH)) - *offset);
3720 }
3721
3722 static void
3723 i40e_stat_update_48(struct i40e_hw *hw,
3724                    uint32_t hireg,
3725                    uint32_t loreg,
3726                    bool offset_loaded,
3727                    uint64_t *offset,
3728                    uint64_t *stat)
3729 {
3730         uint64_t new_data;
3731
3732         new_data = (uint64_t)I40E_READ_REG(hw, loreg);
3733         new_data |= ((uint64_t)(I40E_READ_REG(hw, hireg) &
3734                         I40E_16_BIT_MASK)) << I40E_32_BIT_WIDTH;
3735
3736         if (!offset_loaded)
3737                 *offset = new_data;
3738
3739         if (new_data >= *offset)
3740                 *stat = new_data - *offset;
3741         else
3742                 *stat = (uint64_t)((new_data +
3743                         ((uint64_t)1 << I40E_48_BIT_WIDTH)) - *offset);
3744
3745         *stat &= I40E_48_BIT_MASK;
3746 }
3747
3748 /* Disable IRQ0 */
3749 void
3750 i40e_pf_disable_irq0(struct i40e_hw *hw)
3751 {
3752         /* Disable all interrupt types */
3753         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, 0);
3754         I40E_WRITE_FLUSH(hw);
3755 }
3756
3757 /* Enable IRQ0 */
3758 void
3759 i40e_pf_enable_irq0(struct i40e_hw *hw)
3760 {
3761         I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0,
3762                 I40E_PFINT_DYN_CTL0_INTENA_MASK |
3763                 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
3764                 I40E_PFINT_DYN_CTL0_ITR_INDX_MASK);
3765         I40E_WRITE_FLUSH(hw);
3766 }
3767
3768 static void
3769 i40e_pf_config_irq0(struct i40e_hw *hw)
3770 {
3771         /* read pending request and disable first */
3772         i40e_pf_disable_irq0(hw);
3773         I40E_WRITE_REG(hw, I40E_PFINT_ICR0_ENA, I40E_PFINT_ICR0_ENA_MASK);
3774         I40E_WRITE_REG(hw, I40E_PFINT_STAT_CTL0,
3775                 I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_MASK);
3776
3777         /* Link no queues with irq0 */
3778         I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0,
3779                 I40E_PFINT_LNKLST0_FIRSTQ_INDX_MASK);
3780 }
3781
3782 static void
3783 i40e_dev_handle_vfr_event(struct rte_eth_dev *dev)
3784 {
3785         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3786         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
3787         int i;
3788         uint16_t abs_vf_id;
3789         uint32_t index, offset, val;
3790
3791         if (!pf->vfs)
3792                 return;
3793         /**
3794          * Try to find which VF trigger a reset, use absolute VF id to access
3795          * since the reg is global register.
3796          */
3797         for (i = 0; i < pf->vf_num; i++) {
3798                 abs_vf_id = hw->func_caps.vf_base_id + i;
3799                 index = abs_vf_id / I40E_UINT32_BIT_SIZE;
3800                 offset = abs_vf_id % I40E_UINT32_BIT_SIZE;
3801                 val = I40E_READ_REG(hw, I40E_GLGEN_VFLRSTAT(index));
3802                 /* VFR event occured */
3803                 if (val & (0x1 << offset)) {
3804                         int ret;
3805
3806                         /* Clear the event first */
3807                         I40E_WRITE_REG(hw, I40E_GLGEN_VFLRSTAT(index),
3808                                                         (0x1 << offset));
3809                         PMD_DRV_LOG(INFO, "VF %u reset occured", abs_vf_id);
3810                         /**
3811                          * Only notify a VF reset event occured,
3812                          * don't trigger another SW reset
3813                          */
3814                         ret = i40e_pf_host_vf_reset(&pf->vfs[i], 0);
3815                         if (ret != I40E_SUCCESS)
3816                                 PMD_DRV_LOG(ERR, "Failed to do VF reset");
3817                 }
3818         }
3819 }
3820
3821 static void
3822 i40e_dev_handle_aq_msg(struct rte_eth_dev *dev)
3823 {
3824         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3825         struct i40e_arq_event_info info;
3826         uint16_t pending, opcode;
3827         int ret;
3828
3829         info.buf_len = I40E_AQ_BUF_SZ;
3830         info.msg_buf = rte_zmalloc("msg_buffer", info.buf_len, 0);
3831         if (!info.msg_buf) {
3832                 PMD_DRV_LOG(ERR, "Failed to allocate mem");
3833                 return;
3834         }
3835
3836         pending = 1;
3837         while (pending) {
3838                 ret = i40e_clean_arq_element(hw, &info, &pending);
3839
3840                 if (ret != I40E_SUCCESS) {
3841                         PMD_DRV_LOG(INFO, "Failed to read msg from AdminQ, "
3842                                     "aq_err: %u", hw->aq.asq_last_status);
3843                         break;
3844                 }
3845                 opcode = rte_le_to_cpu_16(info.desc.opcode);
3846
3847                 switch (opcode) {
3848                 case i40e_aqc_opc_send_msg_to_pf:
3849                         /* Refer to i40e_aq_send_msg_to_pf() for argument layout*/
3850                         i40e_pf_host_handle_vf_msg(dev,
3851                                         rte_le_to_cpu_16(info.desc.retval),
3852                                         rte_le_to_cpu_32(info.desc.cookie_high),
3853                                         rte_le_to_cpu_32(info.desc.cookie_low),
3854                                         info.msg_buf,
3855                                         info.msg_len);
3856                         break;
3857                 default:
3858                         PMD_DRV_LOG(ERR, "Request %u is not supported yet",
3859                                     opcode);
3860                         break;
3861                 }
3862         }
3863         rte_free(info.msg_buf);
3864 }
3865
3866 /*
3867  * Interrupt handler is registered as the alarm callback for handling LSC
3868  * interrupt in a definite of time, in order to wait the NIC into a stable
3869  * state. Currently it waits 1 sec in i40e for the link up interrupt, and
3870  * no need for link down interrupt.
3871  */
3872 static void
3873 i40e_dev_interrupt_delayed_handler(void *param)
3874 {
3875         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
3876         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3877         uint32_t icr0;
3878
3879         /* read interrupt causes again */
3880         icr0 = I40E_READ_REG(hw, I40E_PFINT_ICR0);
3881
3882 #ifdef RTE_LIBRTE_I40E_DEBUG_DRIVER
3883         if (icr0 & I40E_PFINT_ICR0_ECC_ERR_MASK)
3884                 PMD_DRV_LOG(ERR, "ICR0: unrecoverable ECC error\n");
3885         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK)
3886                 PMD_DRV_LOG(ERR, "ICR0: malicious programming detected\n");
3887         if (icr0 & I40E_PFINT_ICR0_GRST_MASK)
3888                 PMD_DRV_LOG(INFO, "ICR0: global reset requested\n");
3889         if (icr0 & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)
3890                 PMD_DRV_LOG(INFO, "ICR0: PCI exception\n activated\n");
3891         if (icr0 & I40E_PFINT_ICR0_STORM_DETECT_MASK)
3892                 PMD_DRV_LOG(INFO, "ICR0: a change in the storm control "
3893                                                                 "state\n");
3894         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK)
3895                 PMD_DRV_LOG(ERR, "ICR0: HMC error\n");
3896         if (icr0 & I40E_PFINT_ICR0_PE_CRITERR_MASK)
3897                 PMD_DRV_LOG(ERR, "ICR0: protocol engine critical error\n");
3898 #endif /* RTE_LIBRTE_I40E_DEBUG_DRIVER */
3899
3900         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
3901                 PMD_DRV_LOG(INFO, "INT:VF reset detected\n");
3902                 i40e_dev_handle_vfr_event(dev);
3903         }
3904         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
3905                 PMD_DRV_LOG(INFO, "INT:ADMINQ event\n");
3906                 i40e_dev_handle_aq_msg(dev);
3907         }
3908
3909         /* handle the link up interrupt in an alarm callback */
3910         i40e_dev_link_update(dev, 0);
3911         _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
3912
3913         i40e_pf_enable_irq0(hw);
3914         rte_intr_enable(&(dev->pci_dev->intr_handle));
3915 }
3916
3917 /**
3918  * Interrupt handler triggered by NIC  for handling
3919  * specific interrupt.
3920  *
3921  * @param handle
3922  *  Pointer to interrupt handle.
3923  * @param param
3924  *  The address of parameter (struct rte_eth_dev *) regsitered before.
3925  *
3926  * @return
3927  *  void
3928  */
3929 static void
3930 i40e_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
3931                            void *param)
3932 {
3933         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
3934         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3935         uint32_t icr0;
3936
3937         /* Disable interrupt */
3938         i40e_pf_disable_irq0(hw);
3939
3940         /* read out interrupt causes */
3941         icr0 = I40E_READ_REG(hw, I40E_PFINT_ICR0);
3942
3943         /* No interrupt event indicated */
3944         if (!(icr0 & I40E_PFINT_ICR0_INTEVENT_MASK)) {
3945                 PMD_DRV_LOG(INFO, "No interrupt event");
3946                 goto done;
3947         }
3948 #ifdef RTE_LIBRTE_I40E_DEBUG_DRIVER
3949         if (icr0 & I40E_PFINT_ICR0_ECC_ERR_MASK)
3950                 PMD_DRV_LOG(ERR, "ICR0: unrecoverable ECC error");
3951         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK)
3952                 PMD_DRV_LOG(ERR, "ICR0: malicious programming detected");
3953         if (icr0 & I40E_PFINT_ICR0_GRST_MASK)
3954                 PMD_DRV_LOG(INFO, "ICR0: global reset requested");
3955         if (icr0 & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)
3956                 PMD_DRV_LOG(INFO, "ICR0: PCI exception activated");
3957         if (icr0 & I40E_PFINT_ICR0_STORM_DETECT_MASK)
3958                 PMD_DRV_LOG(INFO, "ICR0: a change in the storm control state");
3959         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK)
3960                 PMD_DRV_LOG(ERR, "ICR0: HMC error");
3961         if (icr0 & I40E_PFINT_ICR0_PE_CRITERR_MASK)
3962                 PMD_DRV_LOG(ERR, "ICR0: protocol engine critical error");
3963 #endif /* RTE_LIBRTE_I40E_DEBUG_DRIVER */
3964
3965         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
3966                 PMD_DRV_LOG(INFO, "ICR0: VF reset detected");
3967                 i40e_dev_handle_vfr_event(dev);
3968         }
3969         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
3970                 PMD_DRV_LOG(INFO, "ICR0: adminq event");
3971                 i40e_dev_handle_aq_msg(dev);
3972         }
3973
3974         /* Link Status Change interrupt */
3975         if (icr0 & I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK) {
3976 #define I40E_US_PER_SECOND 1000000
3977                 struct rte_eth_link link;
3978
3979                 PMD_DRV_LOG(INFO, "ICR0: link status changed\n");
3980                 memset(&link, 0, sizeof(link));
3981                 rte_i40e_dev_atomic_read_link_status(dev, &link);
3982                 i40e_dev_link_update(dev, 0);
3983
3984                 /*
3985                  * For link up interrupt, it needs to wait 1 second to let the
3986                  * hardware be a stable state. Otherwise several consecutive
3987                  * interrupts can be observed.
3988                  * For link down interrupt, no need to wait.
3989                  */
3990                 if (!link.link_status && rte_eal_alarm_set(I40E_US_PER_SECOND,
3991                         i40e_dev_interrupt_delayed_handler, (void *)dev) >= 0)
3992                         return;
3993                 else
3994                         _rte_eth_dev_callback_process(dev,
3995                                 RTE_ETH_EVENT_INTR_LSC);
3996         }
3997
3998 done:
3999         /* Enable interrupt */
4000         i40e_pf_enable_irq0(hw);
4001         rte_intr_enable(&(dev->pci_dev->intr_handle));
4002 }
4003
4004 static int
4005 i40e_add_macvlan_filters(struct i40e_vsi *vsi,
4006                          struct i40e_macvlan_filter *filter,
4007                          int total)
4008 {
4009         int ele_num, ele_buff_size;
4010         int num, actual_num, i;
4011         uint16_t flags;
4012         int ret = I40E_SUCCESS;
4013         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
4014         struct i40e_aqc_add_macvlan_element_data *req_list;
4015
4016         if (filter == NULL  || total == 0)
4017                 return I40E_ERR_PARAM;
4018         ele_num = hw->aq.asq_buf_size / sizeof(*req_list);
4019         ele_buff_size = hw->aq.asq_buf_size;
4020
4021         req_list = rte_zmalloc("macvlan_add", ele_buff_size, 0);
4022         if (req_list == NULL) {
4023                 PMD_DRV_LOG(ERR, "Fail to allocate memory");
4024                 return I40E_ERR_NO_MEMORY;
4025         }
4026
4027         num = 0;
4028         do {
4029                 actual_num = (num + ele_num > total) ? (total - num) : ele_num;
4030                 memset(req_list, 0, ele_buff_size);
4031
4032                 for (i = 0; i < actual_num; i++) {
4033                         (void)rte_memcpy(req_list[i].mac_addr,
4034                                 &filter[num + i].macaddr, ETH_ADDR_LEN);
4035                         req_list[i].vlan_tag =
4036                                 rte_cpu_to_le_16(filter[num + i].vlan_id);
4037
4038                         switch (filter[num + i].filter_type) {
4039                         case RTE_MAC_PERFECT_MATCH:
4040                                 flags = I40E_AQC_MACVLAN_ADD_PERFECT_MATCH |
4041                                         I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
4042                                 break;
4043                         case RTE_MACVLAN_PERFECT_MATCH:
4044                                 flags = I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
4045                                 break;
4046                         case RTE_MAC_HASH_MATCH:
4047                                 flags = I40E_AQC_MACVLAN_ADD_HASH_MATCH |
4048                                         I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
4049                                 break;
4050                         case RTE_MACVLAN_HASH_MATCH:
4051                                 flags = I40E_AQC_MACVLAN_ADD_HASH_MATCH;
4052                                 break;
4053                         default:
4054                                 PMD_DRV_LOG(ERR, "Invalid MAC match type\n");
4055                                 ret = I40E_ERR_PARAM;
4056                                 goto DONE;
4057                         }
4058
4059                         req_list[i].queue_number = 0;
4060
4061                         req_list[i].flags = rte_cpu_to_le_16(flags);
4062                 }
4063
4064                 ret = i40e_aq_add_macvlan(hw, vsi->seid, req_list,
4065                                                 actual_num, NULL);
4066                 if (ret != I40E_SUCCESS) {
4067                         PMD_DRV_LOG(ERR, "Failed to add macvlan filter");
4068                         goto DONE;
4069                 }
4070                 num += actual_num;
4071         } while (num < total);
4072
4073 DONE:
4074         rte_free(req_list);
4075         return ret;
4076 }
4077
4078 static int
4079 i40e_remove_macvlan_filters(struct i40e_vsi *vsi,
4080                             struct i40e_macvlan_filter *filter,
4081                             int total)
4082 {
4083         int ele_num, ele_buff_size;
4084         int num, actual_num, i;
4085         uint16_t flags;
4086         int ret = I40E_SUCCESS;
4087         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
4088         struct i40e_aqc_remove_macvlan_element_data *req_list;
4089
4090         if (filter == NULL  || total == 0)
4091                 return I40E_ERR_PARAM;
4092
4093         ele_num = hw->aq.asq_buf_size / sizeof(*req_list);
4094         ele_buff_size = hw->aq.asq_buf_size;
4095
4096         req_list = rte_zmalloc("macvlan_remove", ele_buff_size, 0);
4097         if (req_list == NULL) {
4098                 PMD_DRV_LOG(ERR, "Fail to allocate memory");
4099                 return I40E_ERR_NO_MEMORY;
4100         }
4101
4102         num = 0;
4103         do {
4104                 actual_num = (num + ele_num > total) ? (total - num) : ele_num;
4105                 memset(req_list, 0, ele_buff_size);
4106
4107                 for (i = 0; i < actual_num; i++) {
4108                         (void)rte_memcpy(req_list[i].mac_addr,
4109                                 &filter[num + i].macaddr, ETH_ADDR_LEN);
4110                         req_list[i].vlan_tag =
4111                                 rte_cpu_to_le_16(filter[num + i].vlan_id);
4112
4113                         switch (filter[num + i].filter_type) {
4114                         case RTE_MAC_PERFECT_MATCH:
4115                                 flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
4116                                         I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
4117                                 break;
4118                         case RTE_MACVLAN_PERFECT_MATCH:
4119                                 flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
4120                                 break;
4121                         case RTE_MAC_HASH_MATCH:
4122                                 flags = I40E_AQC_MACVLAN_DEL_HASH_MATCH |
4123                                         I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
4124                                 break;
4125                         case RTE_MACVLAN_HASH_MATCH:
4126                                 flags = I40E_AQC_MACVLAN_DEL_HASH_MATCH;
4127                                 break;
4128                         default:
4129                                 PMD_DRV_LOG(ERR, "Invalid MAC filter type\n");
4130                                 ret = I40E_ERR_PARAM;
4131                                 goto DONE;
4132                         }
4133                         req_list[i].flags = rte_cpu_to_le_16(flags);
4134                 }
4135
4136                 ret = i40e_aq_remove_macvlan(hw, vsi->seid, req_list,
4137                                                 actual_num, NULL);
4138                 if (ret != I40E_SUCCESS) {
4139                         PMD_DRV_LOG(ERR, "Failed to remove macvlan filter");
4140                         goto DONE;
4141                 }
4142                 num += actual_num;
4143         } while (num < total);
4144
4145 DONE:
4146         rte_free(req_list);
4147         return ret;
4148 }
4149
4150 /* Find out specific MAC filter */
4151 static struct i40e_mac_filter *
4152 i40e_find_mac_filter(struct i40e_vsi *vsi,
4153                          struct ether_addr *macaddr)
4154 {
4155         struct i40e_mac_filter *f;
4156
4157         TAILQ_FOREACH(f, &vsi->mac_list, next) {
4158                 if (is_same_ether_addr(macaddr, &f->mac_info.mac_addr))
4159                         return f;
4160         }
4161
4162         return NULL;
4163 }
4164
4165 static bool
4166 i40e_find_vlan_filter(struct i40e_vsi *vsi,
4167                          uint16_t vlan_id)
4168 {
4169         uint32_t vid_idx, vid_bit;
4170
4171         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
4172         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
4173
4174         if (vsi->vfta[vid_idx] & vid_bit)
4175                 return 1;
4176         else
4177                 return 0;
4178 }
4179
4180 static void
4181 i40e_set_vlan_filter(struct i40e_vsi *vsi,
4182                          uint16_t vlan_id, bool on)
4183 {
4184         uint32_t vid_idx, vid_bit;
4185
4186         /* VFTA is 32-bits size array, each element contains 32 vlan bits, Find the
4187          *  element first, then find the bits it belongs to
4188          */
4189         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
4190         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
4191
4192         if (on)
4193                 vsi->vfta[vid_idx] |= vid_bit;
4194         else
4195                 vsi->vfta[vid_idx] &= ~vid_bit;
4196 }
4197
4198 /**
4199  * Find all vlan options for specific mac addr,
4200  * return with actual vlan found.
4201  */
4202 static inline int
4203 i40e_find_all_vlan_for_mac(struct i40e_vsi *vsi,
4204                            struct i40e_macvlan_filter *mv_f,
4205                            int num, struct ether_addr *addr)
4206 {
4207         int i;
4208         uint32_t j, k;
4209
4210         /**
4211          * Not to use i40e_find_vlan_filter to decrease the loop time,
4212          * although the code looks complex.
4213           */
4214         if (num < vsi->vlan_num)
4215                 return I40E_ERR_PARAM;
4216
4217         i = 0;
4218         for (j = 0; j < I40E_VFTA_SIZE; j++) {
4219                 if (vsi->vfta[j]) {
4220                         for (k = 0; k < I40E_UINT32_BIT_SIZE; k++) {
4221                                 if (vsi->vfta[j] & (1 << k)) {
4222                                         if (i > num - 1) {
4223                                                 PMD_DRV_LOG(ERR, "vlan number "
4224                                                             "not match");
4225                                                 return I40E_ERR_PARAM;
4226                                         }
4227                                         (void)rte_memcpy(&mv_f[i].macaddr,
4228                                                         addr, ETH_ADDR_LEN);
4229                                         mv_f[i].vlan_id =
4230                                                 j * I40E_UINT32_BIT_SIZE + k;
4231                                         i++;
4232                                 }
4233                         }
4234                 }
4235         }
4236         return I40E_SUCCESS;
4237 }
4238
4239 static inline int
4240 i40e_find_all_mac_for_vlan(struct i40e_vsi *vsi,
4241                            struct i40e_macvlan_filter *mv_f,
4242                            int num,
4243                            uint16_t vlan)
4244 {
4245         int i = 0;
4246         struct i40e_mac_filter *f;
4247
4248         if (num < vsi->mac_num)
4249                 return I40E_ERR_PARAM;
4250
4251         TAILQ_FOREACH(f, &vsi->mac_list, next) {
4252                 if (i > num - 1) {
4253                         PMD_DRV_LOG(ERR, "buffer number not match");
4254                         return I40E_ERR_PARAM;
4255                 }
4256                 (void)rte_memcpy(&mv_f[i].macaddr, &f->mac_info.mac_addr,
4257                                 ETH_ADDR_LEN);
4258                 mv_f[i].vlan_id = vlan;
4259                 mv_f[i].filter_type = f->mac_info.filter_type;
4260                 i++;
4261         }
4262
4263         return I40E_SUCCESS;
4264 }
4265
4266 static int
4267 i40e_vsi_remove_all_macvlan_filter(struct i40e_vsi *vsi)
4268 {
4269         int i, num;
4270         struct i40e_mac_filter *f;
4271         struct i40e_macvlan_filter *mv_f;
4272         int ret = I40E_SUCCESS;
4273
4274         if (vsi == NULL || vsi->mac_num == 0)
4275                 return I40E_ERR_PARAM;
4276
4277         /* Case that no vlan is set */
4278         if (vsi->vlan_num == 0)
4279                 num = vsi->mac_num;
4280         else
4281                 num = vsi->mac_num * vsi->vlan_num;
4282
4283         mv_f = rte_zmalloc("macvlan_data", num * sizeof(*mv_f), 0);
4284         if (mv_f == NULL) {
4285                 PMD_DRV_LOG(ERR, "failed to allocate memory");
4286                 return I40E_ERR_NO_MEMORY;
4287         }
4288
4289         i = 0;
4290         if (vsi->vlan_num == 0) {
4291                 TAILQ_FOREACH(f, &vsi->mac_list, next) {
4292                         (void)rte_memcpy(&mv_f[i].macaddr,
4293                                 &f->mac_info.mac_addr, ETH_ADDR_LEN);
4294                         mv_f[i].vlan_id = 0;
4295                         i++;
4296                 }
4297         } else {
4298                 TAILQ_FOREACH(f, &vsi->mac_list, next) {
4299                         ret = i40e_find_all_vlan_for_mac(vsi,&mv_f[i],
4300                                         vsi->vlan_num, &f->mac_info.mac_addr);
4301                         if (ret != I40E_SUCCESS)
4302                                 goto DONE;
4303                         i += vsi->vlan_num;
4304                 }
4305         }
4306
4307         ret = i40e_remove_macvlan_filters(vsi, mv_f, num);
4308 DONE:
4309         rte_free(mv_f);
4310
4311         return ret;
4312 }
4313
4314 int
4315 i40e_vsi_add_vlan(struct i40e_vsi *vsi, uint16_t vlan)
4316 {
4317         struct i40e_macvlan_filter *mv_f;
4318         int mac_num;
4319         int ret = I40E_SUCCESS;
4320
4321         if (!vsi || vlan > ETHER_MAX_VLAN_ID)
4322                 return I40E_ERR_PARAM;
4323
4324         /* If it's already set, just return */
4325         if (i40e_find_vlan_filter(vsi,vlan))
4326                 return I40E_SUCCESS;
4327
4328         mac_num = vsi->mac_num;
4329
4330         if (mac_num == 0) {
4331                 PMD_DRV_LOG(ERR, "Error! VSI doesn't have a mac addr");
4332                 return I40E_ERR_PARAM;
4333         }
4334
4335         mv_f = rte_zmalloc("macvlan_data", mac_num * sizeof(*mv_f), 0);
4336
4337         if (mv_f == NULL) {
4338                 PMD_DRV_LOG(ERR, "failed to allocate memory");
4339                 return I40E_ERR_NO_MEMORY;
4340         }
4341
4342         ret = i40e_find_all_mac_for_vlan(vsi, mv_f, mac_num, vlan);
4343
4344         if (ret != I40E_SUCCESS)
4345                 goto DONE;
4346
4347         ret = i40e_add_macvlan_filters(vsi, mv_f, mac_num);
4348
4349         if (ret != I40E_SUCCESS)
4350                 goto DONE;
4351
4352         i40e_set_vlan_filter(vsi, vlan, 1);
4353
4354         vsi->vlan_num++;
4355         ret = I40E_SUCCESS;
4356 DONE:
4357         rte_free(mv_f);
4358         return ret;
4359 }
4360
4361 int
4362 i40e_vsi_delete_vlan(struct i40e_vsi *vsi, uint16_t vlan)
4363 {
4364         struct i40e_macvlan_filter *mv_f;
4365         int mac_num;
4366         int ret = I40E_SUCCESS;
4367
4368         /**
4369          * Vlan 0 is the generic filter for untagged packets
4370          * and can't be removed.
4371          */
4372         if (!vsi || vlan == 0 || vlan > ETHER_MAX_VLAN_ID)
4373                 return I40E_ERR_PARAM;
4374
4375         /* If can't find it, just return */
4376         if (!i40e_find_vlan_filter(vsi, vlan))
4377                 return I40E_ERR_PARAM;
4378
4379         mac_num = vsi->mac_num;
4380
4381         if (mac_num == 0) {
4382                 PMD_DRV_LOG(ERR, "Error! VSI doesn't have a mac addr");
4383                 return I40E_ERR_PARAM;
4384         }
4385
4386         mv_f = rte_zmalloc("macvlan_data", mac_num * sizeof(*mv_f), 0);
4387
4388         if (mv_f == NULL) {
4389                 PMD_DRV_LOG(ERR, "failed to allocate memory");
4390                 return I40E_ERR_NO_MEMORY;
4391         }
4392
4393         ret = i40e_find_all_mac_for_vlan(vsi, mv_f, mac_num, vlan);
4394
4395         if (ret != I40E_SUCCESS)
4396                 goto DONE;
4397
4398         ret = i40e_remove_macvlan_filters(vsi, mv_f, mac_num);
4399
4400         if (ret != I40E_SUCCESS)
4401                 goto DONE;
4402
4403         /* This is last vlan to remove, replace all mac filter with vlan 0 */
4404         if (vsi->vlan_num == 1) {
4405                 ret = i40e_find_all_mac_for_vlan(vsi, mv_f, mac_num, 0);
4406                 if (ret != I40E_SUCCESS)
4407                         goto DONE;
4408
4409                 ret = i40e_add_macvlan_filters(vsi, mv_f, mac_num);
4410                 if (ret != I40E_SUCCESS)
4411                         goto DONE;
4412         }
4413
4414         i40e_set_vlan_filter(vsi, vlan, 0);
4415
4416         vsi->vlan_num--;
4417         ret = I40E_SUCCESS;
4418 DONE:
4419         rte_free(mv_f);
4420         return ret;
4421 }
4422
4423 int
4424 i40e_vsi_add_mac(struct i40e_vsi *vsi, struct i40e_mac_filter_info *mac_filter)
4425 {
4426         struct i40e_mac_filter *f;
4427         struct i40e_macvlan_filter *mv_f;
4428         int i, vlan_num = 0;
4429         int ret = I40E_SUCCESS;
4430
4431         /* If it's add and we've config it, return */
4432         f = i40e_find_mac_filter(vsi, &mac_filter->mac_addr);
4433         if (f != NULL)
4434                 return I40E_SUCCESS;
4435         if ((mac_filter->filter_type == RTE_MACVLAN_PERFECT_MATCH) ||
4436                 (mac_filter->filter_type == RTE_MACVLAN_HASH_MATCH)) {
4437
4438                 /**
4439                  * If vlan_num is 0, that's the first time to add mac,
4440                  * set mask for vlan_id 0.
4441                  */
4442                 if (vsi->vlan_num == 0) {
4443                         i40e_set_vlan_filter(vsi, 0, 1);
4444                         vsi->vlan_num = 1;
4445                 }
4446                 vlan_num = vsi->vlan_num;
4447         } else if ((mac_filter->filter_type == RTE_MAC_PERFECT_MATCH) ||
4448                         (mac_filter->filter_type == RTE_MAC_HASH_MATCH))
4449                 vlan_num = 1;
4450
4451         mv_f = rte_zmalloc("macvlan_data", vlan_num * sizeof(*mv_f), 0);
4452         if (mv_f == NULL) {
4453                 PMD_DRV_LOG(ERR, "failed to allocate memory");
4454                 return I40E_ERR_NO_MEMORY;
4455         }
4456
4457         for (i = 0; i < vlan_num; i++) {
4458                 mv_f[i].filter_type = mac_filter->filter_type;
4459                 (void)rte_memcpy(&mv_f[i].macaddr, &mac_filter->mac_addr,
4460                                 ETH_ADDR_LEN);
4461         }
4462
4463         if (mac_filter->filter_type == RTE_MACVLAN_PERFECT_MATCH ||
4464                 mac_filter->filter_type == RTE_MACVLAN_HASH_MATCH) {
4465                 ret = i40e_find_all_vlan_for_mac(vsi, mv_f, vlan_num,
4466                                         &mac_filter->mac_addr);
4467                 if (ret != I40E_SUCCESS)
4468                         goto DONE;
4469         }
4470
4471         ret = i40e_add_macvlan_filters(vsi, mv_f, vlan_num);
4472         if (ret != I40E_SUCCESS)
4473                 goto DONE;
4474
4475         /* Add the mac addr into mac list */
4476         f = rte_zmalloc("macv_filter", sizeof(*f), 0);
4477         if (f == NULL) {
4478                 PMD_DRV_LOG(ERR, "failed to allocate memory");
4479                 ret = I40E_ERR_NO_MEMORY;
4480                 goto DONE;
4481         }
4482         (void)rte_memcpy(&f->mac_info.mac_addr, &mac_filter->mac_addr,
4483                         ETH_ADDR_LEN);
4484         f->mac_info.filter_type = mac_filter->filter_type;
4485         TAILQ_INSERT_TAIL(&vsi->mac_list, f, next);
4486         vsi->mac_num++;
4487
4488         ret = I40E_SUCCESS;
4489 DONE:
4490         rte_free(mv_f);
4491
4492         return ret;
4493 }
4494
4495 int
4496 i40e_vsi_delete_mac(struct i40e_vsi *vsi, struct ether_addr *addr)
4497 {
4498         struct i40e_mac_filter *f;
4499         struct i40e_macvlan_filter *mv_f;
4500         int i, vlan_num;
4501         enum rte_mac_filter_type filter_type;
4502         int ret = I40E_SUCCESS;
4503
4504         /* Can't find it, return an error */
4505         f = i40e_find_mac_filter(vsi, addr);
4506         if (f == NULL)
4507                 return I40E_ERR_PARAM;
4508
4509         vlan_num = vsi->vlan_num;
4510         filter_type = f->mac_info.filter_type;
4511         if (filter_type == RTE_MACVLAN_PERFECT_MATCH ||
4512                 filter_type == RTE_MACVLAN_HASH_MATCH) {
4513                 if (vlan_num == 0) {
4514                         PMD_DRV_LOG(ERR, "VLAN number shouldn't be 0\n");
4515                         return I40E_ERR_PARAM;
4516                 }
4517         } else if (filter_type == RTE_MAC_PERFECT_MATCH ||
4518                         filter_type == RTE_MAC_HASH_MATCH)
4519                 vlan_num = 1;
4520
4521         mv_f = rte_zmalloc("macvlan_data", vlan_num * sizeof(*mv_f), 0);
4522         if (mv_f == NULL) {
4523                 PMD_DRV_LOG(ERR, "failed to allocate memory");
4524                 return I40E_ERR_NO_MEMORY;
4525         }
4526
4527         for (i = 0; i < vlan_num; i++) {
4528                 mv_f[i].filter_type = filter_type;
4529                 (void)rte_memcpy(&mv_f[i].macaddr, &f->mac_info.mac_addr,
4530                                 ETH_ADDR_LEN);
4531         }
4532         if (filter_type == RTE_MACVLAN_PERFECT_MATCH ||
4533                         filter_type == RTE_MACVLAN_HASH_MATCH) {
4534                 ret = i40e_find_all_vlan_for_mac(vsi, mv_f, vlan_num, addr);
4535                 if (ret != I40E_SUCCESS)
4536                         goto DONE;
4537         }
4538
4539         ret = i40e_remove_macvlan_filters(vsi, mv_f, vlan_num);
4540         if (ret != I40E_SUCCESS)
4541                 goto DONE;
4542
4543         /* Remove the mac addr into mac list */
4544         TAILQ_REMOVE(&vsi->mac_list, f, next);
4545         rte_free(f);
4546         vsi->mac_num--;
4547
4548         ret = I40E_SUCCESS;
4549 DONE:
4550         rte_free(mv_f);
4551         return ret;
4552 }
4553
4554 /* Configure hash enable flags for RSS */
4555 uint64_t
4556 i40e_config_hena(uint64_t flags)
4557 {
4558         uint64_t hena = 0;
4559
4560         if (!flags)
4561                 return hena;
4562
4563         if (flags & ETH_RSS_NONF_IPV4_UDP)
4564                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
4565         if (flags & ETH_RSS_NONF_IPV4_TCP)
4566                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP;
4567         if (flags & ETH_RSS_NONF_IPV4_SCTP)
4568                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP;
4569         if (flags & ETH_RSS_NONF_IPV4_OTHER)
4570                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER;
4571         if (flags & ETH_RSS_FRAG_IPV4)
4572                 hena |= 1ULL << I40E_FILTER_PCTYPE_FRAG_IPV4;
4573         if (flags & ETH_RSS_NONF_IPV6_UDP)
4574                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_UDP;
4575         if (flags & ETH_RSS_NONF_IPV6_TCP)
4576                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP;
4577         if (flags & ETH_RSS_NONF_IPV6_SCTP)
4578                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP;
4579         if (flags & ETH_RSS_NONF_IPV6_OTHER)
4580                 hena |= 1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER;
4581         if (flags & ETH_RSS_FRAG_IPV6)
4582                 hena |= 1ULL << I40E_FILTER_PCTYPE_FRAG_IPV6;
4583         if (flags & ETH_RSS_L2_PAYLOAD)
4584                 hena |= 1ULL << I40E_FILTER_PCTYPE_L2_PAYLOAD;
4585
4586         return hena;
4587 }
4588
4589 /* Parse the hash enable flags */
4590 uint64_t
4591 i40e_parse_hena(uint64_t flags)
4592 {
4593         uint64_t rss_hf = 0;
4594
4595         if (!flags)
4596                 return rss_hf;
4597
4598         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP))
4599                 rss_hf |= ETH_RSS_NONF_IPV4_UDP;
4600         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP))
4601                 rss_hf |= ETH_RSS_NONF_IPV4_TCP;
4602         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP))
4603                 rss_hf |= ETH_RSS_NONF_IPV4_SCTP;
4604         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER))
4605                 rss_hf |= ETH_RSS_NONF_IPV4_OTHER;
4606         if (flags & (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV4))
4607                 rss_hf |= ETH_RSS_FRAG_IPV4;
4608         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_UDP))
4609                 rss_hf |= ETH_RSS_NONF_IPV6_UDP;
4610         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP))
4611                 rss_hf |= ETH_RSS_NONF_IPV6_TCP;
4612         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP))
4613                 rss_hf |= ETH_RSS_NONF_IPV6_SCTP;
4614         if (flags & (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER))
4615                 rss_hf |= ETH_RSS_NONF_IPV6_OTHER;
4616         if (flags & (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV6))
4617                 rss_hf |= ETH_RSS_FRAG_IPV6;
4618         if (flags & (1ULL << I40E_FILTER_PCTYPE_L2_PAYLOAD))
4619                 rss_hf |= ETH_RSS_L2_PAYLOAD;
4620
4621         return rss_hf;
4622 }
4623
4624 /* Disable RSS */
4625 static void
4626 i40e_pf_disable_rss(struct i40e_pf *pf)
4627 {
4628         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
4629         uint64_t hena;
4630
4631         hena = (uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(0));
4632         hena |= ((uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(1))) << 32;
4633         hena &= ~I40E_RSS_HENA_ALL;
4634         I40E_WRITE_REG(hw, I40E_PFQF_HENA(0), (uint32_t)hena);
4635         I40E_WRITE_REG(hw, I40E_PFQF_HENA(1), (uint32_t)(hena >> 32));
4636         I40E_WRITE_FLUSH(hw);
4637 }
4638
4639 static int
4640 i40e_hw_rss_hash_set(struct i40e_hw *hw, struct rte_eth_rss_conf *rss_conf)
4641 {
4642         uint32_t *hash_key;
4643         uint8_t hash_key_len;
4644         uint64_t rss_hf;
4645         uint16_t i;
4646         uint64_t hena;
4647
4648         hash_key = (uint32_t *)(rss_conf->rss_key);
4649         hash_key_len = rss_conf->rss_key_len;
4650         if (hash_key != NULL && hash_key_len >=
4651                 (I40E_PFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t)) {
4652                 /* Fill in RSS hash key */
4653                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
4654                         I40E_WRITE_REG(hw, I40E_PFQF_HKEY(i), hash_key[i]);
4655         }
4656
4657         rss_hf = rss_conf->rss_hf;
4658         hena = (uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(0));
4659         hena |= ((uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(1))) << 32;
4660         hena &= ~I40E_RSS_HENA_ALL;
4661         hena |= i40e_config_hena(rss_hf);
4662         I40E_WRITE_REG(hw, I40E_PFQF_HENA(0), (uint32_t)hena);
4663         I40E_WRITE_REG(hw, I40E_PFQF_HENA(1), (uint32_t)(hena >> 32));
4664         I40E_WRITE_FLUSH(hw);
4665
4666         return 0;
4667 }
4668
4669 static int
4670 i40e_dev_rss_hash_update(struct rte_eth_dev *dev,
4671                          struct rte_eth_rss_conf *rss_conf)
4672 {
4673         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4674         uint64_t rss_hf = rss_conf->rss_hf & I40E_RSS_OFFLOAD_ALL;
4675         uint64_t hena;
4676
4677         hena = (uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(0));
4678         hena |= ((uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(1))) << 32;
4679         if (!(hena & I40E_RSS_HENA_ALL)) { /* RSS disabled */
4680                 if (rss_hf != 0) /* Enable RSS */
4681                         return -EINVAL;
4682                 return 0; /* Nothing to do */
4683         }
4684         /* RSS enabled */
4685         if (rss_hf == 0) /* Disable RSS */
4686                 return -EINVAL;
4687
4688         return i40e_hw_rss_hash_set(hw, rss_conf);
4689 }
4690
4691 static int
4692 i40e_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
4693                            struct rte_eth_rss_conf *rss_conf)
4694 {
4695         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4696         uint32_t *hash_key = (uint32_t *)(rss_conf->rss_key);
4697         uint64_t hena;
4698         uint16_t i;
4699
4700         if (hash_key != NULL) {
4701                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
4702                         hash_key[i] = I40E_READ_REG(hw, I40E_PFQF_HKEY(i));
4703                 rss_conf->rss_key_len = i * sizeof(uint32_t);
4704         }
4705         hena = (uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(0));
4706         hena |= ((uint64_t)I40E_READ_REG(hw, I40E_PFQF_HENA(1))) << 32;
4707         rss_conf->rss_hf = i40e_parse_hena(hena);
4708
4709         return 0;
4710 }
4711
4712 static int
4713 i40e_dev_get_filter_type(uint16_t filter_type, uint16_t *flag)
4714 {
4715         switch (filter_type) {
4716         case RTE_TUNNEL_FILTER_IMAC_IVLAN:
4717                 *flag = I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN;
4718                 break;
4719         case RTE_TUNNEL_FILTER_IMAC_IVLAN_TENID:
4720                 *flag = I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID;
4721                 break;
4722         case RTE_TUNNEL_FILTER_IMAC_TENID:
4723                 *flag = I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID;
4724                 break;
4725         case RTE_TUNNEL_FILTER_OMAC_TENID_IMAC:
4726                 *flag = I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC;
4727                 break;
4728         case ETH_TUNNEL_FILTER_IMAC:
4729                 *flag = I40E_AQC_ADD_CLOUD_FILTER_IMAC;
4730                 break;
4731         default:
4732                 PMD_DRV_LOG(ERR, "invalid tunnel filter type");
4733                 return -EINVAL;
4734         }
4735
4736         return 0;
4737 }
4738
4739 static int
4740 i40e_dev_tunnel_filter_set(struct i40e_pf *pf,
4741                         struct rte_eth_tunnel_filter_conf *tunnel_filter,
4742                         uint8_t add)
4743 {
4744         uint16_t ip_type;
4745         uint8_t tun_type = 0;
4746         int val, ret = 0;
4747         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
4748         struct i40e_vsi *vsi = pf->main_vsi;
4749         struct i40e_aqc_add_remove_cloud_filters_element_data  *cld_filter;
4750         struct i40e_aqc_add_remove_cloud_filters_element_data  *pfilter;
4751
4752         cld_filter = rte_zmalloc("tunnel_filter",
4753                 sizeof(struct i40e_aqc_add_remove_cloud_filters_element_data),
4754                 0);
4755
4756         if (NULL == cld_filter) {
4757                 PMD_DRV_LOG(ERR, "Failed to alloc memory.");
4758                 return -EINVAL;
4759         }
4760         pfilter = cld_filter;
4761
4762         (void)rte_memcpy(&pfilter->outer_mac, tunnel_filter->outer_mac,
4763                         sizeof(struct ether_addr));
4764         (void)rte_memcpy(&pfilter->inner_mac, tunnel_filter->inner_mac,
4765                         sizeof(struct ether_addr));
4766
4767         pfilter->inner_vlan = tunnel_filter->inner_vlan;
4768         if (tunnel_filter->ip_type == RTE_TUNNEL_IPTYPE_IPV4) {
4769                 ip_type = I40E_AQC_ADD_CLOUD_FLAGS_IPV4;
4770                 (void)rte_memcpy(&pfilter->ipaddr.v4.data,
4771                                 &tunnel_filter->ip_addr,
4772                                 sizeof(pfilter->ipaddr.v4.data));
4773         } else {
4774                 ip_type = I40E_AQC_ADD_CLOUD_FLAGS_IPV6;
4775                 (void)rte_memcpy(&pfilter->ipaddr.v6.data,
4776                                 &tunnel_filter->ip_addr,
4777                                 sizeof(pfilter->ipaddr.v6.data));
4778         }
4779
4780         /* check tunneled type */
4781         switch (tunnel_filter->tunnel_type) {
4782         case RTE_TUNNEL_TYPE_VXLAN:
4783                 tun_type = I40E_AQC_ADD_CLOUD_TNL_TYPE_XVLAN;
4784                 break;
4785         default:
4786                 /* Other tunnel types is not supported. */
4787                 PMD_DRV_LOG(ERR, "tunnel type is not supported.");
4788                 rte_free(cld_filter);
4789                 return -EINVAL;
4790         }
4791
4792         val = i40e_dev_get_filter_type(tunnel_filter->filter_type,
4793                                                 &pfilter->flags);
4794         if (val < 0) {
4795                 rte_free(cld_filter);
4796                 return -EINVAL;
4797         }
4798
4799         pfilter->flags |= I40E_AQC_ADD_CLOUD_FLAGS_TO_QUEUE | ip_type |
4800                 (tun_type << I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT);
4801         pfilter->tenant_id = tunnel_filter->tenant_id;
4802         pfilter->queue_number = tunnel_filter->queue_id;
4803
4804         if (add)
4805                 ret = i40e_aq_add_cloud_filters(hw, vsi->seid, cld_filter, 1);
4806         else
4807                 ret = i40e_aq_remove_cloud_filters(hw, vsi->seid,
4808                                                 cld_filter, 1);
4809
4810         rte_free(cld_filter);
4811         return ret;
4812 }
4813
4814 static int
4815 i40e_get_vxlan_port_idx(struct i40e_pf *pf, uint16_t port)
4816 {
4817         uint8_t i;
4818
4819         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
4820                 if (pf->vxlan_ports[i] == port)
4821                         return i;
4822         }
4823
4824         return -1;
4825 }
4826
4827 static int
4828 i40e_add_vxlan_port(struct i40e_pf *pf, uint16_t port)
4829 {
4830         int  idx, ret;
4831         uint8_t filter_idx;
4832         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
4833
4834         idx = i40e_get_vxlan_port_idx(pf, port);
4835
4836         /* Check if port already exists */
4837         if (idx >= 0) {
4838                 PMD_DRV_LOG(ERR, "Port %d already offloaded", port);
4839                 return -EINVAL;
4840         }
4841
4842         /* Now check if there is space to add the new port */
4843         idx = i40e_get_vxlan_port_idx(pf, 0);
4844         if (idx < 0) {
4845                 PMD_DRV_LOG(ERR, "Maximum number of UDP ports reached,"
4846                         "not adding port %d", port);
4847                 return -ENOSPC;
4848         }
4849
4850         ret =  i40e_aq_add_udp_tunnel(hw, port, I40E_AQC_TUNNEL_TYPE_VXLAN,
4851                                         &filter_idx, NULL);
4852         if (ret < 0) {
4853                 PMD_DRV_LOG(ERR, "Failed to add VXLAN UDP port %d", port);
4854                 return -1;
4855         }
4856
4857         PMD_DRV_LOG(INFO, "Added port %d with AQ command with index %d",
4858                          port,  filter_idx);
4859
4860         /* New port: add it and mark its index in the bitmap */
4861         pf->vxlan_ports[idx] = port;
4862         pf->vxlan_bitmap |= (1 << idx);
4863
4864         if (!(pf->flags & I40E_FLAG_VXLAN))
4865                 pf->flags |= I40E_FLAG_VXLAN;
4866
4867         return 0;
4868 }
4869
4870 static int
4871 i40e_del_vxlan_port(struct i40e_pf *pf, uint16_t port)
4872 {
4873         int idx;
4874         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
4875
4876         if (!(pf->flags & I40E_FLAG_VXLAN)) {
4877                 PMD_DRV_LOG(ERR, "VXLAN UDP port was not configured.");
4878                 return -EINVAL;
4879         }
4880
4881         idx = i40e_get_vxlan_port_idx(pf, port);
4882
4883         if (idx < 0) {
4884                 PMD_DRV_LOG(ERR, "Port %d doesn't exist", port);
4885                 return -EINVAL;
4886         }
4887
4888         if (i40e_aq_del_udp_tunnel(hw, idx, NULL) < 0) {
4889                 PMD_DRV_LOG(ERR, "Failed to delete VXLAN UDP port %d", port);
4890                 return -1;
4891         }
4892
4893         PMD_DRV_LOG(INFO, "Deleted port %d with AQ command with index %d",
4894                         port, idx);
4895
4896         pf->vxlan_ports[idx] = 0;
4897         pf->vxlan_bitmap &= ~(1 << idx);
4898
4899         if (!pf->vxlan_bitmap)
4900                 pf->flags &= ~I40E_FLAG_VXLAN;
4901
4902         return 0;
4903 }
4904
4905 /* Add UDP tunneling port */
4906 static int
4907 i40e_dev_udp_tunnel_add(struct rte_eth_dev *dev,
4908                         struct rte_eth_udp_tunnel *udp_tunnel)
4909 {
4910         int ret = 0;
4911         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
4912
4913         if (udp_tunnel == NULL)
4914                 return -EINVAL;
4915
4916         switch (udp_tunnel->prot_type) {
4917         case RTE_TUNNEL_TYPE_VXLAN:
4918                 ret = i40e_add_vxlan_port(pf, udp_tunnel->udp_port);
4919                 break;
4920
4921         case RTE_TUNNEL_TYPE_GENEVE:
4922         case RTE_TUNNEL_TYPE_TEREDO:
4923                 PMD_DRV_LOG(ERR, "Tunnel type is not supported now.");
4924                 ret = -1;
4925                 break;
4926
4927         default:
4928                 PMD_DRV_LOG(ERR, "Invalid tunnel type");
4929                 ret = -1;
4930                 break;
4931         }
4932
4933         return ret;
4934 }
4935
4936 /* Remove UDP tunneling port */
4937 static int
4938 i40e_dev_udp_tunnel_del(struct rte_eth_dev *dev,
4939                         struct rte_eth_udp_tunnel *udp_tunnel)
4940 {
4941         int ret = 0;
4942         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
4943
4944         if (udp_tunnel == NULL)
4945                 return -EINVAL;
4946
4947         switch (udp_tunnel->prot_type) {
4948         case RTE_TUNNEL_TYPE_VXLAN:
4949                 ret = i40e_del_vxlan_port(pf, udp_tunnel->udp_port);
4950                 break;
4951         case RTE_TUNNEL_TYPE_GENEVE:
4952         case RTE_TUNNEL_TYPE_TEREDO:
4953                 PMD_DRV_LOG(ERR, "Tunnel type is not supported now.");
4954                 ret = -1;
4955                 break;
4956         default:
4957                 PMD_DRV_LOG(ERR, "Invalid tunnel type");
4958                 ret = -1;
4959                 break;
4960         }
4961
4962         return ret;
4963 }
4964
4965 /* Calculate the maximum number of contiguous PF queues that are configured */
4966 static int
4967 i40e_pf_calc_configured_queues_num(struct i40e_pf *pf)
4968 {
4969         struct rte_eth_dev_data *data = pf->dev_data;
4970         int i, num;
4971         struct i40e_rx_queue *rxq;
4972
4973         num = 0;
4974         for (i = 0; i < pf->lan_nb_qps; i++) {
4975                 rxq = data->rx_queues[i];
4976                 if (rxq && rxq->q_set)
4977                         num++;
4978                 else
4979                         break;
4980         }
4981
4982         return num;
4983 }
4984
4985 /* Configure RSS */
4986 static int
4987 i40e_pf_config_rss(struct i40e_pf *pf)
4988 {
4989         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
4990         struct rte_eth_rss_conf rss_conf;
4991         uint32_t i, lut = 0;
4992         uint16_t j, num;
4993
4994         /*
4995          * If both VMDQ and RSS enabled, not all of PF queues are configured.
4996          * It's necessary to calulate the actual PF queues that are configured.
4997          */
4998         if (pf->dev_data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG) {
4999                 num = i40e_pf_calc_configured_queues_num(pf);
5000                 num = i40e_align_floor(num);
5001         } else
5002                 num = i40e_align_floor(pf->dev_data->nb_rx_queues);
5003
5004         PMD_INIT_LOG(INFO, "Max of contiguous %u PF queues are configured",
5005                         num);
5006
5007         if (num == 0) {
5008                 PMD_INIT_LOG(ERR, "No PF queues are configured to enable RSS");
5009                 return -ENOTSUP;
5010         }
5011
5012         for (i = 0, j = 0; i < hw->func_caps.rss_table_size; i++, j++) {
5013                 if (j == num)
5014                         j = 0;
5015                 lut = (lut << 8) | (j & ((0x1 <<
5016                         hw->func_caps.rss_table_entry_width) - 1));
5017                 if ((i & 3) == 3)
5018                         I40E_WRITE_REG(hw, I40E_PFQF_HLUT(i >> 2), lut);
5019         }
5020
5021         rss_conf = pf->dev_data->dev_conf.rx_adv_conf.rss_conf;
5022         if ((rss_conf.rss_hf & I40E_RSS_OFFLOAD_ALL) == 0) {
5023                 i40e_pf_disable_rss(pf);
5024                 return 0;
5025         }
5026         if (rss_conf.rss_key == NULL || rss_conf.rss_key_len <
5027                 (I40E_PFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t)) {
5028                 /* Calculate the default hash key */
5029                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
5030                         rss_key_default[i] = (uint32_t)rte_rand();
5031                 rss_conf.rss_key = (uint8_t *)rss_key_default;
5032                 rss_conf.rss_key_len = (I40E_PFQF_HKEY_MAX_INDEX + 1) *
5033                                                         sizeof(uint32_t);
5034         }
5035
5036         return i40e_hw_rss_hash_set(hw, &rss_conf);
5037 }
5038
5039 static int
5040 i40e_tunnel_filter_param_check(struct i40e_pf *pf,
5041                         struct rte_eth_tunnel_filter_conf *filter)
5042 {
5043         if (pf == NULL || filter == NULL) {
5044                 PMD_DRV_LOG(ERR, "Invalid parameter");
5045                 return -EINVAL;
5046         }
5047
5048         if (filter->queue_id >= pf->dev_data->nb_rx_queues) {
5049                 PMD_DRV_LOG(ERR, "Invalid queue ID");
5050                 return -EINVAL;
5051         }
5052
5053         if (filter->inner_vlan > ETHER_MAX_VLAN_ID) {
5054                 PMD_DRV_LOG(ERR, "Invalid inner VLAN ID");
5055                 return -EINVAL;
5056         }
5057
5058         if ((filter->filter_type & ETH_TUNNEL_FILTER_OMAC) &&
5059                 (is_zero_ether_addr(filter->outer_mac))) {
5060                 PMD_DRV_LOG(ERR, "Cannot add NULL outer MAC address");
5061                 return -EINVAL;
5062         }
5063
5064         if ((filter->filter_type & ETH_TUNNEL_FILTER_IMAC) &&
5065                 (is_zero_ether_addr(filter->inner_mac))) {
5066                 PMD_DRV_LOG(ERR, "Cannot add NULL inner MAC address");
5067                 return -EINVAL;
5068         }
5069
5070         return 0;
5071 }
5072
5073 static int
5074 i40e_tunnel_filter_handle(struct rte_eth_dev *dev, enum rte_filter_op filter_op,
5075                         void *arg)
5076 {
5077         struct rte_eth_tunnel_filter_conf *filter;
5078         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
5079         int ret = I40E_SUCCESS;
5080
5081         filter = (struct rte_eth_tunnel_filter_conf *)(arg);
5082
5083         if (i40e_tunnel_filter_param_check(pf, filter) < 0)
5084                 return I40E_ERR_PARAM;
5085
5086         switch (filter_op) {
5087         case RTE_ETH_FILTER_NOP:
5088                 if (!(pf->flags & I40E_FLAG_VXLAN))
5089                         ret = I40E_NOT_SUPPORTED;
5090         case RTE_ETH_FILTER_ADD:
5091                 ret = i40e_dev_tunnel_filter_set(pf, filter, 1);
5092                 break;
5093         case RTE_ETH_FILTER_DELETE:
5094                 ret = i40e_dev_tunnel_filter_set(pf, filter, 0);
5095                 break;
5096         default:
5097                 PMD_DRV_LOG(ERR, "unknown operation %u", filter_op);
5098                 ret = I40E_ERR_PARAM;
5099                 break;
5100         }
5101
5102         return ret;
5103 }
5104
5105 static int
5106 i40e_pf_config_mq_rx(struct i40e_pf *pf)
5107 {
5108         int ret = 0;
5109         enum rte_eth_rx_mq_mode mq_mode = pf->dev_data->dev_conf.rxmode.mq_mode;
5110
5111         if (mq_mode & ETH_MQ_RX_DCB_FLAG) {
5112                 PMD_INIT_LOG(ERR, "i40e doesn't support DCB yet");
5113                 return -ENOTSUP;
5114         }
5115
5116         /* RSS setup */
5117         if (mq_mode & ETH_MQ_RX_RSS_FLAG)
5118                 ret = i40e_pf_config_rss(pf);
5119         else
5120                 i40e_pf_disable_rss(pf);
5121
5122         return ret;
5123 }
5124
5125 /*
5126  * Configure ethertype filter, which can director packet by filtering
5127  * with mac address and ether_type or only ether_type
5128  */
5129 static int
5130 i40e_ethertype_filter_set(struct i40e_pf *pf,
5131                         struct rte_eth_ethertype_filter *filter,
5132                         bool add)
5133 {
5134         struct i40e_hw *hw = I40E_PF_TO_HW(pf);
5135         struct i40e_control_filter_stats stats;
5136         uint16_t flags = 0;
5137         int ret;
5138
5139         if (filter->queue >= pf->dev_data->nb_rx_queues) {
5140                 PMD_DRV_LOG(ERR, "Invalid queue ID");
5141                 return -EINVAL;
5142         }
5143         if (filter->ether_type == ETHER_TYPE_IPv4 ||
5144                 filter->ether_type == ETHER_TYPE_IPv6) {
5145                 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in"
5146                         " control packet filter.", filter->ether_type);
5147                 return -EINVAL;
5148         }
5149         if (filter->ether_type == ETHER_TYPE_VLAN)
5150                 PMD_DRV_LOG(WARNING, "filter vlan ether_type in first tag is"
5151                         " not supported.");
5152
5153         if (!(filter->flags & RTE_ETHTYPE_FLAGS_MAC))
5154                 flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_IGNORE_MAC;
5155         if (filter->flags & RTE_ETHTYPE_FLAGS_DROP)
5156                 flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_DROP;
5157         flags |= I40E_AQC_ADD_CONTROL_PACKET_FLAGS_TO_QUEUE;
5158
5159         memset(&stats, 0, sizeof(stats));
5160         ret = i40e_aq_add_rem_control_packet_filter(hw,
5161                         filter->mac_addr.addr_bytes,
5162                         filter->ether_type, flags,
5163                         pf->main_vsi->seid,
5164                         filter->queue, add, &stats, NULL);
5165
5166         PMD_DRV_LOG(INFO, "add/rem control packet filter, return %d,"
5167                          " mac_etype_used = %u, etype_used = %u,"
5168                          " mac_etype_free = %u, etype_free = %u\n",
5169                          ret, stats.mac_etype_used, stats.etype_used,
5170                          stats.mac_etype_free, stats.etype_free);
5171         if (ret < 0)
5172                 return -ENOSYS;
5173         return 0;
5174 }
5175
5176 /*
5177  * Handle operations for ethertype filter.
5178  */
5179 static int
5180 i40e_ethertype_filter_handle(struct rte_eth_dev *dev,
5181                                 enum rte_filter_op filter_op,
5182                                 void *arg)
5183 {
5184         struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
5185         int ret = 0;
5186
5187         if (filter_op == RTE_ETH_FILTER_NOP)
5188                 return ret;
5189
5190         if (arg == NULL) {
5191                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
5192                             filter_op);
5193                 return -EINVAL;
5194         }
5195
5196         switch (filter_op) {
5197         case RTE_ETH_FILTER_ADD:
5198                 ret = i40e_ethertype_filter_set(pf,
5199                         (struct rte_eth_ethertype_filter *)arg,
5200                         TRUE);
5201                 break;
5202         case RTE_ETH_FILTER_DELETE:
5203                 ret = i40e_ethertype_filter_set(pf,
5204                         (struct rte_eth_ethertype_filter *)arg,
5205                         FALSE);
5206                 break;
5207         default:
5208                 PMD_DRV_LOG(ERR, "unsupported operation %u\n", filter_op);
5209                 ret = -ENOSYS;
5210                 break;
5211         }
5212         return ret;
5213 }
5214
5215 static int
5216 i40e_dev_filter_ctrl(struct rte_eth_dev *dev,
5217                      enum rte_filter_type filter_type,
5218                      enum rte_filter_op filter_op,
5219                      void *arg)
5220 {
5221         int ret = 0;
5222
5223         if (dev == NULL)
5224                 return -EINVAL;
5225
5226         switch (filter_type) {
5227         case RTE_ETH_FILTER_MACVLAN:
5228                 ret = i40e_mac_filter_handle(dev, filter_op, arg);
5229                 break;
5230         case RTE_ETH_FILTER_ETHERTYPE:
5231                 ret = i40e_ethertype_filter_handle(dev, filter_op, arg);
5232                 break;
5233         case RTE_ETH_FILTER_TUNNEL:
5234                 ret = i40e_tunnel_filter_handle(dev, filter_op, arg);
5235                 break;
5236         case RTE_ETH_FILTER_FDIR:
5237                 ret = i40e_fdir_ctrl_func(dev, filter_op, arg);
5238                 break;
5239         default:
5240                 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported",
5241                                                         filter_type);
5242                 ret = -EINVAL;
5243                 break;
5244         }
5245
5246         return ret;
5247 }
5248
5249 enum i40e_filter_pctype
5250 i40e_flowtype_to_pctype(enum rte_eth_flow_type flow_type)
5251 {
5252         static const enum i40e_filter_pctype pctype_table[] = {
5253                 [RTE_ETH_FLOW_TYPE_UDPV4] = I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
5254                 [RTE_ETH_FLOW_TYPE_TCPV4] = I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
5255                 [RTE_ETH_FLOW_TYPE_SCTPV4] = I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
5256                 [RTE_ETH_FLOW_TYPE_IPV4_OTHER] =
5257                                         I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
5258                 [RTE_ETH_FLOW_TYPE_FRAG_IPV4] =
5259                                         I40E_FILTER_PCTYPE_FRAG_IPV4,
5260                 [RTE_ETH_FLOW_TYPE_UDPV6] = I40E_FILTER_PCTYPE_NONF_IPV6_UDP,
5261                 [RTE_ETH_FLOW_TYPE_TCPV6] = I40E_FILTER_PCTYPE_NONF_IPV6_TCP,
5262                 [RTE_ETH_FLOW_TYPE_SCTPV6] = I40E_FILTER_PCTYPE_NONF_IPV6_SCTP,
5263                 [RTE_ETH_FLOW_TYPE_IPV6_OTHER] =
5264                                         I40E_FILTER_PCTYPE_NONF_IPV6_OTHER,
5265                 [RTE_ETH_FLOW_TYPE_FRAG_IPV6] =
5266                                         I40E_FILTER_PCTYPE_FRAG_IPV6,
5267         };
5268
5269         return pctype_table[flow_type];
5270 }
5271
5272 enum rte_eth_flow_type
5273 i40e_pctype_to_flowtype(enum i40e_filter_pctype pctype)
5274 {
5275         static const enum rte_eth_flow_type flowtype_table[] = {
5276                 [I40E_FILTER_PCTYPE_NONF_IPV4_UDP] = RTE_ETH_FLOW_TYPE_UDPV4,
5277                 [I40E_FILTER_PCTYPE_NONF_IPV4_TCP] = RTE_ETH_FLOW_TYPE_TCPV4,
5278                 [I40E_FILTER_PCTYPE_NONF_IPV4_SCTP] = RTE_ETH_FLOW_TYPE_SCTPV4,
5279                 [I40E_FILTER_PCTYPE_NONF_IPV4_OTHER] =
5280                                         RTE_ETH_FLOW_TYPE_IPV4_OTHER,
5281                 [I40E_FILTER_PCTYPE_FRAG_IPV4] =
5282                                         RTE_ETH_FLOW_TYPE_FRAG_IPV4,
5283                 [I40E_FILTER_PCTYPE_NONF_IPV6_UDP] = RTE_ETH_FLOW_TYPE_UDPV6,
5284                 [I40E_FILTER_PCTYPE_NONF_IPV6_TCP] = RTE_ETH_FLOW_TYPE_TCPV6,
5285                 [I40E_FILTER_PCTYPE_NONF_IPV6_SCTP] = RTE_ETH_FLOW_TYPE_SCTPV6,
5286                 [I40E_FILTER_PCTYPE_NONF_IPV6_OTHER] =
5287                                         RTE_ETH_FLOW_TYPE_IPV6_OTHER,
5288                 [I40E_FILTER_PCTYPE_FRAG_IPV6] =
5289                                         RTE_ETH_FLOW_TYPE_FRAG_IPV6,
5290         };
5291
5292         return flowtype_table[pctype];
5293 }