lib/librte_pmd_ixgbe/ixgbe_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <inttypes.h>
  42 #include <rte_byteorder.h>
  43 #include <rte_common.h>
  44 #include <rte_cycles.h>
  45
  46 #include <rte_interrupts.h>
  47 #include <rte_log.h>
  48 #include <rte_debug.h>
  49 #include <rte_pci.h>
  50 #include <rte_atomic.h>
  51 #include <rte_branch_prediction.h>
  52 #include <rte_memory.h>
  53 #include <rte_memzone.h>
  54 #include <rte_tailq.h>
  55 #include <rte_eal.h>
  56 #include <rte_alarm.h>
  57 #include <rte_ether.h>
  58 #include <rte_ethdev.h>
  59 #include <rte_atomic.h>
  60 #include <rte_malloc.h>
  61 #include <rte_random.h>
  62 #include <rte_dev.h>
  63
  64 #include "ixgbe_logs.h"
  65 #include "ixgbe/ixgbe_api.h"
  66 #include "ixgbe/ixgbe_vf.h"
  67 #include "ixgbe/ixgbe_common.h"
  68 #include "ixgbe_ethdev.h"
  69 #include "ixgbe_bypass.h"
  70
  71 /*
  72  * High threshold controlling when to start sending XOFF frames. Must be at
  73  * least 8 bytes less than receive packet buffer size. This value is in units
  74  * of 1024 bytes.
  75  */
  76 #define IXGBE_FC_HI    0x80
  77
  78 /*
  79  * Low threshold controlling when to start sending XON frames. This value is
  80  * in units of 1024 bytes.
  81  */
  82 #define IXGBE_FC_LO    0x40
  83
  84 /* Timer value included in XOFF frames. */
  85 #define IXGBE_FC_PAUSE 0x680
  86
  87 #define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */
  88 #define IXGBE_LINK_UP_CHECK_TIMEOUT   1000 /* ms */
  89 #define IXGBE_VMDQ_NUM_UC_MAC         4096 /* Maximum nb. of UC MAC addr. */
  90
  91
  92 #define IXGBEVF_PMD_NAME "rte_ixgbevf_pmd" /* PMD name */
  93
  94 #define IXGBE_QUEUE_STAT_COUNTERS (sizeof(hw_stats->qprc) / sizeof(hw_stats->qprc[0]))
  95
  96 static int eth_ixgbe_dev_init(struct eth_driver *eth_drv,
  97                 struct rte_eth_dev *eth_dev);
  98 static int  ixgbe_dev_configure(struct rte_eth_dev *dev);
  99 static int  ixgbe_dev_start(struct rte_eth_dev *dev);
 100 static void ixgbe_dev_stop(struct rte_eth_dev *dev);
 101 static int  ixgbe_dev_set_link_up(struct rte_eth_dev *dev);
 102 static int  ixgbe_dev_set_link_down(struct rte_eth_dev *dev);
 103 static void ixgbe_dev_close(struct rte_eth_dev *dev);
 104 static void ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev);
 105 static void ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev);
 106 static void ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev);
 107 static void ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev);
 108 static int ixgbe_dev_link_update(struct rte_eth_dev *dev,
 109                                 int wait_to_complete);
 110 static void ixgbe_dev_stats_get(struct rte_eth_dev *dev,
 111                                 struct rte_eth_stats *stats);
 112 static void ixgbe_dev_stats_reset(struct rte_eth_dev *dev);
 113 static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 114                                              uint16_t queue_id,
 115                                              uint8_t stat_idx,
 116                                              uint8_t is_rx);
 117 static void ixgbe_dev_info_get(struct rte_eth_dev *dev,
 118                                 struct rte_eth_dev_info *dev_info);
 119 static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev,
 120                 uint16_t vlan_id, int on);
 121 static void ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 122 static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev,
 123                 uint16_t queue, bool on);
 124 static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue,
 125                 int on);
 126 static void ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 127 static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue);
 128 static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue);
 129 static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 130 static void ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 131
 132 static int ixgbe_dev_led_on(struct rte_eth_dev *dev);
 133 static int ixgbe_dev_led_off(struct rte_eth_dev *dev);
 134 static int  ixgbe_flow_ctrl_set(struct rte_eth_dev *dev,
 135                 struct rte_eth_fc_conf *fc_conf);
 136 static int ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 137                 struct rte_eth_pfc_conf *pfc_conf);
 138 static int ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
 139                 struct rte_eth_rss_reta *reta_conf);
 140 static int ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
 141                 struct rte_eth_rss_reta *reta_conf);
 142 static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev);
 143 static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev);
 144 static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev);
 145 static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev);
 146 static void ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle,
 147                 void *param);
 148 static void ixgbe_dev_interrupt_delayed_handler(void *param);
 149 static void ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 150                 uint32_t index, uint32_t pool);
 151 static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index);
 152 static void ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config);
 153
 154 /* For Virtual Function support */
 155 static int eth_ixgbevf_dev_init(struct eth_driver *eth_drv,
 156                 struct rte_eth_dev *eth_dev);
 157 static int  ixgbevf_dev_configure(struct rte_eth_dev *dev);
 158 static int  ixgbevf_dev_start(struct rte_eth_dev *dev);
 159 static void ixgbevf_dev_stop(struct rte_eth_dev *dev);
 160 static void ixgbevf_dev_close(struct rte_eth_dev *dev);
 161 static void ixgbevf_intr_disable(struct ixgbe_hw *hw);
 162 static void ixgbevf_dev_stats_get(struct rte_eth_dev *dev,
 163                 struct rte_eth_stats *stats);
 164 static void ixgbevf_dev_stats_reset(struct rte_eth_dev *dev);
 165 static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev,
 166                 uint16_t vlan_id, int on);
 167 static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev,
 168                 uint16_t queue, int on);
 169 static void ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 170 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 171
 172 /* For Eth VMDQ APIs support */
 173 static int ixgbe_uc_hash_table_set(struct rte_eth_dev *dev, struct
 174                 ether_addr* mac_addr,uint8_t on);
 175 static int ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev,uint8_t on);
 176 static int  ixgbe_set_pool_rx_mode(struct rte_eth_dev *dev,  uint16_t pool,
 177                 uint16_t rx_mask, uint8_t on);
 178 static int ixgbe_set_pool_rx(struct rte_eth_dev *dev,uint16_t pool,uint8_t on);
 179 static int ixgbe_set_pool_tx(struct rte_eth_dev *dev,uint16_t pool,uint8_t on);
 180 static int ixgbe_set_pool_vlan_filter(struct rte_eth_dev *dev, uint16_t vlan,
 181                 uint64_t pool_mask,uint8_t vlan_on);
 182 static int ixgbe_mirror_rule_set(struct rte_eth_dev *dev,
 183                 struct rte_eth_vmdq_mirror_conf *mirror_conf,
 184                 uint8_t rule_id, uint8_t on);
 185 static int ixgbe_mirror_rule_reset(struct rte_eth_dev *dev,
 186                 uint8_t rule_id);
 187
 188 static void ixgbevf_add_mac_addr(struct rte_eth_dev *dev,
 189                                  struct ether_addr *mac_addr,
 190                                  uint32_t index, uint32_t pool);
 191 static void ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index);
 192
 193 /*
 194  * Define VF Stats MACRO for Non "cleared on read" register
 195  */
 196 #define UPDATE_VF_STAT(reg, last, cur)                          \
 197 {                                                               \
 198         u32 latest = IXGBE_READ_REG(hw, reg);                   \
 199         cur += latest - last;                                   \
 200         last = latest;                                          \
 201 }
 202
 203 #define UPDATE_VF_STAT_36BIT(lsb, msb, last, cur)                \
 204 {                                                                \
 205         u64 new_lsb = IXGBE_READ_REG(hw, lsb);                   \
 206         u64 new_msb = IXGBE_READ_REG(hw, msb);                   \
 207         u64 latest = ((new_msb << 32) | new_lsb);                \
 208         cur += (0x1000000000LL + latest - last) & 0xFFFFFFFFFLL; \
 209         last = latest;                                           \
 210 }
 211
 212 #define IXGBE_SET_HWSTRIP(h, q) do{\
 213                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 214                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 215                 (h)->bitmap[idx] |= 1 << bit;\
 216         }while(0)
 217
 218 #define IXGBE_CLEAR_HWSTRIP(h, q) do{\
 219                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 220                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 221                 (h)->bitmap[idx] &= ~(1 << bit);\
 222         }while(0)
 223
 224 #define IXGBE_GET_HWSTRIP(h, q, r) do{\
 225                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 226                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 227                 (r) = (h)->bitmap[idx] >> bit & 1;\
 228         }while(0)
 229
 230 /*
 231  * The set of PCI devices this driver supports
 232  */
 233 static struct rte_pci_id pci_id_ixgbe_map[] = {
 234
 235 #define RTE_PCI_DEV_ID_DECL_IXGBE(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 236 #include "rte_pci_dev_ids.h"
 237
 238 { .vendor_id = 0, /* sentinel */ },
 239 };
 240
 241
 242 /*
 243  * The set of PCI devices this driver supports (for 82599 VF)
 244  */
 245 static struct rte_pci_id pci_id_ixgbevf_map[] = {
 246
 247 #define RTE_PCI_DEV_ID_DECL_IXGBEVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 248 #include "rte_pci_dev_ids.h"
 249 { .vendor_id = 0, /* sentinel */ },
 250
 251 };
 252
 253 static struct eth_dev_ops ixgbe_eth_dev_ops = {
 254         .dev_configure        = ixgbe_dev_configure,
 255         .dev_start            = ixgbe_dev_start,
 256         .dev_stop             = ixgbe_dev_stop,
 257         .dev_set_link_up    = ixgbe_dev_set_link_up,
 258         .dev_set_link_down  = ixgbe_dev_set_link_down,
 259         .dev_close            = ixgbe_dev_close,
 260         .promiscuous_enable   = ixgbe_dev_promiscuous_enable,
 261         .promiscuous_disable  = ixgbe_dev_promiscuous_disable,
 262         .allmulticast_enable  = ixgbe_dev_allmulticast_enable,
 263         .allmulticast_disable = ixgbe_dev_allmulticast_disable,
 264         .link_update          = ixgbe_dev_link_update,
 265         .stats_get            = ixgbe_dev_stats_get,
 266         .stats_reset          = ixgbe_dev_stats_reset,
 267         .queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set,
 268         .dev_infos_get        = ixgbe_dev_info_get,
 269         .vlan_filter_set      = ixgbe_vlan_filter_set,
 270         .vlan_tpid_set        = ixgbe_vlan_tpid_set,
 271         .vlan_offload_set     = ixgbe_vlan_offload_set,
 272         .vlan_strip_queue_set = ixgbe_vlan_strip_queue_set,
 273         .rx_queue_start       = ixgbe_dev_rx_queue_start,
 274         .rx_queue_stop        = ixgbe_dev_rx_queue_stop,
 275         .tx_queue_start       = ixgbe_dev_tx_queue_start,
 276         .tx_queue_stop        = ixgbe_dev_tx_queue_stop,
 277         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 278         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 279         .rx_queue_count       = ixgbe_dev_rx_queue_count,
 280         .rx_descriptor_done   = ixgbe_dev_rx_descriptor_done,
 281         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 282         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 283         .dev_led_on           = ixgbe_dev_led_on,
 284         .dev_led_off          = ixgbe_dev_led_off,
 285         .flow_ctrl_set        = ixgbe_flow_ctrl_set,
 286         .priority_flow_ctrl_set = ixgbe_priority_flow_ctrl_set,
 287         .mac_addr_add         = ixgbe_add_rar,
 288         .mac_addr_remove      = ixgbe_remove_rar,
 289         .uc_hash_table_set    = ixgbe_uc_hash_table_set,
 290         .uc_all_hash_table_set  = ixgbe_uc_all_hash_table_set,
 291         .mirror_rule_set      = ixgbe_mirror_rule_set,
 292         .mirror_rule_reset    = ixgbe_mirror_rule_reset,
 293         .set_vf_rx_mode       = ixgbe_set_pool_rx_mode,
 294         .set_vf_rx            = ixgbe_set_pool_rx,
 295         .set_vf_tx            = ixgbe_set_pool_tx,
 296         .set_vf_vlan_filter   = ixgbe_set_pool_vlan_filter,
 297         .fdir_add_signature_filter    = ixgbe_fdir_add_signature_filter,
 298         .fdir_update_signature_filter = ixgbe_fdir_update_signature_filter,
 299         .fdir_remove_signature_filter = ixgbe_fdir_remove_signature_filter,
 300         .fdir_infos_get               = ixgbe_fdir_info_get,
 301         .fdir_add_perfect_filter      = ixgbe_fdir_add_perfect_filter,
 302         .fdir_update_perfect_filter   = ixgbe_fdir_update_perfect_filter,
 303         .fdir_remove_perfect_filter   = ixgbe_fdir_remove_perfect_filter,
 304         .fdir_set_masks               = ixgbe_fdir_set_masks,
 305         .reta_update          = ixgbe_dev_rss_reta_update,
 306         .reta_query           = ixgbe_dev_rss_reta_query,
 307 #ifdef RTE_NIC_BYPASS
 308         .bypass_init          = ixgbe_bypass_init,
 309         .bypass_state_set     = ixgbe_bypass_state_store,
 310         .bypass_state_show    = ixgbe_bypass_state_show,
 311         .bypass_event_set     = ixgbe_bypass_event_store,
 312         .bypass_event_show    = ixgbe_bypass_event_show,
 313         .bypass_wd_timeout_set  = ixgbe_bypass_wd_timeout_store,
 314         .bypass_wd_timeout_show = ixgbe_bypass_wd_timeout_show,
 315         .bypass_ver_show      = ixgbe_bypass_ver_show,
 316         .bypass_wd_reset      = ixgbe_bypass_wd_reset,
 317 #endif /* RTE_NIC_BYPASS */
 318         .rss_hash_update      = ixgbe_dev_rss_hash_update,
 319         .rss_hash_conf_get    = ixgbe_dev_rss_hash_conf_get,
 320 };
 321
 322 /*
 323  * dev_ops for virtual function, bare necessities for basic vf
 324  * operation have been implemented
 325  */
 326 static struct eth_dev_ops ixgbevf_eth_dev_ops = {
 327
 328         .dev_configure        = ixgbevf_dev_configure,
 329         .dev_start            = ixgbevf_dev_start,
 330         .dev_stop             = ixgbevf_dev_stop,
 331         .link_update          = ixgbe_dev_link_update,
 332         .stats_get            = ixgbevf_dev_stats_get,
 333         .stats_reset          = ixgbevf_dev_stats_reset,
 334         .dev_close            = ixgbevf_dev_close,
 335         .dev_infos_get        = ixgbe_dev_info_get,
 336         .vlan_filter_set      = ixgbevf_vlan_filter_set,
 337         .vlan_strip_queue_set = ixgbevf_vlan_strip_queue_set,
 338         .vlan_offload_set     = ixgbevf_vlan_offload_set,
 339         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 340         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 341         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 342         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 343         .mac_addr_add         = ixgbevf_add_mac_addr,
 344         .mac_addr_remove      = ixgbevf_remove_mac_addr,
 345 };
 346
 347 /**
 348  * Atomically reads the link status information from global
 349  * structure rte_eth_dev.
 350  *
 351  * @param dev
 352  *   - Pointer to the structure rte_eth_dev to read from.
 353  *   - Pointer to the buffer to be saved with the link status.
 354  *
 355  * @return
 356  *   - On success, zero.
 357  *   - On failure, negative value.
 358  */
 359 static inline int
 360 rte_ixgbe_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 361                                 struct rte_eth_link *link)
 362 {
 363         struct rte_eth_link *dst = link;
 364         struct rte_eth_link *src = &(dev->data->dev_link);
 365
 366         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 367                                         *(uint64_t *)src) == 0)
 368                 return -1;
 369
 370         return 0;
 371 }
 372
 373 /**
 374  * Atomically writes the link status information into global
 375  * structure rte_eth_dev.
 376  *
 377  * @param dev
 378  *   - Pointer to the structure rte_eth_dev to read from.
 379  *   - Pointer to the buffer to be saved with the link status.
 380  *
 381  * @return
 382  *   - On success, zero.
 383  *   - On failure, negative value.
 384  */
 385 static inline int
 386 rte_ixgbe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 387                                 struct rte_eth_link *link)
 388 {
 389         struct rte_eth_link *dst = &(dev->data->dev_link);
 390         struct rte_eth_link *src = link;
 391
 392         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 393                                         *(uint64_t *)src) == 0)
 394                 return -1;
 395
 396         return 0;
 397 }
 398
 399 /*
 400  * This function is the same as ixgbe_is_sfp() in ixgbe/ixgbe.h.
 401  */
 402 static inline int
 403 ixgbe_is_sfp(struct ixgbe_hw *hw)
 404 {
 405         switch (hw->phy.type) {
 406         case ixgbe_phy_sfp_avago:
 407         case ixgbe_phy_sfp_ftl:
 408         case ixgbe_phy_sfp_intel:
 409         case ixgbe_phy_sfp_unknown:
 410         case ixgbe_phy_sfp_passive_tyco:
 411         case ixgbe_phy_sfp_passive_unknown:
 412                 return 1;
 413         default:
 414                 return 0;
 415         }
 416 }
 417
 418 static inline int32_t
 419 ixgbe_pf_reset_hw(struct ixgbe_hw *hw)
 420 {
 421         uint32_t ctrl_ext;
 422         int32_t status;
 423
 424         status = ixgbe_reset_hw(hw);
 425
 426         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 427         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 428         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
 429         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 430         IXGBE_WRITE_FLUSH(hw);
 431
 432         return status;
 433 }
 434
 435 static inline void
 436 ixgbe_enable_intr(struct rte_eth_dev *dev)
 437 {
 438         struct ixgbe_interrupt *intr =
 439                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 440         struct ixgbe_hw *hw =
 441                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 442
 443         IXGBE_WRITE_REG(hw, IXGBE_EIMS, intr->mask);
 444         IXGBE_WRITE_FLUSH(hw);
 445 }
 446
 447 /*
 448  * This function is based on ixgbe_disable_intr() in ixgbe/ixgbe.h.
 449  */
 450 static void
 451 ixgbe_disable_intr(struct ixgbe_hw *hw)
 452 {
 453         PMD_INIT_FUNC_TRACE();
 454
 455         if (hw->mac.type == ixgbe_mac_82598EB) {
 456                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ~0);
 457         } else {
 458                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xFFFF0000);
 459                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), ~0);
 460                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), ~0);
 461         }
 462         IXGBE_WRITE_FLUSH(hw);
 463 }
 464
 465 /*
 466  * This function resets queue statistics mapping registers.
 467  * From Niantic datasheet, Initialization of Statistics section:
 468  * "...if software requires the queue counters, the RQSMR and TQSM registers
 469  * must be re-programmed following a device reset.
 470  */
 471 static void
 472 ixgbe_reset_qstat_mappings(struct ixgbe_hw *hw)
 473 {
 474         uint32_t i;
 475
 476         for(i = 0; i != IXGBE_NB_STAT_MAPPING_REGS; i++) {
 477                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), 0);
 478                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), 0);
 479         }
 480 }
 481
 482
 483 static int
 484 ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 485                                   uint16_t queue_id,
 486                                   uint8_t stat_idx,
 487                                   uint8_t is_rx)
 488 {
 489 #define QSM_REG_NB_BITS_PER_QMAP_FIELD 8
 490 #define NB_QMAP_FIELDS_PER_QSM_REG 4
 491 #define QMAP_FIELD_RESERVED_BITS_MASK 0x0f
 492
 493         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 494         struct ixgbe_stat_mapping_registers *stat_mappings =
 495                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(eth_dev->data->dev_private);
 496         uint32_t qsmr_mask = 0;
 497         uint32_t clearing_mask = QMAP_FIELD_RESERVED_BITS_MASK;
 498         uint32_t q_map;
 499         uint8_t n, offset;
 500
 501         if ((hw->mac.type != ixgbe_mac_82599EB) && (hw->mac.type != ixgbe_mac_X540))
 502                 return -ENOSYS;
 503
 504         PMD_INIT_LOG(INFO, "Setting port %d, %s queue_id %d to stat index %d\n",
 505                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx);
 506
 507         n = (uint8_t)(queue_id / NB_QMAP_FIELDS_PER_QSM_REG);
 508         if (n >= IXGBE_NB_STAT_MAPPING_REGS) {
 509                 PMD_INIT_LOG(ERR, "Nb of stat mapping registers exceeded\n");
 510                 return -EIO;
 511         }
 512         offset = (uint8_t)(queue_id % NB_QMAP_FIELDS_PER_QSM_REG);
 513
 514         /* Now clear any previous stat_idx set */
 515         clearing_mask <<= (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 516         if (!is_rx)
 517                 stat_mappings->tqsm[n] &= ~clearing_mask;
 518         else
 519                 stat_mappings->rqsmr[n] &= ~clearing_mask;
 520
 521         q_map = (uint32_t)stat_idx;
 522         q_map &= QMAP_FIELD_RESERVED_BITS_MASK;
 523         qsmr_mask = q_map << (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 524         if (!is_rx)
 525                 stat_mappings->tqsm[n] |= qsmr_mask;
 526         else
 527                 stat_mappings->rqsmr[n] |= qsmr_mask;
 528
 529         PMD_INIT_LOG(INFO, "Set port %d, %s queue_id %d to stat index %d\n"
 530                      "%s[%d] = 0x%08x\n",
 531                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx,
 532                      is_rx ? "RQSMR" : "TQSM",n, is_rx ? stat_mappings->rqsmr[n] : stat_mappings->tqsm[n]);
 533
 534         /* Now write the mapping in the appropriate register */
 535         if (is_rx) {
 536                 PMD_INIT_LOG(INFO, "Write 0x%x to RX IXGBE stat mapping reg:%d\n",
 537                              stat_mappings->rqsmr[n], n);
 538                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(n), stat_mappings->rqsmr[n]);
 539         }
 540         else {
 541                 PMD_INIT_LOG(INFO, "Write 0x%x to TX IXGBE stat mapping reg:%d\n",
 542                              stat_mappings->tqsm[n], n);
 543                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(n), stat_mappings->tqsm[n]);
 544         }
 545         return 0;
 546 }
 547
 548 static void
 549 ixgbe_restore_statistics_mapping(struct rte_eth_dev * dev)
 550 {
 551         struct ixgbe_stat_mapping_registers *stat_mappings =
 552                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(dev->data->dev_private);
 553         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 554         int i;
 555
 556         /* write whatever was in stat mapping table to the NIC */
 557         for (i = 0; i < IXGBE_NB_STAT_MAPPING_REGS; i++) {
 558                 /* rx */
 559                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), stat_mappings->rqsmr[i]);
 560
 561                 /* tx */
 562                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), stat_mappings->tqsm[i]);
 563         }
 564 }
 565
 566 static void
 567 ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config)
 568 {
 569         uint8_t i;
 570         struct ixgbe_dcb_tc_config *tc;
 571         uint8_t dcb_max_tc = IXGBE_DCB_MAX_TRAFFIC_CLASS;
 572
 573         dcb_config->num_tcs.pg_tcs = dcb_max_tc;
 574         dcb_config->num_tcs.pfc_tcs = dcb_max_tc;
 575         for (i = 0; i < dcb_max_tc; i++) {
 576                 tc = &dcb_config->tc_config[i];
 577                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_id = i;
 578                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
 579                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 580                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_id = i;
 581                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
 582                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 583                 tc->pfc = ixgbe_dcb_pfc_disabled;
 584         }
 585
 586         /* Initialize default user to priority mapping, UPx->TC0 */
 587         tc = &dcb_config->tc_config[0];
 588         tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
 589         tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
 590         for (i = 0; i< IXGBE_DCB_MAX_BW_GROUP; i++) {
 591                 dcb_config->bw_percentage[IXGBE_DCB_TX_CONFIG][i] = 100;
 592                 dcb_config->bw_percentage[IXGBE_DCB_RX_CONFIG][i] = 100;
 593         }
 594         dcb_config->rx_pba_cfg = ixgbe_dcb_pba_equal;
 595         dcb_config->pfc_mode_enable = false;
 596         dcb_config->vt_mode = true;
 597         dcb_config->round_robin_enable = false;
 598         /* support all DCB capabilities in 82599 */
 599         dcb_config->support.capabilities = 0xFF;
 600
 601         /*we only support 4 Tcs for X540*/
 602         if (hw->mac.type == ixgbe_mac_X540) {
 603                 dcb_config->num_tcs.pg_tcs = 4;
 604                 dcb_config->num_tcs.pfc_tcs = 4;
 605         }
 606 }
 607
 608 /*
 609  * Ensure that all locks are released before first NVM or PHY access
 610  */
 611 static void
 612 ixgbe_swfw_lock_reset(struct ixgbe_hw *hw)
 613 {
 614         uint16_t mask;
 615
 616         /*
 617          * Phy lock should not fail in this early stage. If this is the case,
 618          * it is due to an improper exit of the application.
 619          * So force the release of the faulty lock. Release of common lock
 620          * is done automatically by swfw_sync function.
 621          */
 622         mask = IXGBE_GSSR_PHY0_SM << hw->bus.func;
 623         if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) {
 624                    DEBUGOUT1("SWFW phy%d lock released", hw->bus.func);
 625         }
 626         ixgbe_release_swfw_semaphore(hw, mask);
 627
 628         /*
 629          * These ones are more tricky since they are common to all ports; but
 630          * swfw_sync retries last long enough (1s) to be almost sure that if
 631          * lock can not be taken it is due to an improper lock of the
 632          * semaphore.
 633          */
 634         mask = IXGBE_GSSR_EEP_SM | IXGBE_GSSR_MAC_CSR_SM | IXGBE_GSSR_SW_MNG_SM;
 635         if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) {
 636                    DEBUGOUT("SWFW common locks released");
 637         }
 638         ixgbe_release_swfw_semaphore(hw, mask);
 639 }
 640
 641 /*
 642  * This function is based on code in ixgbe_attach() in ixgbe/ixgbe.c.
 643  * It returns 0 on success.
 644  */
 645 static int
 646 eth_ixgbe_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 647                      struct rte_eth_dev *eth_dev)
 648 {
 649         struct rte_pci_device *pci_dev;
 650         struct ixgbe_hw *hw =
 651                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 652         struct ixgbe_vfta * shadow_vfta =
 653                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 654         struct ixgbe_hwstrip *hwstrip =
 655                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 656         struct ixgbe_dcb_config *dcb_config =
 657                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private);
 658         uint32_t ctrl_ext;
 659         uint16_t csum;
 660         int diag, i;
 661
 662         PMD_INIT_FUNC_TRACE();
 663
 664         eth_dev->dev_ops = &ixgbe_eth_dev_ops;
 665         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
 666         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
 667
 668         /* for secondary processes, we don't initialise any further as primary
 669          * has already done this work. Only check we don't need a different
 670          * RX function */
 671         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 672                 if (eth_dev->data->scattered_rx)
 673                         eth_dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
 674                 return 0;
 675         }
 676         pci_dev = eth_dev->pci_dev;
 677
 678         /* Vendor and Device ID need to be set before init of shared code */
 679         hw->device_id = pci_dev->id.device_id;
 680         hw->vendor_id = pci_dev->id.vendor_id;
 681         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 682 #ifdef RTE_LIBRTE_IXGBE_ALLOW_UNSUPPORTED_SFP
 683         hw->allow_unsupported_sfp = 1;
 684 #endif
 685
 686         /* Initialize the shared code */
 687 #ifdef RTE_NIC_BYPASS
 688         diag = ixgbe_bypass_init_shared_code(hw);
 689 #else
 690         diag = ixgbe_init_shared_code(hw);
 691 #endif /* RTE_NIC_BYPASS */
 692
 693         if (diag != IXGBE_SUCCESS) {
 694                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
 695                 return -EIO;
 696         }
 697
 698         /* pick up the PCI bus settings for reporting later */
 699         ixgbe_get_bus_info(hw);
 700
 701         /* Unlock any pending hardware semaphore */
 702         ixgbe_swfw_lock_reset(hw);
 703
 704         /* Initialize DCB configuration*/
 705         memset(dcb_config, 0, sizeof(struct ixgbe_dcb_config));
 706         ixgbe_dcb_init(hw,dcb_config);
 707         /* Get Hardware Flow Control setting */
 708         hw->fc.requested_mode = ixgbe_fc_full;
 709         hw->fc.current_mode = ixgbe_fc_full;
 710         hw->fc.pause_time = IXGBE_FC_PAUSE;
 711         for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
 712                 hw->fc.low_water[i] = IXGBE_FC_LO;
 713                 hw->fc.high_water[i] = IXGBE_FC_HI;
 714         }
 715         hw->fc.send_xon = 1;
 716
 717         /* Make sure we have a good EEPROM before we read from it */
 718         diag = ixgbe_validate_eeprom_checksum(hw, &csum);
 719         if (diag != IXGBE_SUCCESS) {
 720                 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag);
 721                 return -EIO;
 722         }
 723
 724 #ifdef RTE_NIC_BYPASS
 725         diag = ixgbe_bypass_init_hw(hw);
 726 #else
 727         diag = ixgbe_init_hw(hw);
 728 #endif /* RTE_NIC_BYPASS */
 729
 730         /*
 731          * Devices with copper phys will fail to initialise if ixgbe_init_hw()
 732          * is called too soon after the kernel driver unbinding/binding occurs.
 733          * The failure occurs in ixgbe_identify_phy_generic() for all devices,
 734          * but for non-copper devies, ixgbe_identify_sfp_module_generic() is
 735          * also called. See ixgbe_identify_phy_82599(). The reason for the
 736          * failure is not known, and only occuts when virtualisation features
 737          * are disabled in the bios. A delay of 100ms  was found to be enough by
 738          * trial-and-error, and is doubled to be safe.
 739          */
 740         if (diag && (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper)) {
 741                 rte_delay_ms(200);
 742                 diag = ixgbe_init_hw(hw);
 743         }
 744
 745         if (diag == IXGBE_ERR_EEPROM_VERSION) {
 746                 PMD_INIT_LOG(ERR, "This device is a pre-production adapter/"
 747                     "LOM.  Please be aware there may be issues associated "
 748                     "with your hardware.\n If you are experiencing problems "
 749                     "please contact your Intel or hardware representative "
 750                     "who provided you with this hardware.\n");
 751         } else if (diag == IXGBE_ERR_SFP_NOT_SUPPORTED)
 752                 PMD_INIT_LOG(ERR, "Unsupported SFP+ Module\n");
 753         if (diag) {
 754                 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag);
 755                 return -EIO;
 756         }
 757
 758         /* disable interrupt */
 759         ixgbe_disable_intr(hw);
 760
 761         /* reset mappings for queue statistics hw counters*/
 762         ixgbe_reset_qstat_mappings(hw);
 763
 764         /* Allocate memory for storing MAC addresses */
 765         eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 766                         hw->mac.num_rar_entries, 0);
 767         if (eth_dev->data->mac_addrs == NULL) {
 768                 PMD_INIT_LOG(ERR,
 769                         "Failed to allocate %u bytes needed to store "
 770                         "MAC addresses",
 771                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 772                 return -ENOMEM;
 773         }
 774         /* Copy the permanent MAC address */
 775         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 776                         &eth_dev->data->mac_addrs[0]);
 777
 778         /* Allocate memory for storing hash filter MAC addresses */
 779         eth_dev->data->hash_mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 780                         IXGBE_VMDQ_NUM_UC_MAC, 0);
 781         if (eth_dev->data->hash_mac_addrs == NULL) {
 782                 PMD_INIT_LOG(ERR,
 783                         "Failed to allocate %d bytes needed to store MAC addresses",
 784                         ETHER_ADDR_LEN * IXGBE_VMDQ_NUM_UC_MAC);
 785                 return -ENOMEM;
 786         }
 787
 788         /* initialize the vfta */
 789         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 790
 791         /* initialize the hw strip bitmap*/
 792         memset(hwstrip, 0, sizeof(*hwstrip));
 793
 794         /* initialize PF if max_vfs not zero */
 795         ixgbe_pf_host_init(eth_dev);
 796
 797         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 798         /* let hardware know driver is loaded */
 799         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 800         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 801         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
 802         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 803         IXGBE_WRITE_FLUSH(hw);
 804
 805         if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
 806                 PMD_INIT_LOG(DEBUG,
 807                              "MAC: %d, PHY: %d, SFP+: %d<n",
 808                              (int) hw->mac.type, (int) hw->phy.type,
 809                              (int) hw->phy.sfp_type);
 810         else
 811                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d\n",
 812                              (int) hw->mac.type, (int) hw->phy.type);
 813
 814         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
 815                         eth_dev->data->port_id, pci_dev->id.vendor_id,
 816                         pci_dev->id.device_id);
 817
 818         rte_intr_callback_register(&(pci_dev->intr_handle),
 819                 ixgbe_dev_interrupt_handler, (void *)eth_dev);
 820
 821         /* enable uio intr after callback register */
 822         rte_intr_enable(&(pci_dev->intr_handle));
 823
 824         /* enable support intr */
 825         ixgbe_enable_intr(eth_dev);
 826
 827         return 0;
 828 }
 829
 830
 831 /*
 832  * Negotiate mailbox API version with the PF.
 833  * After reset API version is always set to the basic one (ixgbe_mbox_api_10).
 834  * Then we try to negotiate starting with the most recent one.
 835  * If all negotiation attempts fail, then we will proceed with
 836  * the default one (ixgbe_mbox_api_10).
 837  */
 838 static void
 839 ixgbevf_negotiate_api(struct ixgbe_hw *hw)
 840 {
 841         int32_t i;
 842
 843         /* start with highest supported, proceed down */
 844         static const enum ixgbe_pfvf_api_rev sup_ver[] = {
 845                 ixgbe_mbox_api_11,
 846                 ixgbe_mbox_api_10,
 847         };
 848
 849         for (i = 0;
 850                         i != RTE_DIM(sup_ver) &&
 851                         ixgbevf_negotiate_api_version(hw, sup_ver[i]) != 0;
 852                         i++)
 853                 ;
 854 }
 855
 856 static void
 857 generate_random_mac_addr(struct ether_addr *mac_addr)
 858 {
 859         uint64_t random;
 860
 861         /* Set Organizationally Unique Identifier (OUI) prefix. */
 862         mac_addr->addr_bytes[0] = 0x00;
 863         mac_addr->addr_bytes[1] = 0x09;
 864         mac_addr->addr_bytes[2] = 0xC0;
 865         /* Force indication of locally assigned MAC address. */
 866         mac_addr->addr_bytes[0] |= ETHER_LOCAL_ADMIN_ADDR;
 867         /* Generate the last 3 bytes of the MAC address with a random number. */
 868         random = rte_rand();
 869         memcpy(&mac_addr->addr_bytes[3], &random, 3);
 870 }
 871
 872 /*
 873  * Virtual Function device init
 874  */
 875 static int
 876 eth_ixgbevf_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 877                      struct rte_eth_dev *eth_dev)
 878 {
 879         int diag;
 880         uint32_t tc, tcs;
 881         struct rte_pci_device *pci_dev;
 882         struct ixgbe_hw *hw =
 883                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 884         struct ixgbe_vfta * shadow_vfta =
 885                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 886         struct ixgbe_hwstrip *hwstrip =
 887                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 888         struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr;
 889
 890         PMD_INIT_LOG(DEBUG, "eth_ixgbevf_dev_init");
 891
 892         eth_dev->dev_ops = &ixgbevf_eth_dev_ops;
 893         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
 894         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
 895
 896         /* for secondary processes, we don't initialise any further as primary
 897          * has already done this work. Only check we don't need a different
 898          * RX function */
 899         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 900                 if (eth_dev->data->scattered_rx)
 901                         eth_dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
 902                 return 0;
 903         }
 904
 905         pci_dev = eth_dev->pci_dev;
 906
 907         hw->device_id = pci_dev->id.device_id;
 908         hw->vendor_id = pci_dev->id.vendor_id;
 909         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 910
 911         /* initialize the vfta */
 912         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 913
 914         /* initialize the hw strip bitmap*/
 915         memset(hwstrip, 0, sizeof(*hwstrip));
 916
 917         /* Initialize the shared code */
 918         diag = ixgbe_init_shared_code(hw);
 919         if (diag != IXGBE_SUCCESS) {
 920                 PMD_INIT_LOG(ERR, "Shared code init failed for ixgbevf: %d", diag);
 921                 return -EIO;
 922         }
 923
 924         /* init_mailbox_params */
 925         hw->mbx.ops.init_params(hw);
 926
 927         /* Disable the interrupts for VF */
 928         ixgbevf_intr_disable(hw);
 929
 930         hw->mac.num_rar_entries = 128; /* The MAX of the underlying PF */
 931         diag = hw->mac.ops.reset_hw(hw);
 932
 933         /*
 934          * The VF reset operation returns the IXGBE_ERR_INVALID_MAC_ADDR when
 935          * the underlying PF driver has not assigned a MAC address to the VF.
 936          * In this case, assign a random MAC address.
 937          */
 938         if ((diag != IXGBE_SUCCESS) && (diag != IXGBE_ERR_INVALID_MAC_ADDR)) {
 939                 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
 940                 return (diag);
 941         }
 942
 943         /* negotiate mailbox API version to use with the PF. */
 944         ixgbevf_negotiate_api(hw);
 945
 946         /* Get Rx/Tx queue count via mailbox, which is ready after reset_hw */
 947         ixgbevf_get_queues(hw, &tcs, &tc);
 948
 949         /* Allocate memory for storing MAC addresses */
 950         eth_dev->data->mac_addrs = rte_zmalloc("ixgbevf", ETHER_ADDR_LEN *
 951                         hw->mac.num_rar_entries, 0);
 952         if (eth_dev->data->mac_addrs == NULL) {
 953                 PMD_INIT_LOG(ERR,
 954                         "Failed to allocate %u bytes needed to store "
 955                         "MAC addresses",
 956                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 957                 return -ENOMEM;
 958         }
 959
 960         /* Generate a random MAC address, if none was assigned by PF. */
 961         if (is_zero_ether_addr(perm_addr)) {
 962                 generate_random_mac_addr(perm_addr);
 963                 diag = ixgbe_set_rar_vf(hw, 1, perm_addr->addr_bytes, 0, 1);
 964                 if (diag) {
 965                         rte_free(eth_dev->data->mac_addrs);
 966                         eth_dev->data->mac_addrs = NULL;
 967                         return diag;
 968                 }
 969                 RTE_LOG(INFO, PMD,
 970                         "\tVF MAC address not assigned by Host PF\n"
 971                         "\tAssign randomly generated MAC address "
 972                         "%02x:%02x:%02x:%02x:%02x:%02x\n",
 973                         perm_addr->addr_bytes[0],
 974                         perm_addr->addr_bytes[1],
 975                         perm_addr->addr_bytes[2],
 976                         perm_addr->addr_bytes[3],
 977                         perm_addr->addr_bytes[4],
 978                         perm_addr->addr_bytes[5]);
 979         }
 980
 981         /* Copy the permanent MAC address */
 982         ether_addr_copy(perm_addr, &eth_dev->data->mac_addrs[0]);
 983
 984         /* reset the hardware with the new settings */
 985         diag = hw->mac.ops.start_hw(hw);
 986         switch (diag) {
 987                 case  0:
 988                         break;
 989
 990                 default:
 991                         PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
 992                         return (-EIO);
 993         }
 994
 995         PMD_INIT_LOG(DEBUG, "\nport %d vendorID=0x%x deviceID=0x%x mac.type=%s\n",
 996                          eth_dev->data->port_id, pci_dev->id.vendor_id, pci_dev->id.device_id,
 997                          "ixgbe_mac_82599_vf");
 998
 999         return 0;
1000 }
1001
1002 static struct eth_driver rte_ixgbe_pmd = {
1003         {
1004                 .name = "rte_ixgbe_pmd",
1005                 .id_table = pci_id_ixgbe_map,
1006                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
1007         },
1008         .eth_dev_init = eth_ixgbe_dev_init,
1009         .dev_private_size = sizeof(struct ixgbe_adapter),
1010 };
1011
1012 /*
1013  * virtual function driver struct
1014  */
1015 static struct eth_driver rte_ixgbevf_pmd = {
1016         {
1017                 .name = "rte_ixgbevf_pmd",
1018                 .id_table = pci_id_ixgbevf_map,
1019                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
1020         },
1021         .eth_dev_init = eth_ixgbevf_dev_init,
1022         .dev_private_size = sizeof(struct ixgbe_adapter),
1023 };
1024
1025 /*
1026  * Driver initialization routine.
1027  * Invoked once at EAL init time.
1028  * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
1029  */
1030 static int
1031 rte_ixgbe_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
1032 {
1033         PMD_INIT_FUNC_TRACE();
1034
1035         rte_eth_driver_register(&rte_ixgbe_pmd);
1036         return 0;
1037 }
1038
1039 /*
1040  * VF Driver initialization routine.
1041  * Invoked one at EAL init time.
1042  * Register itself as the [Virtual Poll Mode] Driver of PCI niantic devices.
1043  */
1044 static int
1045 rte_ixgbevf_pmd_init(const char *name __rte_unused, const char *param __rte_unused)
1046 {
1047         DEBUGFUNC("rte_ixgbevf_pmd_init");
1048
1049         rte_eth_driver_register(&rte_ixgbevf_pmd);
1050         return (0);
1051 }
1052
1053 static int
1054 ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1055 {
1056         struct ixgbe_hw *hw =
1057                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1058         struct ixgbe_vfta * shadow_vfta =
1059                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1060         uint32_t vfta;
1061         uint32_t vid_idx;
1062         uint32_t vid_bit;
1063
1064         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
1065         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
1066         vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(vid_idx));
1067         if (on)
1068                 vfta |= vid_bit;
1069         else
1070                 vfta &= ~vid_bit;
1071         IXGBE_WRITE_REG(hw, IXGBE_VFTA(vid_idx), vfta);
1072
1073         /* update local VFTA copy */
1074         shadow_vfta->vfta[vid_idx] = vfta;
1075
1076         return 0;
1077 }
1078
1079 static void
1080 ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
1081 {
1082         if (on)
1083                 ixgbe_vlan_hw_strip_enable(dev, queue);
1084         else
1085                 ixgbe_vlan_hw_strip_disable(dev, queue);
1086 }
1087
1088 static void
1089 ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
1090 {
1091         struct ixgbe_hw *hw =
1092                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1093
1094         /* Only the high 16-bits is valid */
1095         IXGBE_WRITE_REG(hw, IXGBE_EXVET, tpid << 16);
1096 }
1097
1098 void
1099 ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev)
1100 {
1101         struct ixgbe_hw *hw =
1102                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1103         uint32_t vlnctrl;
1104
1105         PMD_INIT_FUNC_TRACE();
1106
1107         /* Filter Table Disable */
1108         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1109         vlnctrl &= ~IXGBE_VLNCTRL_VFE;
1110
1111         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
1112 }
1113
1114 void
1115 ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1116 {
1117         struct ixgbe_hw *hw =
1118                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1119         struct ixgbe_vfta * shadow_vfta =
1120                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1121         uint32_t vlnctrl;
1122         uint16_t i;
1123
1124         PMD_INIT_FUNC_TRACE();
1125
1126         /* Filter Table Enable */
1127         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1128         vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
1129         vlnctrl |= IXGBE_VLNCTRL_VFE;
1130
1131         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
1132
1133         /* write whatever is in local vfta copy */
1134         for (i = 0; i < IXGBE_VFTA_SIZE; i++)
1135                 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), shadow_vfta->vfta[i]);
1136 }
1137
1138 static void
1139 ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on)
1140 {
1141         struct ixgbe_hwstrip *hwstrip =
1142                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(dev->data->dev_private);
1143
1144         if(queue >= IXGBE_MAX_RX_QUEUE_NUM)
1145                 return;
1146
1147         if (on)
1148                 IXGBE_SET_HWSTRIP(hwstrip, queue);
1149         else
1150                 IXGBE_CLEAR_HWSTRIP(hwstrip, queue);
1151 }
1152
1153 static void
1154 ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue)
1155 {
1156         struct ixgbe_hw *hw =
1157                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1158         uint32_t ctrl;
1159
1160         PMD_INIT_FUNC_TRACE();
1161
1162         if (hw->mac.type == ixgbe_mac_82598EB) {
1163                 /* No queue level support */
1164                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
1165                 return;
1166         }
1167         else {
1168                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1169                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1170                 ctrl &= ~IXGBE_RXDCTL_VME;
1171                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
1172         }
1173         /* record those setting for HW strip per queue */
1174         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 0);
1175 }
1176
1177 static void
1178 ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue)
1179 {
1180         struct ixgbe_hw *hw =
1181                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1182         uint32_t ctrl;
1183
1184         PMD_INIT_FUNC_TRACE();
1185
1186         if (hw->mac.type == ixgbe_mac_82598EB) {
1187                 /* No queue level supported */
1188                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
1189                 return;
1190         }
1191         else {
1192                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1193                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1194                 ctrl |= IXGBE_RXDCTL_VME;
1195                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
1196         }
1197         /* record those setting for HW strip per queue */
1198         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 1);
1199 }
1200
1201 void
1202 ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev)
1203 {
1204         struct ixgbe_hw *hw =
1205                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1206         uint32_t ctrl;
1207         uint16_t i;
1208
1209         PMD_INIT_FUNC_TRACE();
1210
1211         if (hw->mac.type == ixgbe_mac_82598EB) {
1212                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1213                 ctrl &= ~IXGBE_VLNCTRL_VME;
1214                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1215         }
1216         else {
1217                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1218                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1219                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1220                         ctrl &= ~IXGBE_RXDCTL_VME;
1221                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1222
1223                         /* record those setting for HW strip per queue */
1224                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 0);
1225                 }
1226         }
1227 }
1228
1229 void
1230 ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev)
1231 {
1232         struct ixgbe_hw *hw =
1233                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1234         uint32_t ctrl;
1235         uint16_t i;
1236
1237         PMD_INIT_FUNC_TRACE();
1238
1239         if (hw->mac.type == ixgbe_mac_82598EB) {
1240                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1241                 ctrl |= IXGBE_VLNCTRL_VME;
1242                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1243         }
1244         else {
1245                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1246                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1247                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1248                         ctrl |= IXGBE_RXDCTL_VME;
1249                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1250
1251                         /* record those setting for HW strip per queue */
1252                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 1);
1253                 }
1254         }
1255 }
1256
1257 static void
1258 ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1259 {
1260         struct ixgbe_hw *hw =
1261                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1262         uint32_t ctrl;
1263
1264         PMD_INIT_FUNC_TRACE();
1265
1266         /* DMATXCTRL: Geric Double VLAN Disable */
1267         ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1268         ctrl &= ~IXGBE_DMATXCTL_GDV;
1269         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1270
1271         /* CTRL_EXT: Global Double VLAN Disable */
1272         ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1273         ctrl &= ~IXGBE_EXTENDED_VLAN;
1274         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1275
1276 }
1277
1278 static void
1279 ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1280 {
1281         struct ixgbe_hw *hw =
1282                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1283         uint32_t ctrl;
1284
1285         PMD_INIT_FUNC_TRACE();
1286
1287         /* DMATXCTRL: Geric Double VLAN Enable */
1288         ctrl  = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1289         ctrl |= IXGBE_DMATXCTL_GDV;
1290         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1291
1292         /* CTRL_EXT: Global Double VLAN Enable */
1293         ctrl  = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1294         ctrl |= IXGBE_EXTENDED_VLAN;
1295         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1296
1297         /*
1298          * VET EXT field in the EXVET register = 0x8100 by default
1299          * So no need to change. Same to VT field of DMATXCTL register
1300          */
1301 }
1302
1303 static void
1304 ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1305 {
1306         if(mask & ETH_VLAN_STRIP_MASK){
1307                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1308                         ixgbe_vlan_hw_strip_enable_all(dev);
1309                 else
1310                         ixgbe_vlan_hw_strip_disable_all(dev);
1311         }
1312
1313         if(mask & ETH_VLAN_FILTER_MASK){
1314                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1315                         ixgbe_vlan_hw_filter_enable(dev);
1316                 else
1317                         ixgbe_vlan_hw_filter_disable(dev);
1318         }
1319
1320         if(mask & ETH_VLAN_EXTEND_MASK){
1321                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1322                         ixgbe_vlan_hw_extend_enable(dev);
1323                 else
1324                         ixgbe_vlan_hw_extend_disable(dev);
1325         }
1326 }
1327
1328 static void
1329 ixgbe_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1330 {
1331         struct ixgbe_hw *hw =
1332                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1333         /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
1334         uint32_t vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1335         vlanctrl |= IXGBE_VLNCTRL_VFE ; /* enable vlan filters */
1336         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
1337 }
1338
1339 static int
1340 ixgbe_dev_configure(struct rte_eth_dev *dev)
1341 {
1342         struct ixgbe_interrupt *intr =
1343                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1344
1345         PMD_INIT_FUNC_TRACE();
1346
1347         /* set flag to update link status after init */
1348         intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1349
1350         return 0;
1351 }
1352
1353 /*
1354  * Configure device link speed and setup link.
1355  * It returns 0 on success.
1356  */
1357 static int
1358 ixgbe_dev_start(struct rte_eth_dev *dev)
1359 {
1360         struct ixgbe_hw *hw =
1361                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1362         int err, link_up = 0, negotiate = 0;
1363         uint32_t speed = 0;
1364         int mask = 0;
1365         int status;
1366
1367         PMD_INIT_FUNC_TRACE();
1368
1369         /* IXGBE devices don't support half duplex */
1370         if ((dev->data->dev_conf.link_duplex != ETH_LINK_AUTONEG_DUPLEX) &&
1371                         (dev->data->dev_conf.link_duplex != ETH_LINK_FULL_DUPLEX)) {
1372                 PMD_INIT_LOG(ERR, "Invalid link_duplex (%hu) for port %hhu\n",
1373                                 dev->data->dev_conf.link_duplex,
1374                                 dev->data->port_id);
1375                 return -EINVAL;
1376         }
1377
1378         /* stop adapter */
1379         hw->adapter_stopped = FALSE;
1380         ixgbe_stop_adapter(hw);
1381
1382         /* reinitialize adapter
1383          * this calls reset and start */
1384         status = ixgbe_pf_reset_hw(hw);
1385         if (status != 0)
1386                 return -1;
1387         hw->mac.ops.start_hw(hw);
1388
1389         /* configure PF module if SRIOV enabled */
1390         ixgbe_pf_host_configure(dev);
1391
1392         /* initialize transmission unit */
1393         ixgbe_dev_tx_init(dev);
1394
1395         /* This can fail when allocating mbufs for descriptor rings */
1396         err = ixgbe_dev_rx_init(dev);
1397         if (err) {
1398                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware\n");
1399                 goto error;
1400         }
1401
1402         ixgbe_dev_rxtx_start(dev);
1403
1404         if (ixgbe_is_sfp(hw) && hw->phy.multispeed_fiber) {
1405                 err = hw->mac.ops.setup_sfp(hw);
1406                 if (err)
1407                         goto error;
1408         }
1409
1410         /* Turn on the laser */
1411         ixgbe_enable_tx_laser(hw);
1412
1413         /* Skip link setup if loopback mode is enabled for 82599. */
1414         if (hw->mac.type == ixgbe_mac_82599EB &&
1415                         dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_82599_TX_RX)
1416                 goto skip_link_setup;
1417
1418         err = ixgbe_check_link(hw, &speed, &link_up, 0);
1419         if (err)
1420                 goto error;
1421         err = ixgbe_get_link_capabilities(hw, &speed, &negotiate);
1422         if (err)
1423                 goto error;
1424
1425         switch(dev->data->dev_conf.link_speed) {
1426         case ETH_LINK_SPEED_AUTONEG:
1427                 speed = (hw->mac.type != ixgbe_mac_82598EB) ?
1428                                 IXGBE_LINK_SPEED_82599_AUTONEG :
1429                                 IXGBE_LINK_SPEED_82598_AUTONEG;
1430                 break;
1431         case ETH_LINK_SPEED_100:
1432                 /*
1433                  * Invalid for 82598 but error will be detected by
1434                  * ixgbe_setup_link()
1435                  */
1436                 speed = IXGBE_LINK_SPEED_100_FULL;
1437                 break;
1438         case ETH_LINK_SPEED_1000:
1439                 speed = IXGBE_LINK_SPEED_1GB_FULL;
1440                 break;
1441         case ETH_LINK_SPEED_10000:
1442                 speed = IXGBE_LINK_SPEED_10GB_FULL;
1443                 break;
1444         default:
1445                 PMD_INIT_LOG(ERR, "Invalid link_speed (%hu) for port %hhu\n",
1446                                 dev->data->dev_conf.link_speed,
1447                                 dev->data->port_id);
1448                 goto error;
1449         }
1450
1451         err = ixgbe_setup_link(hw, speed, negotiate, link_up);
1452         if (err)
1453                 goto error;
1454
1455 skip_link_setup:
1456
1457         /* check if lsc interrupt is enabled */
1458         if (dev->data->dev_conf.intr_conf.lsc != 0)
1459                 ixgbe_dev_lsc_interrupt_setup(dev);
1460
1461         /* resume enabled intr since hw reset */
1462         ixgbe_enable_intr(dev);
1463
1464         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
1465                 ETH_VLAN_EXTEND_MASK;
1466         ixgbe_vlan_offload_set(dev, mask);
1467
1468         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
1469                 /* Enable vlan filtering for VMDq */
1470                 ixgbe_vmdq_vlan_hw_filter_enable(dev);
1471         }
1472
1473         /* Configure DCB hw */
1474         ixgbe_configure_dcb(dev);
1475
1476         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) {
1477                 err = ixgbe_fdir_configure(dev);
1478                 if (err)
1479                         goto error;
1480         }
1481
1482         ixgbe_restore_statistics_mapping(dev);
1483
1484         return (0);
1485
1486 error:
1487         PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err);
1488         ixgbe_dev_clear_queues(dev);
1489         return -EIO;
1490 }
1491
1492 /*
1493  * Stop device: disable rx and tx functions to allow for reconfiguring.
1494  */
1495 static void
1496 ixgbe_dev_stop(struct rte_eth_dev *dev)
1497 {
1498         struct rte_eth_link link;
1499         struct ixgbe_hw *hw =
1500                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1501         struct ixgbe_vf_info *vfinfo =
1502                 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private);
1503         int vf;
1504
1505         PMD_INIT_FUNC_TRACE();
1506
1507         /* disable interrupts */
1508         ixgbe_disable_intr(hw);
1509
1510         /* reset the NIC */
1511         ixgbe_pf_reset_hw(hw);
1512         hw->adapter_stopped = FALSE;
1513
1514         /* stop adapter */
1515         ixgbe_stop_adapter(hw);
1516
1517         for (vf = 0; vfinfo != NULL &&
1518                      vf < dev->pci_dev->max_vfs; vf++)
1519                 vfinfo[vf].clear_to_send = false;
1520
1521         /* Turn off the laser */
1522         ixgbe_disable_tx_laser(hw);
1523
1524         ixgbe_dev_clear_queues(dev);
1525
1526         /* Clear recorded link status */
1527         memset(&link, 0, sizeof(link));
1528         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1529 }
1530
1531 /*
1532  * Set device link up: enable tx laser.
1533  */
1534 static int
1535 ixgbe_dev_set_link_up(struct rte_eth_dev *dev)
1536 {
1537         struct ixgbe_hw *hw =
1538                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1539         if (hw->mac.type == ixgbe_mac_82599EB) {
1540 #ifdef RTE_NIC_BYPASS
1541                 if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) {
1542                         /* Not suported in bypass mode */
1543                         PMD_INIT_LOG(ERR,
1544                                 "\nSet link up is not supported "
1545                                 "by device id 0x%x\n",
1546                                 hw->device_id);
1547                         return -ENOTSUP;
1548                 }
1549 #endif
1550                 /* Turn on the laser */
1551                 ixgbe_enable_tx_laser(hw);
1552                 return 0;
1553         }
1554
1555         PMD_INIT_LOG(ERR, "\nSet link up is not supported by device id 0x%x\n",
1556                 hw->device_id);
1557         return -ENOTSUP;
1558 }
1559
1560 /*
1561  * Set device link down: disable tx laser.
1562  */
1563 static int
1564 ixgbe_dev_set_link_down(struct rte_eth_dev *dev)
1565 {
1566         struct ixgbe_hw *hw =
1567                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1568         if (hw->mac.type == ixgbe_mac_82599EB) {
1569 #ifdef RTE_NIC_BYPASS
1570                 if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) {
1571                         /* Not suported in bypass mode */
1572                         PMD_INIT_LOG(ERR,
1573                                 "\nSet link down is not supported "
1574                                 "by device id 0x%x\n",
1575                                  hw->device_id);
1576                         return -ENOTSUP;
1577                 }
1578 #endif
1579                 /* Turn off the laser */
1580                 ixgbe_disable_tx_laser(hw);
1581                 return 0;
1582         }
1583
1584         PMD_INIT_LOG(ERR,
1585                 "\nSet link down is not supported by device id 0x%x\n",
1586                  hw->device_id);
1587         return -ENOTSUP;
1588 }
1589
1590 /*
1591  * Reest and stop device.
1592  */
1593 static void
1594 ixgbe_dev_close(struct rte_eth_dev *dev)
1595 {
1596         struct ixgbe_hw *hw =
1597                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1598
1599         PMD_INIT_FUNC_TRACE();
1600
1601         ixgbe_pf_reset_hw(hw);
1602
1603         ixgbe_dev_stop(dev);
1604         hw->adapter_stopped = 1;
1605
1606         ixgbe_disable_pcie_master(hw);
1607
1608         /* reprogram the RAR[0] in case user changed it. */
1609         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
1610 }
1611
1612 /*
1613  * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c
1614  */
1615 static void
1616 ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1617 {
1618         struct ixgbe_hw *hw =
1619                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1620         struct ixgbe_hw_stats *hw_stats =
1621                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1622         uint32_t bprc, lxon, lxoff, total;
1623         uint64_t total_missed_rx, total_qbrc, total_qprc;
1624         unsigned i;
1625
1626         total_missed_rx = 0;
1627         total_qbrc = 0;
1628         total_qprc = 0;
1629
1630         hw_stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
1631         hw_stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
1632         hw_stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
1633         hw_stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
1634
1635         for (i = 0; i < 8; i++) {
1636                 uint32_t mp;
1637                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
1638                 /* global total per queue */
1639                 hw_stats->mpc[i] += mp;
1640                 /* Running comprehensive total for stats display */
1641                 total_missed_rx += hw_stats->mpc[i];
1642                 if (hw->mac.type == ixgbe_mac_82598EB)
1643                         hw_stats->rnbc[i] +=
1644                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
1645                 hw_stats->pxontxc[i] +=
1646                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
1647                 hw_stats->pxonrxc[i] +=
1648                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
1649                 hw_stats->pxofftxc[i] +=
1650                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
1651                 hw_stats->pxoffrxc[i] +=
1652                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
1653                 hw_stats->pxon2offc[i] +=
1654                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
1655         }
1656         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1657                 hw_stats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
1658                 hw_stats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
1659                 hw_stats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
1660                 hw_stats->qbrc[i] +=
1661                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32);
1662                 hw_stats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
1663                 hw_stats->qbtc[i] +=
1664                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32);
1665                 hw_stats->qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
1666
1667                 total_qprc += hw_stats->qprc[i];
1668                 total_qbrc += hw_stats->qbrc[i];
1669         }
1670         hw_stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
1671         hw_stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
1672         hw_stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
1673
1674         /* Note that gprc counts missed packets */
1675         hw_stats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
1676
1677         if (hw->mac.type != ixgbe_mac_82598EB) {
1678                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
1679                 hw_stats->gorc += ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
1680                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
1681                 hw_stats->gotc += ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
1682                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
1683                 hw_stats->tor += ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
1684                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
1685                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
1686         } else {
1687                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
1688                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
1689                 /* 82598 only has a counter in the high register */
1690                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
1691                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
1692                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
1693         }
1694
1695         /*
1696          * Workaround: mprc hardware is incorrectly counting
1697          * broadcasts, so for now we subtract those.
1698          */
1699         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
1700         hw_stats->bprc += bprc;
1701         hw_stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
1702         if (hw->mac.type == ixgbe_mac_82598EB)
1703                 hw_stats->mprc -= bprc;
1704
1705         hw_stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
1706         hw_stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
1707         hw_stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
1708         hw_stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
1709         hw_stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
1710         hw_stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
1711
1712         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
1713         hw_stats->lxontxc += lxon;
1714         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
1715         hw_stats->lxofftxc += lxoff;
1716         total = lxon + lxoff;
1717
1718         hw_stats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
1719         hw_stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
1720         hw_stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
1721         hw_stats->gptc -= total;
1722         hw_stats->mptc -= total;
1723         hw_stats->ptc64 -= total;
1724         hw_stats->gotc -= total * ETHER_MIN_LEN;
1725
1726         hw_stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
1727         hw_stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
1728         hw_stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
1729         hw_stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
1730         hw_stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
1731         hw_stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
1732         hw_stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
1733         hw_stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
1734         hw_stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
1735         hw_stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
1736         hw_stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
1737         hw_stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
1738         hw_stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
1739         hw_stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
1740         hw_stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
1741         hw_stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);
1742         hw_stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
1743         hw_stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
1744         /* Only read FCOE on 82599 */
1745         if (hw->mac.type != ixgbe_mac_82598EB) {
1746                 hw_stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
1747                 hw_stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
1748                 hw_stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
1749                 hw_stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
1750                 hw_stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
1751         }
1752
1753         if (stats == NULL)
1754                 return;
1755
1756         /* Fill out the rte_eth_stats statistics structure */
1757         stats->ipackets = total_qprc;
1758         stats->ibytes = total_qbrc;
1759         stats->opackets = hw_stats->gptc;
1760         stats->obytes = hw_stats->gotc;
1761         stats->imcasts = hw_stats->mprc;
1762
1763         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1764                 stats->q_ipackets[i] = hw_stats->qprc[i];
1765                 stats->q_opackets[i] = hw_stats->qptc[i];
1766                 stats->q_ibytes[i] = hw_stats->qbrc[i];
1767                 stats->q_obytes[i] = hw_stats->qbtc[i];
1768                 stats->q_errors[i] = hw_stats->qprdc[i];
1769         }
1770
1771         /* Rx Errors */
1772         stats->ierrors = total_missed_rx + hw_stats->crcerrs +
1773                 hw_stats->rlec;
1774
1775         stats->oerrors  = 0;
1776
1777         /* XON/XOFF pause frames */
1778         stats->tx_pause_xon  = hw_stats->lxontxc;
1779         stats->rx_pause_xon  = hw_stats->lxonrxc;
1780         stats->tx_pause_xoff = hw_stats->lxofftxc;
1781         stats->rx_pause_xoff = hw_stats->lxoffrxc;
1782
1783         /* Flow Director Stats registers */
1784         hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
1785         hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
1786         stats->fdirmatch = hw_stats->fdirmatch;
1787         stats->fdirmiss = hw_stats->fdirmiss;
1788 }
1789
1790 static void
1791 ixgbe_dev_stats_reset(struct rte_eth_dev *dev)
1792 {
1793         struct ixgbe_hw_stats *stats =
1794                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1795
1796         /* HW registers are cleared on read */
1797         ixgbe_dev_stats_get(dev, NULL);
1798
1799         /* Reset software totals */
1800         memset(stats, 0, sizeof(*stats));
1801 }
1802
1803 static void
1804 ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1805 {
1806         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1807         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1808                           IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1809
1810         /* Good Rx packet, include VF loopback */
1811         UPDATE_VF_STAT(IXGBE_VFGPRC,
1812             hw_stats->last_vfgprc, hw_stats->vfgprc);
1813
1814         /* Good Rx octets, include VF loopback */
1815         UPDATE_VF_STAT_36BIT(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
1816             hw_stats->last_vfgorc, hw_stats->vfgorc);
1817
1818         /* Good Tx packet, include VF loopback */
1819         UPDATE_VF_STAT(IXGBE_VFGPTC,
1820             hw_stats->last_vfgptc, hw_stats->vfgptc);
1821
1822         /* Good Tx octets, include VF loopback */
1823         UPDATE_VF_STAT_36BIT(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
1824             hw_stats->last_vfgotc, hw_stats->vfgotc);
1825
1826         /* Rx Multicst Packet */
1827         UPDATE_VF_STAT(IXGBE_VFMPRC,
1828             hw_stats->last_vfmprc, hw_stats->vfmprc);
1829
1830         if (stats == NULL)
1831                 return;
1832
1833         memset(stats, 0, sizeof(*stats));
1834         stats->ipackets = hw_stats->vfgprc;
1835         stats->ibytes = hw_stats->vfgorc;
1836         stats->opackets = hw_stats->vfgptc;
1837         stats->obytes = hw_stats->vfgotc;
1838         stats->imcasts = hw_stats->vfmprc;
1839 }
1840
1841 static void
1842 ixgbevf_dev_stats_reset(struct rte_eth_dev *dev)
1843 {
1844         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1845                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1846
1847         /* Sync HW register to the last stats */
1848         ixgbevf_dev_stats_get(dev, NULL);
1849
1850         /* reset HW current stats*/
1851         hw_stats->vfgprc = 0;
1852         hw_stats->vfgorc = 0;
1853         hw_stats->vfgptc = 0;
1854         hw_stats->vfgotc = 0;
1855         hw_stats->vfmprc = 0;
1856
1857 }
1858
1859 static void
1860 ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1861 {
1862         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1863
1864         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
1865         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
1866         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL register */
1867         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */
1868         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
1869         dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC;
1870         dev_info->max_vfs = dev->pci_dev->max_vfs;
1871         if (hw->mac.type == ixgbe_mac_82598EB)
1872                 dev_info->max_vmdq_pools = ETH_16_POOLS;
1873         else
1874                 dev_info->max_vmdq_pools = ETH_64_POOLS;
1875         dev_info->rx_offload_capa =
1876                 DEV_RX_OFFLOAD_VLAN_STRIP |
1877                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1878                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1879                 DEV_RX_OFFLOAD_TCP_CKSUM;
1880         dev_info->tx_offload_capa =
1881                 DEV_TX_OFFLOAD_VLAN_INSERT |
1882                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1883                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1884                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1885                 DEV_TX_OFFLOAD_SCTP_CKSUM;
1886 }
1887
1888 /* return 0 means link status changed, -1 means not changed */
1889 static int
1890 ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1891 {
1892         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1893         struct rte_eth_link link, old;
1894         ixgbe_link_speed link_speed;
1895         int link_up;
1896         int diag;
1897
1898         link.link_status = 0;
1899         link.link_speed = 0;
1900         link.link_duplex = 0;
1901         memset(&old, 0, sizeof(old));
1902         rte_ixgbe_dev_atomic_read_link_status(dev, &old);
1903
1904         /* check if it needs to wait to complete, if lsc interrupt is enabled */
1905         if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
1906                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 0);
1907         else
1908                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 1);
1909         if (diag != 0) {
1910                 link.link_speed = ETH_LINK_SPEED_100;
1911                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1912                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1913                 if (link.link_status == old.link_status)
1914                         return -1;
1915                 return 0;
1916         }
1917
1918         if (link_up == 0) {
1919                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1920                 if (link.link_status == old.link_status)
1921                         return -1;
1922                 return 0;
1923         }
1924         link.link_status = 1;
1925         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1926
1927         switch (link_speed) {
1928         default:
1929         case IXGBE_LINK_SPEED_UNKNOWN:
1930                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1931                 link.link_speed = ETH_LINK_SPEED_100;
1932                 break;
1933
1934         case IXGBE_LINK_SPEED_100_FULL:
1935                 link.link_speed = ETH_LINK_SPEED_100;
1936                 break;
1937
1938         case IXGBE_LINK_SPEED_1GB_FULL:
1939                 link.link_speed = ETH_LINK_SPEED_1000;
1940                 break;
1941
1942         case IXGBE_LINK_SPEED_10GB_FULL:
1943                 link.link_speed = ETH_LINK_SPEED_10000;
1944                 break;
1945         }
1946         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1947
1948         if (link.link_status == old.link_status)
1949                 return -1;
1950
1951         return 0;
1952 }
1953
1954 static void
1955 ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev)
1956 {
1957         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1958         uint32_t fctrl;
1959
1960         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1961         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1962         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1963 }
1964
1965 static void
1966 ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev)
1967 {
1968         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1969         uint32_t fctrl;
1970
1971         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1972         fctrl &= (~IXGBE_FCTRL_UPE);
1973         if (dev->data->all_multicast == 1)
1974                 fctrl |= IXGBE_FCTRL_MPE;
1975         else
1976                 fctrl &= (~IXGBE_FCTRL_MPE);
1977         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1978 }
1979
1980 static void
1981 ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev)
1982 {
1983         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1984         uint32_t fctrl;
1985
1986         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1987         fctrl |= IXGBE_FCTRL_MPE;
1988         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1989 }
1990
1991 static void
1992 ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev)
1993 {
1994         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1995         uint32_t fctrl;
1996
1997         if (dev->data->promiscuous == 1)
1998                 return; /* must remain in all_multicast mode */
1999
2000         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2001         fctrl &= (~IXGBE_FCTRL_MPE);
2002         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2003 }
2004
2005 /**
2006  * It clears the interrupt causes and enables the interrupt.
2007  * It will be called once only during nic initialized.
2008  *
2009  * @param dev
2010  *  Pointer to struct rte_eth_dev.
2011  *
2012  * @return
2013  *  - On success, zero.
2014  *  - On failure, a negative value.
2015  */
2016 static int
2017 ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev)
2018 {
2019         struct ixgbe_interrupt *intr =
2020                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2021
2022         ixgbe_dev_link_status_print(dev);
2023         intr->mask |= IXGBE_EICR_LSC;
2024
2025         return 0;
2026 }
2027
2028 /*
2029  * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update.
2030  *
2031  * @param dev
2032  *  Pointer to struct rte_eth_dev.
2033  *
2034  * @return
2035  *  - On success, zero.
2036  *  - On failure, a negative value.
2037  */
2038 static int
2039 ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
2040 {
2041         uint32_t eicr;
2042         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2043         struct ixgbe_interrupt *intr =
2044                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2045
2046         /* clear all cause mask */
2047         ixgbe_disable_intr(hw);
2048
2049         /* read-on-clear nic registers here */
2050         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
2051         PMD_DRV_LOG(INFO, "eicr %x", eicr);
2052
2053         intr->flags = 0;
2054         if (eicr & IXGBE_EICR_LSC) {
2055                 /* set flag for async link update */
2056                 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
2057         }
2058
2059         if (eicr & IXGBE_EICR_MAILBOX)
2060                 intr->flags |= IXGBE_FLAG_MAILBOX;
2061
2062         return 0;
2063 }
2064
2065 /**
2066  * It gets and then prints the link status.
2067  *
2068  * @param dev
2069  *  Pointer to struct rte_eth_dev.
2070  *
2071  * @return
2072  *  - On success, zero.
2073  *  - On failure, a negative value.
2074  */
2075 static void
2076 ixgbe_dev_link_status_print(struct rte_eth_dev *dev)
2077 {
2078         struct rte_eth_link link;
2079
2080         memset(&link, 0, sizeof(link));
2081         rte_ixgbe_dev_atomic_read_link_status(dev, &link);
2082         if (link.link_status) {
2083                 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
2084                                         (int)(dev->data->port_id),
2085                                         (unsigned)link.link_speed,
2086                         link.link_duplex == ETH_LINK_FULL_DUPLEX ?
2087                                         "full-duplex" : "half-duplex");
2088         } else {
2089                 PMD_INIT_LOG(INFO, " Port %d: Link Down",
2090                                 (int)(dev->data->port_id));
2091         }
2092         PMD_INIT_LOG(INFO, "PCI Address: %04d:%02d:%02d:%d",
2093                                 dev->pci_dev->addr.domain,
2094                                 dev->pci_dev->addr.bus,
2095                                 dev->pci_dev->addr.devid,
2096                                 dev->pci_dev->addr.function);
2097 }
2098
2099 /*
2100  * It executes link_update after knowing an interrupt occurred.
2101  *
2102  * @param dev
2103  *  Pointer to struct rte_eth_dev.
2104  *
2105  * @return
2106  *  - On success, zero.
2107  *  - On failure, a negative value.
2108  */
2109 static int
2110 ixgbe_dev_interrupt_action(struct rte_eth_dev *dev)
2111 {
2112         struct ixgbe_interrupt *intr =
2113                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2114         int64_t timeout;
2115         struct rte_eth_link link;
2116         int intr_enable_delay = false;
2117
2118         PMD_DRV_LOG(DEBUG, "intr action type %d\n", intr->flags);
2119
2120         if (intr->flags & IXGBE_FLAG_MAILBOX) {
2121                 ixgbe_pf_mbx_process(dev);
2122                 intr->flags &= ~IXGBE_FLAG_MAILBOX;
2123         }
2124
2125         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
2126                 /* get the link status before link update, for predicting later */
2127                 memset(&link, 0, sizeof(link));
2128                 rte_ixgbe_dev_atomic_read_link_status(dev, &link);
2129
2130                 ixgbe_dev_link_update(dev, 0);
2131
2132                 /* likely to up */
2133                 if (!link.link_status)
2134                         /* handle it 1 sec later, wait it being stable */
2135                         timeout = IXGBE_LINK_UP_CHECK_TIMEOUT;
2136                 /* likely to down */
2137                 else
2138                         /* handle it 4 sec later, wait it being stable */
2139                         timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT;
2140
2141                 ixgbe_dev_link_status_print(dev);
2142
2143                 intr_enable_delay = true;
2144         }
2145
2146         if (intr_enable_delay) {
2147                 if (rte_eal_alarm_set(timeout * 1000,
2148                                       ixgbe_dev_interrupt_delayed_handler, (void*)dev) < 0)
2149                         PMD_DRV_LOG(ERR, "Error setting alarm");
2150         } else {
2151                 PMD_DRV_LOG(DEBUG, "enable intr immediately");
2152                 ixgbe_enable_intr(dev);
2153                 rte_intr_enable(&(dev->pci_dev->intr_handle));
2154         }
2155
2156
2157         return 0;
2158 }
2159
2160 /**
2161  * Interrupt handler which shall be registered for alarm callback for delayed
2162  * handling specific interrupt to wait for the stable nic state. As the
2163  * NIC interrupt state is not stable for ixgbe after link is just down,
2164  * it needs to wait 4 seconds to get the stable status.
2165  *
2166  * @param handle
2167  *  Pointer to interrupt handle.
2168  * @param param
2169  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2170  *
2171  * @return
2172  *  void
2173  */
2174 static void
2175 ixgbe_dev_interrupt_delayed_handler(void *param)
2176 {
2177         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2178         struct ixgbe_interrupt *intr =
2179                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2180         struct ixgbe_hw *hw =
2181                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2182         uint32_t eicr;
2183
2184         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
2185         if (eicr & IXGBE_EICR_MAILBOX)
2186                 ixgbe_pf_mbx_process(dev);
2187
2188         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
2189                 ixgbe_dev_link_update(dev, 0);
2190                 intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
2191                 ixgbe_dev_link_status_print(dev);
2192                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
2193         }
2194
2195         PMD_DRV_LOG(DEBUG, "enable intr in delayed handler S[%08x]\n", eicr);
2196         ixgbe_enable_intr(dev);
2197         rte_intr_enable(&(dev->pci_dev->intr_handle));
2198 }
2199
2200 /**
2201  * Interrupt handler triggered by NIC  for handling
2202  * specific interrupt.
2203  *
2204  * @param handle
2205  *  Pointer to interrupt handle.
2206  * @param param
2207  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2208  *
2209  * @return
2210  *  void
2211  */
2212 static void
2213 ixgbe_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
2214                                                         void *param)
2215 {
2216         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2217         ixgbe_dev_interrupt_get_status(dev);
2218         ixgbe_dev_interrupt_action(dev);
2219 }
2220
2221 static int
2222 ixgbe_dev_led_on(struct rte_eth_dev *dev)
2223 {
2224         struct ixgbe_hw *hw;
2225
2226         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2227         return (ixgbe_led_on(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
2228 }
2229
2230 static int
2231 ixgbe_dev_led_off(struct rte_eth_dev *dev)
2232 {
2233         struct ixgbe_hw *hw;
2234
2235         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2236         return (ixgbe_led_off(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
2237 }
2238
2239 static int
2240 ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2241 {
2242         struct ixgbe_hw *hw;
2243         int err;
2244         uint32_t rx_buf_size;
2245         uint32_t max_high_water;
2246         uint32_t mflcn;
2247         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
2248                 ixgbe_fc_none,
2249                 ixgbe_fc_rx_pause,
2250                 ixgbe_fc_tx_pause,
2251                 ixgbe_fc_full
2252         };
2253
2254         PMD_INIT_FUNC_TRACE();
2255
2256         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2257         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0));
2258         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
2259
2260         /*
2261          * At least reserve one Ethernet frame for watermark
2262          * high_water/low_water in kilo bytes for ixgbe
2263          */
2264         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
2265         if ((fc_conf->high_water > max_high_water) ||
2266                 (fc_conf->high_water < fc_conf->low_water)) {
2267                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
2268                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
2269                 return (-EINVAL);
2270         }
2271
2272         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[fc_conf->mode];
2273         hw->fc.pause_time     = fc_conf->pause_time;
2274         hw->fc.high_water[0]  = fc_conf->high_water;
2275         hw->fc.low_water[0]   = fc_conf->low_water;
2276         hw->fc.send_xon       = fc_conf->send_xon;
2277
2278         err = ixgbe_fc_enable(hw);
2279
2280         /* Not negotiated is not an error case */
2281         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) {
2282
2283                 /* check if we want to forward MAC frames - driver doesn't have native
2284                  * capability to do that, so we'll write the registers ourselves */
2285
2286                 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN);
2287
2288                 /* set or clear MFLCN.PMCF bit depending on configuration */
2289                 if (fc_conf->mac_ctrl_frame_fwd != 0)
2290                         mflcn |= IXGBE_MFLCN_PMCF;
2291                 else
2292                         mflcn &= ~IXGBE_MFLCN_PMCF;
2293
2294                 IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn);
2295                 IXGBE_WRITE_FLUSH(hw);
2296
2297                 return 0;
2298         }
2299
2300         PMD_INIT_LOG(ERR, "ixgbe_fc_enable = 0x%x \n", err);
2301         return -EIO;
2302 }
2303
2304 /**
2305  *  ixgbe_pfc_enable_generic - Enable flow control
2306  *  @hw: pointer to hardware structure
2307  *  @tc_num: traffic class number
2308  *  Enable flow control according to the current settings.
2309  */
2310 static int
2311 ixgbe_dcb_pfc_enable_generic(struct ixgbe_hw *hw,uint8_t tc_num)
2312 {
2313         int ret_val = 0;
2314         uint32_t mflcn_reg, fccfg_reg;
2315         uint32_t reg;
2316         uint32_t fcrtl, fcrth;
2317         uint8_t i;
2318         uint8_t nb_rx_en;
2319
2320         /* Validate the water mark configuration */
2321         if (!hw->fc.pause_time) {
2322                 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2323                 goto out;
2324         }
2325
2326         /* Low water mark of zero causes XOFF floods */
2327         if (hw->fc.current_mode & ixgbe_fc_tx_pause) {
2328                  /* High/Low water can not be 0 */
2329                 if( (!hw->fc.high_water[tc_num])|| (!hw->fc.low_water[tc_num])) {
2330                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
2331                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2332                         goto out;
2333                 }
2334
2335                 if(hw->fc.low_water[tc_num] >= hw->fc.high_water[tc_num]) {
2336                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
2337                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2338                         goto out;
2339                 }
2340         }
2341         /* Negotiate the fc mode to use */
2342         ixgbe_fc_autoneg(hw);
2343
2344         /* Disable any previous flow control settings */
2345         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
2346         mflcn_reg &= ~(IXGBE_MFLCN_RPFCE_SHIFT | IXGBE_MFLCN_RFCE|IXGBE_MFLCN_RPFCE);
2347
2348         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
2349         fccfg_reg &= ~(IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY);
2350
2351         switch (hw->fc.current_mode) {
2352         case ixgbe_fc_none:
2353                 /*
2354                  * If the count of enabled RX Priority Flow control >1,
2355                  * and the TX pause can not be disabled
2356                  */
2357                 nb_rx_en = 0;
2358                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
2359                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
2360                         if (reg & IXGBE_FCRTH_FCEN)
2361                                 nb_rx_en++;
2362                 }
2363                 if (nb_rx_en > 1)
2364                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2365                 break;
2366         case ixgbe_fc_rx_pause:
2367                 /*
2368                  * Rx Flow control is enabled and Tx Flow control is
2369                  * disabled by software override. Since there really
2370                  * isn't a way to advertise that we are capable of RX
2371                  * Pause ONLY, we will advertise that we support both
2372                  * symmetric and asymmetric Rx PAUSE.  Later, we will
2373                  * disable the adapter's ability to send PAUSE frames.
2374                  */
2375                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
2376                 /*
2377                  * If the count of enabled RX Priority Flow control >1,
2378                  * and the TX pause can not be disabled
2379                  */
2380                 nb_rx_en = 0;
2381                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
2382                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
2383                         if (reg & IXGBE_FCRTH_FCEN)
2384                                 nb_rx_en++;
2385                 }
2386                 if (nb_rx_en > 1)
2387                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2388                 break;
2389         case ixgbe_fc_tx_pause:
2390                 /*
2391                  * Tx Flow control is enabled, and Rx Flow control is
2392                  * disabled by software override.
2393                  */
2394                 fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2395                 break;
2396         case ixgbe_fc_full:
2397                 /* Flow control (both Rx and Tx) is enabled by SW override. */
2398                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
2399                 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
2400                 break;
2401         default:
2402                 DEBUGOUT("Flow control param set incorrectly\n");
2403                 ret_val = IXGBE_ERR_CONFIG;
2404                 goto out;
2405                 break;
2406         }
2407
2408         /* Set 802.3x based flow control settings. */
2409         mflcn_reg |= IXGBE_MFLCN_DPF;
2410         IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg);
2411         IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg);
2412
2413         /* Set up and enable Rx high/low water mark thresholds, enable XON. */
2414         if ((hw->fc.current_mode & ixgbe_fc_tx_pause) &&
2415                 hw->fc.high_water[tc_num]) {
2416                 fcrtl = (hw->fc.low_water[tc_num] << 10) | IXGBE_FCRTL_XONE;
2417                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), fcrtl);
2418                 fcrth = (hw->fc.high_water[tc_num] << 10) | IXGBE_FCRTH_FCEN;
2419         } else {
2420                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), 0);
2421                 /*
2422                  * In order to prevent Tx hangs when the internal Tx
2423                  * switch is enabled we must set the high water mark
2424                  * to the maximum FCRTH value.  This allows the Tx
2425                  * switch to function even under heavy Rx workloads.
2426                  */
2427                 fcrth = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)) - 32;
2428         }
2429         IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(tc_num), fcrth);
2430
2431         /* Configure pause time (2 TCs per register) */
2432         reg = hw->fc.pause_time * 0x00010001;
2433         for (i = 0; i < (IXGBE_DCB_MAX_TRAFFIC_CLASS / 2); i++)
2434                 IXGBE_WRITE_REG(hw, IXGBE_FCTTV(i), reg);
2435
2436         /* Configure flow control refresh threshold value */
2437         IXGBE_WRITE_REG(hw, IXGBE_FCRTV, hw->fc.pause_time / 2);
2438
2439 out:
2440         return ret_val;
2441 }
2442
2443 static int
2444 ixgbe_dcb_pfc_enable(struct rte_eth_dev *dev,uint8_t tc_num)
2445 {
2446         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2447         int32_t ret_val = IXGBE_NOT_IMPLEMENTED;
2448
2449         if(hw->mac.type != ixgbe_mac_82598EB) {
2450                 ret_val = ixgbe_dcb_pfc_enable_generic(hw,tc_num);
2451         }
2452         return ret_val;
2453 }
2454
2455 static int
2456 ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf)
2457 {
2458         int err;
2459         uint32_t rx_buf_size;
2460         uint32_t max_high_water;
2461         uint8_t tc_num;
2462         uint8_t  map[IXGBE_DCB_MAX_USER_PRIORITY] = { 0 };
2463         struct ixgbe_hw *hw =
2464                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2465         struct ixgbe_dcb_config *dcb_config =
2466                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
2467
2468         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
2469                 ixgbe_fc_none,
2470                 ixgbe_fc_rx_pause,
2471                 ixgbe_fc_tx_pause,
2472                 ixgbe_fc_full
2473         };
2474
2475         PMD_INIT_FUNC_TRACE();
2476
2477         ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
2478         tc_num = map[pfc_conf->priority];
2479         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num));
2480         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
2481         /*
2482          * At least reserve one Ethernet frame for watermark
2483          * high_water/low_water in kilo bytes for ixgbe
2484          */
2485         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
2486         if ((pfc_conf->fc.high_water > max_high_water) ||
2487                 (pfc_conf->fc.high_water <= pfc_conf->fc.low_water)) {
2488                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
2489                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
2490                 return (-EINVAL);
2491         }
2492
2493         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[pfc_conf->fc.mode];
2494         hw->fc.pause_time = pfc_conf->fc.pause_time;
2495         hw->fc.send_xon = pfc_conf->fc.send_xon;
2496         hw->fc.low_water[tc_num] =  pfc_conf->fc.low_water;
2497         hw->fc.high_water[tc_num] = pfc_conf->fc.high_water;
2498
2499         err = ixgbe_dcb_pfc_enable(dev,tc_num);
2500
2501         /* Not negotiated is not an error case */
2502         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED))
2503                 return 0;
2504
2505         PMD_INIT_LOG(ERR, "ixgbe_dcb_pfc_enable = 0x%x \n", err);
2506         return -EIO;
2507 }
2508
2509 static int
2510 ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
2511                                 struct rte_eth_rss_reta *reta_conf)
2512 {
2513         uint8_t i,j,mask;
2514         uint32_t reta;
2515         struct ixgbe_hw *hw =
2516                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2517
2518         PMD_INIT_FUNC_TRACE();
2519         /*
2520         * Update Redirection Table RETA[n],n=0...31,The redirection table has
2521         * 128-entries in 32 registers
2522          */
2523         for(i = 0; i < ETH_RSS_RETA_NUM_ENTRIES; i += 4) {
2524                 if (i < ETH_RSS_RETA_NUM_ENTRIES/2)
2525                         mask = (uint8_t)((reta_conf->mask_lo >> i) & 0xF);
2526                 else
2527                         mask = (uint8_t)((reta_conf->mask_hi >>
2528                                 (i - ETH_RSS_RETA_NUM_ENTRIES/2)) & 0xF);
2529                 if (mask != 0) {
2530                         reta = 0;
2531                         if (mask != 0xF)
2532                                 reta = IXGBE_READ_REG(hw,IXGBE_RETA(i >> 2));
2533
2534                         for (j = 0; j < 4; j++) {
2535                                 if (mask & (0x1 << j)) {
2536                                         if (mask != 0xF)
2537                                                 reta &= ~(0xFF << 8 * j);
2538                                         reta |= reta_conf->reta[i + j] << 8*j;
2539                                 }
2540                         }
2541                         IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2),reta);
2542                 }
2543         }
2544
2545         return 0;
2546 }
2547
2548 static int
2549 ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
2550                                 struct rte_eth_rss_reta *reta_conf)
2551 {
2552         uint8_t i,j,mask;
2553         uint32_t reta;
2554         struct ixgbe_hw *hw =
2555                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2556
2557         PMD_INIT_FUNC_TRACE();
2558         /*
2559          * Read Redirection Table RETA[n],n=0...31,The redirection table has
2560          * 128-entries in 32 registers
2561          */
2562         for(i = 0; i < ETH_RSS_RETA_NUM_ENTRIES; i += 4) {
2563                 if (i < ETH_RSS_RETA_NUM_ENTRIES/2)
2564                         mask = (uint8_t)((reta_conf->mask_lo >> i) & 0xF);
2565                 else
2566                         mask = (uint8_t)((reta_conf->mask_hi >>
2567                                 (i - ETH_RSS_RETA_NUM_ENTRIES/2)) & 0xF);
2568
2569                 if (mask != 0) {
2570                         reta = IXGBE_READ_REG(hw,IXGBE_RETA(i >> 2));
2571                         for (j = 0; j < 4; j++) {
2572                                 if (mask & (0x1 << j))
2573                                         reta_conf->reta[i + j] =
2574                                                 (uint8_t)((reta >> 8 * j) & 0xFF);
2575                         }
2576                 }
2577         }
2578
2579         return 0;
2580 }
2581
2582 static void
2583 ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
2584                                 uint32_t index, uint32_t pool)
2585 {
2586         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2587         uint32_t enable_addr = 1;
2588
2589         ixgbe_set_rar(hw, index, mac_addr->addr_bytes, pool, enable_addr);
2590 }
2591
2592 static void
2593 ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
2594 {
2595         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2596
2597         ixgbe_clear_rar(hw, index);
2598 }
2599
2600 /*
2601  * Virtual Function operations
2602  */
2603 static void
2604 ixgbevf_intr_disable(struct ixgbe_hw *hw)
2605 {
2606         PMD_INIT_LOG(DEBUG, "ixgbevf_intr_disable");
2607
2608         /* Clear interrupt mask to stop from interrupts being generated */
2609         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, IXGBE_VF_IRQ_CLEAR_MASK);
2610
2611         IXGBE_WRITE_FLUSH(hw);
2612 }
2613
2614 static int
2615 ixgbevf_dev_configure(struct rte_eth_dev *dev)
2616 {
2617         struct rte_eth_conf* conf = &dev->data->dev_conf;
2618
2619         PMD_INIT_LOG(DEBUG, "\nConfigured Virtual Function port id: %d\n",
2620                 dev->data->port_id);
2621
2622         /*
2623          * VF has no ability to enable/disable HW CRC
2624          * Keep the persistent behavior the same as Host PF
2625          */
2626 #ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC
2627         if (!conf->rxmode.hw_strip_crc) {
2628                 PMD_INIT_LOG(INFO, "VF can't disable HW CRC Strip\n");
2629                 conf->rxmode.hw_strip_crc = 1;
2630         }
2631 #else
2632         if (conf->rxmode.hw_strip_crc) {
2633                 PMD_INIT_LOG(INFO, "VF can't enable HW CRC Strip\n");
2634                 conf->rxmode.hw_strip_crc = 0;
2635         }
2636 #endif
2637
2638         return 0;
2639 }
2640
2641 static int
2642 ixgbevf_dev_start(struct rte_eth_dev *dev)
2643 {
2644         struct ixgbe_hw *hw =
2645                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2646         int err, mask = 0;
2647
2648         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_start");
2649
2650         hw->mac.ops.reset_hw(hw);
2651
2652         /* negotiate mailbox API version to use with the PF. */
2653         ixgbevf_negotiate_api(hw);
2654
2655         ixgbevf_dev_tx_init(dev);
2656
2657         /* This can fail when allocating mbufs for descriptor rings */
2658         err = ixgbevf_dev_rx_init(dev);
2659         if (err) {
2660                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware (%d)\n", err);
2661                 ixgbe_dev_clear_queues(dev);
2662                 return err;
2663         }
2664
2665         /* Set vfta */
2666         ixgbevf_set_vfta_all(dev,1);
2667
2668         /* Set HW strip */
2669         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
2670                 ETH_VLAN_EXTEND_MASK;
2671         ixgbevf_vlan_offload_set(dev, mask);
2672
2673         ixgbevf_dev_rxtx_start(dev);
2674
2675         return 0;
2676 }
2677
2678 static void
2679 ixgbevf_dev_stop(struct rte_eth_dev *dev)
2680 {
2681         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2682
2683         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_stop");
2684
2685         hw->adapter_stopped = TRUE;
2686         ixgbe_stop_adapter(hw);
2687
2688         /*
2689           * Clear what we set, but we still keep shadow_vfta to
2690           * restore after device starts
2691           */
2692         ixgbevf_set_vfta_all(dev,0);
2693
2694         ixgbe_dev_clear_queues(dev);
2695 }
2696
2697 static void
2698 ixgbevf_dev_close(struct rte_eth_dev *dev)
2699 {
2700         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2701
2702         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_close");
2703
2704         ixgbe_reset_hw(hw);
2705
2706         ixgbevf_dev_stop(dev);
2707
2708         /* reprogram the RAR[0] in case user changed it. */
2709         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
2710 }
2711
2712 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on)
2713 {
2714         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2715         struct ixgbe_vfta * shadow_vfta =
2716                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2717         int i = 0, j = 0, vfta = 0, mask = 1;
2718
2719         for (i = 0; i < IXGBE_VFTA_SIZE; i++){
2720                 vfta = shadow_vfta->vfta[i];
2721                 if(vfta){
2722                         mask = 1;
2723                         for (j = 0; j < 32; j++){
2724                                 if(vfta & mask)
2725                                         ixgbe_set_vfta(hw, (i<<5)+j, 0, on);
2726                                 mask<<=1;
2727                         }
2728                 }
2729         }
2730
2731 }
2732
2733 static int
2734 ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2735 {
2736         struct ixgbe_hw *hw =
2737                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2738         struct ixgbe_vfta * shadow_vfta =
2739                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2740         uint32_t vid_idx = 0;
2741         uint32_t vid_bit = 0;
2742         int ret = 0;
2743
2744         PMD_INIT_FUNC_TRACE();
2745
2746         /* vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf */
2747         ret = ixgbe_set_vfta(hw, vlan_id, 0, !!on);
2748         if(ret){
2749                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
2750                 return ret;
2751         }
2752         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
2753         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
2754
2755         /* Save what we set and retore it after device reset */
2756         if (on)
2757                 shadow_vfta->vfta[vid_idx] |= vid_bit;
2758         else
2759                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
2760
2761         return 0;
2762 }
2763
2764 static void
2765 ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
2766 {
2767         struct ixgbe_hw *hw =
2768                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2769         uint32_t ctrl;
2770
2771         PMD_INIT_FUNC_TRACE();
2772
2773         if(queue >= hw->mac.max_rx_queues)
2774                 return;
2775
2776         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
2777         if(on)
2778                 ctrl |= IXGBE_RXDCTL_VME;
2779         else
2780                 ctrl &= ~IXGBE_RXDCTL_VME;
2781         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
2782
2783         ixgbe_vlan_hw_strip_bitmap_set( dev, queue, on);
2784 }
2785
2786 static void
2787 ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2788 {
2789         struct ixgbe_hw *hw =
2790                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2791         uint16_t i;
2792         int on = 0;
2793
2794         /* VF function only support hw strip feature, others are not support */
2795         if(mask & ETH_VLAN_STRIP_MASK){
2796                 on = !!(dev->data->dev_conf.rxmode.hw_vlan_strip);
2797
2798                 for(i=0; i < hw->mac.max_rx_queues; i++)
2799                         ixgbevf_vlan_strip_queue_set(dev,i,on);
2800         }
2801 }
2802
2803 static int
2804 ixgbe_vmdq_mode_check(struct ixgbe_hw *hw)
2805 {
2806         uint32_t reg_val;
2807
2808         /* we only need to do this if VMDq is enabled */
2809         reg_val = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
2810         if (!(reg_val & IXGBE_VT_CTL_VT_ENABLE)) {
2811                 PMD_INIT_LOG(ERR, "VMDq must be enabled for this setting\n");
2812                 return (-1);
2813         }
2814
2815         return 0;
2816 }
2817
2818 static uint32_t
2819 ixgbe_uta_vector(struct ixgbe_hw *hw, struct ether_addr* uc_addr)
2820 {
2821         uint32_t vector = 0;
2822         switch (hw->mac.mc_filter_type) {
2823         case 0:   /* use bits [47:36] of the address */
2824                 vector = ((uc_addr->addr_bytes[4] >> 4) |
2825                         (((uint16_t)uc_addr->addr_bytes[5]) << 4));
2826                 break;
2827         case 1:   /* use bits [46:35] of the address */
2828                 vector = ((uc_addr->addr_bytes[4] >> 3) |
2829                         (((uint16_t)uc_addr->addr_bytes[5]) << 5));
2830                 break;
2831         case 2:   /* use bits [45:34] of the address */
2832                 vector = ((uc_addr->addr_bytes[4] >> 2) |
2833                         (((uint16_t)uc_addr->addr_bytes[5]) << 6));
2834                 break;
2835         case 3:   /* use bits [43:32] of the address */
2836                 vector = ((uc_addr->addr_bytes[4]) |
2837                         (((uint16_t)uc_addr->addr_bytes[5]) << 8));
2838                 break;
2839         default:  /* Invalid mc_filter_type */
2840                 break;
2841         }
2842
2843         /* vector can only be 12-bits or boundary will be exceeded */
2844         vector &= 0xFFF;
2845         return vector;
2846 }
2847
2848 static int
2849 ixgbe_uc_hash_table_set(struct rte_eth_dev *dev,struct ether_addr* mac_addr,
2850                                uint8_t on)
2851 {
2852         uint32_t vector;
2853         uint32_t uta_idx;
2854         uint32_t reg_val;
2855         uint32_t uta_shift;
2856         uint32_t rc;
2857         const uint32_t ixgbe_uta_idx_mask = 0x7F;
2858         const uint32_t ixgbe_uta_bit_shift = 5;
2859         const uint32_t ixgbe_uta_bit_mask = (0x1 << ixgbe_uta_bit_shift) - 1;
2860         const uint32_t bit1 = 0x1;
2861
2862         struct ixgbe_hw *hw =
2863                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2864         struct ixgbe_uta_info *uta_info =
2865                 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private);
2866
2867         /* The UTA table only exists on 82599 hardware and newer */
2868         if (hw->mac.type < ixgbe_mac_82599EB)
2869                 return (-ENOTSUP);
2870
2871         vector = ixgbe_uta_vector(hw,mac_addr);
2872         uta_idx = (vector >> ixgbe_uta_bit_shift) & ixgbe_uta_idx_mask;
2873         uta_shift = vector & ixgbe_uta_bit_mask;
2874
2875         rc = ((uta_info->uta_shadow[uta_idx] >> uta_shift & bit1) != 0);
2876         if(rc == on)
2877                 return 0;
2878
2879         reg_val = IXGBE_READ_REG(hw, IXGBE_UTA(uta_idx));
2880         if (on) {
2881                 uta_info->uta_in_use++;
2882                 reg_val |= (bit1 << uta_shift);
2883                 uta_info->uta_shadow[uta_idx] |= (bit1 << uta_shift);
2884         } else {
2885                 uta_info->uta_in_use--;
2886                 reg_val &= ~(bit1 << uta_shift);
2887                 uta_info->uta_shadow[uta_idx] &= ~(bit1 << uta_shift);
2888         }
2889
2890         IXGBE_WRITE_REG(hw, IXGBE_UTA(uta_idx), reg_val);
2891
2892         if (uta_info->uta_in_use > 0)
2893                 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL,
2894                                 IXGBE_MCSTCTRL_MFE | hw->mac.mc_filter_type);
2895         else
2896                 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL,hw->mac.mc_filter_type);
2897
2898         return 0;
2899 }
2900
2901 static int
2902 ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on)
2903 {
2904         int i;
2905         struct ixgbe_hw *hw =
2906                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2907         struct ixgbe_uta_info *uta_info =
2908                 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private);
2909
2910         /* The UTA table only exists on 82599 hardware and newer */
2911         if (hw->mac.type < ixgbe_mac_82599EB)
2912                 return (-ENOTSUP);
2913
2914         if(on) {
2915                 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
2916                         uta_info->uta_shadow[i] = ~0;
2917                         IXGBE_WRITE_REG(hw, IXGBE_UTA(i), ~0);
2918                 }
2919         } else {
2920                 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
2921                         uta_info->uta_shadow[i] = 0;
2922                         IXGBE_WRITE_REG(hw, IXGBE_UTA(i), 0);
2923                 }
2924         }
2925         return 0;
2926
2927 }
2928 static int
2929 ixgbe_set_pool_rx_mode(struct rte_eth_dev *dev, uint16_t pool,
2930                                uint16_t rx_mask, uint8_t on)
2931 {
2932         int val = 0;
2933
2934         struct ixgbe_hw *hw =
2935                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2936         uint32_t vmolr = IXGBE_READ_REG(hw, IXGBE_VMOLR(pool));
2937
2938         if (hw->mac.type == ixgbe_mac_82598EB) {
2939                 PMD_INIT_LOG(ERR, "setting VF receive mode set should be done"
2940                         " on 82599 hardware and newer\n");
2941                 return (-ENOTSUP);
2942         }
2943         if (ixgbe_vmdq_mode_check(hw) < 0)
2944                 return (-ENOTSUP);
2945
2946         if (rx_mask & ETH_VMDQ_ACCEPT_UNTAG )
2947                 val |= IXGBE_VMOLR_AUPE;
2948         if (rx_mask & ETH_VMDQ_ACCEPT_HASH_MC )
2949                 val |= IXGBE_VMOLR_ROMPE;
2950         if (rx_mask & ETH_VMDQ_ACCEPT_HASH_UC)
2951                 val |= IXGBE_VMOLR_ROPE;
2952         if (rx_mask & ETH_VMDQ_ACCEPT_BROADCAST)
2953                 val |= IXGBE_VMOLR_BAM;
2954         if (rx_mask & ETH_VMDQ_ACCEPT_MULTICAST)
2955                 val |= IXGBE_VMOLR_MPE;
2956
2957         if (on)
2958                 vmolr |= val;
2959         else
2960                 vmolr &= ~val;
2961
2962         IXGBE_WRITE_REG(hw, IXGBE_VMOLR(pool), vmolr);
2963
2964         return 0;
2965 }
2966
2967 static int
2968 ixgbe_set_pool_rx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on)
2969 {
2970         uint32_t reg,addr;
2971         uint32_t val;
2972         const uint8_t bit1 = 0x1;
2973
2974         struct ixgbe_hw *hw =
2975                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2976
2977         if (ixgbe_vmdq_mode_check(hw) < 0)
2978                 return (-ENOTSUP);
2979
2980         addr = IXGBE_VFRE(pool >= ETH_64_POOLS/2);
2981         reg = IXGBE_READ_REG(hw, addr);
2982         val = bit1 << pool;
2983
2984         if (on)
2985                 reg |= val;
2986         else
2987                 reg &= ~val;
2988
2989         IXGBE_WRITE_REG(hw, addr,reg);
2990
2991         return 0;
2992 }
2993
2994 static int
2995 ixgbe_set_pool_tx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on)
2996 {
2997         uint32_t reg,addr;
2998         uint32_t val;
2999         const uint8_t bit1 = 0x1;
3000
3001         struct ixgbe_hw *hw =
3002                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3003
3004         if (ixgbe_vmdq_mode_check(hw) < 0)
3005                 return (-ENOTSUP);
3006
3007         addr = IXGBE_VFTE(pool >= ETH_64_POOLS/2);
3008         reg = IXGBE_READ_REG(hw, addr);
3009         val = bit1 << pool;
3010
3011         if (on)
3012                 reg |= val;
3013         else
3014                 reg &= ~val;
3015
3016         IXGBE_WRITE_REG(hw, addr,reg);
3017
3018         return 0;
3019 }
3020
3021 static int
3022 ixgbe_set_pool_vlan_filter(struct rte_eth_dev *dev, uint16_t vlan,
3023                         uint64_t pool_mask, uint8_t vlan_on)
3024 {
3025         int ret = 0;
3026         uint16_t pool_idx;
3027         struct ixgbe_hw *hw =
3028                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3029
3030         if (ixgbe_vmdq_mode_check(hw) < 0)
3031                 return (-ENOTSUP);
3032         for (pool_idx = 0; pool_idx < ETH_64_POOLS; pool_idx++) {
3033                 if (pool_mask & ((uint64_t)(1ULL << pool_idx)))
3034                         ret = hw->mac.ops.set_vfta(hw,vlan,pool_idx,vlan_on);
3035                         if (ret < 0)
3036                                 return ret;
3037         }
3038
3039         return ret;
3040 }
3041
3042 static int
3043 ixgbe_mirror_rule_set(struct rte_eth_dev *dev,
3044                         struct rte_eth_vmdq_mirror_conf *mirror_conf,
3045                         uint8_t rule_id, uint8_t on)
3046 {
3047         uint32_t mr_ctl,vlvf;
3048         uint32_t mp_lsb = 0;
3049         uint32_t mv_msb = 0;
3050         uint32_t mv_lsb = 0;
3051         uint32_t mp_msb = 0;
3052         uint8_t i = 0;
3053         int reg_index = 0;
3054         uint64_t vlan_mask = 0;
3055
3056         const uint8_t pool_mask_offset = 32;
3057         const uint8_t vlan_mask_offset = 32;
3058         const uint8_t dst_pool_offset = 8;
3059         const uint8_t rule_mr_offset  = 4;
3060         const uint8_t mirror_rule_mask= 0x0F;
3061
3062         struct ixgbe_mirror_info *mr_info =
3063                         (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private));
3064         struct ixgbe_hw *hw =
3065                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3066
3067         if (ixgbe_vmdq_mode_check(hw) < 0)
3068                 return (-ENOTSUP);
3069
3070         /* Check if vlan mask is valid */
3071         if ((mirror_conf->rule_type_mask & ETH_VMDQ_VLAN_MIRROR) && (on)) {
3072                 if (mirror_conf->vlan.vlan_mask == 0)
3073                         return (-EINVAL);
3074         }
3075
3076         /* Check if vlan id is valid and find conresponding VLAN ID index in VLVF */
3077         if (mirror_conf->rule_type_mask & ETH_VMDQ_VLAN_MIRROR) {
3078                 for (i = 0;i < IXGBE_VLVF_ENTRIES; i++) {
3079                         if (mirror_conf->vlan.vlan_mask & (1ULL << i)) {
3080                                 /* search vlan id related pool vlan filter index */
3081                                 reg_index = ixgbe_find_vlvf_slot(hw,
3082                                                 mirror_conf->vlan.vlan_id[i]);
3083                                 if(reg_index < 0)
3084                                         return (-EINVAL);
3085                                 vlvf = IXGBE_READ_REG(hw, IXGBE_VLVF(reg_index));
3086                                 if ((vlvf & IXGBE_VLVF_VIEN) &&
3087                                         ((vlvf & IXGBE_VLVF_VLANID_MASK)
3088                                                 == mirror_conf->vlan.vlan_id[i]))
3089                                         vlan_mask |= (1ULL << reg_index);
3090                                 else
3091                                         return (-EINVAL);
3092                         }
3093                 }
3094
3095                 if (on) {
3096                         mv_lsb = vlan_mask & 0xFFFFFFFF;
3097                         mv_msb = vlan_mask >> vlan_mask_offset;
3098
3099                         mr_info->mr_conf[rule_id].vlan.vlan_mask =
3100                                                 mirror_conf->vlan.vlan_mask;
3101                         for(i = 0 ;i < ETH_VMDQ_MAX_VLAN_FILTERS; i++) {
3102                                 if(mirror_conf->vlan.vlan_mask & (1ULL << i))
3103                                         mr_info->mr_conf[rule_id].vlan.vlan_id[i] =
3104                                                 mirror_conf->vlan.vlan_id[i];
3105                         }
3106                 } else {
3107                         mv_lsb = 0;
3108                         mv_msb = 0;
3109                         mr_info->mr_conf[rule_id].vlan.vlan_mask = 0;
3110                         for(i = 0 ;i < ETH_VMDQ_MAX_VLAN_FILTERS; i++)
3111                                 mr_info->mr_conf[rule_id].vlan.vlan_id[i] = 0;
3112                 }
3113         }
3114
3115         /*
3116          * if enable pool mirror, write related pool mask register,if disable
3117          * pool mirror, clear PFMRVM register
3118          */
3119         if (mirror_conf->rule_type_mask & ETH_VMDQ_POOL_MIRROR) {
3120                 if (on) {
3121                         mp_lsb = mirror_conf->pool_mask & 0xFFFFFFFF;
3122                         mp_msb = mirror_conf->pool_mask >> pool_mask_offset;
3123                         mr_info->mr_conf[rule_id].pool_mask =
3124                                         mirror_conf->pool_mask;
3125
3126                 } else {
3127                         mp_lsb = 0;
3128                         mp_msb = 0;
3129                         mr_info->mr_conf[rule_id].pool_mask = 0;
3130                 }
3131         }
3132
3133         /* read  mirror control register and recalculate it */
3134         mr_ctl = IXGBE_READ_REG(hw,IXGBE_MRCTL(rule_id));
3135
3136         if (on) {
3137                 mr_ctl |= mirror_conf->rule_type_mask;
3138                 mr_ctl &= mirror_rule_mask;
3139                 mr_ctl |= mirror_conf->dst_pool << dst_pool_offset;
3140         } else
3141                 mr_ctl &= ~(mirror_conf->rule_type_mask & mirror_rule_mask);
3142
3143         mr_info->mr_conf[rule_id].rule_type_mask = (uint8_t)(mr_ctl & mirror_rule_mask);
3144         mr_info->mr_conf[rule_id].dst_pool = mirror_conf->dst_pool;
3145
3146         /* write mirrror control  register */
3147         IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl);
3148
3149         /* write pool mirrror control  register */
3150         if (mirror_conf->rule_type_mask & ETH_VMDQ_POOL_MIRROR) {
3151                 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), mp_lsb);
3152                 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset),
3153                                 mp_msb);
3154         }
3155         /* write VLAN mirrror control  register */
3156         if (mirror_conf->rule_type_mask & ETH_VMDQ_VLAN_MIRROR) {
3157                 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), mv_lsb);
3158                 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset),
3159                                 mv_msb);
3160         }
3161
3162         return 0;
3163 }
3164
3165 static int
3166 ixgbe_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t rule_id)
3167 {
3168         int mr_ctl = 0;
3169         uint32_t lsb_val = 0;
3170         uint32_t msb_val = 0;
3171         const uint8_t rule_mr_offset = 4;
3172
3173         struct ixgbe_hw *hw =
3174                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3175         struct ixgbe_mirror_info *mr_info =
3176                 (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private));
3177
3178         if (ixgbe_vmdq_mode_check(hw) < 0)
3179                 return (-ENOTSUP);
3180
3181         memset(&mr_info->mr_conf[rule_id], 0,
3182                 sizeof(struct rte_eth_vmdq_mirror_conf));
3183
3184         /* clear PFVMCTL register */
3185         IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl);
3186
3187         /* clear pool mask register */
3188         IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), lsb_val);
3189         IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset), msb_val);
3190
3191         /* clear vlan mask register */
3192         IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), lsb_val);
3193         IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset), msb_val);
3194
3195         return 0;
3196 }
3197
3198 static void
3199 ixgbevf_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
3200                      __attribute__((unused)) uint32_t index,
3201                      __attribute__((unused)) uint32_t pool)
3202 {
3203         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3204         int diag;
3205
3206         /*
3207          * On a 82599 VF, adding again the same MAC addr is not an idempotent
3208          * operation. Trap this case to avoid exhausting the [very limited]
3209          * set of PF resources used to store VF MAC addresses.
3210          */
3211         if (memcmp(hw->mac.perm_addr, mac_addr, sizeof(struct ether_addr)) == 0)
3212                 return;
3213         diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes);
3214         if (diag == 0)
3215                 return;
3216         PMD_DRV_LOG(ERR, "Unable to add MAC address - diag=%d", diag);
3217 }
3218
3219 static void
3220 ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index)
3221 {
3222         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3223         struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr;
3224         struct ether_addr *mac_addr;
3225         uint32_t i;
3226         int diag;
3227
3228         /*
3229          * The IXGBE_VF_SET_MACVLAN command of the ixgbe-pf driver does
3230          * not support the deletion of a given MAC address.
3231          * Instead, it imposes to delete all MAC addresses, then to add again
3232          * all MAC addresses with the exception of the one to be deleted.
3233          */
3234         (void) ixgbevf_set_uc_addr_vf(hw, 0, NULL);
3235
3236         /*
3237          * Add again all MAC addresses, with the exception of the deleted one
3238          * and of the permanent MAC address.
3239          */
3240         for (i = 0, mac_addr = dev->data->mac_addrs;
3241              i < hw->mac.num_rar_entries; i++, mac_addr++) {
3242                 /* Skip the deleted MAC address */
3243                 if (i == index)
3244                         continue;
3245                 /* Skip NULL MAC addresses */
3246                 if (is_zero_ether_addr(mac_addr))
3247                         continue;
3248                 /* Skip the permanent MAC address */
3249                 if (memcmp(perm_addr, mac_addr, sizeof(struct ether_addr)) == 0)
3250                         continue;
3251                 diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes);
3252                 if (diag != 0)
3253                         PMD_DRV_LOG(ERR,
3254                                     "Adding again MAC address "
3255                                     "%02x:%02x:%02x:%02x:%02x:%02x failed "
3256                                     "diag=%d",
3257                                     mac_addr->addr_bytes[0],
3258                                     mac_addr->addr_bytes[1],
3259                                     mac_addr->addr_bytes[2],
3260                                     mac_addr->addr_bytes[3],
3261                                     mac_addr->addr_bytes[4],
3262                                     mac_addr->addr_bytes[5],
3263                                     diag);
3264         }
3265 }
3266
3267 static struct rte_driver rte_ixgbe_driver = {
3268         .type = PMD_PDEV,
3269         .init = rte_ixgbe_pmd_init,
3270 };
3271
3272 static struct rte_driver rte_ixgbevf_driver = {
3273         .type = PMD_PDEV,
3274         .init = rte_ixgbevf_pmd_init,
3275 };
3276
3277 PMD_REGISTER_DRIVER(rte_ixgbe_driver);
3278 PMD_REGISTER_DRIVER(rte_ixgbevf_driver);