lib/librte_pmd_ixgbe/ixgbe_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <inttypes.h>
  42 #include <rte_byteorder.h>
  43 #include <rte_common.h>
  44 #include <rte_cycles.h>
  45
  46 #include <rte_interrupts.h>
  47 #include <rte_log.h>
  48 #include <rte_debug.h>
  49 #include <rte_pci.h>
  50 #include <rte_atomic.h>
  51 #include <rte_branch_prediction.h>
  52 #include <rte_memory.h>
  53 #include <rte_memzone.h>
  54 #include <rte_tailq.h>
  55 #include <rte_eal.h>
  56 #include <rte_alarm.h>
  57 #include <rte_ether.h>
  58 #include <rte_ethdev.h>
  59 #include <rte_atomic.h>
  60 #include <rte_malloc.h>
  61 #include <rte_random.h>
  62 #include <rte_dev.h>
  63
  64 #include "ixgbe_logs.h"
  65 #include "ixgbe/ixgbe_api.h"
  66 #include "ixgbe/ixgbe_vf.h"
  67 #include "ixgbe/ixgbe_common.h"
  68 #include "ixgbe_ethdev.h"
  69 #include "ixgbe_bypass.h"
  70
  71 /*
  72  * High threshold controlling when to start sending XOFF frames. Must be at
  73  * least 8 bytes less than receive packet buffer size. This value is in units
  74  * of 1024 bytes.
  75  */
  76 #define IXGBE_FC_HI    0x80
  77
  78 /*
  79  * Low threshold controlling when to start sending XON frames. This value is
  80  * in units of 1024 bytes.
  81  */
  82 #define IXGBE_FC_LO    0x40
  83
  84 /* Timer value included in XOFF frames. */
  85 #define IXGBE_FC_PAUSE 0x680
  86
  87 #define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */
  88 #define IXGBE_LINK_UP_CHECK_TIMEOUT   1000 /* ms */
  89 #define IXGBE_VMDQ_NUM_UC_MAC         4096 /* Maximum nb. of UC MAC addr. */
  90
  91
  92 #define IXGBEVF_PMD_NAME "rte_ixgbevf_pmd" /* PMD name */
  93
  94 #define IXGBE_QUEUE_STAT_COUNTERS (sizeof(hw_stats->qprc) / sizeof(hw_stats->qprc[0]))
  95
  96 static int eth_ixgbe_dev_init(struct eth_driver *eth_drv,
  97                 struct rte_eth_dev *eth_dev);
  98 static int  ixgbe_dev_configure(struct rte_eth_dev *dev);
  99 static int  ixgbe_dev_start(struct rte_eth_dev *dev);
 100 static void ixgbe_dev_stop(struct rte_eth_dev *dev);
 101 static void ixgbe_dev_close(struct rte_eth_dev *dev);
 102 static void ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev);
 103 static void ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev);
 104 static void ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev);
 105 static void ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev);
 106 static int ixgbe_dev_link_update(struct rte_eth_dev *dev,
 107                                 int wait_to_complete);
 108 static void ixgbe_dev_stats_get(struct rte_eth_dev *dev,
 109                                 struct rte_eth_stats *stats);
 110 static void ixgbe_dev_stats_reset(struct rte_eth_dev *dev);
 111 static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 112                                              uint16_t queue_id,
 113                                              uint8_t stat_idx,
 114                                              uint8_t is_rx);
 115 static void ixgbe_dev_info_get(struct rte_eth_dev *dev,
 116                                 struct rte_eth_dev_info *dev_info);
 117 static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev,
 118                 uint16_t vlan_id, int on);
 119 static void ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 120 static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev,
 121                 uint16_t queue, bool on);
 122 static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue,
 123                 int on);
 124 static void ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 125 static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue);
 126 static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue);
 127 static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 128 static void ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 129
 130 static int ixgbe_dev_led_on(struct rte_eth_dev *dev);
 131 static int ixgbe_dev_led_off(struct rte_eth_dev *dev);
 132 static int  ixgbe_flow_ctrl_set(struct rte_eth_dev *dev,
 133                 struct rte_eth_fc_conf *fc_conf);
 134 static int ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 135                 struct rte_eth_pfc_conf *pfc_conf);
 136 static int ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
 137                 struct rte_eth_rss_reta *reta_conf);
 138 static int ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
 139                 struct rte_eth_rss_reta *reta_conf);
 140 static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev);
 141 static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev);
 142 static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev);
 143 static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev);
 144 static void ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle,
 145                 void *param);
 146 static void ixgbe_dev_interrupt_delayed_handler(void *param);
 147 static void ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 148                 uint32_t index, uint32_t pool);
 149 static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index);
 150 static void ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config);
 151
 152 /* For Virtual Function support */
 153 static int eth_ixgbevf_dev_init(struct eth_driver *eth_drv,
 154                 struct rte_eth_dev *eth_dev);
 155 static int  ixgbevf_dev_configure(struct rte_eth_dev *dev);
 156 static int  ixgbevf_dev_start(struct rte_eth_dev *dev);
 157 static void ixgbevf_dev_stop(struct rte_eth_dev *dev);
 158 static void ixgbevf_dev_close(struct rte_eth_dev *dev);
 159 static void ixgbevf_intr_disable(struct ixgbe_hw *hw);
 160 static void ixgbevf_dev_stats_get(struct rte_eth_dev *dev,
 161                 struct rte_eth_stats *stats);
 162 static void ixgbevf_dev_stats_reset(struct rte_eth_dev *dev);
 163 static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev,
 164                 uint16_t vlan_id, int on);
 165 static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev,
 166                 uint16_t queue, int on);
 167 static void ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 168 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 169
 170 /* For Eth VMDQ APIs support */
 171 static int ixgbe_uc_hash_table_set(struct rte_eth_dev *dev, struct
 172                 ether_addr* mac_addr,uint8_t on);
 173 static int ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev,uint8_t on);
 174 static int  ixgbe_set_pool_rx_mode(struct rte_eth_dev *dev,  uint16_t pool,
 175                 uint16_t rx_mask, uint8_t on);
 176 static int ixgbe_set_pool_rx(struct rte_eth_dev *dev,uint16_t pool,uint8_t on);
 177 static int ixgbe_set_pool_tx(struct rte_eth_dev *dev,uint16_t pool,uint8_t on);
 178 static int ixgbe_set_pool_vlan_filter(struct rte_eth_dev *dev, uint16_t vlan,
 179                 uint64_t pool_mask,uint8_t vlan_on);
 180 static int ixgbe_mirror_rule_set(struct rte_eth_dev *dev,
 181                 struct rte_eth_vmdq_mirror_conf *mirror_conf,
 182                 uint8_t rule_id, uint8_t on);
 183 static int ixgbe_mirror_rule_reset(struct rte_eth_dev *dev,
 184                 uint8_t rule_id);
 185
 186 static void ixgbevf_add_mac_addr(struct rte_eth_dev *dev,
 187                                  struct ether_addr *mac_addr,
 188                                  uint32_t index, uint32_t pool);
 189 static void ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index);
 190
 191 /*
 192  * Define VF Stats MACRO for Non "cleared on read" register
 193  */
 194 #define UPDATE_VF_STAT(reg, last, cur)                          \
 195 {                                                               \
 196         u32 latest = IXGBE_READ_REG(hw, reg);                   \
 197         cur += latest - last;                                   \
 198         last = latest;                                          \
 199 }
 200
 201 #define UPDATE_VF_STAT_36BIT(lsb, msb, last, cur)                \
 202 {                                                                \
 203         u64 new_lsb = IXGBE_READ_REG(hw, lsb);                   \
 204         u64 new_msb = IXGBE_READ_REG(hw, msb);                   \
 205         u64 latest = ((new_msb << 32) | new_lsb);                \
 206         cur += (0x1000000000LL + latest - last) & 0xFFFFFFFFFLL; \
 207         last = latest;                                           \
 208 }
 209
 210 #define IXGBE_SET_HWSTRIP(h, q) do{\
 211                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 212                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 213                 (h)->bitmap[idx] |= 1 << bit;\
 214         }while(0)
 215
 216 #define IXGBE_CLEAR_HWSTRIP(h, q) do{\
 217                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 218                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 219                 (h)->bitmap[idx] &= ~(1 << bit);\
 220         }while(0)
 221
 222 #define IXGBE_GET_HWSTRIP(h, q, r) do{\
 223                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 224                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 225                 (r) = (h)->bitmap[idx] >> bit & 1;\
 226         }while(0)
 227
 228 /*
 229  * The set of PCI devices this driver supports
 230  */
 231 static struct rte_pci_id pci_id_ixgbe_map[] = {
 232
 233 #define RTE_PCI_DEV_ID_DECL_IXGBE(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 234 #include "rte_pci_dev_ids.h"
 235
 236 { .vendor_id = 0, /* sentinel */ },
 237 };
 238
 239
 240 /*
 241  * The set of PCI devices this driver supports (for 82599 VF)
 242  */
 243 static struct rte_pci_id pci_id_ixgbevf_map[] = {
 244
 245 #define RTE_PCI_DEV_ID_DECL_IXGBEVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 246 #include "rte_pci_dev_ids.h"
 247 { .vendor_id = 0, /* sentinel */ },
 248
 249 };
 250
 251 static struct eth_dev_ops ixgbe_eth_dev_ops = {
 252         .dev_configure        = ixgbe_dev_configure,
 253         .dev_start            = ixgbe_dev_start,
 254         .dev_stop             = ixgbe_dev_stop,
 255         .dev_close            = ixgbe_dev_close,
 256         .promiscuous_enable   = ixgbe_dev_promiscuous_enable,
 257         .promiscuous_disable  = ixgbe_dev_promiscuous_disable,
 258         .allmulticast_enable  = ixgbe_dev_allmulticast_enable,
 259         .allmulticast_disable = ixgbe_dev_allmulticast_disable,
 260         .link_update          = ixgbe_dev_link_update,
 261         .stats_get            = ixgbe_dev_stats_get,
 262         .stats_reset          = ixgbe_dev_stats_reset,
 263         .queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set,
 264         .dev_infos_get        = ixgbe_dev_info_get,
 265         .vlan_filter_set      = ixgbe_vlan_filter_set,
 266         .vlan_tpid_set        = ixgbe_vlan_tpid_set,
 267         .vlan_offload_set     = ixgbe_vlan_offload_set,
 268         .vlan_strip_queue_set = ixgbe_vlan_strip_queue_set,
 269         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 270         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 271         .rx_queue_count       = ixgbe_dev_rx_queue_count,
 272         .rx_descriptor_done   = ixgbe_dev_rx_descriptor_done,
 273         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 274         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 275         .dev_led_on           = ixgbe_dev_led_on,
 276         .dev_led_off          = ixgbe_dev_led_off,
 277         .flow_ctrl_set        = ixgbe_flow_ctrl_set,
 278         .priority_flow_ctrl_set = ixgbe_priority_flow_ctrl_set,
 279         .mac_addr_add         = ixgbe_add_rar,
 280         .mac_addr_remove      = ixgbe_remove_rar,
 281         .uc_hash_table_set    = ixgbe_uc_hash_table_set,
 282         .uc_all_hash_table_set  = ixgbe_uc_all_hash_table_set,
 283         .mirror_rule_set        = ixgbe_mirror_rule_set,
 284         .mirror_rule_reset      = ixgbe_mirror_rule_reset,
 285         .set_vf_rx_mode       = ixgbe_set_pool_rx_mode,
 286         .set_vf_rx            = ixgbe_set_pool_rx,
 287         .set_vf_tx            = ixgbe_set_pool_tx,
 288         .set_vf_vlan_filter   = ixgbe_set_pool_vlan_filter,
 289         .fdir_add_signature_filter    = ixgbe_fdir_add_signature_filter,
 290         .fdir_update_signature_filter = ixgbe_fdir_update_signature_filter,
 291         .fdir_remove_signature_filter = ixgbe_fdir_remove_signature_filter,
 292         .fdir_infos_get               = ixgbe_fdir_info_get,
 293         .fdir_add_perfect_filter      = ixgbe_fdir_add_perfect_filter,
 294         .fdir_update_perfect_filter   = ixgbe_fdir_update_perfect_filter,
 295         .fdir_remove_perfect_filter   = ixgbe_fdir_remove_perfect_filter,
 296         .fdir_set_masks               = ixgbe_fdir_set_masks,
 297         .reta_update          = ixgbe_dev_rss_reta_update,
 298         .reta_query           = ixgbe_dev_rss_reta_query,
 299 #ifdef RTE_NIC_BYPASS
 300         .bypass_init          = ixgbe_bypass_init,
 301         .bypass_state_set     = ixgbe_bypass_state_store,
 302         .bypass_state_show    = ixgbe_bypass_state_show,
 303         .bypass_event_set     = ixgbe_bypass_event_store,
 304         .bypass_event_show    = ixgbe_bypass_event_show,
 305         .bypass_wd_timeout_set  = ixgbe_bypass_wd_timeout_store,
 306         .bypass_wd_timeout_show = ixgbe_bypass_wd_timeout_show,
 307         .bypass_ver_show      = ixgbe_bypass_ver_show,
 308         .bypass_wd_reset      = ixgbe_bypass_wd_reset,
 309 #endif /* RTE_NIC_BYPASS */
 310         .rss_hash_update      = ixgbe_dev_rss_hash_update,
 311         .rss_hash_conf_get    = ixgbe_dev_rss_hash_conf_get,
 312 };
 313
 314 /*
 315  * dev_ops for virtual function, bare necessities for basic vf
 316  * operation have been implemented
 317  */
 318 static struct eth_dev_ops ixgbevf_eth_dev_ops = {
 319
 320         .dev_configure        = ixgbevf_dev_configure,
 321         .dev_start            = ixgbevf_dev_start,
 322         .dev_stop             = ixgbevf_dev_stop,
 323         .link_update          = ixgbe_dev_link_update,
 324         .stats_get            = ixgbevf_dev_stats_get,
 325         .stats_reset          = ixgbevf_dev_stats_reset,
 326         .dev_close            = ixgbevf_dev_close,
 327         .dev_infos_get        = ixgbe_dev_info_get,
 328         .vlan_filter_set      = ixgbevf_vlan_filter_set,
 329         .vlan_strip_queue_set = ixgbevf_vlan_strip_queue_set,
 330         .vlan_offload_set     = ixgbevf_vlan_offload_set,
 331         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 332         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 333         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 334         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 335         .mac_addr_add         = ixgbevf_add_mac_addr,
 336         .mac_addr_remove      = ixgbevf_remove_mac_addr,
 337 };
 338
 339 /**
 340  * Atomically reads the link status information from global
 341  * structure rte_eth_dev.
 342  *
 343  * @param dev
 344  *   - Pointer to the structure rte_eth_dev to read from.
 345  *   - Pointer to the buffer to be saved with the link status.
 346  *
 347  * @return
 348  *   - On success, zero.
 349  *   - On failure, negative value.
 350  */
 351 static inline int
 352 rte_ixgbe_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 353                                 struct rte_eth_link *link)
 354 {
 355         struct rte_eth_link *dst = link;
 356         struct rte_eth_link *src = &(dev->data->dev_link);
 357
 358         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 359                                         *(uint64_t *)src) == 0)
 360                 return -1;
 361
 362         return 0;
 363 }
 364
 365 /**
 366  * Atomically writes the link status information into global
 367  * structure rte_eth_dev.
 368  *
 369  * @param dev
 370  *   - Pointer to the structure rte_eth_dev to read from.
 371  *   - Pointer to the buffer to be saved with the link status.
 372  *
 373  * @return
 374  *   - On success, zero.
 375  *   - On failure, negative value.
 376  */
 377 static inline int
 378 rte_ixgbe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 379                                 struct rte_eth_link *link)
 380 {
 381         struct rte_eth_link *dst = &(dev->data->dev_link);
 382         struct rte_eth_link *src = link;
 383
 384         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 385                                         *(uint64_t *)src) == 0)
 386                 return -1;
 387
 388         return 0;
 389 }
 390
 391 /*
 392  * This function is the same as ixgbe_is_sfp() in ixgbe/ixgbe.h.
 393  */
 394 static inline int
 395 ixgbe_is_sfp(struct ixgbe_hw *hw)
 396 {
 397         switch (hw->phy.type) {
 398         case ixgbe_phy_sfp_avago:
 399         case ixgbe_phy_sfp_ftl:
 400         case ixgbe_phy_sfp_intel:
 401         case ixgbe_phy_sfp_unknown:
 402         case ixgbe_phy_sfp_passive_tyco:
 403         case ixgbe_phy_sfp_passive_unknown:
 404                 return 1;
 405         default:
 406                 return 0;
 407         }
 408 }
 409
 410 static inline int32_t
 411 ixgbe_pf_reset_hw(struct ixgbe_hw *hw)
 412 {
 413         uint32_t ctrl_ext;
 414         int32_t status;
 415
 416         status = ixgbe_reset_hw(hw);
 417
 418         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 419         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 420         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
 421         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 422         IXGBE_WRITE_FLUSH(hw);
 423
 424         return status;
 425 }
 426
 427 static inline void
 428 ixgbe_enable_intr(struct rte_eth_dev *dev)
 429 {
 430         struct ixgbe_interrupt *intr =
 431                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 432         struct ixgbe_hw *hw =
 433                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 434
 435         IXGBE_WRITE_REG(hw, IXGBE_EIMS, intr->mask);
 436         IXGBE_WRITE_FLUSH(hw);
 437 }
 438
 439 /*
 440  * This function is based on ixgbe_disable_intr() in ixgbe/ixgbe.h.
 441  */
 442 static void
 443 ixgbe_disable_intr(struct ixgbe_hw *hw)
 444 {
 445         PMD_INIT_FUNC_TRACE();
 446
 447         if (hw->mac.type == ixgbe_mac_82598EB) {
 448                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ~0);
 449         } else {
 450                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xFFFF0000);
 451                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), ~0);
 452                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), ~0);
 453         }
 454         IXGBE_WRITE_FLUSH(hw);
 455 }
 456
 457 /*
 458  * This function resets queue statistics mapping registers.
 459  * From Niantic datasheet, Initialization of Statistics section:
 460  * "...if software requires the queue counters, the RQSMR and TQSM registers
 461  * must be re-programmed following a device reset.
 462  */
 463 static void
 464 ixgbe_reset_qstat_mappings(struct ixgbe_hw *hw)
 465 {
 466         uint32_t i;
 467
 468         for(i = 0; i != IXGBE_NB_STAT_MAPPING_REGS; i++) {
 469                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), 0);
 470                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), 0);
 471         }
 472 }
 473
 474
 475 static int
 476 ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 477                                   uint16_t queue_id,
 478                                   uint8_t stat_idx,
 479                                   uint8_t is_rx)
 480 {
 481 #define QSM_REG_NB_BITS_PER_QMAP_FIELD 8
 482 #define NB_QMAP_FIELDS_PER_QSM_REG 4
 483 #define QMAP_FIELD_RESERVED_BITS_MASK 0x0f
 484
 485         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 486         struct ixgbe_stat_mapping_registers *stat_mappings =
 487                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(eth_dev->data->dev_private);
 488         uint32_t qsmr_mask = 0;
 489         uint32_t clearing_mask = QMAP_FIELD_RESERVED_BITS_MASK;
 490         uint32_t q_map;
 491         uint8_t n, offset;
 492
 493         if ((hw->mac.type != ixgbe_mac_82599EB) && (hw->mac.type != ixgbe_mac_X540))
 494                 return -ENOSYS;
 495
 496         PMD_INIT_LOG(INFO, "Setting port %d, %s queue_id %d to stat index %d\n",
 497                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx);
 498
 499         n = (uint8_t)(queue_id / NB_QMAP_FIELDS_PER_QSM_REG);
 500         if (n >= IXGBE_NB_STAT_MAPPING_REGS) {
 501                 PMD_INIT_LOG(ERR, "Nb of stat mapping registers exceeded\n");
 502                 return -EIO;
 503         }
 504         offset = (uint8_t)(queue_id % NB_QMAP_FIELDS_PER_QSM_REG);
 505
 506         /* Now clear any previous stat_idx set */
 507         clearing_mask <<= (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 508         if (!is_rx)
 509                 stat_mappings->tqsm[n] &= ~clearing_mask;
 510         else
 511                 stat_mappings->rqsmr[n] &= ~clearing_mask;
 512
 513         q_map = (uint32_t)stat_idx;
 514         q_map &= QMAP_FIELD_RESERVED_BITS_MASK;
 515         qsmr_mask = q_map << (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 516         if (!is_rx)
 517                 stat_mappings->tqsm[n] |= qsmr_mask;
 518         else
 519                 stat_mappings->rqsmr[n] |= qsmr_mask;
 520
 521         PMD_INIT_LOG(INFO, "Set port %d, %s queue_id %d to stat index %d\n"
 522                      "%s[%d] = 0x%08x\n",
 523                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx,
 524                      is_rx ? "RQSMR" : "TQSM",n, is_rx ? stat_mappings->rqsmr[n] : stat_mappings->tqsm[n]);
 525
 526         /* Now write the mapping in the appropriate register */
 527         if (is_rx) {
 528                 PMD_INIT_LOG(INFO, "Write 0x%x to RX IXGBE stat mapping reg:%d\n",
 529                              stat_mappings->rqsmr[n], n);
 530                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(n), stat_mappings->rqsmr[n]);
 531         }
 532         else {
 533                 PMD_INIT_LOG(INFO, "Write 0x%x to TX IXGBE stat mapping reg:%d\n",
 534                              stat_mappings->tqsm[n], n);
 535                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(n), stat_mappings->tqsm[n]);
 536         }
 537         return 0;
 538 }
 539
 540 static void
 541 ixgbe_restore_statistics_mapping(struct rte_eth_dev * dev)
 542 {
 543         struct ixgbe_stat_mapping_registers *stat_mappings =
 544                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(dev->data->dev_private);
 545         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 546         int i;
 547
 548         /* write whatever was in stat mapping table to the NIC */
 549         for (i = 0; i < IXGBE_NB_STAT_MAPPING_REGS; i++) {
 550                 /* rx */
 551                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), stat_mappings->rqsmr[i]);
 552
 553                 /* tx */
 554                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), stat_mappings->tqsm[i]);
 555         }
 556 }
 557
 558 static void
 559 ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config)
 560 {
 561         uint8_t i;
 562         struct ixgbe_dcb_tc_config *tc;
 563         uint8_t dcb_max_tc = IXGBE_DCB_MAX_TRAFFIC_CLASS;
 564
 565         dcb_config->num_tcs.pg_tcs = dcb_max_tc;
 566         dcb_config->num_tcs.pfc_tcs = dcb_max_tc;
 567         for (i = 0; i < dcb_max_tc; i++) {
 568                 tc = &dcb_config->tc_config[i];
 569                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_id = i;
 570                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
 571                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 572                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_id = i;
 573                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
 574                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 575                 tc->pfc = ixgbe_dcb_pfc_disabled;
 576         }
 577
 578         /* Initialize default user to priority mapping, UPx->TC0 */
 579         tc = &dcb_config->tc_config[0];
 580         tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
 581         tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
 582         for (i = 0; i< IXGBE_DCB_MAX_BW_GROUP; i++) {
 583                 dcb_config->bw_percentage[IXGBE_DCB_TX_CONFIG][i] = 100;
 584                 dcb_config->bw_percentage[IXGBE_DCB_RX_CONFIG][i] = 100;
 585         }
 586         dcb_config->rx_pba_cfg = ixgbe_dcb_pba_equal;
 587         dcb_config->pfc_mode_enable = false;
 588         dcb_config->vt_mode = true;
 589         dcb_config->round_robin_enable = false;
 590         /* support all DCB capabilities in 82599 */
 591         dcb_config->support.capabilities = 0xFF;
 592
 593         /*we only support 4 Tcs for X540*/
 594         if (hw->mac.type == ixgbe_mac_X540) {
 595                 dcb_config->num_tcs.pg_tcs = 4;
 596                 dcb_config->num_tcs.pfc_tcs = 4;
 597         }
 598 }
 599
 600 /*
 601  * Ensure that all locks are released before first NVM or PHY access
 602  */
 603 static void
 604 ixgbe_swfw_lock_reset(struct ixgbe_hw *hw)
 605 {
 606         uint16_t mask;
 607
 608         /*
 609          * Phy lock should not fail in this early stage. If this is the case,
 610          * it is due to an improper exit of the application.
 611          * So force the release of the faulty lock. Release of common lock
 612          * is done automatically by swfw_sync function.
 613          */
 614         mask = IXGBE_GSSR_PHY0_SM << hw->bus.func;
 615         if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) {
 616                    DEBUGOUT1("SWFW phy%d lock released", hw->bus.func);
 617         }
 618         ixgbe_release_swfw_semaphore(hw, mask);
 619
 620         /*
 621          * These ones are more tricky since they are common to all ports; but
 622          * swfw_sync retries last long enough (1s) to be almost sure that if
 623          * lock can not be taken it is due to an improper lock of the
 624          * semaphore.
 625          */
 626         mask = IXGBE_GSSR_EEP_SM | IXGBE_GSSR_MAC_CSR_SM | IXGBE_GSSR_SW_MNG_SM;
 627         if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) {
 628                    DEBUGOUT("SWFW common locks released");
 629         }
 630         ixgbe_release_swfw_semaphore(hw, mask);
 631 }
 632
 633 /*
 634  * This function is based on code in ixgbe_attach() in ixgbe/ixgbe.c.
 635  * It returns 0 on success.
 636  */
 637 static int
 638 eth_ixgbe_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 639                      struct rte_eth_dev *eth_dev)
 640 {
 641         struct rte_pci_device *pci_dev;
 642         struct ixgbe_hw *hw =
 643                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 644         struct ixgbe_vfta * shadow_vfta =
 645                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 646         struct ixgbe_hwstrip *hwstrip =
 647                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 648         struct ixgbe_dcb_config *dcb_config =
 649                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private);
 650         uint32_t ctrl_ext;
 651         uint16_t csum;
 652         int diag, i;
 653
 654         PMD_INIT_FUNC_TRACE();
 655
 656         eth_dev->dev_ops = &ixgbe_eth_dev_ops;
 657         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
 658         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
 659
 660         /* for secondary processes, we don't initialise any further as primary
 661          * has already done this work. Only check we don't need a different
 662          * RX function */
 663         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 664                 if (eth_dev->data->scattered_rx)
 665                         eth_dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
 666                 return 0;
 667         }
 668         pci_dev = eth_dev->pci_dev;
 669
 670         /* Vendor and Device ID need to be set before init of shared code */
 671         hw->device_id = pci_dev->id.device_id;
 672         hw->vendor_id = pci_dev->id.vendor_id;
 673         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 674 #ifdef RTE_LIBRTE_IXGBE_ALLOW_UNSUPPORTED_SFP
 675         hw->allow_unsupported_sfp = 1;
 676 #endif
 677
 678         /* Initialize the shared code */
 679 #ifdef RTE_NIC_BYPASS
 680         diag = ixgbe_bypass_init_shared_code(hw);
 681 #else
 682         diag = ixgbe_init_shared_code(hw);
 683 #endif /* RTE_NIC_BYPASS */
 684
 685         if (diag != IXGBE_SUCCESS) {
 686                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
 687                 return -EIO;
 688         }
 689
 690         /* pick up the PCI bus settings for reporting later */
 691         ixgbe_get_bus_info(hw);
 692
 693         /* Unlock any pending hardware semaphore */
 694         ixgbe_swfw_lock_reset(hw);
 695
 696         /* Initialize DCB configuration*/
 697         memset(dcb_config, 0, sizeof(struct ixgbe_dcb_config));
 698         ixgbe_dcb_init(hw,dcb_config);
 699         /* Get Hardware Flow Control setting */
 700         hw->fc.requested_mode = ixgbe_fc_full;
 701         hw->fc.current_mode = ixgbe_fc_full;
 702         hw->fc.pause_time = IXGBE_FC_PAUSE;
 703         for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
 704                 hw->fc.low_water[i] = IXGBE_FC_LO;
 705                 hw->fc.high_water[i] = IXGBE_FC_HI;
 706         }
 707         hw->fc.send_xon = 1;
 708
 709         /* Make sure we have a good EEPROM before we read from it */
 710         diag = ixgbe_validate_eeprom_checksum(hw, &csum);
 711         if (diag != IXGBE_SUCCESS) {
 712                 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag);
 713                 return -EIO;
 714         }
 715
 716 #ifdef RTE_NIC_BYPASS
 717         diag = ixgbe_bypass_init_hw(hw);
 718 #else
 719         diag = ixgbe_init_hw(hw);
 720 #endif /* RTE_NIC_BYPASS */
 721
 722         /*
 723          * Devices with copper phys will fail to initialise if ixgbe_init_hw()
 724          * is called too soon after the kernel driver unbinding/binding occurs.
 725          * The failure occurs in ixgbe_identify_phy_generic() for all devices,
 726          * but for non-copper devies, ixgbe_identify_sfp_module_generic() is
 727          * also called. See ixgbe_identify_phy_82599(). The reason for the
 728          * failure is not known, and only occuts when virtualisation features
 729          * are disabled in the bios. A delay of 100ms  was found to be enough by
 730          * trial-and-error, and is doubled to be safe.
 731          */
 732         if (diag && (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper)) {
 733                 rte_delay_ms(200);
 734                 diag = ixgbe_init_hw(hw);
 735         }
 736
 737         if (diag == IXGBE_ERR_EEPROM_VERSION) {
 738                 PMD_INIT_LOG(ERR, "This device is a pre-production adapter/"
 739                     "LOM.  Please be aware there may be issues associated "
 740                     "with your hardware.\n If you are experiencing problems "
 741                     "please contact your Intel or hardware representative "
 742                     "who provided you with this hardware.\n");
 743         } else if (diag == IXGBE_ERR_SFP_NOT_SUPPORTED)
 744                 PMD_INIT_LOG(ERR, "Unsupported SFP+ Module\n");
 745         if (diag) {
 746                 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag);
 747                 return -EIO;
 748         }
 749
 750         /* disable interrupt */
 751         ixgbe_disable_intr(hw);
 752
 753         /* reset mappings for queue statistics hw counters*/
 754         ixgbe_reset_qstat_mappings(hw);
 755
 756         /* Allocate memory for storing MAC addresses */
 757         eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 758                         hw->mac.num_rar_entries, 0);
 759         if (eth_dev->data->mac_addrs == NULL) {
 760                 PMD_INIT_LOG(ERR,
 761                         "Failed to allocate %u bytes needed to store "
 762                         "MAC addresses",
 763                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 764                 return -ENOMEM;
 765         }
 766         /* Copy the permanent MAC address */
 767         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 768                         &eth_dev->data->mac_addrs[0]);
 769
 770         /* Allocate memory for storing hash filter MAC addresses */
 771         eth_dev->data->hash_mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 772                         IXGBE_VMDQ_NUM_UC_MAC, 0);
 773         if (eth_dev->data->hash_mac_addrs == NULL) {
 774                 PMD_INIT_LOG(ERR,
 775                         "Failed to allocate %d bytes needed to store MAC addresses",
 776                         ETHER_ADDR_LEN * IXGBE_VMDQ_NUM_UC_MAC);
 777                 return -ENOMEM;
 778         }
 779
 780         /* initialize the vfta */
 781         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 782
 783         /* initialize the hw strip bitmap*/
 784         memset(hwstrip, 0, sizeof(*hwstrip));
 785
 786         /* initialize PF if max_vfs not zero */
 787         ixgbe_pf_host_init(eth_dev);
 788
 789         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 790         /* let hardware know driver is loaded */
 791         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 792         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 793         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
 794         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 795         IXGBE_WRITE_FLUSH(hw);
 796
 797         if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
 798                 PMD_INIT_LOG(DEBUG,
 799                              "MAC: %d, PHY: %d, SFP+: %d<n",
 800                              (int) hw->mac.type, (int) hw->phy.type,
 801                              (int) hw->phy.sfp_type);
 802         else
 803                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d\n",
 804                              (int) hw->mac.type, (int) hw->phy.type);
 805
 806         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
 807                         eth_dev->data->port_id, pci_dev->id.vendor_id,
 808                         pci_dev->id.device_id);
 809
 810         rte_intr_callback_register(&(pci_dev->intr_handle),
 811                 ixgbe_dev_interrupt_handler, (void *)eth_dev);
 812
 813         /* enable uio intr after callback register */
 814         rte_intr_enable(&(pci_dev->intr_handle));
 815
 816         /* enable support intr */
 817         ixgbe_enable_intr(eth_dev);
 818
 819         return 0;
 820 }
 821
 822
 823 /*
 824  * Negotiate mailbox API version with the PF.
 825  * After reset API version is always set to the basic one (ixgbe_mbox_api_10).
 826  * Then we try to negotiate starting with the most recent one.
 827  * If all negotiation attempts fail, then we will proceed with
 828  * the default one (ixgbe_mbox_api_10).
 829  */
 830 static void
 831 ixgbevf_negotiate_api(struct ixgbe_hw *hw)
 832 {
 833         int32_t i;
 834
 835         /* start with highest supported, proceed down */
 836         static const enum ixgbe_pfvf_api_rev sup_ver[] = {
 837                 ixgbe_mbox_api_11,
 838                 ixgbe_mbox_api_10,
 839         };
 840
 841         for (i = 0;
 842                         i != RTE_DIM(sup_ver) &&
 843                         ixgbevf_negotiate_api_version(hw, sup_ver[i]) != 0;
 844                         i++)
 845                 ;
 846 }
 847
 848 static void
 849 generate_random_mac_addr(struct ether_addr *mac_addr)
 850 {
 851         uint64_t random;
 852
 853         /* Set Organizationally Unique Identifier (OUI) prefix. */
 854         mac_addr->addr_bytes[0] = 0x00;
 855         mac_addr->addr_bytes[1] = 0x09;
 856         mac_addr->addr_bytes[2] = 0xC0;
 857         /* Force indication of locally assigned MAC address. */
 858         mac_addr->addr_bytes[0] |= ETHER_LOCAL_ADMIN_ADDR;
 859         /* Generate the last 3 bytes of the MAC address with a random number. */
 860         random = rte_rand();
 861         memcpy(&mac_addr->addr_bytes[3], &random, 3);
 862 }
 863
 864 /*
 865  * Virtual Function device init
 866  */
 867 static int
 868 eth_ixgbevf_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 869                      struct rte_eth_dev *eth_dev)
 870 {
 871         int diag;
 872         uint32_t tc, tcs;
 873         struct rte_pci_device *pci_dev;
 874         struct ixgbe_hw *hw =
 875                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 876         struct ixgbe_vfta * shadow_vfta =
 877                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 878         struct ixgbe_hwstrip *hwstrip =
 879                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 880         struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr;
 881
 882         PMD_INIT_LOG(DEBUG, "eth_ixgbevf_dev_init");
 883
 884         eth_dev->dev_ops = &ixgbevf_eth_dev_ops;
 885         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
 886         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
 887
 888         /* for secondary processes, we don't initialise any further as primary
 889          * has already done this work. Only check we don't need a different
 890          * RX function */
 891         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 892                 if (eth_dev->data->scattered_rx)
 893                         eth_dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
 894                 return 0;
 895         }
 896
 897         pci_dev = eth_dev->pci_dev;
 898
 899         hw->device_id = pci_dev->id.device_id;
 900         hw->vendor_id = pci_dev->id.vendor_id;
 901         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 902
 903         /* initialize the vfta */
 904         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 905
 906         /* initialize the hw strip bitmap*/
 907         memset(hwstrip, 0, sizeof(*hwstrip));
 908
 909         /* Initialize the shared code */
 910         diag = ixgbe_init_shared_code(hw);
 911         if (diag != IXGBE_SUCCESS) {
 912                 PMD_INIT_LOG(ERR, "Shared code init failed for ixgbevf: %d", diag);
 913                 return -EIO;
 914         }
 915
 916         /* init_mailbox_params */
 917         hw->mbx.ops.init_params(hw);
 918
 919         /* Disable the interrupts for VF */
 920         ixgbevf_intr_disable(hw);
 921
 922         hw->mac.num_rar_entries = 128; /* The MAX of the underlying PF */
 923         diag = hw->mac.ops.reset_hw(hw);
 924
 925         /*
 926          * The VF reset operation returns the IXGBE_ERR_INVALID_MAC_ADDR when
 927          * the underlying PF driver has not assigned a MAC address to the VF.
 928          * In this case, assign a random MAC address.
 929          */
 930         if ((diag != IXGBE_SUCCESS) && (diag != IXGBE_ERR_INVALID_MAC_ADDR)) {
 931                 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
 932                 return (diag);
 933         }
 934
 935         /* negotiate mailbox API version to use with the PF. */
 936         ixgbevf_negotiate_api(hw);
 937
 938         /* Get Rx/Tx queue count via mailbox, which is ready after reset_hw */
 939         ixgbevf_get_queues(hw, &tcs, &tc);
 940
 941         /* Allocate memory for storing MAC addresses */
 942         eth_dev->data->mac_addrs = rte_zmalloc("ixgbevf", ETHER_ADDR_LEN *
 943                         hw->mac.num_rar_entries, 0);
 944         if (eth_dev->data->mac_addrs == NULL) {
 945                 PMD_INIT_LOG(ERR,
 946                         "Failed to allocate %u bytes needed to store "
 947                         "MAC addresses",
 948                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 949                 return -ENOMEM;
 950         }
 951
 952         /* Generate a random MAC address, if none was assigned by PF. */
 953         if (is_zero_ether_addr(perm_addr)) {
 954                 generate_random_mac_addr(perm_addr);
 955                 diag = ixgbe_set_rar_vf(hw, 1, perm_addr->addr_bytes, 0, 1);
 956                 if (diag) {
 957                         rte_free(eth_dev->data->mac_addrs);
 958                         eth_dev->data->mac_addrs = NULL;
 959                         return diag;
 960                 }
 961                 RTE_LOG(INFO, PMD,
 962                         "\tVF MAC address not assigned by Host PF\n"
 963                         "\tAssign randomly generated MAC address "
 964                         "%02x:%02x:%02x:%02x:%02x:%02x\n",
 965                         perm_addr->addr_bytes[0],
 966                         perm_addr->addr_bytes[1],
 967                         perm_addr->addr_bytes[2],
 968                         perm_addr->addr_bytes[3],
 969                         perm_addr->addr_bytes[4],
 970                         perm_addr->addr_bytes[5]);
 971         }
 972
 973         /* Copy the permanent MAC address */
 974         ether_addr_copy(perm_addr, &eth_dev->data->mac_addrs[0]);
 975
 976         /* reset the hardware with the new settings */
 977         diag = hw->mac.ops.start_hw(hw);
 978         switch (diag) {
 979                 case  0:
 980                         break;
 981
 982                 default:
 983                         PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
 984                         return (-EIO);
 985         }
 986
 987         PMD_INIT_LOG(DEBUG, "\nport %d vendorID=0x%x deviceID=0x%x mac.type=%s\n",
 988                          eth_dev->data->port_id, pci_dev->id.vendor_id, pci_dev->id.device_id,
 989                          "ixgbe_mac_82599_vf");
 990
 991         return 0;
 992 }
 993
 994 static struct eth_driver rte_ixgbe_pmd = {
 995         {
 996                 .name = "rte_ixgbe_pmd",
 997                 .id_table = pci_id_ixgbe_map,
 998                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
 999         },
1000         .eth_dev_init = eth_ixgbe_dev_init,
1001         .dev_private_size = sizeof(struct ixgbe_adapter),
1002 };
1003
1004 /*
1005  * virtual function driver struct
1006  */
1007 static struct eth_driver rte_ixgbevf_pmd = {
1008         {
1009                 .name = "rte_ixgbevf_pmd",
1010                 .id_table = pci_id_ixgbevf_map,
1011                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
1012         },
1013         .eth_dev_init = eth_ixgbevf_dev_init,
1014         .dev_private_size = sizeof(struct ixgbe_adapter),
1015 };
1016
1017 /*
1018  * Driver initialization routine.
1019  * Invoked once at EAL init time.
1020  * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
1021  */
1022 static int
1023 rte_ixgbe_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
1024 {
1025         PMD_INIT_FUNC_TRACE();
1026
1027         rte_eth_driver_register(&rte_ixgbe_pmd);
1028         return 0;
1029 }
1030
1031 /*
1032  * VF Driver initialization routine.
1033  * Invoked one at EAL init time.
1034  * Register itself as the [Virtual Poll Mode] Driver of PCI niantic devices.
1035  */
1036 static int
1037 rte_ixgbevf_pmd_init(const char *name __rte_unused, const char *param __rte_unused)
1038 {
1039         DEBUGFUNC("rte_ixgbevf_pmd_init");
1040
1041         rte_eth_driver_register(&rte_ixgbevf_pmd);
1042         return (0);
1043 }
1044
1045 static int
1046 ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1047 {
1048         struct ixgbe_hw *hw =
1049                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1050         struct ixgbe_vfta * shadow_vfta =
1051                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1052         uint32_t vfta;
1053         uint32_t vid_idx;
1054         uint32_t vid_bit;
1055
1056         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
1057         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
1058         vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(vid_idx));
1059         if (on)
1060                 vfta |= vid_bit;
1061         else
1062                 vfta &= ~vid_bit;
1063         IXGBE_WRITE_REG(hw, IXGBE_VFTA(vid_idx), vfta);
1064
1065         /* update local VFTA copy */
1066         shadow_vfta->vfta[vid_idx] = vfta;
1067
1068         return 0;
1069 }
1070
1071 static void
1072 ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
1073 {
1074         if (on)
1075                 ixgbe_vlan_hw_strip_enable(dev, queue);
1076         else
1077                 ixgbe_vlan_hw_strip_disable(dev, queue);
1078 }
1079
1080 static void
1081 ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
1082 {
1083         struct ixgbe_hw *hw =
1084                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1085
1086         /* Only the high 16-bits is valid */
1087         IXGBE_WRITE_REG(hw, IXGBE_EXVET, tpid << 16);
1088 }
1089
1090 void
1091 ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev)
1092 {
1093         struct ixgbe_hw *hw =
1094                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1095         uint32_t vlnctrl;
1096
1097         PMD_INIT_FUNC_TRACE();
1098
1099         /* Filter Table Disable */
1100         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1101         vlnctrl &= ~IXGBE_VLNCTRL_VFE;
1102
1103         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
1104 }
1105
1106 void
1107 ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1108 {
1109         struct ixgbe_hw *hw =
1110                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1111         struct ixgbe_vfta * shadow_vfta =
1112                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1113         uint32_t vlnctrl;
1114         uint16_t i;
1115
1116         PMD_INIT_FUNC_TRACE();
1117
1118         /* Filter Table Enable */
1119         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1120         vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
1121         vlnctrl |= IXGBE_VLNCTRL_VFE;
1122
1123         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
1124
1125         /* write whatever is in local vfta copy */
1126         for (i = 0; i < IXGBE_VFTA_SIZE; i++)
1127                 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), shadow_vfta->vfta[i]);
1128 }
1129
1130 static void
1131 ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on)
1132 {
1133         struct ixgbe_hwstrip *hwstrip =
1134                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(dev->data->dev_private);
1135
1136         if(queue >= IXGBE_MAX_RX_QUEUE_NUM)
1137                 return;
1138
1139         if (on)
1140                 IXGBE_SET_HWSTRIP(hwstrip, queue);
1141         else
1142                 IXGBE_CLEAR_HWSTRIP(hwstrip, queue);
1143 }
1144
1145 static void
1146 ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue)
1147 {
1148         struct ixgbe_hw *hw =
1149                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1150         uint32_t ctrl;
1151
1152         PMD_INIT_FUNC_TRACE();
1153
1154         if (hw->mac.type == ixgbe_mac_82598EB) {
1155                 /* No queue level support */
1156                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
1157                 return;
1158         }
1159         else {
1160                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1161                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1162                 ctrl &= ~IXGBE_RXDCTL_VME;
1163                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
1164         }
1165         /* record those setting for HW strip per queue */
1166         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 0);
1167 }
1168
1169 static void
1170 ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue)
1171 {
1172         struct ixgbe_hw *hw =
1173                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1174         uint32_t ctrl;
1175
1176         PMD_INIT_FUNC_TRACE();
1177
1178         if (hw->mac.type == ixgbe_mac_82598EB) {
1179                 /* No queue level supported */
1180                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
1181                 return;
1182         }
1183         else {
1184                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1185                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1186                 ctrl |= IXGBE_RXDCTL_VME;
1187                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
1188         }
1189         /* record those setting for HW strip per queue */
1190         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 1);
1191 }
1192
1193 void
1194 ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev)
1195 {
1196         struct ixgbe_hw *hw =
1197                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1198         uint32_t ctrl;
1199         uint16_t i;
1200
1201         PMD_INIT_FUNC_TRACE();
1202
1203         if (hw->mac.type == ixgbe_mac_82598EB) {
1204                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1205                 ctrl &= ~IXGBE_VLNCTRL_VME;
1206                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1207         }
1208         else {
1209                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1210                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1211                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1212                         ctrl &= ~IXGBE_RXDCTL_VME;
1213                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1214
1215                         /* record those setting for HW strip per queue */
1216                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 0);
1217                 }
1218         }
1219 }
1220
1221 void
1222 ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev)
1223 {
1224         struct ixgbe_hw *hw =
1225                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1226         uint32_t ctrl;
1227         uint16_t i;
1228
1229         PMD_INIT_FUNC_TRACE();
1230
1231         if (hw->mac.type == ixgbe_mac_82598EB) {
1232                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1233                 ctrl |= IXGBE_VLNCTRL_VME;
1234                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1235         }
1236         else {
1237                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1238                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1239                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1240                         ctrl |= IXGBE_RXDCTL_VME;
1241                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1242
1243                         /* record those setting for HW strip per queue */
1244                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 1);
1245                 }
1246         }
1247 }
1248
1249 static void
1250 ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1251 {
1252         struct ixgbe_hw *hw =
1253                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1254         uint32_t ctrl;
1255
1256         PMD_INIT_FUNC_TRACE();
1257
1258         /* DMATXCTRL: Geric Double VLAN Disable */
1259         ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1260         ctrl &= ~IXGBE_DMATXCTL_GDV;
1261         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1262
1263         /* CTRL_EXT: Global Double VLAN Disable */
1264         ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1265         ctrl &= ~IXGBE_EXTENDED_VLAN;
1266         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1267
1268 }
1269
1270 static void
1271 ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1272 {
1273         struct ixgbe_hw *hw =
1274                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1275         uint32_t ctrl;
1276
1277         PMD_INIT_FUNC_TRACE();
1278
1279         /* DMATXCTRL: Geric Double VLAN Enable */
1280         ctrl  = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1281         ctrl |= IXGBE_DMATXCTL_GDV;
1282         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1283
1284         /* CTRL_EXT: Global Double VLAN Enable */
1285         ctrl  = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1286         ctrl |= IXGBE_EXTENDED_VLAN;
1287         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1288
1289         /*
1290          * VET EXT field in the EXVET register = 0x8100 by default
1291          * So no need to change. Same to VT field of DMATXCTL register
1292          */
1293 }
1294
1295 static void
1296 ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1297 {
1298         if(mask & ETH_VLAN_STRIP_MASK){
1299                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1300                         ixgbe_vlan_hw_strip_enable_all(dev);
1301                 else
1302                         ixgbe_vlan_hw_strip_disable_all(dev);
1303         }
1304
1305         if(mask & ETH_VLAN_FILTER_MASK){
1306                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1307                         ixgbe_vlan_hw_filter_enable(dev);
1308                 else
1309                         ixgbe_vlan_hw_filter_disable(dev);
1310         }
1311
1312         if(mask & ETH_VLAN_EXTEND_MASK){
1313                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1314                         ixgbe_vlan_hw_extend_enable(dev);
1315                 else
1316                         ixgbe_vlan_hw_extend_disable(dev);
1317         }
1318 }
1319
1320 static void
1321 ixgbe_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1322 {
1323         struct ixgbe_hw *hw =
1324                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1325         /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
1326         uint32_t vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1327         vlanctrl |= IXGBE_VLNCTRL_VFE ; /* enable vlan filters */
1328         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
1329 }
1330
1331 static int
1332 ixgbe_dev_configure(struct rte_eth_dev *dev)
1333 {
1334         struct ixgbe_interrupt *intr =
1335                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1336
1337         PMD_INIT_FUNC_TRACE();
1338
1339         /* set flag to update link status after init */
1340         intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1341
1342         return 0;
1343 }
1344
1345 /*
1346  * Configure device link speed and setup link.
1347  * It returns 0 on success.
1348  */
1349 static int
1350 ixgbe_dev_start(struct rte_eth_dev *dev)
1351 {
1352         struct ixgbe_hw *hw =
1353                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1354         int err, link_up = 0, negotiate = 0;
1355         uint32_t speed = 0;
1356         int mask = 0;
1357         int status;
1358
1359         PMD_INIT_FUNC_TRACE();
1360
1361         /* IXGBE devices don't support half duplex */
1362         if ((dev->data->dev_conf.link_duplex != ETH_LINK_AUTONEG_DUPLEX) &&
1363                         (dev->data->dev_conf.link_duplex != ETH_LINK_FULL_DUPLEX)) {
1364                 PMD_INIT_LOG(ERR, "Invalid link_duplex (%hu) for port %hhu\n",
1365                                 dev->data->dev_conf.link_duplex,
1366                                 dev->data->port_id);
1367                 return -EINVAL;
1368         }
1369
1370         /* stop adapter */
1371         hw->adapter_stopped = FALSE;
1372         ixgbe_stop_adapter(hw);
1373
1374         /* reinitialize adapter
1375          * this calls reset and start */
1376         status = ixgbe_pf_reset_hw(hw);
1377         if (status != 0)
1378                 return -1;
1379         hw->mac.ops.start_hw(hw);
1380
1381         /* configure PF module if SRIOV enabled */
1382         ixgbe_pf_host_configure(dev);
1383
1384         /* initialize transmission unit */
1385         ixgbe_dev_tx_init(dev);
1386
1387         /* This can fail when allocating mbufs for descriptor rings */
1388         err = ixgbe_dev_rx_init(dev);
1389         if (err) {
1390                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware\n");
1391                 goto error;
1392         }
1393
1394         ixgbe_dev_rxtx_start(dev);
1395
1396         if (ixgbe_is_sfp(hw) && hw->phy.multispeed_fiber) {
1397                 err = hw->mac.ops.setup_sfp(hw);
1398                 if (err)
1399                         goto error;
1400         }
1401
1402         /* Turn on the laser */
1403         ixgbe_enable_tx_laser(hw);
1404
1405         /* Skip link setup if loopback mode is enabled for 82599. */
1406         if (hw->mac.type == ixgbe_mac_82599EB &&
1407                         dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_82599_TX_RX)
1408                 goto skip_link_setup;
1409
1410         err = ixgbe_check_link(hw, &speed, &link_up, 0);
1411         if (err)
1412                 goto error;
1413         err = ixgbe_get_link_capabilities(hw, &speed, &negotiate);
1414         if (err)
1415                 goto error;
1416
1417         switch(dev->data->dev_conf.link_speed) {
1418         case ETH_LINK_SPEED_AUTONEG:
1419                 speed = (hw->mac.type != ixgbe_mac_82598EB) ?
1420                                 IXGBE_LINK_SPEED_82599_AUTONEG :
1421                                 IXGBE_LINK_SPEED_82598_AUTONEG;
1422                 break;
1423         case ETH_LINK_SPEED_100:
1424                 /*
1425                  * Invalid for 82598 but error will be detected by
1426                  * ixgbe_setup_link()
1427                  */
1428                 speed = IXGBE_LINK_SPEED_100_FULL;
1429                 break;
1430         case ETH_LINK_SPEED_1000:
1431                 speed = IXGBE_LINK_SPEED_1GB_FULL;
1432                 break;
1433         case ETH_LINK_SPEED_10000:
1434                 speed = IXGBE_LINK_SPEED_10GB_FULL;
1435                 break;
1436         default:
1437                 PMD_INIT_LOG(ERR, "Invalid link_speed (%hu) for port %hhu\n",
1438                                 dev->data->dev_conf.link_speed,
1439                                 dev->data->port_id);
1440                 goto error;
1441         }
1442
1443         err = ixgbe_setup_link(hw, speed, negotiate, link_up);
1444         if (err)
1445                 goto error;
1446
1447 skip_link_setup:
1448
1449         /* check if lsc interrupt is enabled */
1450         if (dev->data->dev_conf.intr_conf.lsc != 0)
1451                 ixgbe_dev_lsc_interrupt_setup(dev);
1452
1453         /* resume enabled intr since hw reset */
1454         ixgbe_enable_intr(dev);
1455
1456         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
1457                 ETH_VLAN_EXTEND_MASK;
1458         ixgbe_vlan_offload_set(dev, mask);
1459
1460         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
1461                 /* Enable vlan filtering for VMDq */
1462                 ixgbe_vmdq_vlan_hw_filter_enable(dev);
1463         }
1464
1465         /* Configure DCB hw */
1466         ixgbe_configure_dcb(dev);
1467
1468         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) {
1469                 err = ixgbe_fdir_configure(dev);
1470                 if (err)
1471                         goto error;
1472         }
1473
1474         ixgbe_restore_statistics_mapping(dev);
1475
1476         return (0);
1477
1478 error:
1479         PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err);
1480         ixgbe_dev_clear_queues(dev);
1481         return -EIO;
1482 }
1483
1484 /*
1485  * Stop device: disable rx and tx functions to allow for reconfiguring.
1486  */
1487 static void
1488 ixgbe_dev_stop(struct rte_eth_dev *dev)
1489 {
1490         struct rte_eth_link link;
1491         struct ixgbe_hw *hw =
1492                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1493         struct ixgbe_vf_info *vfinfo =
1494                 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private);
1495         int vf;
1496
1497         PMD_INIT_FUNC_TRACE();
1498
1499         /* disable interrupts */
1500         ixgbe_disable_intr(hw);
1501
1502         /* reset the NIC */
1503         ixgbe_pf_reset_hw(hw);
1504         hw->adapter_stopped = FALSE;
1505
1506         /* stop adapter */
1507         ixgbe_stop_adapter(hw);
1508
1509         for (vf = 0; vfinfo != NULL &&
1510                      vf < dev->pci_dev->max_vfs; vf++)
1511                 vfinfo[vf].clear_to_send = false;
1512
1513         /* Turn off the laser */
1514         ixgbe_disable_tx_laser(hw);
1515
1516         ixgbe_dev_clear_queues(dev);
1517
1518         /* Clear recorded link status */
1519         memset(&link, 0, sizeof(link));
1520         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1521 }
1522
1523 /*
1524  * Reest and stop device.
1525  */
1526 static void
1527 ixgbe_dev_close(struct rte_eth_dev *dev)
1528 {
1529         struct ixgbe_hw *hw =
1530                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1531
1532         PMD_INIT_FUNC_TRACE();
1533
1534         ixgbe_pf_reset_hw(hw);
1535
1536         ixgbe_dev_stop(dev);
1537         hw->adapter_stopped = 1;
1538
1539         ixgbe_disable_pcie_master(hw);
1540
1541         /* reprogram the RAR[0] in case user changed it. */
1542         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
1543 }
1544
1545 /*
1546  * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c
1547  */
1548 static void
1549 ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1550 {
1551         struct ixgbe_hw *hw =
1552                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1553         struct ixgbe_hw_stats *hw_stats =
1554                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1555         uint32_t bprc, lxon, lxoff, total;
1556         uint64_t total_missed_rx, total_qbrc, total_qprc;
1557         unsigned i;
1558
1559         total_missed_rx = 0;
1560         total_qbrc = 0;
1561         total_qprc = 0;
1562
1563         hw_stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
1564         hw_stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
1565         hw_stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
1566         hw_stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
1567
1568         for (i = 0; i < 8; i++) {
1569                 uint32_t mp;
1570                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
1571                 /* global total per queue */
1572                 hw_stats->mpc[i] += mp;
1573                 /* Running comprehensive total for stats display */
1574                 total_missed_rx += hw_stats->mpc[i];
1575                 if (hw->mac.type == ixgbe_mac_82598EB)
1576                         hw_stats->rnbc[i] +=
1577                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
1578                 hw_stats->pxontxc[i] +=
1579                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
1580                 hw_stats->pxonrxc[i] +=
1581                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
1582                 hw_stats->pxofftxc[i] +=
1583                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
1584                 hw_stats->pxoffrxc[i] +=
1585                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
1586                 hw_stats->pxon2offc[i] +=
1587                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
1588         }
1589         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1590                 hw_stats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
1591                 hw_stats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
1592                 hw_stats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
1593                 hw_stats->qbrc[i] +=
1594                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32);
1595                 hw_stats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
1596                 hw_stats->qbtc[i] +=
1597                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32);
1598                 hw_stats->qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
1599
1600                 total_qprc += hw_stats->qprc[i];
1601                 total_qbrc += hw_stats->qbrc[i];
1602         }
1603         hw_stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
1604         hw_stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
1605         hw_stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
1606
1607         /* Note that gprc counts missed packets */
1608         hw_stats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
1609
1610         if (hw->mac.type != ixgbe_mac_82598EB) {
1611                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
1612                 hw_stats->gorc += ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
1613                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
1614                 hw_stats->gotc += ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
1615                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
1616                 hw_stats->tor += ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
1617                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
1618                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
1619         } else {
1620                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
1621                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
1622                 /* 82598 only has a counter in the high register */
1623                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
1624                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
1625                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
1626         }
1627
1628         /*
1629          * Workaround: mprc hardware is incorrectly counting
1630          * broadcasts, so for now we subtract those.
1631          */
1632         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
1633         hw_stats->bprc += bprc;
1634         hw_stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
1635         if (hw->mac.type == ixgbe_mac_82598EB)
1636                 hw_stats->mprc -= bprc;
1637
1638         hw_stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
1639         hw_stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
1640         hw_stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
1641         hw_stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
1642         hw_stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
1643         hw_stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
1644
1645         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
1646         hw_stats->lxontxc += lxon;
1647         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
1648         hw_stats->lxofftxc += lxoff;
1649         total = lxon + lxoff;
1650
1651         hw_stats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
1652         hw_stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
1653         hw_stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
1654         hw_stats->gptc -= total;
1655         hw_stats->mptc -= total;
1656         hw_stats->ptc64 -= total;
1657         hw_stats->gotc -= total * ETHER_MIN_LEN;
1658
1659         hw_stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
1660         hw_stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
1661         hw_stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
1662         hw_stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
1663         hw_stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
1664         hw_stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
1665         hw_stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
1666         hw_stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
1667         hw_stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
1668         hw_stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
1669         hw_stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
1670         hw_stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
1671         hw_stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
1672         hw_stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
1673         hw_stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
1674         hw_stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);
1675         hw_stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
1676         hw_stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
1677         /* Only read FCOE on 82599 */
1678         if (hw->mac.type != ixgbe_mac_82598EB) {
1679                 hw_stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
1680                 hw_stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
1681                 hw_stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
1682                 hw_stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
1683                 hw_stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
1684         }
1685
1686         if (stats == NULL)
1687                 return;
1688
1689         /* Fill out the rte_eth_stats statistics structure */
1690         stats->ipackets = total_qprc;
1691         stats->ibytes = total_qbrc;
1692         stats->opackets = hw_stats->gptc;
1693         stats->obytes = hw_stats->gotc;
1694         stats->imcasts = hw_stats->mprc;
1695
1696         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1697                 stats->q_ipackets[i] = hw_stats->qprc[i];
1698                 stats->q_opackets[i] = hw_stats->qptc[i];
1699                 stats->q_ibytes[i] = hw_stats->qbrc[i];
1700                 stats->q_obytes[i] = hw_stats->qbtc[i];
1701                 stats->q_errors[i] = hw_stats->qprdc[i];
1702         }
1703
1704         /* Rx Errors */
1705         stats->ierrors = total_missed_rx + hw_stats->crcerrs +
1706                 hw_stats->rlec;
1707
1708         stats->oerrors  = 0;
1709
1710         /* XON/XOFF pause frames */
1711         stats->tx_pause_xon  = hw_stats->lxontxc;
1712         stats->rx_pause_xon  = hw_stats->lxonrxc;
1713         stats->tx_pause_xoff = hw_stats->lxofftxc;
1714         stats->rx_pause_xoff = hw_stats->lxoffrxc;
1715
1716         /* Flow Director Stats registers */
1717         hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
1718         hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
1719         stats->fdirmatch = hw_stats->fdirmatch;
1720         stats->fdirmiss = hw_stats->fdirmiss;
1721 }
1722
1723 static void
1724 ixgbe_dev_stats_reset(struct rte_eth_dev *dev)
1725 {
1726         struct ixgbe_hw_stats *stats =
1727                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1728
1729         /* HW registers are cleared on read */
1730         ixgbe_dev_stats_get(dev, NULL);
1731
1732         /* Reset software totals */
1733         memset(stats, 0, sizeof(*stats));
1734 }
1735
1736 static void
1737 ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1738 {
1739         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1740         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1741                           IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1742
1743         /* Good Rx packet, include VF loopback */
1744         UPDATE_VF_STAT(IXGBE_VFGPRC,
1745             hw_stats->last_vfgprc, hw_stats->vfgprc);
1746
1747         /* Good Rx octets, include VF loopback */
1748         UPDATE_VF_STAT_36BIT(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
1749             hw_stats->last_vfgorc, hw_stats->vfgorc);
1750
1751         /* Good Tx packet, include VF loopback */
1752         UPDATE_VF_STAT(IXGBE_VFGPTC,
1753             hw_stats->last_vfgptc, hw_stats->vfgptc);
1754
1755         /* Good Tx octets, include VF loopback */
1756         UPDATE_VF_STAT_36BIT(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
1757             hw_stats->last_vfgotc, hw_stats->vfgotc);
1758
1759         /* Rx Multicst Packet */
1760         UPDATE_VF_STAT(IXGBE_VFMPRC,
1761             hw_stats->last_vfmprc, hw_stats->vfmprc);
1762
1763         if (stats == NULL)
1764                 return;
1765
1766         memset(stats, 0, sizeof(*stats));
1767         stats->ipackets = hw_stats->vfgprc;
1768         stats->ibytes = hw_stats->vfgorc;
1769         stats->opackets = hw_stats->vfgptc;
1770         stats->obytes = hw_stats->vfgotc;
1771         stats->imcasts = hw_stats->vfmprc;
1772 }
1773
1774 static void
1775 ixgbevf_dev_stats_reset(struct rte_eth_dev *dev)
1776 {
1777         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1778                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1779
1780         /* Sync HW register to the last stats */
1781         ixgbevf_dev_stats_get(dev, NULL);
1782
1783         /* reset HW current stats*/
1784         hw_stats->vfgprc = 0;
1785         hw_stats->vfgorc = 0;
1786         hw_stats->vfgptc = 0;
1787         hw_stats->vfgotc = 0;
1788         hw_stats->vfmprc = 0;
1789
1790 }
1791
1792 static void
1793 ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1794 {
1795         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1796
1797         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
1798         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
1799         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL register */
1800         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */
1801         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
1802         dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC;
1803         dev_info->max_vfs = dev->pci_dev->max_vfs;
1804         if (hw->mac.type == ixgbe_mac_82598EB)
1805                 dev_info->max_vmdq_pools = ETH_16_POOLS;
1806         else
1807                 dev_info->max_vmdq_pools = ETH_64_POOLS;
1808         dev_info->rx_offload_capa =
1809                 DEV_RX_OFFLOAD_VLAN_STRIP |
1810                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1811                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1812                 DEV_RX_OFFLOAD_TCP_CKSUM;
1813         dev_info->tx_offload_capa =
1814                 DEV_TX_OFFLOAD_VLAN_INSERT |
1815                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1816                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1817                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1818                 DEV_TX_OFFLOAD_SCTP_CKSUM;
1819 }
1820
1821 /* return 0 means link status changed, -1 means not changed */
1822 static int
1823 ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1824 {
1825         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1826         struct rte_eth_link link, old;
1827         ixgbe_link_speed link_speed;
1828         int link_up;
1829         int diag;
1830
1831         link.link_status = 0;
1832         link.link_speed = 0;
1833         link.link_duplex = 0;
1834         memset(&old, 0, sizeof(old));
1835         rte_ixgbe_dev_atomic_read_link_status(dev, &old);
1836
1837         /* check if it needs to wait to complete, if lsc interrupt is enabled */
1838         if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
1839                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 0);
1840         else
1841                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 1);
1842         if (diag != 0) {
1843                 link.link_speed = ETH_LINK_SPEED_100;
1844                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1845                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1846                 if (link.link_status == old.link_status)
1847                         return -1;
1848                 return 0;
1849         }
1850
1851         if (link_up == 0) {
1852                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1853                 if (link.link_status == old.link_status)
1854                         return -1;
1855                 return 0;
1856         }
1857         link.link_status = 1;
1858         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1859
1860         switch (link_speed) {
1861         default:
1862         case IXGBE_LINK_SPEED_UNKNOWN:
1863                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1864                 link.link_speed = ETH_LINK_SPEED_100;
1865                 break;
1866
1867         case IXGBE_LINK_SPEED_100_FULL:
1868                 link.link_speed = ETH_LINK_SPEED_100;
1869                 break;
1870
1871         case IXGBE_LINK_SPEED_1GB_FULL:
1872                 link.link_speed = ETH_LINK_SPEED_1000;
1873                 break;
1874
1875         case IXGBE_LINK_SPEED_10GB_FULL:
1876                 link.link_speed = ETH_LINK_SPEED_10000;
1877                 break;
1878         }
1879         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1880
1881         if (link.link_status == old.link_status)
1882                 return -1;
1883
1884         return 0;
1885 }
1886
1887 static void
1888 ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev)
1889 {
1890         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1891         uint32_t fctrl;
1892
1893         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1894         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1895         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1896 }
1897
1898 static void
1899 ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev)
1900 {
1901         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1902         uint32_t fctrl;
1903
1904         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1905         fctrl &= (~IXGBE_FCTRL_UPE);
1906         if (dev->data->all_multicast == 1)
1907                 fctrl |= IXGBE_FCTRL_MPE;
1908         else
1909                 fctrl &= (~IXGBE_FCTRL_MPE);
1910         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1911 }
1912
1913 static void
1914 ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev)
1915 {
1916         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1917         uint32_t fctrl;
1918
1919         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1920         fctrl |= IXGBE_FCTRL_MPE;
1921         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1922 }
1923
1924 static void
1925 ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev)
1926 {
1927         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1928         uint32_t fctrl;
1929
1930         if (dev->data->promiscuous == 1)
1931                 return; /* must remain in all_multicast mode */
1932
1933         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1934         fctrl &= (~IXGBE_FCTRL_MPE);
1935         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1936 }
1937
1938 /**
1939  * It clears the interrupt causes and enables the interrupt.
1940  * It will be called once only during nic initialized.
1941  *
1942  * @param dev
1943  *  Pointer to struct rte_eth_dev.
1944  *
1945  * @return
1946  *  - On success, zero.
1947  *  - On failure, a negative value.
1948  */
1949 static int
1950 ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev)
1951 {
1952         struct ixgbe_interrupt *intr =
1953                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1954
1955         ixgbe_dev_link_status_print(dev);
1956         intr->mask |= IXGBE_EICR_LSC;
1957
1958         return 0;
1959 }
1960
1961 /*
1962  * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update.
1963  *
1964  * @param dev
1965  *  Pointer to struct rte_eth_dev.
1966  *
1967  * @return
1968  *  - On success, zero.
1969  *  - On failure, a negative value.
1970  */
1971 static int
1972 ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
1973 {
1974         uint32_t eicr;
1975         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1976         struct ixgbe_interrupt *intr =
1977                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1978
1979         /* clear all cause mask */
1980         ixgbe_disable_intr(hw);
1981
1982         /* read-on-clear nic registers here */
1983         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1984         PMD_DRV_LOG(INFO, "eicr %x", eicr);
1985
1986         intr->flags = 0;
1987         if (eicr & IXGBE_EICR_LSC) {
1988                 /* set flag for async link update */
1989                 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1990         }
1991
1992         if (eicr & IXGBE_EICR_MAILBOX)
1993                 intr->flags |= IXGBE_FLAG_MAILBOX;
1994
1995         return 0;
1996 }
1997
1998 /**
1999  * It gets and then prints the link status.
2000  *
2001  * @param dev
2002  *  Pointer to struct rte_eth_dev.
2003  *
2004  * @return
2005  *  - On success, zero.
2006  *  - On failure, a negative value.
2007  */
2008 static void
2009 ixgbe_dev_link_status_print(struct rte_eth_dev *dev)
2010 {
2011         struct rte_eth_link link;
2012
2013         memset(&link, 0, sizeof(link));
2014         rte_ixgbe_dev_atomic_read_link_status(dev, &link);
2015         if (link.link_status) {
2016                 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
2017                                         (int)(dev->data->port_id),
2018                                         (unsigned)link.link_speed,
2019                         link.link_duplex == ETH_LINK_FULL_DUPLEX ?
2020                                         "full-duplex" : "half-duplex");
2021         } else {
2022                 PMD_INIT_LOG(INFO, " Port %d: Link Down",
2023                                 (int)(dev->data->port_id));
2024         }
2025         PMD_INIT_LOG(INFO, "PCI Address: %04d:%02d:%02d:%d",
2026                                 dev->pci_dev->addr.domain,
2027                                 dev->pci_dev->addr.bus,
2028                                 dev->pci_dev->addr.devid,
2029                                 dev->pci_dev->addr.function);
2030 }
2031
2032 /*
2033  * It executes link_update after knowing an interrupt occurred.
2034  *
2035  * @param dev
2036  *  Pointer to struct rte_eth_dev.
2037  *
2038  * @return
2039  *  - On success, zero.
2040  *  - On failure, a negative value.
2041  */
2042 static int
2043 ixgbe_dev_interrupt_action(struct rte_eth_dev *dev)
2044 {
2045         struct ixgbe_interrupt *intr =
2046                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2047         int64_t timeout;
2048         struct rte_eth_link link;
2049         int intr_enable_delay = false;
2050
2051         PMD_DRV_LOG(DEBUG, "intr action type %d\n", intr->flags);
2052
2053         if (intr->flags & IXGBE_FLAG_MAILBOX) {
2054                 ixgbe_pf_mbx_process(dev);
2055                 intr->flags &= ~IXGBE_FLAG_MAILBOX;
2056         }
2057
2058         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
2059                 /* get the link status before link update, for predicting later */
2060                 memset(&link, 0, sizeof(link));
2061                 rte_ixgbe_dev_atomic_read_link_status(dev, &link);
2062
2063                 ixgbe_dev_link_update(dev, 0);
2064
2065                 /* likely to up */
2066                 if (!link.link_status)
2067                         /* handle it 1 sec later, wait it being stable */
2068                         timeout = IXGBE_LINK_UP_CHECK_TIMEOUT;
2069                 /* likely to down */
2070                 else
2071                         /* handle it 4 sec later, wait it being stable */
2072                         timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT;
2073
2074                 ixgbe_dev_link_status_print(dev);
2075
2076                 intr_enable_delay = true;
2077         }
2078
2079         if (intr_enable_delay) {
2080                 if (rte_eal_alarm_set(timeout * 1000,
2081                                       ixgbe_dev_interrupt_delayed_handler, (void*)dev) < 0)
2082                         PMD_DRV_LOG(ERR, "Error setting alarm");
2083         } else {
2084                 PMD_DRV_LOG(DEBUG, "enable intr immediately");
2085                 ixgbe_enable_intr(dev);
2086                 rte_intr_enable(&(dev->pci_dev->intr_handle));
2087         }
2088
2089
2090         return 0;
2091 }
2092
2093 /**
2094  * Interrupt handler which shall be registered for alarm callback for delayed
2095  * handling specific interrupt to wait for the stable nic state. As the
2096  * NIC interrupt state is not stable for ixgbe after link is just down,
2097  * it needs to wait 4 seconds to get the stable status.
2098  *
2099  * @param handle
2100  *  Pointer to interrupt handle.
2101  * @param param
2102  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2103  *
2104  * @return
2105  *  void
2106  */
2107 static void
2108 ixgbe_dev_interrupt_delayed_handler(void *param)
2109 {
2110         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2111         struct ixgbe_interrupt *intr =
2112                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2113         struct ixgbe_hw *hw =
2114                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2115         uint32_t eicr;
2116
2117         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
2118         if (eicr & IXGBE_EICR_MAILBOX)
2119                 ixgbe_pf_mbx_process(dev);
2120
2121         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
2122                 ixgbe_dev_link_update(dev, 0);
2123                 intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
2124                 ixgbe_dev_link_status_print(dev);
2125                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
2126         }
2127
2128         PMD_DRV_LOG(DEBUG, "enable intr in delayed handler S[%08x]\n", eicr);
2129         ixgbe_enable_intr(dev);
2130         rte_intr_enable(&(dev->pci_dev->intr_handle));
2131 }
2132
2133 /**
2134  * Interrupt handler triggered by NIC  for handling
2135  * specific interrupt.
2136  *
2137  * @param handle
2138  *  Pointer to interrupt handle.
2139  * @param param
2140  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2141  *
2142  * @return
2143  *  void
2144  */
2145 static void
2146 ixgbe_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
2147                                                         void *param)
2148 {
2149         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2150         ixgbe_dev_interrupt_get_status(dev);
2151         ixgbe_dev_interrupt_action(dev);
2152 }
2153
2154 static int
2155 ixgbe_dev_led_on(struct rte_eth_dev *dev)
2156 {
2157         struct ixgbe_hw *hw;
2158
2159         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2160         return (ixgbe_led_on(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
2161 }
2162
2163 static int
2164 ixgbe_dev_led_off(struct rte_eth_dev *dev)
2165 {
2166         struct ixgbe_hw *hw;
2167
2168         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2169         return (ixgbe_led_off(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
2170 }
2171
2172 static int
2173 ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2174 {
2175         struct ixgbe_hw *hw;
2176         int err;
2177         uint32_t rx_buf_size;
2178         uint32_t max_high_water;
2179         uint32_t mflcn;
2180         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
2181                 ixgbe_fc_none,
2182                 ixgbe_fc_rx_pause,
2183                 ixgbe_fc_tx_pause,
2184                 ixgbe_fc_full
2185         };
2186
2187         PMD_INIT_FUNC_TRACE();
2188
2189         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2190         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0));
2191         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
2192
2193         /*
2194          * At least reserve one Ethernet frame for watermark
2195          * high_water/low_water in kilo bytes for ixgbe
2196          */
2197         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
2198         if ((fc_conf->high_water > max_high_water) ||
2199                 (fc_conf->high_water < fc_conf->low_water)) {
2200                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
2201                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
2202                 return (-EINVAL);
2203         }
2204
2205         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[fc_conf->mode];
2206         hw->fc.pause_time     = fc_conf->pause_time;
2207         hw->fc.high_water[0]  = fc_conf->high_water;
2208         hw->fc.low_water[0]   = fc_conf->low_water;
2209         hw->fc.send_xon       = fc_conf->send_xon;
2210
2211         err = ixgbe_fc_enable(hw);
2212
2213         /* Not negotiated is not an error case */
2214         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) {
2215
2216                 /* check if we want to forward MAC frames - driver doesn't have native
2217                  * capability to do that, so we'll write the registers ourselves */
2218
2219                 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN);
2220
2221                 /* set or clear MFLCN.PMCF bit depending on configuration */
2222                 if (fc_conf->mac_ctrl_frame_fwd != 0)
2223                         mflcn |= IXGBE_MFLCN_PMCF;
2224                 else
2225                         mflcn &= ~IXGBE_MFLCN_PMCF;
2226
2227                 IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn);
2228                 IXGBE_WRITE_FLUSH(hw);
2229
2230                 return 0;
2231         }
2232
2233         PMD_INIT_LOG(ERR, "ixgbe_fc_enable = 0x%x \n", err);
2234         return -EIO;
2235 }
2236
2237 /**
2238  *  ixgbe_pfc_enable_generic - Enable flow control
2239  *  @hw: pointer to hardware structure
2240  *  @tc_num: traffic class number
2241  *  Enable flow control according to the current settings.
2242  */
2243 static int
2244 ixgbe_dcb_pfc_enable_generic(struct ixgbe_hw *hw,uint8_t tc_num)
2245 {
2246         int ret_val = 0;
2247         uint32_t mflcn_reg, fccfg_reg;
2248         uint32_t reg;
2249         uint32_t fcrtl, fcrth;
2250         uint8_t i;
2251         uint8_t nb_rx_en;
2252
2253         /* Validate the water mark configuration */
2254         if (!hw->fc.pause_time) {
2255                 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2256                 goto out;
2257         }
2258
2259         /* Low water mark of zero causes XOFF floods */
2260         if (hw->fc.current_mode & ixgbe_fc_tx_pause) {
2261                  /* High/Low water can not be 0 */
2262                 if( (!hw->fc.high_water[tc_num])|| (!hw->fc.low_water[tc_num])) {
2263                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
2264                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2265                         goto out;
2266                 }
2267
2268                 if(hw->fc.low_water[tc_num] >= hw->fc.high_water[tc_num]) {
2269                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
2270                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2271                         goto out;
2272                 }
2273         }
2274         /* Negotiate the fc mode to use */
2275         ixgbe_fc_autoneg(hw);
2276
2277         /* Disable any previous flow control settings */
2278         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
2279         mflcn_reg &= ~(IXGBE_MFLCN_RPFCE_SHIFT | IXGBE_MFLCN_RFCE|IXGBE_MFLCN_RPFCE);
2280
2281         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
2282         fccfg_reg &= ~(IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY);
2283
2284         switch (hw->fc.current_mode) {
2285         case ixgbe_fc_none:
2286                 /*
2287                  * If the count of enabled RX Priority Flow control >1,
2288                  * and the TX pause can not be disabled
2289                  */
2290                 nb_rx_en = 0;
2291                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
2292                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
2293                         if (reg & IXGBE_FCRTH_FCEN)
2294                                 nb_rx_en++;
2295                 }
2296                 if (nb_rx_en > 1)
2297                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2298                 break;
2299         case ixgbe_fc_rx_pause:
2300                 /*
2301                  * Rx Flow control is enabled and Tx Flow control is
2302                  * disabled by software override. Since there really
2303                  * isn't a way to advertise that we are capable of RX
2304                  * Pause ONLY, we will advertise that we support both
2305                  * symmetric and asymmetric Rx PAUSE.  Later, we will
2306                  * disable the adapter's ability to send PAUSE frames.
2307                  */
2308                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
2309                 /*
2310                  * If the count of enabled RX Priority Flow control >1,
2311                  * and the TX pause can not be disabled
2312                  */
2313                 nb_rx_en = 0;
2314                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
2315                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
2316                         if (reg & IXGBE_FCRTH_FCEN)
2317                                 nb_rx_en++;
2318                 }
2319                 if (nb_rx_en > 1)
2320                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2321                 break;
2322         case ixgbe_fc_tx_pause:
2323                 /*
2324                  * Tx Flow control is enabled, and Rx Flow control is
2325                  * disabled by software override.
2326                  */
2327                 fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2328                 break;
2329         case ixgbe_fc_full:
2330                 /* Flow control (both Rx and Tx) is enabled by SW override. */
2331                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
2332                 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
2333                 break;
2334         default:
2335                 DEBUGOUT("Flow control param set incorrectly\n");
2336                 ret_val = IXGBE_ERR_CONFIG;
2337                 goto out;
2338                 break;
2339         }
2340
2341         /* Set 802.3x based flow control settings. */
2342         mflcn_reg |= IXGBE_MFLCN_DPF;
2343         IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg);
2344         IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg);
2345
2346         /* Set up and enable Rx high/low water mark thresholds, enable XON. */
2347         if ((hw->fc.current_mode & ixgbe_fc_tx_pause) &&
2348                 hw->fc.high_water[tc_num]) {
2349                 fcrtl = (hw->fc.low_water[tc_num] << 10) | IXGBE_FCRTL_XONE;
2350                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), fcrtl);
2351                 fcrth = (hw->fc.high_water[tc_num] << 10) | IXGBE_FCRTH_FCEN;
2352         } else {
2353                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), 0);
2354                 /*
2355                  * In order to prevent Tx hangs when the internal Tx
2356                  * switch is enabled we must set the high water mark
2357                  * to the maximum FCRTH value.  This allows the Tx
2358                  * switch to function even under heavy Rx workloads.
2359                  */
2360                 fcrth = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)) - 32;
2361         }
2362         IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(tc_num), fcrth);
2363
2364         /* Configure pause time (2 TCs per register) */
2365         reg = hw->fc.pause_time * 0x00010001;
2366         for (i = 0; i < (IXGBE_DCB_MAX_TRAFFIC_CLASS / 2); i++)
2367                 IXGBE_WRITE_REG(hw, IXGBE_FCTTV(i), reg);
2368
2369         /* Configure flow control refresh threshold value */
2370         IXGBE_WRITE_REG(hw, IXGBE_FCRTV, hw->fc.pause_time / 2);
2371
2372 out:
2373         return ret_val;
2374 }
2375
2376 static int
2377 ixgbe_dcb_pfc_enable(struct rte_eth_dev *dev,uint8_t tc_num)
2378 {
2379         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2380         int32_t ret_val = IXGBE_NOT_IMPLEMENTED;
2381
2382         if(hw->mac.type != ixgbe_mac_82598EB) {
2383                 ret_val = ixgbe_dcb_pfc_enable_generic(hw,tc_num);
2384         }
2385         return ret_val;
2386 }
2387
2388 static int
2389 ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf)
2390 {
2391         int err;
2392         uint32_t rx_buf_size;
2393         uint32_t max_high_water;
2394         uint8_t tc_num;
2395         uint8_t  map[IXGBE_DCB_MAX_USER_PRIORITY] = { 0 };
2396         struct ixgbe_hw *hw =
2397                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2398         struct ixgbe_dcb_config *dcb_config =
2399                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
2400
2401         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
2402                 ixgbe_fc_none,
2403                 ixgbe_fc_rx_pause,
2404                 ixgbe_fc_tx_pause,
2405                 ixgbe_fc_full
2406         };
2407
2408         PMD_INIT_FUNC_TRACE();
2409
2410         ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
2411         tc_num = map[pfc_conf->priority];
2412         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num));
2413         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
2414         /*
2415          * At least reserve one Ethernet frame for watermark
2416          * high_water/low_water in kilo bytes for ixgbe
2417          */
2418         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
2419         if ((pfc_conf->fc.high_water > max_high_water) ||
2420                 (pfc_conf->fc.high_water <= pfc_conf->fc.low_water)) {
2421                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
2422                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
2423                 return (-EINVAL);
2424         }
2425
2426         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[pfc_conf->fc.mode];
2427         hw->fc.pause_time = pfc_conf->fc.pause_time;
2428         hw->fc.send_xon = pfc_conf->fc.send_xon;
2429         hw->fc.low_water[tc_num] =  pfc_conf->fc.low_water;
2430         hw->fc.high_water[tc_num] = pfc_conf->fc.high_water;
2431
2432         err = ixgbe_dcb_pfc_enable(dev,tc_num);
2433
2434         /* Not negotiated is not an error case */
2435         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED))
2436                 return 0;
2437
2438         PMD_INIT_LOG(ERR, "ixgbe_dcb_pfc_enable = 0x%x \n", err);
2439         return -EIO;
2440 }
2441
2442 static int
2443 ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
2444                                 struct rte_eth_rss_reta *reta_conf)
2445 {
2446         uint8_t i,j,mask;
2447         uint32_t reta;
2448         struct ixgbe_hw *hw =
2449                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2450
2451         PMD_INIT_FUNC_TRACE();
2452         /*
2453         * Update Redirection Table RETA[n],n=0...31,The redirection table has
2454         * 128-entries in 32 registers
2455          */
2456         for(i = 0; i < ETH_RSS_RETA_NUM_ENTRIES; i += 4) {
2457                 if (i < ETH_RSS_RETA_NUM_ENTRIES/2)
2458                         mask = (uint8_t)((reta_conf->mask_lo >> i) & 0xF);
2459                 else
2460                         mask = (uint8_t)((reta_conf->mask_hi >>
2461                                 (i - ETH_RSS_RETA_NUM_ENTRIES/2)) & 0xF);
2462                 if (mask != 0) {
2463                         reta = 0;
2464                         if (mask != 0xF)
2465                                 reta = IXGBE_READ_REG(hw,IXGBE_RETA(i >> 2));
2466
2467                         for (j = 0; j < 4; j++) {
2468                                 if (mask & (0x1 << j)) {
2469                                         if (mask != 0xF)
2470                                                 reta &= ~(0xFF << 8 * j);
2471                                         reta |= reta_conf->reta[i + j] << 8*j;
2472                                 }
2473                         }
2474                         IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2),reta);
2475                 }
2476         }
2477
2478         return 0;
2479 }
2480
2481 static int
2482 ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
2483                                 struct rte_eth_rss_reta *reta_conf)
2484 {
2485         uint8_t i,j,mask;
2486         uint32_t reta;
2487         struct ixgbe_hw *hw =
2488                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2489
2490         PMD_INIT_FUNC_TRACE();
2491         /*
2492          * Read Redirection Table RETA[n],n=0...31,The redirection table has
2493          * 128-entries in 32 registers
2494          */
2495         for(i = 0; i < ETH_RSS_RETA_NUM_ENTRIES; i += 4) {
2496                 if (i < ETH_RSS_RETA_NUM_ENTRIES/2)
2497                         mask = (uint8_t)((reta_conf->mask_lo >> i) & 0xF);
2498                 else
2499                         mask = (uint8_t)((reta_conf->mask_hi >>
2500                                 (i - ETH_RSS_RETA_NUM_ENTRIES/2)) & 0xF);
2501
2502                 if (mask != 0) {
2503                         reta = IXGBE_READ_REG(hw,IXGBE_RETA(i >> 2));
2504                         for (j = 0; j < 4; j++) {
2505                                 if (mask & (0x1 << j))
2506                                         reta_conf->reta[i + j] =
2507                                                 (uint8_t)((reta >> 8 * j) & 0xFF);
2508                         }
2509                 }
2510         }
2511
2512         return 0;
2513 }
2514
2515 static void
2516 ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
2517                                 uint32_t index, uint32_t pool)
2518 {
2519         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2520         uint32_t enable_addr = 1;
2521
2522         ixgbe_set_rar(hw, index, mac_addr->addr_bytes, pool, enable_addr);
2523 }
2524
2525 static void
2526 ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
2527 {
2528         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2529
2530         ixgbe_clear_rar(hw, index);
2531 }
2532
2533 /*
2534  * Virtual Function operations
2535  */
2536 static void
2537 ixgbevf_intr_disable(struct ixgbe_hw *hw)
2538 {
2539         PMD_INIT_LOG(DEBUG, "ixgbevf_intr_disable");
2540
2541         /* Clear interrupt mask to stop from interrupts being generated */
2542         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, IXGBE_VF_IRQ_CLEAR_MASK);
2543
2544         IXGBE_WRITE_FLUSH(hw);
2545 }
2546
2547 static int
2548 ixgbevf_dev_configure(struct rte_eth_dev *dev)
2549 {
2550         struct rte_eth_conf* conf = &dev->data->dev_conf;
2551
2552         PMD_INIT_LOG(DEBUG, "\nConfigured Virtual Function port id: %d\n",
2553                 dev->data->port_id);
2554
2555         /*
2556          * VF has no ability to enable/disable HW CRC
2557          * Keep the persistent behavior the same as Host PF
2558          */
2559 #ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC
2560         if (!conf->rxmode.hw_strip_crc) {
2561                 PMD_INIT_LOG(INFO, "VF can't disable HW CRC Strip\n");
2562                 conf->rxmode.hw_strip_crc = 1;
2563         }
2564 #else
2565         if (conf->rxmode.hw_strip_crc) {
2566                 PMD_INIT_LOG(INFO, "VF can't enable HW CRC Strip\n");
2567                 conf->rxmode.hw_strip_crc = 0;
2568         }
2569 #endif
2570
2571         return 0;
2572 }
2573
2574 static int
2575 ixgbevf_dev_start(struct rte_eth_dev *dev)
2576 {
2577         struct ixgbe_hw *hw =
2578                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2579         int err, mask = 0;
2580
2581         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_start");
2582
2583         hw->mac.ops.reset_hw(hw);
2584
2585         /* negotiate mailbox API version to use with the PF. */
2586         ixgbevf_negotiate_api(hw);
2587
2588         ixgbevf_dev_tx_init(dev);
2589
2590         /* This can fail when allocating mbufs for descriptor rings */
2591         err = ixgbevf_dev_rx_init(dev);
2592         if (err) {
2593                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware (%d)\n", err);
2594                 ixgbe_dev_clear_queues(dev);
2595                 return err;
2596         }
2597
2598         /* Set vfta */
2599         ixgbevf_set_vfta_all(dev,1);
2600
2601         /* Set HW strip */
2602         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
2603                 ETH_VLAN_EXTEND_MASK;
2604         ixgbevf_vlan_offload_set(dev, mask);
2605
2606         ixgbevf_dev_rxtx_start(dev);
2607
2608         return 0;
2609 }
2610
2611 static void
2612 ixgbevf_dev_stop(struct rte_eth_dev *dev)
2613 {
2614         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2615
2616         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_stop");
2617
2618         hw->adapter_stopped = TRUE;
2619         ixgbe_stop_adapter(hw);
2620
2621         /*
2622           * Clear what we set, but we still keep shadow_vfta to
2623           * restore after device starts
2624           */
2625         ixgbevf_set_vfta_all(dev,0);
2626
2627         ixgbe_dev_clear_queues(dev);
2628 }
2629
2630 static void
2631 ixgbevf_dev_close(struct rte_eth_dev *dev)
2632 {
2633         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2634
2635         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_close");
2636
2637         ixgbe_reset_hw(hw);
2638
2639         ixgbevf_dev_stop(dev);
2640
2641         /* reprogram the RAR[0] in case user changed it. */
2642         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
2643 }
2644
2645 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on)
2646 {
2647         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2648         struct ixgbe_vfta * shadow_vfta =
2649                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2650         int i = 0, j = 0, vfta = 0, mask = 1;
2651
2652         for (i = 0; i < IXGBE_VFTA_SIZE; i++){
2653                 vfta = shadow_vfta->vfta[i];
2654                 if(vfta){
2655                         mask = 1;
2656                         for (j = 0; j < 32; j++){
2657                                 if(vfta & mask)
2658                                         ixgbe_set_vfta(hw, (i<<5)+j, 0, on);
2659                                 mask<<=1;
2660                         }
2661                 }
2662         }
2663
2664 }
2665
2666 static int
2667 ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2668 {
2669         struct ixgbe_hw *hw =
2670                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2671         struct ixgbe_vfta * shadow_vfta =
2672                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2673         uint32_t vid_idx = 0;
2674         uint32_t vid_bit = 0;
2675         int ret = 0;
2676
2677         PMD_INIT_FUNC_TRACE();
2678
2679         /* vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf */
2680         ret = ixgbe_set_vfta(hw, vlan_id, 0, !!on);
2681         if(ret){
2682                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
2683                 return ret;
2684         }
2685         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
2686         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
2687
2688         /* Save what we set and retore it after device reset */
2689         if (on)
2690                 shadow_vfta->vfta[vid_idx] |= vid_bit;
2691         else
2692                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
2693
2694         return 0;
2695 }
2696
2697 static void
2698 ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
2699 {
2700         struct ixgbe_hw *hw =
2701                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2702         uint32_t ctrl;
2703
2704         PMD_INIT_FUNC_TRACE();
2705
2706         if(queue >= hw->mac.max_rx_queues)
2707                 return;
2708
2709         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
2710         if(on)
2711                 ctrl |= IXGBE_RXDCTL_VME;
2712         else
2713                 ctrl &= ~IXGBE_RXDCTL_VME;
2714         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
2715
2716         ixgbe_vlan_hw_strip_bitmap_set( dev, queue, on);
2717 }
2718
2719 static void
2720 ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2721 {
2722         struct ixgbe_hw *hw =
2723                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2724         uint16_t i;
2725         int on = 0;
2726
2727         /* VF function only support hw strip feature, others are not support */
2728         if(mask & ETH_VLAN_STRIP_MASK){
2729                 on = !!(dev->data->dev_conf.rxmode.hw_vlan_strip);
2730
2731                 for(i=0; i < hw->mac.max_rx_queues; i++)
2732                         ixgbevf_vlan_strip_queue_set(dev,i,on);
2733         }
2734 }
2735
2736 static int
2737 ixgbe_vmdq_mode_check(struct ixgbe_hw *hw)
2738 {
2739         uint32_t reg_val;
2740
2741         /* we only need to do this if VMDq is enabled */
2742         reg_val = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
2743         if (!(reg_val & IXGBE_VT_CTL_VT_ENABLE)) {
2744                 PMD_INIT_LOG(ERR, "VMDq must be enabled for this setting\n");
2745                 return (-1);
2746         }
2747
2748         return 0;
2749 }
2750
2751 static uint32_t
2752 ixgbe_uta_vector(struct ixgbe_hw *hw, struct ether_addr* uc_addr)
2753 {
2754         uint32_t vector = 0;
2755         switch (hw->mac.mc_filter_type) {
2756         case 0:   /* use bits [47:36] of the address */
2757                 vector = ((uc_addr->addr_bytes[4] >> 4) |
2758                         (((uint16_t)uc_addr->addr_bytes[5]) << 4));
2759                 break;
2760         case 1:   /* use bits [46:35] of the address */
2761                 vector = ((uc_addr->addr_bytes[4] >> 3) |
2762                         (((uint16_t)uc_addr->addr_bytes[5]) << 5));
2763                 break;
2764         case 2:   /* use bits [45:34] of the address */
2765                 vector = ((uc_addr->addr_bytes[4] >> 2) |
2766                         (((uint16_t)uc_addr->addr_bytes[5]) << 6));
2767                 break;
2768         case 3:   /* use bits [43:32] of the address */
2769                 vector = ((uc_addr->addr_bytes[4]) |
2770                         (((uint16_t)uc_addr->addr_bytes[5]) << 8));
2771                 break;
2772         default:  /* Invalid mc_filter_type */
2773                 break;
2774         }
2775
2776         /* vector can only be 12-bits or boundary will be exceeded */
2777         vector &= 0xFFF;
2778         return vector;
2779 }
2780
2781 static int
2782 ixgbe_uc_hash_table_set(struct rte_eth_dev *dev,struct ether_addr* mac_addr,
2783                                uint8_t on)
2784 {
2785         uint32_t vector;
2786         uint32_t uta_idx;
2787         uint32_t reg_val;
2788         uint32_t uta_shift;
2789         uint32_t rc;
2790         const uint32_t ixgbe_uta_idx_mask = 0x7F;
2791         const uint32_t ixgbe_uta_bit_shift = 5;
2792         const uint32_t ixgbe_uta_bit_mask = (0x1 << ixgbe_uta_bit_shift) - 1;
2793         const uint32_t bit1 = 0x1;
2794
2795         struct ixgbe_hw *hw =
2796                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2797         struct ixgbe_uta_info *uta_info =
2798                 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private);
2799
2800         /* The UTA table only exists on 82599 hardware and newer */
2801         if (hw->mac.type < ixgbe_mac_82599EB)
2802                 return (-ENOTSUP);
2803
2804         vector = ixgbe_uta_vector(hw,mac_addr);
2805         uta_idx = (vector >> ixgbe_uta_bit_shift) & ixgbe_uta_idx_mask;
2806         uta_shift = vector & ixgbe_uta_bit_mask;
2807
2808         rc = ((uta_info->uta_shadow[uta_idx] >> uta_shift & bit1) != 0);
2809         if(rc == on)
2810                 return 0;
2811
2812         reg_val = IXGBE_READ_REG(hw, IXGBE_UTA(uta_idx));
2813         if (on) {
2814                 uta_info->uta_in_use++;
2815                 reg_val |= (bit1 << uta_shift);
2816                 uta_info->uta_shadow[uta_idx] |= (bit1 << uta_shift);
2817         } else {
2818                 uta_info->uta_in_use--;
2819                 reg_val &= ~(bit1 << uta_shift);
2820                 uta_info->uta_shadow[uta_idx] &= ~(bit1 << uta_shift);
2821         }
2822
2823         IXGBE_WRITE_REG(hw, IXGBE_UTA(uta_idx), reg_val);
2824
2825         if (uta_info->uta_in_use > 0)
2826                 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL,
2827                                 IXGBE_MCSTCTRL_MFE | hw->mac.mc_filter_type);
2828         else
2829                 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL,hw->mac.mc_filter_type);
2830
2831         return 0;
2832 }
2833
2834 static int
2835 ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on)
2836 {
2837         int i;
2838         struct ixgbe_hw *hw =
2839                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2840         struct ixgbe_uta_info *uta_info =
2841                 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private);
2842
2843         /* The UTA table only exists on 82599 hardware and newer */
2844         if (hw->mac.type < ixgbe_mac_82599EB)
2845                 return (-ENOTSUP);
2846
2847         if(on) {
2848                 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
2849                         uta_info->uta_shadow[i] = ~0;
2850                         IXGBE_WRITE_REG(hw, IXGBE_UTA(i), ~0);
2851                 }
2852         } else {
2853                 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
2854                         uta_info->uta_shadow[i] = 0;
2855                         IXGBE_WRITE_REG(hw, IXGBE_UTA(i), 0);
2856                 }
2857         }
2858         return 0;
2859
2860 }
2861 static int
2862 ixgbe_set_pool_rx_mode(struct rte_eth_dev *dev, uint16_t pool,
2863                                uint16_t rx_mask, uint8_t on)
2864 {
2865         int val = 0;
2866
2867         struct ixgbe_hw *hw =
2868                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2869         uint32_t vmolr = IXGBE_READ_REG(hw, IXGBE_VMOLR(pool));
2870
2871         if (hw->mac.type == ixgbe_mac_82598EB) {
2872                 PMD_INIT_LOG(ERR, "setting VF receive mode set should be done"
2873                         " on 82599 hardware and newer\n");
2874                 return (-ENOTSUP);
2875         }
2876         if (ixgbe_vmdq_mode_check(hw) < 0)
2877                 return (-ENOTSUP);
2878
2879         if (rx_mask & ETH_VMDQ_ACCEPT_UNTAG )
2880                 val |= IXGBE_VMOLR_AUPE;
2881         if (rx_mask & ETH_VMDQ_ACCEPT_HASH_MC )
2882                 val |= IXGBE_VMOLR_ROMPE;
2883         if (rx_mask & ETH_VMDQ_ACCEPT_HASH_UC)
2884                 val |= IXGBE_VMOLR_ROPE;
2885         if (rx_mask & ETH_VMDQ_ACCEPT_BROADCAST)
2886                 val |= IXGBE_VMOLR_BAM;
2887         if (rx_mask & ETH_VMDQ_ACCEPT_MULTICAST)
2888                 val |= IXGBE_VMOLR_MPE;
2889
2890         if (on)
2891                 vmolr |= val;
2892         else
2893                 vmolr &= ~val;
2894
2895         IXGBE_WRITE_REG(hw, IXGBE_VMOLR(pool), vmolr);
2896
2897         return 0;
2898 }
2899
2900 static int
2901 ixgbe_set_pool_rx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on)
2902 {
2903         uint32_t reg,addr;
2904         uint32_t val;
2905         const uint8_t bit1 = 0x1;
2906
2907         struct ixgbe_hw *hw =
2908                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2909
2910         if (ixgbe_vmdq_mode_check(hw) < 0)
2911                 return (-ENOTSUP);
2912
2913         addr = IXGBE_VFRE(pool >= ETH_64_POOLS/2);
2914         reg = IXGBE_READ_REG(hw, addr);
2915         val = bit1 << pool;
2916
2917         if (on)
2918                 reg |= val;
2919         else
2920                 reg &= ~val;
2921
2922         IXGBE_WRITE_REG(hw, addr,reg);
2923
2924         return 0;
2925 }
2926
2927 static int
2928 ixgbe_set_pool_tx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on)
2929 {
2930         uint32_t reg,addr;
2931         uint32_t val;
2932         const uint8_t bit1 = 0x1;
2933
2934         struct ixgbe_hw *hw =
2935                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2936
2937         if (ixgbe_vmdq_mode_check(hw) < 0)
2938                 return (-ENOTSUP);
2939
2940         addr = IXGBE_VFTE(pool >= ETH_64_POOLS/2);
2941         reg = IXGBE_READ_REG(hw, addr);
2942         val = bit1 << pool;
2943
2944         if (on)
2945                 reg |= val;
2946         else
2947                 reg &= ~val;
2948
2949         IXGBE_WRITE_REG(hw, addr,reg);
2950
2951         return 0;
2952 }
2953
2954 static int
2955 ixgbe_set_pool_vlan_filter(struct rte_eth_dev *dev, uint16_t vlan,
2956                         uint64_t pool_mask, uint8_t vlan_on)
2957 {
2958         int ret = 0;
2959         uint16_t pool_idx;
2960         struct ixgbe_hw *hw =
2961                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2962
2963         if (ixgbe_vmdq_mode_check(hw) < 0)
2964                 return (-ENOTSUP);
2965         for (pool_idx = 0; pool_idx < ETH_64_POOLS; pool_idx++) {
2966                 if (pool_mask & ((uint64_t)(1ULL << pool_idx)))
2967                         ret = hw->mac.ops.set_vfta(hw,vlan,pool_idx,vlan_on);
2968                         if (ret < 0)
2969                                 return ret;
2970         }
2971
2972         return ret;
2973 }
2974
2975 static int
2976 ixgbe_mirror_rule_set(struct rte_eth_dev *dev,
2977                         struct rte_eth_vmdq_mirror_conf *mirror_conf,
2978                         uint8_t rule_id, uint8_t on)
2979 {
2980         uint32_t mr_ctl,vlvf;
2981         uint32_t mp_lsb = 0;
2982         uint32_t mv_msb = 0;
2983         uint32_t mv_lsb = 0;
2984         uint32_t mp_msb = 0;
2985         uint8_t i = 0;
2986         int reg_index = 0;
2987         uint64_t vlan_mask = 0;
2988
2989         const uint8_t pool_mask_offset = 32;
2990         const uint8_t vlan_mask_offset = 32;
2991         const uint8_t dst_pool_offset = 8;
2992         const uint8_t rule_mr_offset  = 4;
2993         const uint8_t mirror_rule_mask= 0x0F;
2994
2995         struct ixgbe_mirror_info *mr_info =
2996                         (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private));
2997         struct ixgbe_hw *hw =
2998                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2999
3000         if (ixgbe_vmdq_mode_check(hw) < 0)
3001                 return (-ENOTSUP);
3002
3003         /* Check if vlan mask is valid */
3004         if ((mirror_conf->rule_type_mask & ETH_VMDQ_VLAN_MIRROR) && (on)) {
3005                 if (mirror_conf->vlan.vlan_mask == 0)
3006                         return (-EINVAL);
3007         }
3008
3009         /* Check if vlan id is valid and find conresponding VLAN ID index in VLVF */
3010         if (mirror_conf->rule_type_mask & ETH_VMDQ_VLAN_MIRROR) {
3011                 for (i = 0;i < IXGBE_VLVF_ENTRIES; i++) {
3012                         if (mirror_conf->vlan.vlan_mask & (1ULL << i)) {
3013                                 /* search vlan id related pool vlan filter index */
3014                                 reg_index = ixgbe_find_vlvf_slot(hw,
3015                                                 mirror_conf->vlan.vlan_id[i]);
3016                                 if(reg_index < 0)
3017                                         return (-EINVAL);
3018                                 vlvf = IXGBE_READ_REG(hw, IXGBE_VLVF(reg_index));
3019                                 if ((vlvf & IXGBE_VLVF_VIEN) &&
3020                                         ((vlvf & IXGBE_VLVF_VLANID_MASK)
3021                                                 == mirror_conf->vlan.vlan_id[i]))
3022                                         vlan_mask |= (1ULL << reg_index);
3023                                 else
3024                                         return (-EINVAL);
3025                         }
3026                 }
3027
3028                 if (on) {
3029                         mv_lsb = vlan_mask & 0xFFFFFFFF;
3030                         mv_msb = vlan_mask >> vlan_mask_offset;
3031
3032                         mr_info->mr_conf[rule_id].vlan.vlan_mask =
3033                                                 mirror_conf->vlan.vlan_mask;
3034                         for(i = 0 ;i < ETH_VMDQ_MAX_VLAN_FILTERS; i++) {
3035                                 if(mirror_conf->vlan.vlan_mask & (1ULL << i))
3036                                         mr_info->mr_conf[rule_id].vlan.vlan_id[i] =
3037                                                 mirror_conf->vlan.vlan_id[i];
3038                         }
3039                 } else {
3040                         mv_lsb = 0;
3041                         mv_msb = 0;
3042                         mr_info->mr_conf[rule_id].vlan.vlan_mask = 0;
3043                         for(i = 0 ;i < ETH_VMDQ_MAX_VLAN_FILTERS; i++)
3044                                 mr_info->mr_conf[rule_id].vlan.vlan_id[i] = 0;
3045                 }
3046         }
3047
3048         /*
3049          * if enable pool mirror, write related pool mask register,if disable
3050          * pool mirror, clear PFMRVM register
3051          */
3052         if (mirror_conf->rule_type_mask & ETH_VMDQ_POOL_MIRROR) {
3053                 if (on) {
3054                         mp_lsb = mirror_conf->pool_mask & 0xFFFFFFFF;
3055                         mp_msb = mirror_conf->pool_mask >> pool_mask_offset;
3056                         mr_info->mr_conf[rule_id].pool_mask =
3057                                         mirror_conf->pool_mask;
3058
3059                 } else {
3060                         mp_lsb = 0;
3061                         mp_msb = 0;
3062                         mr_info->mr_conf[rule_id].pool_mask = 0;
3063                 }
3064         }
3065
3066         /* read  mirror control register and recalculate it */
3067         mr_ctl = IXGBE_READ_REG(hw,IXGBE_MRCTL(rule_id));
3068
3069         if (on) {
3070                 mr_ctl |= mirror_conf->rule_type_mask;
3071                 mr_ctl &= mirror_rule_mask;
3072                 mr_ctl |= mirror_conf->dst_pool << dst_pool_offset;
3073         } else
3074                 mr_ctl &= ~(mirror_conf->rule_type_mask & mirror_rule_mask);
3075
3076         mr_info->mr_conf[rule_id].rule_type_mask = (uint8_t)(mr_ctl & mirror_rule_mask);
3077         mr_info->mr_conf[rule_id].dst_pool = mirror_conf->dst_pool;
3078
3079         /* write mirrror control  register */
3080         IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl);
3081
3082         /* write pool mirrror control  register */
3083         if (mirror_conf->rule_type_mask & ETH_VMDQ_POOL_MIRROR) {
3084                 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), mp_lsb);
3085                 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset),
3086                                 mp_msb);
3087         }
3088         /* write VLAN mirrror control  register */
3089         if (mirror_conf->rule_type_mask & ETH_VMDQ_VLAN_MIRROR) {
3090                 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), mv_lsb);
3091                 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset),
3092                                 mv_msb);
3093         }
3094
3095         return 0;
3096 }
3097
3098 static int
3099 ixgbe_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t rule_id)
3100 {
3101         int mr_ctl = 0;
3102         uint32_t lsb_val = 0;
3103         uint32_t msb_val = 0;
3104         const uint8_t rule_mr_offset = 4;
3105
3106         struct ixgbe_hw *hw =
3107                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3108         struct ixgbe_mirror_info *mr_info =
3109                 (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private));
3110
3111         if (ixgbe_vmdq_mode_check(hw) < 0)
3112                 return (-ENOTSUP);
3113
3114         memset(&mr_info->mr_conf[rule_id], 0,
3115                 sizeof(struct rte_eth_vmdq_mirror_conf));
3116
3117         /* clear PFVMCTL register */
3118         IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl);
3119
3120         /* clear pool mask register */
3121         IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), lsb_val);
3122         IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset), msb_val);
3123
3124         /* clear vlan mask register */
3125         IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), lsb_val);
3126         IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset), msb_val);
3127
3128         return 0;
3129 }
3130
3131 static void
3132 ixgbevf_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
3133                      __attribute__((unused)) uint32_t index,
3134                      __attribute__((unused)) uint32_t pool)
3135 {
3136         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3137         int diag;
3138
3139         /*
3140          * On a 82599 VF, adding again the same MAC addr is not an idempotent
3141          * operation. Trap this case to avoid exhausting the [very limited]
3142          * set of PF resources used to store VF MAC addresses.
3143          */
3144         if (memcmp(hw->mac.perm_addr, mac_addr, sizeof(struct ether_addr)) == 0)
3145                 return;
3146         diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes);
3147         if (diag == 0)
3148                 return;
3149         PMD_DRV_LOG(ERR, "Unable to add MAC address - diag=%d", diag);
3150 }
3151
3152 static void
3153 ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index)
3154 {
3155         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3156         struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr;
3157         struct ether_addr *mac_addr;
3158         uint32_t i;
3159         int diag;
3160
3161         /*
3162          * The IXGBE_VF_SET_MACVLAN command of the ixgbe-pf driver does
3163          * not support the deletion of a given MAC address.
3164          * Instead, it imposes to delete all MAC addresses, then to add again
3165          * all MAC addresses with the exception of the one to be deleted.
3166          */
3167         (void) ixgbevf_set_uc_addr_vf(hw, 0, NULL);
3168
3169         /*
3170          * Add again all MAC addresses, with the exception of the deleted one
3171          * and of the permanent MAC address.
3172          */
3173         for (i = 0, mac_addr = dev->data->mac_addrs;
3174              i < hw->mac.num_rar_entries; i++, mac_addr++) {
3175                 /* Skip the deleted MAC address */
3176                 if (i == index)
3177                         continue;
3178                 /* Skip NULL MAC addresses */
3179                 if (is_zero_ether_addr(mac_addr))
3180                         continue;
3181                 /* Skip the permanent MAC address */
3182                 if (memcmp(perm_addr, mac_addr, sizeof(struct ether_addr)) == 0)
3183                         continue;
3184                 diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes);
3185                 if (diag != 0)
3186                         PMD_DRV_LOG(ERR,
3187                                     "Adding again MAC address "
3188                                     "%02x:%02x:%02x:%02x:%02x:%02x failed "
3189                                     "diag=%d",
3190                                     mac_addr->addr_bytes[0],
3191                                     mac_addr->addr_bytes[1],
3192                                     mac_addr->addr_bytes[2],
3193                                     mac_addr->addr_bytes[3],
3194                                     mac_addr->addr_bytes[4],
3195                                     mac_addr->addr_bytes[5],
3196                                     diag);
3197         }
3198 }
3199
3200 static struct rte_driver rte_ixgbe_driver = {
3201         .type = PMD_PDEV,
3202         .init = rte_ixgbe_pmd_init,
3203 };
3204
3205 static struct rte_driver rte_ixgbevf_driver = {
3206         .type = PMD_PDEV,
3207         .init = rte_ixgbevf_pmd_init,
3208 };
3209
3210 PMD_REGISTER_DRIVER(rte_ixgbe_driver);
3211 PMD_REGISTER_DRIVER(rte_ixgbevf_driver);