lib/librte_pmd_ixgbe/ixgbe_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <inttypes.h>
  42 #include <rte_byteorder.h>
  43 #include <rte_common.h>
  44 #include <rte_cycles.h>
  45
  46 #include <rte_interrupts.h>
  47 #include <rte_log.h>
  48 #include <rte_debug.h>
  49 #include <rte_pci.h>
  50 #include <rte_atomic.h>
  51 #include <rte_branch_prediction.h>
  52 #include <rte_memory.h>
  53 #include <rte_memzone.h>
  54 #include <rte_tailq.h>
  55 #include <rte_eal.h>
  56 #include <rte_alarm.h>
  57 #include <rte_ether.h>
  58 #include <rte_ethdev.h>
  59 #include <rte_atomic.h>
  60 #include <rte_malloc.h>
  61 #include <rte_random.h>
  62 #include <rte_dev.h>
  63
  64 #include "ixgbe_logs.h"
  65 #include "ixgbe/ixgbe_api.h"
  66 #include "ixgbe/ixgbe_vf.h"
  67 #include "ixgbe/ixgbe_common.h"
  68 #include "ixgbe_ethdev.h"
  69 #include "ixgbe_bypass.h"
  70
  71 /*
  72  * High threshold controlling when to start sending XOFF frames. Must be at
  73  * least 8 bytes less than receive packet buffer size. This value is in units
  74  * of 1024 bytes.
  75  */
  76 #define IXGBE_FC_HI    0x80
  77
  78 /*
  79  * Low threshold controlling when to start sending XON frames. This value is
  80  * in units of 1024 bytes.
  81  */
  82 #define IXGBE_FC_LO    0x40
  83
  84 /* Timer value included in XOFF frames. */
  85 #define IXGBE_FC_PAUSE 0x680
  86
  87 #define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */
  88 #define IXGBE_LINK_UP_CHECK_TIMEOUT   1000 /* ms */
  89 #define IXGBE_VMDQ_NUM_UC_MAC         4096 /* Maximum nb. of UC MAC addr. */
  90
  91
  92 #define IXGBEVF_PMD_NAME "rte_ixgbevf_pmd" /* PMD name */
  93
  94 #define IXGBE_QUEUE_STAT_COUNTERS (sizeof(hw_stats->qprc) / sizeof(hw_stats->qprc[0]))
  95
  96 static int eth_ixgbe_dev_init(struct eth_driver *eth_drv,
  97                 struct rte_eth_dev *eth_dev);
  98 static int  ixgbe_dev_configure(struct rte_eth_dev *dev);
  99 static int  ixgbe_dev_start(struct rte_eth_dev *dev);
 100 static void ixgbe_dev_stop(struct rte_eth_dev *dev);
 101 static void ixgbe_dev_close(struct rte_eth_dev *dev);
 102 static void ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev);
 103 static void ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev);
 104 static void ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev);
 105 static void ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev);
 106 static int ixgbe_dev_link_update(struct rte_eth_dev *dev,
 107                                 int wait_to_complete);
 108 static void ixgbe_dev_stats_get(struct rte_eth_dev *dev,
 109                                 struct rte_eth_stats *stats);
 110 static void ixgbe_dev_stats_reset(struct rte_eth_dev *dev);
 111 static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 112                                              uint16_t queue_id,
 113                                              uint8_t stat_idx,
 114                                              uint8_t is_rx);
 115 static void ixgbe_dev_info_get(struct rte_eth_dev *dev,
 116                                 struct rte_eth_dev_info *dev_info);
 117 static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev,
 118                 uint16_t vlan_id, int on);
 119 static void ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 120 static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev,
 121                 uint16_t queue, bool on);
 122 static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue,
 123                 int on);
 124 static void ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 125 static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue);
 126 static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue);
 127 static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 128 static void ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 129
 130 static int ixgbe_dev_led_on(struct rte_eth_dev *dev);
 131 static int ixgbe_dev_led_off(struct rte_eth_dev *dev);
 132 static int  ixgbe_flow_ctrl_set(struct rte_eth_dev *dev,
 133                 struct rte_eth_fc_conf *fc_conf);
 134 static int ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 135                 struct rte_eth_pfc_conf *pfc_conf);
 136 static int ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
 137                 struct rte_eth_rss_reta *reta_conf);
 138 static int ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
 139                 struct rte_eth_rss_reta *reta_conf);
 140 static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev);
 141 static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev);
 142 static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev);
 143 static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev);
 144 static void ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle,
 145                 void *param);
 146 static void ixgbe_dev_interrupt_delayed_handler(void *param);
 147 static void ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 148                 uint32_t index, uint32_t pool);
 149 static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index);
 150 static void ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config);
 151
 152 /* For Virtual Function support */
 153 static int eth_ixgbevf_dev_init(struct eth_driver *eth_drv,
 154                 struct rte_eth_dev *eth_dev);
 155 static int  ixgbevf_dev_configure(struct rte_eth_dev *dev);
 156 static int  ixgbevf_dev_start(struct rte_eth_dev *dev);
 157 static void ixgbevf_dev_stop(struct rte_eth_dev *dev);
 158 static void ixgbevf_dev_close(struct rte_eth_dev *dev);
 159 static void ixgbevf_intr_disable(struct ixgbe_hw *hw);
 160 static void ixgbevf_dev_stats_get(struct rte_eth_dev *dev,
 161                 struct rte_eth_stats *stats);
 162 static void ixgbevf_dev_stats_reset(struct rte_eth_dev *dev);
 163 static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev,
 164                 uint16_t vlan_id, int on);
 165 static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev,
 166                 uint16_t queue, int on);
 167 static void ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 168 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 169
 170 /* For Eth VMDQ APIs support */
 171 static int ixgbe_uc_hash_table_set(struct rte_eth_dev *dev, struct
 172                 ether_addr* mac_addr,uint8_t on);
 173 static int ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev,uint8_t on);
 174 static int  ixgbe_set_pool_rx_mode(struct rte_eth_dev *dev,  uint16_t pool,
 175                 uint16_t rx_mask, uint8_t on);
 176 static int ixgbe_set_pool_rx(struct rte_eth_dev *dev,uint16_t pool,uint8_t on);
 177 static int ixgbe_set_pool_tx(struct rte_eth_dev *dev,uint16_t pool,uint8_t on);
 178 static int ixgbe_set_pool_vlan_filter(struct rte_eth_dev *dev, uint16_t vlan,
 179                 uint64_t pool_mask,uint8_t vlan_on);
 180 static int ixgbe_mirror_rule_set(struct rte_eth_dev *dev,
 181                 struct rte_eth_vmdq_mirror_conf *mirror_conf,
 182                 uint8_t rule_id, uint8_t on);
 183 static int ixgbe_mirror_rule_reset(struct rte_eth_dev *dev,
 184                 uint8_t rule_id);
 185
 186 static void ixgbevf_add_mac_addr(struct rte_eth_dev *dev,
 187                                  struct ether_addr *mac_addr,
 188                                  uint32_t index, uint32_t pool);
 189 static void ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index);
 190
 191 /*
 192  * Define VF Stats MACRO for Non "cleared on read" register
 193  */
 194 #define UPDATE_VF_STAT(reg, last, cur)                          \
 195 {                                                               \
 196         u32 latest = IXGBE_READ_REG(hw, reg);                   \
 197         cur += latest - last;                                   \
 198         last = latest;                                          \
 199 }
 200
 201 #define UPDATE_VF_STAT_36BIT(lsb, msb, last, cur)                \
 202 {                                                                \
 203         u64 new_lsb = IXGBE_READ_REG(hw, lsb);                   \
 204         u64 new_msb = IXGBE_READ_REG(hw, msb);                   \
 205         u64 latest = ((new_msb << 32) | new_lsb);                \
 206         cur += (0x1000000000LL + latest - last) & 0xFFFFFFFFFLL; \
 207         last = latest;                                           \
 208 }
 209
 210 #define IXGBE_SET_HWSTRIP(h, q) do{\
 211                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 212                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 213                 (h)->bitmap[idx] |= 1 << bit;\
 214         }while(0)
 215
 216 #define IXGBE_CLEAR_HWSTRIP(h, q) do{\
 217                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 218                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 219                 (h)->bitmap[idx] &= ~(1 << bit);\
 220         }while(0)
 221
 222 #define IXGBE_GET_HWSTRIP(h, q, r) do{\
 223                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 224                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 225                 (r) = (h)->bitmap[idx] >> bit & 1;\
 226         }while(0)
 227
 228 /*
 229  * The set of PCI devices this driver supports
 230  */
 231 static struct rte_pci_id pci_id_ixgbe_map[] = {
 232
 233 #define RTE_PCI_DEV_ID_DECL_IXGBE(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 234 #include "rte_pci_dev_ids.h"
 235
 236 { .vendor_id = 0, /* sentinel */ },
 237 };
 238
 239
 240 /*
 241  * The set of PCI devices this driver supports (for 82599 VF)
 242  */
 243 static struct rte_pci_id pci_id_ixgbevf_map[] = {
 244
 245 #define RTE_PCI_DEV_ID_DECL_IXGBEVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 246 #include "rte_pci_dev_ids.h"
 247 { .vendor_id = 0, /* sentinel */ },
 248
 249 };
 250
 251 static struct eth_dev_ops ixgbe_eth_dev_ops = {
 252         .dev_configure        = ixgbe_dev_configure,
 253         .dev_start            = ixgbe_dev_start,
 254         .dev_stop             = ixgbe_dev_stop,
 255         .dev_close            = ixgbe_dev_close,
 256         .promiscuous_enable   = ixgbe_dev_promiscuous_enable,
 257         .promiscuous_disable  = ixgbe_dev_promiscuous_disable,
 258         .allmulticast_enable  = ixgbe_dev_allmulticast_enable,
 259         .allmulticast_disable = ixgbe_dev_allmulticast_disable,
 260         .link_update          = ixgbe_dev_link_update,
 261         .stats_get            = ixgbe_dev_stats_get,
 262         .stats_reset          = ixgbe_dev_stats_reset,
 263         .queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set,
 264         .dev_infos_get        = ixgbe_dev_info_get,
 265         .vlan_filter_set      = ixgbe_vlan_filter_set,
 266         .vlan_tpid_set        = ixgbe_vlan_tpid_set,
 267         .vlan_offload_set     = ixgbe_vlan_offload_set,
 268         .vlan_strip_queue_set = ixgbe_vlan_strip_queue_set,
 269         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 270         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 271         .rx_queue_count       = ixgbe_dev_rx_queue_count,
 272         .rx_descriptor_done   = ixgbe_dev_rx_descriptor_done,
 273         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 274         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 275         .dev_led_on           = ixgbe_dev_led_on,
 276         .dev_led_off          = ixgbe_dev_led_off,
 277         .flow_ctrl_set        = ixgbe_flow_ctrl_set,
 278         .priority_flow_ctrl_set = ixgbe_priority_flow_ctrl_set,
 279         .mac_addr_add         = ixgbe_add_rar,
 280         .mac_addr_remove      = ixgbe_remove_rar,
 281         .uc_hash_table_set    = ixgbe_uc_hash_table_set,
 282         .uc_all_hash_table_set  = ixgbe_uc_all_hash_table_set,
 283         .mirror_rule_set        = ixgbe_mirror_rule_set,
 284         .mirror_rule_reset      = ixgbe_mirror_rule_reset,
 285         .set_vf_rx_mode       = ixgbe_set_pool_rx_mode,
 286         .set_vf_rx            = ixgbe_set_pool_rx,
 287         .set_vf_tx            = ixgbe_set_pool_tx,
 288         .set_vf_vlan_filter   = ixgbe_set_pool_vlan_filter,
 289         .fdir_add_signature_filter    = ixgbe_fdir_add_signature_filter,
 290         .fdir_update_signature_filter = ixgbe_fdir_update_signature_filter,
 291         .fdir_remove_signature_filter = ixgbe_fdir_remove_signature_filter,
 292         .fdir_infos_get               = ixgbe_fdir_info_get,
 293         .fdir_add_perfect_filter      = ixgbe_fdir_add_perfect_filter,
 294         .fdir_update_perfect_filter   = ixgbe_fdir_update_perfect_filter,
 295         .fdir_remove_perfect_filter   = ixgbe_fdir_remove_perfect_filter,
 296         .fdir_set_masks               = ixgbe_fdir_set_masks,
 297         .reta_update          = ixgbe_dev_rss_reta_update,
 298         .reta_query           = ixgbe_dev_rss_reta_query,
 299 #ifdef RTE_NIC_BYPASS
 300         .bypass_init          = ixgbe_bypass_init,
 301         .bypass_state_set     = ixgbe_bypass_state_store,
 302         .bypass_state_show    = ixgbe_bypass_state_show,
 303         .bypass_event_set     = ixgbe_bypass_event_store,
 304         .bypass_event_show    = ixgbe_bypass_event_show,
 305         .bypass_wd_timeout_set  = ixgbe_bypass_wd_timeout_store,
 306         .bypass_wd_timeout_show = ixgbe_bypass_wd_timeout_show,
 307         .bypass_ver_show      = ixgbe_bypass_ver_show,
 308         .bypass_wd_reset      = ixgbe_bypass_wd_reset,
 309 #endif /* RTE_NIC_BYPASS */
 310 };
 311
 312 /*
 313  * dev_ops for virtual function, bare necessities for basic vf
 314  * operation have been implemented
 315  */
 316 static struct eth_dev_ops ixgbevf_eth_dev_ops = {
 317
 318         .dev_configure        = ixgbevf_dev_configure,
 319         .dev_start            = ixgbevf_dev_start,
 320         .dev_stop             = ixgbevf_dev_stop,
 321         .link_update          = ixgbe_dev_link_update,
 322         .stats_get            = ixgbevf_dev_stats_get,
 323         .stats_reset          = ixgbevf_dev_stats_reset,
 324         .dev_close            = ixgbevf_dev_close,
 325         .dev_infos_get        = ixgbe_dev_info_get,
 326         .vlan_filter_set      = ixgbevf_vlan_filter_set,
 327         .vlan_strip_queue_set = ixgbevf_vlan_strip_queue_set,
 328         .vlan_offload_set     = ixgbevf_vlan_offload_set,
 329         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 330         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 331         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 332         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 333         .mac_addr_add         = ixgbevf_add_mac_addr,
 334         .mac_addr_remove      = ixgbevf_remove_mac_addr,
 335 };
 336
 337 /**
 338  * Atomically reads the link status information from global
 339  * structure rte_eth_dev.
 340  *
 341  * @param dev
 342  *   - Pointer to the structure rte_eth_dev to read from.
 343  *   - Pointer to the buffer to be saved with the link status.
 344  *
 345  * @return
 346  *   - On success, zero.
 347  *   - On failure, negative value.
 348  */
 349 static inline int
 350 rte_ixgbe_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 351                                 struct rte_eth_link *link)
 352 {
 353         struct rte_eth_link *dst = link;
 354         struct rte_eth_link *src = &(dev->data->dev_link);
 355
 356         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 357                                         *(uint64_t *)src) == 0)
 358                 return -1;
 359
 360         return 0;
 361 }
 362
 363 /**
 364  * Atomically writes the link status information into global
 365  * structure rte_eth_dev.
 366  *
 367  * @param dev
 368  *   - Pointer to the structure rte_eth_dev to read from.
 369  *   - Pointer to the buffer to be saved with the link status.
 370  *
 371  * @return
 372  *   - On success, zero.
 373  *   - On failure, negative value.
 374  */
 375 static inline int
 376 rte_ixgbe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 377                                 struct rte_eth_link *link)
 378 {
 379         struct rte_eth_link *dst = &(dev->data->dev_link);
 380         struct rte_eth_link *src = link;
 381
 382         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 383                                         *(uint64_t *)src) == 0)
 384                 return -1;
 385
 386         return 0;
 387 }
 388
 389 /*
 390  * This function is the same as ixgbe_is_sfp() in ixgbe/ixgbe.h.
 391  */
 392 static inline int
 393 ixgbe_is_sfp(struct ixgbe_hw *hw)
 394 {
 395         switch (hw->phy.type) {
 396         case ixgbe_phy_sfp_avago:
 397         case ixgbe_phy_sfp_ftl:
 398         case ixgbe_phy_sfp_intel:
 399         case ixgbe_phy_sfp_unknown:
 400         case ixgbe_phy_sfp_passive_tyco:
 401         case ixgbe_phy_sfp_passive_unknown:
 402                 return 1;
 403         default:
 404                 return 0;
 405         }
 406 }
 407
 408 static inline int32_t
 409 ixgbe_pf_reset_hw(struct ixgbe_hw *hw)
 410 {
 411         uint32_t ctrl_ext;
 412         int32_t status;
 413
 414         status = ixgbe_reset_hw(hw);
 415
 416         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 417         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 418         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
 419         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 420         IXGBE_WRITE_FLUSH(hw);
 421
 422         return status;
 423 }
 424
 425 static inline void
 426 ixgbe_enable_intr(struct rte_eth_dev *dev)
 427 {
 428         struct ixgbe_interrupt *intr =
 429                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 430         struct ixgbe_hw *hw =
 431                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 432
 433         IXGBE_WRITE_REG(hw, IXGBE_EIMS, intr->mask);
 434         IXGBE_WRITE_FLUSH(hw);
 435 }
 436
 437 /*
 438  * This function is based on ixgbe_disable_intr() in ixgbe/ixgbe.h.
 439  */
 440 static void
 441 ixgbe_disable_intr(struct ixgbe_hw *hw)
 442 {
 443         PMD_INIT_FUNC_TRACE();
 444
 445         if (hw->mac.type == ixgbe_mac_82598EB) {
 446                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ~0);
 447         } else {
 448                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xFFFF0000);
 449                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), ~0);
 450                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), ~0);
 451         }
 452         IXGBE_WRITE_FLUSH(hw);
 453 }
 454
 455 /*
 456  * This function resets queue statistics mapping registers.
 457  * From Niantic datasheet, Initialization of Statistics section:
 458  * "...if software requires the queue counters, the RQSMR and TQSM registers
 459  * must be re-programmed following a device reset.
 460  */
 461 static void
 462 ixgbe_reset_qstat_mappings(struct ixgbe_hw *hw)
 463 {
 464         uint32_t i;
 465
 466         for(i = 0; i != IXGBE_NB_STAT_MAPPING_REGS; i++) {
 467                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), 0);
 468                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), 0);
 469         }
 470 }
 471
 472
 473 static int
 474 ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 475                                   uint16_t queue_id,
 476                                   uint8_t stat_idx,
 477                                   uint8_t is_rx)
 478 {
 479 #define QSM_REG_NB_BITS_PER_QMAP_FIELD 8
 480 #define NB_QMAP_FIELDS_PER_QSM_REG 4
 481 #define QMAP_FIELD_RESERVED_BITS_MASK 0x0f
 482
 483         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 484         struct ixgbe_stat_mapping_registers *stat_mappings =
 485                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(eth_dev->data->dev_private);
 486         uint32_t qsmr_mask = 0;
 487         uint32_t clearing_mask = QMAP_FIELD_RESERVED_BITS_MASK;
 488         uint32_t q_map;
 489         uint8_t n, offset;
 490
 491         if ((hw->mac.type != ixgbe_mac_82599EB) && (hw->mac.type != ixgbe_mac_X540))
 492                 return -ENOSYS;
 493
 494         PMD_INIT_LOG(INFO, "Setting port %d, %s queue_id %d to stat index %d\n",
 495                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx);
 496
 497         n = (uint8_t)(queue_id / NB_QMAP_FIELDS_PER_QSM_REG);
 498         if (n >= IXGBE_NB_STAT_MAPPING_REGS) {
 499                 PMD_INIT_LOG(ERR, "Nb of stat mapping registers exceeded\n");
 500                 return -EIO;
 501         }
 502         offset = (uint8_t)(queue_id % NB_QMAP_FIELDS_PER_QSM_REG);
 503
 504         /* Now clear any previous stat_idx set */
 505         clearing_mask <<= (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 506         if (!is_rx)
 507                 stat_mappings->tqsm[n] &= ~clearing_mask;
 508         else
 509                 stat_mappings->rqsmr[n] &= ~clearing_mask;
 510
 511         q_map = (uint32_t)stat_idx;
 512         q_map &= QMAP_FIELD_RESERVED_BITS_MASK;
 513         qsmr_mask = q_map << (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 514         if (!is_rx)
 515                 stat_mappings->tqsm[n] |= qsmr_mask;
 516         else
 517                 stat_mappings->rqsmr[n] |= qsmr_mask;
 518
 519         PMD_INIT_LOG(INFO, "Set port %d, %s queue_id %d to stat index %d\n"
 520                      "%s[%d] = 0x%08x\n",
 521                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx,
 522                      is_rx ? "RQSMR" : "TQSM",n, is_rx ? stat_mappings->rqsmr[n] : stat_mappings->tqsm[n]);
 523
 524         /* Now write the mapping in the appropriate register */
 525         if (is_rx) {
 526                 PMD_INIT_LOG(INFO, "Write 0x%x to RX IXGBE stat mapping reg:%d\n",
 527                              stat_mappings->rqsmr[n], n);
 528                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(n), stat_mappings->rqsmr[n]);
 529         }
 530         else {
 531                 PMD_INIT_LOG(INFO, "Write 0x%x to TX IXGBE stat mapping reg:%d\n",
 532                              stat_mappings->tqsm[n], n);
 533                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(n), stat_mappings->tqsm[n]);
 534         }
 535         return 0;
 536 }
 537
 538 static void
 539 ixgbe_restore_statistics_mapping(struct rte_eth_dev * dev)
 540 {
 541         struct ixgbe_stat_mapping_registers *stat_mappings =
 542                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(dev->data->dev_private);
 543         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 544         int i;
 545
 546         /* write whatever was in stat mapping table to the NIC */
 547         for (i = 0; i < IXGBE_NB_STAT_MAPPING_REGS; i++) {
 548                 /* rx */
 549                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), stat_mappings->rqsmr[i]);
 550
 551                 /* tx */
 552                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), stat_mappings->tqsm[i]);
 553         }
 554 }
 555
 556 static void
 557 ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config)
 558 {
 559         uint8_t i;
 560         struct ixgbe_dcb_tc_config *tc;
 561         uint8_t dcb_max_tc = IXGBE_DCB_MAX_TRAFFIC_CLASS;
 562
 563         dcb_config->num_tcs.pg_tcs = dcb_max_tc;
 564         dcb_config->num_tcs.pfc_tcs = dcb_max_tc;
 565         for (i = 0; i < dcb_max_tc; i++) {
 566                 tc = &dcb_config->tc_config[i];
 567                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_id = i;
 568                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
 569                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 570                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_id = i;
 571                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
 572                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 573                 tc->pfc = ixgbe_dcb_pfc_disabled;
 574         }
 575
 576         /* Initialize default user to priority mapping, UPx->TC0 */
 577         tc = &dcb_config->tc_config[0];
 578         tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
 579         tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
 580         for (i = 0; i< IXGBE_DCB_MAX_BW_GROUP; i++) {
 581                 dcb_config->bw_percentage[IXGBE_DCB_TX_CONFIG][i] = 100;
 582                 dcb_config->bw_percentage[IXGBE_DCB_RX_CONFIG][i] = 100;
 583         }
 584         dcb_config->rx_pba_cfg = ixgbe_dcb_pba_equal;
 585         dcb_config->pfc_mode_enable = false;
 586         dcb_config->vt_mode = true;
 587         dcb_config->round_robin_enable = false;
 588         /* support all DCB capabilities in 82599 */
 589         dcb_config->support.capabilities = 0xFF;
 590
 591         /*we only support 4 Tcs for X540*/
 592         if (hw->mac.type == ixgbe_mac_X540) {
 593                 dcb_config->num_tcs.pg_tcs = 4;
 594                 dcb_config->num_tcs.pfc_tcs = 4;
 595         }
 596 }
 597
 598 /*
 599  * Ensure that all locks are released before first NVM or PHY access
 600  */
 601 static void
 602 ixgbe_swfw_lock_reset(struct ixgbe_hw *hw)
 603 {
 604         uint16_t mask;
 605
 606         /*
 607          * Phy lock should not fail in this early stage. If this is the case,
 608          * it is due to an improper exit of the application.
 609          * So force the release of the faulty lock. Release of common lock
 610          * is done automatically by swfw_sync function.
 611          */
 612         mask = IXGBE_GSSR_PHY0_SM << hw->bus.func;
 613         if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) {
 614                    DEBUGOUT1("SWFW phy%d lock released", hw->bus.func);
 615         }
 616         ixgbe_release_swfw_semaphore(hw, mask);
 617
 618         /*
 619          * These ones are more tricky since they are common to all ports; but
 620          * swfw_sync retries last long enough (1s) to be almost sure that if
 621          * lock can not be taken it is due to an improper lock of the
 622          * semaphore.
 623          */
 624         mask = IXGBE_GSSR_EEP_SM | IXGBE_GSSR_MAC_CSR_SM | IXGBE_GSSR_SW_MNG_SM;
 625         if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) {
 626                    DEBUGOUT("SWFW common locks released");
 627         }
 628         ixgbe_release_swfw_semaphore(hw, mask);
 629 }
 630
 631 /*
 632  * This function is based on code in ixgbe_attach() in ixgbe/ixgbe.c.
 633  * It returns 0 on success.
 634  */
 635 static int
 636 eth_ixgbe_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 637                      struct rte_eth_dev *eth_dev)
 638 {
 639         struct rte_pci_device *pci_dev;
 640         struct ixgbe_hw *hw =
 641                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 642         struct ixgbe_vfta * shadow_vfta =
 643                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 644         struct ixgbe_hwstrip *hwstrip =
 645                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 646         struct ixgbe_dcb_config *dcb_config =
 647                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private);
 648         uint32_t ctrl_ext;
 649         uint16_t csum;
 650         int diag, i;
 651
 652         PMD_INIT_FUNC_TRACE();
 653
 654         eth_dev->dev_ops = &ixgbe_eth_dev_ops;
 655         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
 656         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
 657
 658         /* for secondary processes, we don't initialise any further as primary
 659          * has already done this work. Only check we don't need a different
 660          * RX function */
 661         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 662                 if (eth_dev->data->scattered_rx)
 663                         eth_dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
 664                 return 0;
 665         }
 666         pci_dev = eth_dev->pci_dev;
 667
 668         /* Vendor and Device ID need to be set before init of shared code */
 669         hw->device_id = pci_dev->id.device_id;
 670         hw->vendor_id = pci_dev->id.vendor_id;
 671         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 672 #ifdef RTE_LIBRTE_IXGBE_ALLOW_UNSUPPORTED_SFP
 673         hw->allow_unsupported_sfp = 1;
 674 #endif
 675
 676         /* Initialize the shared code */
 677 #ifdef RTE_NIC_BYPASS
 678         diag = ixgbe_bypass_init_shared_code(hw);
 679 #else
 680         diag = ixgbe_init_shared_code(hw);
 681 #endif /* RTE_NIC_BYPASS */
 682
 683         if (diag != IXGBE_SUCCESS) {
 684                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
 685                 return -EIO;
 686         }
 687
 688         /* pick up the PCI bus settings for reporting later */
 689         ixgbe_get_bus_info(hw);
 690
 691         /* Unlock any pending hardware semaphore */
 692         ixgbe_swfw_lock_reset(hw);
 693
 694         /* Initialize DCB configuration*/
 695         memset(dcb_config, 0, sizeof(struct ixgbe_dcb_config));
 696         ixgbe_dcb_init(hw,dcb_config);
 697         /* Get Hardware Flow Control setting */
 698         hw->fc.requested_mode = ixgbe_fc_full;
 699         hw->fc.current_mode = ixgbe_fc_full;
 700         hw->fc.pause_time = IXGBE_FC_PAUSE;
 701         for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
 702                 hw->fc.low_water[i] = IXGBE_FC_LO;
 703                 hw->fc.high_water[i] = IXGBE_FC_HI;
 704         }
 705         hw->fc.send_xon = 1;
 706
 707         /* Make sure we have a good EEPROM before we read from it */
 708         diag = ixgbe_validate_eeprom_checksum(hw, &csum);
 709         if (diag != IXGBE_SUCCESS) {
 710                 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag);
 711                 return -EIO;
 712         }
 713
 714 #ifdef RTE_NIC_BYPASS
 715         diag = ixgbe_bypass_init_hw(hw);
 716 #else
 717         diag = ixgbe_init_hw(hw);
 718 #endif /* RTE_NIC_BYPASS */
 719
 720         /*
 721          * Devices with copper phys will fail to initialise if ixgbe_init_hw()
 722          * is called too soon after the kernel driver unbinding/binding occurs.
 723          * The failure occurs in ixgbe_identify_phy_generic() for all devices,
 724          * but for non-copper devies, ixgbe_identify_sfp_module_generic() is
 725          * also called. See ixgbe_identify_phy_82599(). The reason for the
 726          * failure is not known, and only occuts when virtualisation features
 727          * are disabled in the bios. A delay of 100ms  was found to be enough by
 728          * trial-and-error, and is doubled to be safe.
 729          */
 730         if (diag && (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper)) {
 731                 rte_delay_ms(200);
 732                 diag = ixgbe_init_hw(hw);
 733         }
 734
 735         if (diag == IXGBE_ERR_EEPROM_VERSION) {
 736                 PMD_INIT_LOG(ERR, "This device is a pre-production adapter/"
 737                     "LOM.  Please be aware there may be issues associated "
 738                     "with your hardware.\n If you are experiencing problems "
 739                     "please contact your Intel or hardware representative "
 740                     "who provided you with this hardware.\n");
 741         } else if (diag == IXGBE_ERR_SFP_NOT_SUPPORTED)
 742                 PMD_INIT_LOG(ERR, "Unsupported SFP+ Module\n");
 743         if (diag) {
 744                 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag);
 745                 return -EIO;
 746         }
 747
 748         /* disable interrupt */
 749         ixgbe_disable_intr(hw);
 750
 751         /* reset mappings for queue statistics hw counters*/
 752         ixgbe_reset_qstat_mappings(hw);
 753
 754         /* Allocate memory for storing MAC addresses */
 755         eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 756                         hw->mac.num_rar_entries, 0);
 757         if (eth_dev->data->mac_addrs == NULL) {
 758                 PMD_INIT_LOG(ERR,
 759                         "Failed to allocate %u bytes needed to store "
 760                         "MAC addresses",
 761                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 762                 return -ENOMEM;
 763         }
 764         /* Copy the permanent MAC address */
 765         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 766                         &eth_dev->data->mac_addrs[0]);
 767
 768         /* Allocate memory for storing hash filter MAC addresses */
 769         eth_dev->data->hash_mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 770                         IXGBE_VMDQ_NUM_UC_MAC, 0);
 771         if (eth_dev->data->hash_mac_addrs == NULL) {
 772                 PMD_INIT_LOG(ERR,
 773                         "Failed to allocate %d bytes needed to store MAC addresses",
 774                         ETHER_ADDR_LEN * IXGBE_VMDQ_NUM_UC_MAC);
 775                 return -ENOMEM;
 776         }
 777
 778         /* initialize the vfta */
 779         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 780
 781         /* initialize the hw strip bitmap*/
 782         memset(hwstrip, 0, sizeof(*hwstrip));
 783
 784         /* initialize PF if max_vfs not zero */
 785         ixgbe_pf_host_init(eth_dev);
 786
 787         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 788         /* let hardware know driver is loaded */
 789         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 790         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 791         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
 792         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 793         IXGBE_WRITE_FLUSH(hw);
 794
 795         if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
 796                 PMD_INIT_LOG(DEBUG,
 797                              "MAC: %d, PHY: %d, SFP+: %d<n",
 798                              (int) hw->mac.type, (int) hw->phy.type,
 799                              (int) hw->phy.sfp_type);
 800         else
 801                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d\n",
 802                              (int) hw->mac.type, (int) hw->phy.type);
 803
 804         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
 805                         eth_dev->data->port_id, pci_dev->id.vendor_id,
 806                         pci_dev->id.device_id);
 807
 808         rte_intr_callback_register(&(pci_dev->intr_handle),
 809                 ixgbe_dev_interrupt_handler, (void *)eth_dev);
 810
 811         /* enable uio intr after callback register */
 812         rte_intr_enable(&(pci_dev->intr_handle));
 813
 814         /* enable support intr */
 815         ixgbe_enable_intr(eth_dev);
 816
 817         return 0;
 818 }
 819
 820
 821 /*
 822  * Negotiate mailbox API version with the PF.
 823  * After reset API version is always set to the basic one (ixgbe_mbox_api_10).
 824  * Then we try to negotiate starting with the most recent one.
 825  * If all negotiation attempts fail, then we will proceed with
 826  * the default one (ixgbe_mbox_api_10).
 827  */
 828 static void
 829 ixgbevf_negotiate_api(struct ixgbe_hw *hw)
 830 {
 831         int32_t i;
 832
 833         /* start with highest supported, proceed down */
 834         static const enum ixgbe_pfvf_api_rev sup_ver[] = {
 835                 ixgbe_mbox_api_11,
 836                 ixgbe_mbox_api_10,
 837         };
 838
 839         for (i = 0;
 840                         i != RTE_DIM(sup_ver) &&
 841                         ixgbevf_negotiate_api_version(hw, sup_ver[i]) != 0;
 842                         i++)
 843                 ;
 844 }
 845
 846 static void
 847 generate_random_mac_addr(struct ether_addr *mac_addr)
 848 {
 849         uint64_t random;
 850
 851         /* Set Organizationally Unique Identifier (OUI) prefix. */
 852         mac_addr->addr_bytes[0] = 0x00;
 853         mac_addr->addr_bytes[1] = 0x09;
 854         mac_addr->addr_bytes[2] = 0xC0;
 855         /* Force indication of locally assigned MAC address. */
 856         mac_addr->addr_bytes[0] |= ETHER_LOCAL_ADMIN_ADDR;
 857         /* Generate the last 3 bytes of the MAC address with a random number. */
 858         random = rte_rand();
 859         memcpy(&mac_addr->addr_bytes[3], &random, 3);
 860 }
 861
 862 /*
 863  * Virtual Function device init
 864  */
 865 static int
 866 eth_ixgbevf_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 867                      struct rte_eth_dev *eth_dev)
 868 {
 869         int diag;
 870         uint32_t tc, tcs;
 871         struct rte_pci_device *pci_dev;
 872         struct ixgbe_hw *hw =
 873                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 874         struct ixgbe_vfta * shadow_vfta =
 875                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 876         struct ixgbe_hwstrip *hwstrip =
 877                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 878         struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr;
 879
 880         PMD_INIT_LOG(DEBUG, "eth_ixgbevf_dev_init");
 881
 882         eth_dev->dev_ops = &ixgbevf_eth_dev_ops;
 883         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
 884         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
 885
 886         /* for secondary processes, we don't initialise any further as primary
 887          * has already done this work. Only check we don't need a different
 888          * RX function */
 889         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 890                 if (eth_dev->data->scattered_rx)
 891                         eth_dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
 892                 return 0;
 893         }
 894
 895         pci_dev = eth_dev->pci_dev;
 896
 897         hw->device_id = pci_dev->id.device_id;
 898         hw->vendor_id = pci_dev->id.vendor_id;
 899         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 900
 901         /* initialize the vfta */
 902         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 903
 904         /* initialize the hw strip bitmap*/
 905         memset(hwstrip, 0, sizeof(*hwstrip));
 906
 907         /* Initialize the shared code */
 908         diag = ixgbe_init_shared_code(hw);
 909         if (diag != IXGBE_SUCCESS) {
 910                 PMD_INIT_LOG(ERR, "Shared code init failed for ixgbevf: %d", diag);
 911                 return -EIO;
 912         }
 913
 914         /* init_mailbox_params */
 915         hw->mbx.ops.init_params(hw);
 916
 917         /* Disable the interrupts for VF */
 918         ixgbevf_intr_disable(hw);
 919
 920         hw->mac.num_rar_entries = 128; /* The MAX of the underlying PF */
 921         diag = hw->mac.ops.reset_hw(hw);
 922
 923         /*
 924          * The VF reset operation returns the IXGBE_ERR_INVALID_MAC_ADDR when
 925          * the underlying PF driver has not assigned a MAC address to the VF.
 926          * In this case, assign a random MAC address.
 927          */
 928         if ((diag != IXGBE_SUCCESS) && (diag != IXGBE_ERR_INVALID_MAC_ADDR)) {
 929                 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
 930                 return (diag);
 931         }
 932
 933         /* negotiate mailbox API version to use with the PF. */
 934         ixgbevf_negotiate_api(hw);
 935
 936         /* Get Rx/Tx queue count via mailbox, which is ready after reset_hw */
 937         ixgbevf_get_queues(hw, &tcs, &tc);
 938
 939         /* Allocate memory for storing MAC addresses */
 940         eth_dev->data->mac_addrs = rte_zmalloc("ixgbevf", ETHER_ADDR_LEN *
 941                         hw->mac.num_rar_entries, 0);
 942         if (eth_dev->data->mac_addrs == NULL) {
 943                 PMD_INIT_LOG(ERR,
 944                         "Failed to allocate %u bytes needed to store "
 945                         "MAC addresses",
 946                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 947                 return -ENOMEM;
 948         }
 949
 950         /* Generate a random MAC address, if none was assigned by PF. */
 951         if (is_zero_ether_addr(perm_addr)) {
 952                 generate_random_mac_addr(perm_addr);
 953                 diag = ixgbe_set_rar_vf(hw, 1, perm_addr->addr_bytes, 0, 1);
 954                 if (diag) {
 955                         rte_free(eth_dev->data->mac_addrs);
 956                         eth_dev->data->mac_addrs = NULL;
 957                         return diag;
 958                 }
 959                 RTE_LOG(INFO, PMD,
 960                         "\tVF MAC address not assigned by Host PF\n"
 961                         "\tAssign randomly generated MAC address "
 962                         "%02x:%02x:%02x:%02x:%02x:%02x\n",
 963                         perm_addr->addr_bytes[0],
 964                         perm_addr->addr_bytes[1],
 965                         perm_addr->addr_bytes[2],
 966                         perm_addr->addr_bytes[3],
 967                         perm_addr->addr_bytes[4],
 968                         perm_addr->addr_bytes[5]);
 969         }
 970
 971         /* Copy the permanent MAC address */
 972         ether_addr_copy(perm_addr, &eth_dev->data->mac_addrs[0]);
 973
 974         /* reset the hardware with the new settings */
 975         diag = hw->mac.ops.start_hw(hw);
 976         switch (diag) {
 977                 case  0:
 978                         break;
 979
 980                 default:
 981                         PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
 982                         return (-EIO);
 983         }
 984
 985         PMD_INIT_LOG(DEBUG, "\nport %d vendorID=0x%x deviceID=0x%x mac.type=%s\n",
 986                          eth_dev->data->port_id, pci_dev->id.vendor_id, pci_dev->id.device_id,
 987                          "ixgbe_mac_82599_vf");
 988
 989         return 0;
 990 }
 991
 992 static struct eth_driver rte_ixgbe_pmd = {
 993         {
 994                 .name = "rte_ixgbe_pmd",
 995                 .id_table = pci_id_ixgbe_map,
 996                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
 997         },
 998         .eth_dev_init = eth_ixgbe_dev_init,
 999         .dev_private_size = sizeof(struct ixgbe_adapter),
1000 };
1001
1002 /*
1003  * virtual function driver struct
1004  */
1005 static struct eth_driver rte_ixgbevf_pmd = {
1006         {
1007                 .name = "rte_ixgbevf_pmd",
1008                 .id_table = pci_id_ixgbevf_map,
1009                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
1010         },
1011         .eth_dev_init = eth_ixgbevf_dev_init,
1012         .dev_private_size = sizeof(struct ixgbe_adapter),
1013 };
1014
1015 /*
1016  * Driver initialization routine.
1017  * Invoked once at EAL init time.
1018  * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
1019  */
1020 static int
1021 rte_ixgbe_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
1022 {
1023         PMD_INIT_FUNC_TRACE();
1024
1025         rte_eth_driver_register(&rte_ixgbe_pmd);
1026         return 0;
1027 }
1028
1029 /*
1030  * VF Driver initialization routine.
1031  * Invoked one at EAL init time.
1032  * Register itself as the [Virtual Poll Mode] Driver of PCI niantic devices.
1033  */
1034 static int
1035 rte_ixgbevf_pmd_init(const char *name __rte_unused, const char *param __rte_unused)
1036 {
1037         DEBUGFUNC("rte_ixgbevf_pmd_init");
1038
1039         rte_eth_driver_register(&rte_ixgbevf_pmd);
1040         return (0);
1041 }
1042
1043 static int
1044 ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1045 {
1046         struct ixgbe_hw *hw =
1047                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1048         struct ixgbe_vfta * shadow_vfta =
1049                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1050         uint32_t vfta;
1051         uint32_t vid_idx;
1052         uint32_t vid_bit;
1053
1054         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
1055         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
1056         vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(vid_idx));
1057         if (on)
1058                 vfta |= vid_bit;
1059         else
1060                 vfta &= ~vid_bit;
1061         IXGBE_WRITE_REG(hw, IXGBE_VFTA(vid_idx), vfta);
1062
1063         /* update local VFTA copy */
1064         shadow_vfta->vfta[vid_idx] = vfta;
1065
1066         return 0;
1067 }
1068
1069 static void
1070 ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
1071 {
1072         if (on)
1073                 ixgbe_vlan_hw_strip_enable(dev, queue);
1074         else
1075                 ixgbe_vlan_hw_strip_disable(dev, queue);
1076 }
1077
1078 static void
1079 ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
1080 {
1081         struct ixgbe_hw *hw =
1082                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1083
1084         /* Only the high 16-bits is valid */
1085         IXGBE_WRITE_REG(hw, IXGBE_EXVET, tpid << 16);
1086 }
1087
1088 void
1089 ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev)
1090 {
1091         struct ixgbe_hw *hw =
1092                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1093         uint32_t vlnctrl;
1094
1095         PMD_INIT_FUNC_TRACE();
1096
1097         /* Filter Table Disable */
1098         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1099         vlnctrl &= ~IXGBE_VLNCTRL_VFE;
1100
1101         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
1102 }
1103
1104 void
1105 ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1106 {
1107         struct ixgbe_hw *hw =
1108                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1109         struct ixgbe_vfta * shadow_vfta =
1110                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1111         uint32_t vlnctrl;
1112         uint16_t i;
1113
1114         PMD_INIT_FUNC_TRACE();
1115
1116         /* Filter Table Enable */
1117         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1118         vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
1119         vlnctrl |= IXGBE_VLNCTRL_VFE;
1120
1121         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
1122
1123         /* write whatever is in local vfta copy */
1124         for (i = 0; i < IXGBE_VFTA_SIZE; i++)
1125                 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), shadow_vfta->vfta[i]);
1126 }
1127
1128 static void
1129 ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on)
1130 {
1131         struct ixgbe_hwstrip *hwstrip =
1132                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(dev->data->dev_private);
1133
1134         if(queue >= IXGBE_MAX_RX_QUEUE_NUM)
1135                 return;
1136
1137         if (on)
1138                 IXGBE_SET_HWSTRIP(hwstrip, queue);
1139         else
1140                 IXGBE_CLEAR_HWSTRIP(hwstrip, queue);
1141 }
1142
1143 static void
1144 ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue)
1145 {
1146         struct ixgbe_hw *hw =
1147                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1148         uint32_t ctrl;
1149
1150         PMD_INIT_FUNC_TRACE();
1151
1152         if (hw->mac.type == ixgbe_mac_82598EB) {
1153                 /* No queue level support */
1154                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
1155                 return;
1156         }
1157         else {
1158                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1159                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1160                 ctrl &= ~IXGBE_RXDCTL_VME;
1161                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
1162         }
1163         /* record those setting for HW strip per queue */
1164         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 0);
1165 }
1166
1167 static void
1168 ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue)
1169 {
1170         struct ixgbe_hw *hw =
1171                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1172         uint32_t ctrl;
1173
1174         PMD_INIT_FUNC_TRACE();
1175
1176         if (hw->mac.type == ixgbe_mac_82598EB) {
1177                 /* No queue level supported */
1178                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
1179                 return;
1180         }
1181         else {
1182                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1183                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1184                 ctrl |= IXGBE_RXDCTL_VME;
1185                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
1186         }
1187         /* record those setting for HW strip per queue */
1188         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 1);
1189 }
1190
1191 void
1192 ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev)
1193 {
1194         struct ixgbe_hw *hw =
1195                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1196         uint32_t ctrl;
1197         uint16_t i;
1198
1199         PMD_INIT_FUNC_TRACE();
1200
1201         if (hw->mac.type == ixgbe_mac_82598EB) {
1202                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1203                 ctrl &= ~IXGBE_VLNCTRL_VME;
1204                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1205         }
1206         else {
1207                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1208                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1209                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1210                         ctrl &= ~IXGBE_RXDCTL_VME;
1211                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1212
1213                         /* record those setting for HW strip per queue */
1214                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 0);
1215                 }
1216         }
1217 }
1218
1219 void
1220 ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev)
1221 {
1222         struct ixgbe_hw *hw =
1223                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1224         uint32_t ctrl;
1225         uint16_t i;
1226
1227         PMD_INIT_FUNC_TRACE();
1228
1229         if (hw->mac.type == ixgbe_mac_82598EB) {
1230                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1231                 ctrl |= IXGBE_VLNCTRL_VME;
1232                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1233         }
1234         else {
1235                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1236                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1237                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1238                         ctrl |= IXGBE_RXDCTL_VME;
1239                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1240
1241                         /* record those setting for HW strip per queue */
1242                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 1);
1243                 }
1244         }
1245 }
1246
1247 static void
1248 ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1249 {
1250         struct ixgbe_hw *hw =
1251                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1252         uint32_t ctrl;
1253
1254         PMD_INIT_FUNC_TRACE();
1255
1256         /* DMATXCTRL: Geric Double VLAN Disable */
1257         ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1258         ctrl &= ~IXGBE_DMATXCTL_GDV;
1259         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1260
1261         /* CTRL_EXT: Global Double VLAN Disable */
1262         ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1263         ctrl &= ~IXGBE_EXTENDED_VLAN;
1264         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1265
1266 }
1267
1268 static void
1269 ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1270 {
1271         struct ixgbe_hw *hw =
1272                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1273         uint32_t ctrl;
1274
1275         PMD_INIT_FUNC_TRACE();
1276
1277         /* DMATXCTRL: Geric Double VLAN Enable */
1278         ctrl  = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1279         ctrl |= IXGBE_DMATXCTL_GDV;
1280         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1281
1282         /* CTRL_EXT: Global Double VLAN Enable */
1283         ctrl  = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1284         ctrl |= IXGBE_EXTENDED_VLAN;
1285         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1286
1287         /*
1288          * VET EXT field in the EXVET register = 0x8100 by default
1289          * So no need to change. Same to VT field of DMATXCTL register
1290          */
1291 }
1292
1293 static void
1294 ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1295 {
1296         if(mask & ETH_VLAN_STRIP_MASK){
1297                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1298                         ixgbe_vlan_hw_strip_enable_all(dev);
1299                 else
1300                         ixgbe_vlan_hw_strip_disable_all(dev);
1301         }
1302
1303         if(mask & ETH_VLAN_FILTER_MASK){
1304                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1305                         ixgbe_vlan_hw_filter_enable(dev);
1306                 else
1307                         ixgbe_vlan_hw_filter_disable(dev);
1308         }
1309
1310         if(mask & ETH_VLAN_EXTEND_MASK){
1311                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1312                         ixgbe_vlan_hw_extend_enable(dev);
1313                 else
1314                         ixgbe_vlan_hw_extend_disable(dev);
1315         }
1316 }
1317
1318 static void
1319 ixgbe_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1320 {
1321         struct ixgbe_hw *hw =
1322                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1323         /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
1324         uint32_t vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1325         vlanctrl |= IXGBE_VLNCTRL_VFE ; /* enable vlan filters */
1326         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
1327 }
1328
1329 static int
1330 ixgbe_dev_configure(struct rte_eth_dev *dev)
1331 {
1332         struct ixgbe_interrupt *intr =
1333                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1334
1335         PMD_INIT_FUNC_TRACE();
1336
1337         /* set flag to update link status after init */
1338         intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1339
1340         return 0;
1341 }
1342
1343 /*
1344  * Configure device link speed and setup link.
1345  * It returns 0 on success.
1346  */
1347 static int
1348 ixgbe_dev_start(struct rte_eth_dev *dev)
1349 {
1350         struct ixgbe_hw *hw =
1351                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1352         int err, link_up = 0, negotiate = 0;
1353         uint32_t speed = 0;
1354         int mask = 0;
1355         int status;
1356
1357         PMD_INIT_FUNC_TRACE();
1358
1359         /* IXGBE devices don't support half duplex */
1360         if ((dev->data->dev_conf.link_duplex != ETH_LINK_AUTONEG_DUPLEX) &&
1361                         (dev->data->dev_conf.link_duplex != ETH_LINK_FULL_DUPLEX)) {
1362                 PMD_INIT_LOG(ERR, "Invalid link_duplex (%hu) for port %hhu\n",
1363                                 dev->data->dev_conf.link_duplex,
1364                                 dev->data->port_id);
1365                 return -EINVAL;
1366         }
1367
1368         /* stop adapter */
1369         hw->adapter_stopped = FALSE;
1370         ixgbe_stop_adapter(hw);
1371
1372         /* reinitialize adapter
1373          * this calls reset and start */
1374         status = ixgbe_pf_reset_hw(hw);
1375         if (status != 0)
1376                 return -1;
1377         hw->mac.ops.start_hw(hw);
1378
1379         /* configure PF module if SRIOV enabled */
1380         ixgbe_pf_host_configure(dev);
1381
1382         /* initialize transmission unit */
1383         ixgbe_dev_tx_init(dev);
1384
1385         /* This can fail when allocating mbufs for descriptor rings */
1386         err = ixgbe_dev_rx_init(dev);
1387         if (err) {
1388                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware\n");
1389                 goto error;
1390         }
1391
1392         ixgbe_dev_rxtx_start(dev);
1393
1394         if (ixgbe_is_sfp(hw) && hw->phy.multispeed_fiber) {
1395                 err = hw->mac.ops.setup_sfp(hw);
1396                 if (err)
1397                         goto error;
1398         }
1399
1400         /* Turn on the laser */
1401         ixgbe_enable_tx_laser(hw);
1402
1403         /* Skip link setup if loopback mode is enabled for 82599. */
1404         if (hw->mac.type == ixgbe_mac_82599EB &&
1405                         dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_82599_TX_RX)
1406                 goto skip_link_setup;
1407
1408         err = ixgbe_check_link(hw, &speed, &link_up, 0);
1409         if (err)
1410                 goto error;
1411         err = ixgbe_get_link_capabilities(hw, &speed, &negotiate);
1412         if (err)
1413                 goto error;
1414
1415         switch(dev->data->dev_conf.link_speed) {
1416         case ETH_LINK_SPEED_AUTONEG:
1417                 speed = (hw->mac.type != ixgbe_mac_82598EB) ?
1418                                 IXGBE_LINK_SPEED_82599_AUTONEG :
1419                                 IXGBE_LINK_SPEED_82598_AUTONEG;
1420                 break;
1421         case ETH_LINK_SPEED_100:
1422                 /*
1423                  * Invalid for 82598 but error will be detected by
1424                  * ixgbe_setup_link()
1425                  */
1426                 speed = IXGBE_LINK_SPEED_100_FULL;
1427                 break;
1428         case ETH_LINK_SPEED_1000:
1429                 speed = IXGBE_LINK_SPEED_1GB_FULL;
1430                 break;
1431         case ETH_LINK_SPEED_10000:
1432                 speed = IXGBE_LINK_SPEED_10GB_FULL;
1433                 break;
1434         default:
1435                 PMD_INIT_LOG(ERR, "Invalid link_speed (%hu) for port %hhu\n",
1436                                 dev->data->dev_conf.link_speed,
1437                                 dev->data->port_id);
1438                 goto error;
1439         }
1440
1441         err = ixgbe_setup_link(hw, speed, negotiate, link_up);
1442         if (err)
1443                 goto error;
1444
1445 skip_link_setup:
1446
1447         /* check if lsc interrupt is enabled */
1448         if (dev->data->dev_conf.intr_conf.lsc != 0)
1449                 ixgbe_dev_lsc_interrupt_setup(dev);
1450
1451         /* resume enabled intr since hw reset */
1452         ixgbe_enable_intr(dev);
1453
1454         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
1455                 ETH_VLAN_EXTEND_MASK;
1456         ixgbe_vlan_offload_set(dev, mask);
1457
1458         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
1459                 /* Enable vlan filtering for VMDq */
1460                 ixgbe_vmdq_vlan_hw_filter_enable(dev);
1461         }
1462
1463         /* Configure DCB hw */
1464         ixgbe_configure_dcb(dev);
1465
1466         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) {
1467                 err = ixgbe_fdir_configure(dev);
1468                 if (err)
1469                         goto error;
1470         }
1471
1472         ixgbe_restore_statistics_mapping(dev);
1473
1474         return (0);
1475
1476 error:
1477         PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err);
1478         ixgbe_dev_clear_queues(dev);
1479         return -EIO;
1480 }
1481
1482 /*
1483  * Stop device: disable rx and tx functions to allow for reconfiguring.
1484  */
1485 static void
1486 ixgbe_dev_stop(struct rte_eth_dev *dev)
1487 {
1488         struct rte_eth_link link;
1489         struct ixgbe_hw *hw =
1490                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1491         struct ixgbe_vf_info *vfinfo =
1492                 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private);
1493         int vf;
1494
1495         PMD_INIT_FUNC_TRACE();
1496
1497         /* disable interrupts */
1498         ixgbe_disable_intr(hw);
1499
1500         /* reset the NIC */
1501         ixgbe_pf_reset_hw(hw);
1502         hw->adapter_stopped = FALSE;
1503
1504         /* stop adapter */
1505         ixgbe_stop_adapter(hw);
1506
1507         for (vf = 0; vfinfo != NULL &&
1508                      vf < dev->pci_dev->max_vfs; vf++)
1509                 vfinfo[vf].clear_to_send = false;
1510
1511         /* Turn off the laser */
1512         ixgbe_disable_tx_laser(hw);
1513
1514         ixgbe_dev_clear_queues(dev);
1515
1516         /* Clear recorded link status */
1517         memset(&link, 0, sizeof(link));
1518         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1519 }
1520
1521 /*
1522  * Reest and stop device.
1523  */
1524 static void
1525 ixgbe_dev_close(struct rte_eth_dev *dev)
1526 {
1527         struct ixgbe_hw *hw =
1528                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1529
1530         PMD_INIT_FUNC_TRACE();
1531
1532         ixgbe_pf_reset_hw(hw);
1533
1534         ixgbe_dev_stop(dev);
1535         hw->adapter_stopped = 1;
1536
1537         ixgbe_disable_pcie_master(hw);
1538
1539         /* reprogram the RAR[0] in case user changed it. */
1540         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
1541 }
1542
1543 /*
1544  * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c
1545  */
1546 static void
1547 ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1548 {
1549         struct ixgbe_hw *hw =
1550                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1551         struct ixgbe_hw_stats *hw_stats =
1552                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1553         uint32_t bprc, lxon, lxoff, total;
1554         uint64_t total_missed_rx, total_qbrc, total_qprc;
1555         unsigned i;
1556
1557         total_missed_rx = 0;
1558         total_qbrc = 0;
1559         total_qprc = 0;
1560
1561         hw_stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
1562         hw_stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
1563         hw_stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
1564         hw_stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
1565
1566         for (i = 0; i < 8; i++) {
1567                 uint32_t mp;
1568                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
1569                 /* global total per queue */
1570                 hw_stats->mpc[i] += mp;
1571                 /* Running comprehensive total for stats display */
1572                 total_missed_rx += hw_stats->mpc[i];
1573                 if (hw->mac.type == ixgbe_mac_82598EB)
1574                         hw_stats->rnbc[i] +=
1575                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
1576                 hw_stats->pxontxc[i] +=
1577                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
1578                 hw_stats->pxonrxc[i] +=
1579                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
1580                 hw_stats->pxofftxc[i] +=
1581                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
1582                 hw_stats->pxoffrxc[i] +=
1583                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
1584                 hw_stats->pxon2offc[i] +=
1585                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
1586         }
1587         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1588                 hw_stats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
1589                 hw_stats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
1590                 hw_stats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
1591                 hw_stats->qbrc[i] +=
1592                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32);
1593                 hw_stats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
1594                 hw_stats->qbtc[i] +=
1595                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32);
1596                 hw_stats->qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
1597
1598                 total_qprc += hw_stats->qprc[i];
1599                 total_qbrc += hw_stats->qbrc[i];
1600         }
1601         hw_stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
1602         hw_stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
1603         hw_stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
1604
1605         /* Note that gprc counts missed packets */
1606         hw_stats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
1607
1608         if (hw->mac.type != ixgbe_mac_82598EB) {
1609                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
1610                 hw_stats->gorc += ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
1611                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
1612                 hw_stats->gotc += ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
1613                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
1614                 hw_stats->tor += ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
1615                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
1616                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
1617         } else {
1618                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
1619                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
1620                 /* 82598 only has a counter in the high register */
1621                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
1622                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
1623                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
1624         }
1625
1626         /*
1627          * Workaround: mprc hardware is incorrectly counting
1628          * broadcasts, so for now we subtract those.
1629          */
1630         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
1631         hw_stats->bprc += bprc;
1632         hw_stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
1633         if (hw->mac.type == ixgbe_mac_82598EB)
1634                 hw_stats->mprc -= bprc;
1635
1636         hw_stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
1637         hw_stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
1638         hw_stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
1639         hw_stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
1640         hw_stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
1641         hw_stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
1642
1643         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
1644         hw_stats->lxontxc += lxon;
1645         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
1646         hw_stats->lxofftxc += lxoff;
1647         total = lxon + lxoff;
1648
1649         hw_stats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
1650         hw_stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
1651         hw_stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
1652         hw_stats->gptc -= total;
1653         hw_stats->mptc -= total;
1654         hw_stats->ptc64 -= total;
1655         hw_stats->gotc -= total * ETHER_MIN_LEN;
1656
1657         hw_stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
1658         hw_stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
1659         hw_stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
1660         hw_stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
1661         hw_stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
1662         hw_stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
1663         hw_stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
1664         hw_stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
1665         hw_stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
1666         hw_stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
1667         hw_stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
1668         hw_stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
1669         hw_stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
1670         hw_stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
1671         hw_stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
1672         hw_stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);
1673         hw_stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
1674         hw_stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
1675         /* Only read FCOE on 82599 */
1676         if (hw->mac.type != ixgbe_mac_82598EB) {
1677                 hw_stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
1678                 hw_stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
1679                 hw_stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
1680                 hw_stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
1681                 hw_stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
1682         }
1683
1684         if (stats == NULL)
1685                 return;
1686
1687         /* Fill out the rte_eth_stats statistics structure */
1688         stats->ipackets = total_qprc;
1689         stats->ibytes = total_qbrc;
1690         stats->opackets = hw_stats->gptc;
1691         stats->obytes = hw_stats->gotc;
1692         stats->imcasts = hw_stats->mprc;
1693
1694         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1695                 stats->q_ipackets[i] = hw_stats->qprc[i];
1696                 stats->q_opackets[i] = hw_stats->qptc[i];
1697                 stats->q_ibytes[i] = hw_stats->qbrc[i];
1698                 stats->q_obytes[i] = hw_stats->qbtc[i];
1699                 stats->q_errors[i] = hw_stats->qprdc[i];
1700         }
1701
1702         /* Rx Errors */
1703         stats->ierrors = total_missed_rx + hw_stats->crcerrs +
1704                 hw_stats->rlec;
1705
1706         stats->oerrors  = 0;
1707
1708         /* XON/XOFF pause frames */
1709         stats->tx_pause_xon  = hw_stats->lxontxc;
1710         stats->rx_pause_xon  = hw_stats->lxonrxc;
1711         stats->tx_pause_xoff = hw_stats->lxofftxc;
1712         stats->rx_pause_xoff = hw_stats->lxoffrxc;
1713
1714         /* Flow Director Stats registers */
1715         hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
1716         hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
1717         stats->fdirmatch = hw_stats->fdirmatch;
1718         stats->fdirmiss = hw_stats->fdirmiss;
1719 }
1720
1721 static void
1722 ixgbe_dev_stats_reset(struct rte_eth_dev *dev)
1723 {
1724         struct ixgbe_hw_stats *stats =
1725                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1726
1727         /* HW registers are cleared on read */
1728         ixgbe_dev_stats_get(dev, NULL);
1729
1730         /* Reset software totals */
1731         memset(stats, 0, sizeof(*stats));
1732 }
1733
1734 static void
1735 ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1736 {
1737         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1738         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1739                           IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1740
1741         /* Good Rx packet, include VF loopback */
1742         UPDATE_VF_STAT(IXGBE_VFGPRC,
1743             hw_stats->last_vfgprc, hw_stats->vfgprc);
1744
1745         /* Good Rx octets, include VF loopback */
1746         UPDATE_VF_STAT_36BIT(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
1747             hw_stats->last_vfgorc, hw_stats->vfgorc);
1748
1749         /* Good Tx packet, include VF loopback */
1750         UPDATE_VF_STAT(IXGBE_VFGPTC,
1751             hw_stats->last_vfgptc, hw_stats->vfgptc);
1752
1753         /* Good Tx octets, include VF loopback */
1754         UPDATE_VF_STAT_36BIT(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
1755             hw_stats->last_vfgotc, hw_stats->vfgotc);
1756
1757         /* Rx Multicst Packet */
1758         UPDATE_VF_STAT(IXGBE_VFMPRC,
1759             hw_stats->last_vfmprc, hw_stats->vfmprc);
1760
1761         if (stats == NULL)
1762                 return;
1763
1764         memset(stats, 0, sizeof(*stats));
1765         stats->ipackets = hw_stats->vfgprc;
1766         stats->ibytes = hw_stats->vfgorc;
1767         stats->opackets = hw_stats->vfgptc;
1768         stats->obytes = hw_stats->vfgotc;
1769         stats->imcasts = hw_stats->vfmprc;
1770 }
1771
1772 static void
1773 ixgbevf_dev_stats_reset(struct rte_eth_dev *dev)
1774 {
1775         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1776                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1777
1778         /* Sync HW register to the last stats */
1779         ixgbevf_dev_stats_get(dev, NULL);
1780
1781         /* reset HW current stats*/
1782         hw_stats->vfgprc = 0;
1783         hw_stats->vfgorc = 0;
1784         hw_stats->vfgptc = 0;
1785         hw_stats->vfgotc = 0;
1786         hw_stats->vfmprc = 0;
1787
1788 }
1789
1790 static void
1791 ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1792 {
1793         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1794
1795         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
1796         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
1797         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL register */
1798         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */
1799         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
1800         dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC;
1801         dev_info->max_vfs = dev->pci_dev->max_vfs;
1802         if (hw->mac.type == ixgbe_mac_82598EB)
1803                 dev_info->max_vmdq_pools = ETH_16_POOLS;
1804         else
1805                 dev_info->max_vmdq_pools = ETH_64_POOLS;
1806         dev_info->rx_offload_capa =
1807                 DEV_RX_OFFLOAD_VLAN_STRIP |
1808                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1809                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1810                 DEV_RX_OFFLOAD_TCP_CKSUM;
1811         dev_info->tx_offload_capa =
1812                 DEV_TX_OFFLOAD_VLAN_INSERT |
1813                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1814                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1815                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1816                 DEV_TX_OFFLOAD_SCTP_CKSUM;
1817 }
1818
1819 /* return 0 means link status changed, -1 means not changed */
1820 static int
1821 ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1822 {
1823         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1824         struct rte_eth_link link, old;
1825         ixgbe_link_speed link_speed;
1826         int link_up;
1827         int diag;
1828
1829         link.link_status = 0;
1830         link.link_speed = 0;
1831         link.link_duplex = 0;
1832         memset(&old, 0, sizeof(old));
1833         rte_ixgbe_dev_atomic_read_link_status(dev, &old);
1834
1835         /* check if it needs to wait to complete, if lsc interrupt is enabled */
1836         if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
1837                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 0);
1838         else
1839                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 1);
1840         if (diag != 0) {
1841                 link.link_speed = ETH_LINK_SPEED_100;
1842                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1843                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1844                 if (link.link_status == old.link_status)
1845                         return -1;
1846                 return 0;
1847         }
1848
1849         if (link_up == 0) {
1850                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1851                 if (link.link_status == old.link_status)
1852                         return -1;
1853                 return 0;
1854         }
1855         link.link_status = 1;
1856         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1857
1858         switch (link_speed) {
1859         default:
1860         case IXGBE_LINK_SPEED_UNKNOWN:
1861                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1862                 link.link_speed = ETH_LINK_SPEED_100;
1863                 break;
1864
1865         case IXGBE_LINK_SPEED_100_FULL:
1866                 link.link_speed = ETH_LINK_SPEED_100;
1867                 break;
1868
1869         case IXGBE_LINK_SPEED_1GB_FULL:
1870                 link.link_speed = ETH_LINK_SPEED_1000;
1871                 break;
1872
1873         case IXGBE_LINK_SPEED_10GB_FULL:
1874                 link.link_speed = ETH_LINK_SPEED_10000;
1875                 break;
1876         }
1877         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1878
1879         if (link.link_status == old.link_status)
1880                 return -1;
1881
1882         return 0;
1883 }
1884
1885 static void
1886 ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev)
1887 {
1888         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1889         uint32_t fctrl;
1890
1891         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1892         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1893         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1894 }
1895
1896 static void
1897 ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev)
1898 {
1899         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1900         uint32_t fctrl;
1901
1902         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1903         fctrl &= (~IXGBE_FCTRL_UPE);
1904         if (dev->data->all_multicast == 1)
1905                 fctrl |= IXGBE_FCTRL_MPE;
1906         else
1907                 fctrl &= (~IXGBE_FCTRL_MPE);
1908         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1909 }
1910
1911 static void
1912 ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev)
1913 {
1914         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1915         uint32_t fctrl;
1916
1917         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1918         fctrl |= IXGBE_FCTRL_MPE;
1919         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1920 }
1921
1922 static void
1923 ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev)
1924 {
1925         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1926         uint32_t fctrl;
1927
1928         if (dev->data->promiscuous == 1)
1929                 return; /* must remain in all_multicast mode */
1930
1931         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1932         fctrl &= (~IXGBE_FCTRL_MPE);
1933         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1934 }
1935
1936 /**
1937  * It clears the interrupt causes and enables the interrupt.
1938  * It will be called once only during nic initialized.
1939  *
1940  * @param dev
1941  *  Pointer to struct rte_eth_dev.
1942  *
1943  * @return
1944  *  - On success, zero.
1945  *  - On failure, a negative value.
1946  */
1947 static int
1948 ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev)
1949 {
1950         struct ixgbe_interrupt *intr =
1951                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1952
1953         ixgbe_dev_link_status_print(dev);
1954         intr->mask |= IXGBE_EICR_LSC;
1955
1956         return 0;
1957 }
1958
1959 /*
1960  * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update.
1961  *
1962  * @param dev
1963  *  Pointer to struct rte_eth_dev.
1964  *
1965  * @return
1966  *  - On success, zero.
1967  *  - On failure, a negative value.
1968  */
1969 static int
1970 ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
1971 {
1972         uint32_t eicr;
1973         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1974         struct ixgbe_interrupt *intr =
1975                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1976
1977         /* clear all cause mask */
1978         ixgbe_disable_intr(hw);
1979
1980         /* read-on-clear nic registers here */
1981         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1982         PMD_DRV_LOG(INFO, "eicr %x", eicr);
1983
1984         intr->flags = 0;
1985         if (eicr & IXGBE_EICR_LSC) {
1986                 /* set flag for async link update */
1987                 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1988         }
1989
1990         if (eicr & IXGBE_EICR_MAILBOX)
1991                 intr->flags |= IXGBE_FLAG_MAILBOX;
1992
1993         return 0;
1994 }
1995
1996 /**
1997  * It gets and then prints the link status.
1998  *
1999  * @param dev
2000  *  Pointer to struct rte_eth_dev.
2001  *
2002  * @return
2003  *  - On success, zero.
2004  *  - On failure, a negative value.
2005  */
2006 static void
2007 ixgbe_dev_link_status_print(struct rte_eth_dev *dev)
2008 {
2009         struct rte_eth_link link;
2010
2011         memset(&link, 0, sizeof(link));
2012         rte_ixgbe_dev_atomic_read_link_status(dev, &link);
2013         if (link.link_status) {
2014                 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
2015                                         (int)(dev->data->port_id),
2016                                         (unsigned)link.link_speed,
2017                         link.link_duplex == ETH_LINK_FULL_DUPLEX ?
2018                                         "full-duplex" : "half-duplex");
2019         } else {
2020                 PMD_INIT_LOG(INFO, " Port %d: Link Down",
2021                                 (int)(dev->data->port_id));
2022         }
2023         PMD_INIT_LOG(INFO, "PCI Address: %04d:%02d:%02d:%d",
2024                                 dev->pci_dev->addr.domain,
2025                                 dev->pci_dev->addr.bus,
2026                                 dev->pci_dev->addr.devid,
2027                                 dev->pci_dev->addr.function);
2028 }
2029
2030 /*
2031  * It executes link_update after knowing an interrupt occurred.
2032  *
2033  * @param dev
2034  *  Pointer to struct rte_eth_dev.
2035  *
2036  * @return
2037  *  - On success, zero.
2038  *  - On failure, a negative value.
2039  */
2040 static int
2041 ixgbe_dev_interrupt_action(struct rte_eth_dev *dev)
2042 {
2043         struct ixgbe_interrupt *intr =
2044                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2045         int64_t timeout;
2046         struct rte_eth_link link;
2047         int intr_enable_delay = false;
2048
2049         PMD_DRV_LOG(DEBUG, "intr action type %d\n", intr->flags);
2050
2051         if (intr->flags & IXGBE_FLAG_MAILBOX) {
2052                 ixgbe_pf_mbx_process(dev);
2053                 intr->flags &= ~IXGBE_FLAG_MAILBOX;
2054         }
2055
2056         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
2057                 /* get the link status before link update, for predicting later */
2058                 memset(&link, 0, sizeof(link));
2059                 rte_ixgbe_dev_atomic_read_link_status(dev, &link);
2060
2061                 ixgbe_dev_link_update(dev, 0);
2062
2063                 /* likely to up */
2064                 if (!link.link_status)
2065                         /* handle it 1 sec later, wait it being stable */
2066                         timeout = IXGBE_LINK_UP_CHECK_TIMEOUT;
2067                 /* likely to down */
2068                 else
2069                         /* handle it 4 sec later, wait it being stable */
2070                         timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT;
2071
2072                 ixgbe_dev_link_status_print(dev);
2073
2074                 intr_enable_delay = true;
2075         }
2076
2077         if (intr_enable_delay) {
2078                 if (rte_eal_alarm_set(timeout * 1000,
2079                                       ixgbe_dev_interrupt_delayed_handler, (void*)dev) < 0)
2080                         PMD_DRV_LOG(ERR, "Error setting alarm");
2081         } else {
2082                 PMD_DRV_LOG(DEBUG, "enable intr immediately");
2083                 ixgbe_enable_intr(dev);
2084                 rte_intr_enable(&(dev->pci_dev->intr_handle));
2085         }
2086
2087
2088         return 0;
2089 }
2090
2091 /**
2092  * Interrupt handler which shall be registered for alarm callback for delayed
2093  * handling specific interrupt to wait for the stable nic state. As the
2094  * NIC interrupt state is not stable for ixgbe after link is just down,
2095  * it needs to wait 4 seconds to get the stable status.
2096  *
2097  * @param handle
2098  *  Pointer to interrupt handle.
2099  * @param param
2100  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2101  *
2102  * @return
2103  *  void
2104  */
2105 static void
2106 ixgbe_dev_interrupt_delayed_handler(void *param)
2107 {
2108         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2109         struct ixgbe_interrupt *intr =
2110                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2111         struct ixgbe_hw *hw =
2112                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2113         uint32_t eicr;
2114
2115         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
2116         if (eicr & IXGBE_EICR_MAILBOX)
2117                 ixgbe_pf_mbx_process(dev);
2118
2119         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
2120                 ixgbe_dev_link_update(dev, 0);
2121                 intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
2122                 ixgbe_dev_link_status_print(dev);
2123                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
2124         }
2125
2126         PMD_DRV_LOG(DEBUG, "enable intr in delayed handler S[%08x]\n", eicr);
2127         ixgbe_enable_intr(dev);
2128         rte_intr_enable(&(dev->pci_dev->intr_handle));
2129 }
2130
2131 /**
2132  * Interrupt handler triggered by NIC  for handling
2133  * specific interrupt.
2134  *
2135  * @param handle
2136  *  Pointer to interrupt handle.
2137  * @param param
2138  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2139  *
2140  * @return
2141  *  void
2142  */
2143 static void
2144 ixgbe_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
2145                                                         void *param)
2146 {
2147         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2148         ixgbe_dev_interrupt_get_status(dev);
2149         ixgbe_dev_interrupt_action(dev);
2150 }
2151
2152 static int
2153 ixgbe_dev_led_on(struct rte_eth_dev *dev)
2154 {
2155         struct ixgbe_hw *hw;
2156
2157         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2158         return (ixgbe_led_on(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
2159 }
2160
2161 static int
2162 ixgbe_dev_led_off(struct rte_eth_dev *dev)
2163 {
2164         struct ixgbe_hw *hw;
2165
2166         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2167         return (ixgbe_led_off(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
2168 }
2169
2170 static int
2171 ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2172 {
2173         struct ixgbe_hw *hw;
2174         int err;
2175         uint32_t rx_buf_size;
2176         uint32_t max_high_water;
2177         uint32_t mflcn;
2178         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
2179                 ixgbe_fc_none,
2180                 ixgbe_fc_rx_pause,
2181                 ixgbe_fc_tx_pause,
2182                 ixgbe_fc_full
2183         };
2184
2185         PMD_INIT_FUNC_TRACE();
2186
2187         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2188         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0));
2189         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
2190
2191         /*
2192          * At least reserve one Ethernet frame for watermark
2193          * high_water/low_water in kilo bytes for ixgbe
2194          */
2195         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
2196         if ((fc_conf->high_water > max_high_water) ||
2197                 (fc_conf->high_water < fc_conf->low_water)) {
2198                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
2199                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
2200                 return (-EINVAL);
2201         }
2202
2203         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[fc_conf->mode];
2204         hw->fc.pause_time     = fc_conf->pause_time;
2205         hw->fc.high_water[0]  = fc_conf->high_water;
2206         hw->fc.low_water[0]   = fc_conf->low_water;
2207         hw->fc.send_xon       = fc_conf->send_xon;
2208
2209         err = ixgbe_fc_enable(hw);
2210
2211         /* Not negotiated is not an error case */
2212         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) {
2213
2214                 /* check if we want to forward MAC frames - driver doesn't have native
2215                  * capability to do that, so we'll write the registers ourselves */
2216
2217                 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN);
2218
2219                 /* set or clear MFLCN.PMCF bit depending on configuration */
2220                 if (fc_conf->mac_ctrl_frame_fwd != 0)
2221                         mflcn |= IXGBE_MFLCN_PMCF;
2222                 else
2223                         mflcn &= ~IXGBE_MFLCN_PMCF;
2224
2225                 IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn);
2226                 IXGBE_WRITE_FLUSH(hw);
2227
2228                 return 0;
2229         }
2230
2231         PMD_INIT_LOG(ERR, "ixgbe_fc_enable = 0x%x \n", err);
2232         return -EIO;
2233 }
2234
2235 /**
2236  *  ixgbe_pfc_enable_generic - Enable flow control
2237  *  @hw: pointer to hardware structure
2238  *  @tc_num: traffic class number
2239  *  Enable flow control according to the current settings.
2240  */
2241 static int
2242 ixgbe_dcb_pfc_enable_generic(struct ixgbe_hw *hw,uint8_t tc_num)
2243 {
2244         int ret_val = 0;
2245         uint32_t mflcn_reg, fccfg_reg;
2246         uint32_t reg;
2247         uint32_t fcrtl, fcrth;
2248         uint8_t i;
2249         uint8_t nb_rx_en;
2250
2251         /* Validate the water mark configuration */
2252         if (!hw->fc.pause_time) {
2253                 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2254                 goto out;
2255         }
2256
2257         /* Low water mark of zero causes XOFF floods */
2258         if (hw->fc.current_mode & ixgbe_fc_tx_pause) {
2259                  /* High/Low water can not be 0 */
2260                 if( (!hw->fc.high_water[tc_num])|| (!hw->fc.low_water[tc_num])) {
2261                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
2262                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2263                         goto out;
2264                 }
2265
2266                 if(hw->fc.low_water[tc_num] >= hw->fc.high_water[tc_num]) {
2267                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
2268                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2269                         goto out;
2270                 }
2271         }
2272         /* Negotiate the fc mode to use */
2273         ixgbe_fc_autoneg(hw);
2274
2275         /* Disable any previous flow control settings */
2276         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
2277         mflcn_reg &= ~(IXGBE_MFLCN_RPFCE_SHIFT | IXGBE_MFLCN_RFCE|IXGBE_MFLCN_RPFCE);
2278
2279         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
2280         fccfg_reg &= ~(IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY);
2281
2282         switch (hw->fc.current_mode) {
2283         case ixgbe_fc_none:
2284                 /*
2285                  * If the count of enabled RX Priority Flow control >1,
2286                  * and the TX pause can not be disabled
2287                  */
2288                 nb_rx_en = 0;
2289                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
2290                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
2291                         if (reg & IXGBE_FCRTH_FCEN)
2292                                 nb_rx_en++;
2293                 }
2294                 if (nb_rx_en > 1)
2295                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2296                 break;
2297         case ixgbe_fc_rx_pause:
2298                 /*
2299                  * Rx Flow control is enabled and Tx Flow control is
2300                  * disabled by software override. Since there really
2301                  * isn't a way to advertise that we are capable of RX
2302                  * Pause ONLY, we will advertise that we support both
2303                  * symmetric and asymmetric Rx PAUSE.  Later, we will
2304                  * disable the adapter's ability to send PAUSE frames.
2305                  */
2306                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
2307                 /*
2308                  * If the count of enabled RX Priority Flow control >1,
2309                  * and the TX pause can not be disabled
2310                  */
2311                 nb_rx_en = 0;
2312                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
2313                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
2314                         if (reg & IXGBE_FCRTH_FCEN)
2315                                 nb_rx_en++;
2316                 }
2317                 if (nb_rx_en > 1)
2318                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2319                 break;
2320         case ixgbe_fc_tx_pause:
2321                 /*
2322                  * Tx Flow control is enabled, and Rx Flow control is
2323                  * disabled by software override.
2324                  */
2325                 fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2326                 break;
2327         case ixgbe_fc_full:
2328                 /* Flow control (both Rx and Tx) is enabled by SW override. */
2329                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
2330                 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
2331                 break;
2332         default:
2333                 DEBUGOUT("Flow control param set incorrectly\n");
2334                 ret_val = IXGBE_ERR_CONFIG;
2335                 goto out;
2336                 break;
2337         }
2338
2339         /* Set 802.3x based flow control settings. */
2340         mflcn_reg |= IXGBE_MFLCN_DPF;
2341         IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg);
2342         IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg);
2343
2344         /* Set up and enable Rx high/low water mark thresholds, enable XON. */
2345         if ((hw->fc.current_mode & ixgbe_fc_tx_pause) &&
2346                 hw->fc.high_water[tc_num]) {
2347                 fcrtl = (hw->fc.low_water[tc_num] << 10) | IXGBE_FCRTL_XONE;
2348                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), fcrtl);
2349                 fcrth = (hw->fc.high_water[tc_num] << 10) | IXGBE_FCRTH_FCEN;
2350         } else {
2351                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), 0);
2352                 /*
2353                  * In order to prevent Tx hangs when the internal Tx
2354                  * switch is enabled we must set the high water mark
2355                  * to the maximum FCRTH value.  This allows the Tx
2356                  * switch to function even under heavy Rx workloads.
2357                  */
2358                 fcrth = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)) - 32;
2359         }
2360         IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(tc_num), fcrth);
2361
2362         /* Configure pause time (2 TCs per register) */
2363         reg = hw->fc.pause_time * 0x00010001;
2364         for (i = 0; i < (IXGBE_DCB_MAX_TRAFFIC_CLASS / 2); i++)
2365                 IXGBE_WRITE_REG(hw, IXGBE_FCTTV(i), reg);
2366
2367         /* Configure flow control refresh threshold value */
2368         IXGBE_WRITE_REG(hw, IXGBE_FCRTV, hw->fc.pause_time / 2);
2369
2370 out:
2371         return ret_val;
2372 }
2373
2374 static int
2375 ixgbe_dcb_pfc_enable(struct rte_eth_dev *dev,uint8_t tc_num)
2376 {
2377         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2378         int32_t ret_val = IXGBE_NOT_IMPLEMENTED;
2379
2380         if(hw->mac.type != ixgbe_mac_82598EB) {
2381                 ret_val = ixgbe_dcb_pfc_enable_generic(hw,tc_num);
2382         }
2383         return ret_val;
2384 }
2385
2386 static int
2387 ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf)
2388 {
2389         int err;
2390         uint32_t rx_buf_size;
2391         uint32_t max_high_water;
2392         uint8_t tc_num;
2393         uint8_t  map[IXGBE_DCB_MAX_USER_PRIORITY] = { 0 };
2394         struct ixgbe_hw *hw =
2395                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2396         struct ixgbe_dcb_config *dcb_config =
2397                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
2398
2399         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
2400                 ixgbe_fc_none,
2401                 ixgbe_fc_rx_pause,
2402                 ixgbe_fc_tx_pause,
2403                 ixgbe_fc_full
2404         };
2405
2406         PMD_INIT_FUNC_TRACE();
2407
2408         ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
2409         tc_num = map[pfc_conf->priority];
2410         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num));
2411         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
2412         /*
2413          * At least reserve one Ethernet frame for watermark
2414          * high_water/low_water in kilo bytes for ixgbe
2415          */
2416         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
2417         if ((pfc_conf->fc.high_water > max_high_water) ||
2418                 (pfc_conf->fc.high_water <= pfc_conf->fc.low_water)) {
2419                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
2420                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
2421                 return (-EINVAL);
2422         }
2423
2424         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[pfc_conf->fc.mode];
2425         hw->fc.pause_time = pfc_conf->fc.pause_time;
2426         hw->fc.send_xon = pfc_conf->fc.send_xon;
2427         hw->fc.low_water[tc_num] =  pfc_conf->fc.low_water;
2428         hw->fc.high_water[tc_num] = pfc_conf->fc.high_water;
2429
2430         err = ixgbe_dcb_pfc_enable(dev,tc_num);
2431
2432         /* Not negotiated is not an error case */
2433         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED))
2434                 return 0;
2435
2436         PMD_INIT_LOG(ERR, "ixgbe_dcb_pfc_enable = 0x%x \n", err);
2437         return -EIO;
2438 }
2439
2440 static int
2441 ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
2442                                 struct rte_eth_rss_reta *reta_conf)
2443 {
2444         uint8_t i,j,mask;
2445         uint32_t reta;
2446         struct ixgbe_hw *hw =
2447                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2448
2449         PMD_INIT_FUNC_TRACE();
2450         /*
2451         * Update Redirection Table RETA[n],n=0...31,The redirection table has
2452         * 128-entries in 32 registers
2453          */
2454         for(i = 0; i < ETH_RSS_RETA_NUM_ENTRIES; i += 4) {
2455                 if (i < ETH_RSS_RETA_NUM_ENTRIES/2)
2456                         mask = (uint8_t)((reta_conf->mask_lo >> i) & 0xF);
2457                 else
2458                         mask = (uint8_t)((reta_conf->mask_hi >>
2459                                 (i - ETH_RSS_RETA_NUM_ENTRIES/2)) & 0xF);
2460                 if (mask != 0) {
2461                         reta = 0;
2462                         if (mask != 0xF)
2463                                 reta = IXGBE_READ_REG(hw,IXGBE_RETA(i >> 2));
2464
2465                         for (j = 0; j < 4; j++) {
2466                                 if (mask & (0x1 << j)) {
2467                                         if (mask != 0xF)
2468                                                 reta &= ~(0xFF << 8 * j);
2469                                         reta |= reta_conf->reta[i + j] << 8*j;
2470                                 }
2471                         }
2472                         IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2),reta);
2473                 }
2474         }
2475
2476         return 0;
2477 }
2478
2479 static int
2480 ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
2481                                 struct rte_eth_rss_reta *reta_conf)
2482 {
2483         uint8_t i,j,mask;
2484         uint32_t reta;
2485         struct ixgbe_hw *hw =
2486                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2487
2488         PMD_INIT_FUNC_TRACE();
2489         /*
2490          * Read Redirection Table RETA[n],n=0...31,The redirection table has
2491          * 128-entries in 32 registers
2492          */
2493         for(i = 0; i < ETH_RSS_RETA_NUM_ENTRIES; i += 4) {
2494                 if (i < ETH_RSS_RETA_NUM_ENTRIES/2)
2495                         mask = (uint8_t)((reta_conf->mask_lo >> i) & 0xF);
2496                 else
2497                         mask = (uint8_t)((reta_conf->mask_hi >>
2498                                 (i - ETH_RSS_RETA_NUM_ENTRIES/2)) & 0xF);
2499
2500                 if (mask != 0) {
2501                         reta = IXGBE_READ_REG(hw,IXGBE_RETA(i >> 2));
2502                         for (j = 0; j < 4; j++) {
2503                                 if (mask & (0x1 << j))
2504                                         reta_conf->reta[i + j] =
2505                                                 (uint8_t)((reta >> 8 * j) & 0xFF);
2506                         }
2507                 }
2508         }
2509
2510         return 0;
2511 }
2512
2513 static void
2514 ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
2515                                 uint32_t index, uint32_t pool)
2516 {
2517         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2518         uint32_t enable_addr = 1;
2519
2520         ixgbe_set_rar(hw, index, mac_addr->addr_bytes, pool, enable_addr);
2521 }
2522
2523 static void
2524 ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
2525 {
2526         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2527
2528         ixgbe_clear_rar(hw, index);
2529 }
2530
2531 /*
2532  * Virtual Function operations
2533  */
2534 static void
2535 ixgbevf_intr_disable(struct ixgbe_hw *hw)
2536 {
2537         PMD_INIT_LOG(DEBUG, "ixgbevf_intr_disable");
2538
2539         /* Clear interrupt mask to stop from interrupts being generated */
2540         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, IXGBE_VF_IRQ_CLEAR_MASK);
2541
2542         IXGBE_WRITE_FLUSH(hw);
2543 }
2544
2545 static int
2546 ixgbevf_dev_configure(struct rte_eth_dev *dev)
2547 {
2548         struct rte_eth_conf* conf = &dev->data->dev_conf;
2549
2550         PMD_INIT_LOG(DEBUG, "\nConfigured Virtual Function port id: %d\n",
2551                 dev->data->port_id);
2552
2553         /*
2554          * VF has no ability to enable/disable HW CRC
2555          * Keep the persistent behavior the same as Host PF
2556          */
2557 #ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC
2558         if (!conf->rxmode.hw_strip_crc) {
2559                 PMD_INIT_LOG(INFO, "VF can't disable HW CRC Strip\n");
2560                 conf->rxmode.hw_strip_crc = 1;
2561         }
2562 #else
2563         if (conf->rxmode.hw_strip_crc) {
2564                 PMD_INIT_LOG(INFO, "VF can't enable HW CRC Strip\n");
2565                 conf->rxmode.hw_strip_crc = 0;
2566         }
2567 #endif
2568
2569         return 0;
2570 }
2571
2572 static int
2573 ixgbevf_dev_start(struct rte_eth_dev *dev)
2574 {
2575         struct ixgbe_hw *hw =
2576                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2577         int err, mask = 0;
2578
2579         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_start");
2580
2581         hw->mac.ops.reset_hw(hw);
2582
2583         /* negotiate mailbox API version to use with the PF. */
2584         ixgbevf_negotiate_api(hw);
2585
2586         ixgbevf_dev_tx_init(dev);
2587
2588         /* This can fail when allocating mbufs for descriptor rings */
2589         err = ixgbevf_dev_rx_init(dev);
2590         if (err) {
2591                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware (%d)\n", err);
2592                 ixgbe_dev_clear_queues(dev);
2593                 return err;
2594         }
2595
2596         /* Set vfta */
2597         ixgbevf_set_vfta_all(dev,1);
2598
2599         /* Set HW strip */
2600         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
2601                 ETH_VLAN_EXTEND_MASK;
2602         ixgbevf_vlan_offload_set(dev, mask);
2603
2604         ixgbevf_dev_rxtx_start(dev);
2605
2606         return 0;
2607 }
2608
2609 static void
2610 ixgbevf_dev_stop(struct rte_eth_dev *dev)
2611 {
2612         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2613
2614         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_stop");
2615
2616         hw->adapter_stopped = TRUE;
2617         ixgbe_stop_adapter(hw);
2618
2619         /*
2620           * Clear what we set, but we still keep shadow_vfta to
2621           * restore after device starts
2622           */
2623         ixgbevf_set_vfta_all(dev,0);
2624
2625         ixgbe_dev_clear_queues(dev);
2626 }
2627
2628 static void
2629 ixgbevf_dev_close(struct rte_eth_dev *dev)
2630 {
2631         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2632
2633         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_close");
2634
2635         ixgbe_reset_hw(hw);
2636
2637         ixgbevf_dev_stop(dev);
2638
2639         /* reprogram the RAR[0] in case user changed it. */
2640         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
2641 }
2642
2643 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on)
2644 {
2645         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2646         struct ixgbe_vfta * shadow_vfta =
2647                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2648         int i = 0, j = 0, vfta = 0, mask = 1;
2649
2650         for (i = 0; i < IXGBE_VFTA_SIZE; i++){
2651                 vfta = shadow_vfta->vfta[i];
2652                 if(vfta){
2653                         mask = 1;
2654                         for (j = 0; j < 32; j++){
2655                                 if(vfta & mask)
2656                                         ixgbe_set_vfta(hw, (i<<5)+j, 0, on);
2657                                 mask<<=1;
2658                         }
2659                 }
2660         }
2661
2662 }
2663
2664 static int
2665 ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2666 {
2667         struct ixgbe_hw *hw =
2668                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2669         struct ixgbe_vfta * shadow_vfta =
2670                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2671         uint32_t vid_idx = 0;
2672         uint32_t vid_bit = 0;
2673         int ret = 0;
2674
2675         PMD_INIT_FUNC_TRACE();
2676
2677         /* vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf */
2678         ret = ixgbe_set_vfta(hw, vlan_id, 0, !!on);
2679         if(ret){
2680                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
2681                 return ret;
2682         }
2683         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
2684         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
2685
2686         /* Save what we set and retore it after device reset */
2687         if (on)
2688                 shadow_vfta->vfta[vid_idx] |= vid_bit;
2689         else
2690                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
2691
2692         return 0;
2693 }
2694
2695 static void
2696 ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
2697 {
2698         struct ixgbe_hw *hw =
2699                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2700         uint32_t ctrl;
2701
2702         PMD_INIT_FUNC_TRACE();
2703
2704         if(queue >= hw->mac.max_rx_queues)
2705                 return;
2706
2707         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
2708         if(on)
2709                 ctrl |= IXGBE_RXDCTL_VME;
2710         else
2711                 ctrl &= ~IXGBE_RXDCTL_VME;
2712         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
2713
2714         ixgbe_vlan_hw_strip_bitmap_set( dev, queue, on);
2715 }
2716
2717 static void
2718 ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2719 {
2720         struct ixgbe_hw *hw =
2721                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2722         uint16_t i;
2723         int on = 0;
2724
2725         /* VF function only support hw strip feature, others are not support */
2726         if(mask & ETH_VLAN_STRIP_MASK){
2727                 on = !!(dev->data->dev_conf.rxmode.hw_vlan_strip);
2728
2729                 for(i=0; i < hw->mac.max_rx_queues; i++)
2730                         ixgbevf_vlan_strip_queue_set(dev,i,on);
2731         }
2732 }
2733
2734 static int
2735 ixgbe_vmdq_mode_check(struct ixgbe_hw *hw)
2736 {
2737         uint32_t reg_val;
2738
2739         /* we only need to do this if VMDq is enabled */
2740         reg_val = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
2741         if (!(reg_val & IXGBE_VT_CTL_VT_ENABLE)) {
2742                 PMD_INIT_LOG(ERR, "VMDq must be enabled for this setting\n");
2743                 return (-1);
2744         }
2745
2746         return 0;
2747 }
2748
2749 static uint32_t
2750 ixgbe_uta_vector(struct ixgbe_hw *hw, struct ether_addr* uc_addr)
2751 {
2752         uint32_t vector = 0;
2753         switch (hw->mac.mc_filter_type) {
2754         case 0:   /* use bits [47:36] of the address */
2755                 vector = ((uc_addr->addr_bytes[4] >> 4) |
2756                         (((uint16_t)uc_addr->addr_bytes[5]) << 4));
2757                 break;
2758         case 1:   /* use bits [46:35] of the address */
2759                 vector = ((uc_addr->addr_bytes[4] >> 3) |
2760                         (((uint16_t)uc_addr->addr_bytes[5]) << 5));
2761                 break;
2762         case 2:   /* use bits [45:34] of the address */
2763                 vector = ((uc_addr->addr_bytes[4] >> 2) |
2764                         (((uint16_t)uc_addr->addr_bytes[5]) << 6));
2765                 break;
2766         case 3:   /* use bits [43:32] of the address */
2767                 vector = ((uc_addr->addr_bytes[4]) |
2768                         (((uint16_t)uc_addr->addr_bytes[5]) << 8));
2769                 break;
2770         default:  /* Invalid mc_filter_type */
2771                 break;
2772         }
2773
2774         /* vector can only be 12-bits or boundary will be exceeded */
2775         vector &= 0xFFF;
2776         return vector;
2777 }
2778
2779 static int
2780 ixgbe_uc_hash_table_set(struct rte_eth_dev *dev,struct ether_addr* mac_addr,
2781                                uint8_t on)
2782 {
2783         uint32_t vector;
2784         uint32_t uta_idx;
2785         uint32_t reg_val;
2786         uint32_t uta_shift;
2787         uint32_t rc;
2788         const uint32_t ixgbe_uta_idx_mask = 0x7F;
2789         const uint32_t ixgbe_uta_bit_shift = 5;
2790         const uint32_t ixgbe_uta_bit_mask = (0x1 << ixgbe_uta_bit_shift) - 1;
2791         const uint32_t bit1 = 0x1;
2792
2793         struct ixgbe_hw *hw =
2794                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2795         struct ixgbe_uta_info *uta_info =
2796                 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private);
2797
2798         /* The UTA table only exists on 82599 hardware and newer */
2799         if (hw->mac.type < ixgbe_mac_82599EB)
2800                 return (-ENOTSUP);
2801
2802         vector = ixgbe_uta_vector(hw,mac_addr);
2803         uta_idx = (vector >> ixgbe_uta_bit_shift) & ixgbe_uta_idx_mask;
2804         uta_shift = vector & ixgbe_uta_bit_mask;
2805
2806         rc = ((uta_info->uta_shadow[uta_idx] >> uta_shift & bit1) != 0);
2807         if(rc == on)
2808                 return 0;
2809
2810         reg_val = IXGBE_READ_REG(hw, IXGBE_UTA(uta_idx));
2811         if (on) {
2812                 uta_info->uta_in_use++;
2813                 reg_val |= (bit1 << uta_shift);
2814                 uta_info->uta_shadow[uta_idx] |= (bit1 << uta_shift);
2815         } else {
2816                 uta_info->uta_in_use--;
2817                 reg_val &= ~(bit1 << uta_shift);
2818                 uta_info->uta_shadow[uta_idx] &= ~(bit1 << uta_shift);
2819         }
2820
2821         IXGBE_WRITE_REG(hw, IXGBE_UTA(uta_idx), reg_val);
2822
2823         if (uta_info->uta_in_use > 0)
2824                 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL,
2825                                 IXGBE_MCSTCTRL_MFE | hw->mac.mc_filter_type);
2826         else
2827                 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL,hw->mac.mc_filter_type);
2828
2829         return 0;
2830 }
2831
2832 static int
2833 ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on)
2834 {
2835         int i;
2836         struct ixgbe_hw *hw =
2837                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2838         struct ixgbe_uta_info *uta_info =
2839                 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private);
2840
2841         /* The UTA table only exists on 82599 hardware and newer */
2842         if (hw->mac.type < ixgbe_mac_82599EB)
2843                 return (-ENOTSUP);
2844
2845         if(on) {
2846                 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
2847                         uta_info->uta_shadow[i] = ~0;
2848                         IXGBE_WRITE_REG(hw, IXGBE_UTA(i), ~0);
2849                 }
2850         } else {
2851                 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
2852                         uta_info->uta_shadow[i] = 0;
2853                         IXGBE_WRITE_REG(hw, IXGBE_UTA(i), 0);
2854                 }
2855         }
2856         return 0;
2857
2858 }
2859 static int
2860 ixgbe_set_pool_rx_mode(struct rte_eth_dev *dev, uint16_t pool,
2861                                uint16_t rx_mask, uint8_t on)
2862 {
2863         int val = 0;
2864
2865         struct ixgbe_hw *hw =
2866                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2867         uint32_t vmolr = IXGBE_READ_REG(hw, IXGBE_VMOLR(pool));
2868
2869         if (hw->mac.type == ixgbe_mac_82598EB) {
2870                 PMD_INIT_LOG(ERR, "setting VF receive mode set should be done"
2871                         " on 82599 hardware and newer\n");
2872                 return (-ENOTSUP);
2873         }
2874         if (ixgbe_vmdq_mode_check(hw) < 0)
2875                 return (-ENOTSUP);
2876
2877         if (rx_mask & ETH_VMDQ_ACCEPT_UNTAG )
2878                 val |= IXGBE_VMOLR_AUPE;
2879         if (rx_mask & ETH_VMDQ_ACCEPT_HASH_MC )
2880                 val |= IXGBE_VMOLR_ROMPE;
2881         if (rx_mask & ETH_VMDQ_ACCEPT_HASH_UC)
2882                 val |= IXGBE_VMOLR_ROPE;
2883         if (rx_mask & ETH_VMDQ_ACCEPT_BROADCAST)
2884                 val |= IXGBE_VMOLR_BAM;
2885         if (rx_mask & ETH_VMDQ_ACCEPT_MULTICAST)
2886                 val |= IXGBE_VMOLR_MPE;
2887
2888         if (on)
2889                 vmolr |= val;
2890         else
2891                 vmolr &= ~val;
2892
2893         IXGBE_WRITE_REG(hw, IXGBE_VMOLR(pool), vmolr);
2894
2895         return 0;
2896 }
2897
2898 static int
2899 ixgbe_set_pool_rx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on)
2900 {
2901         uint32_t reg,addr;
2902         uint32_t val;
2903         const uint8_t bit1 = 0x1;
2904
2905         struct ixgbe_hw *hw =
2906                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2907
2908         if (ixgbe_vmdq_mode_check(hw) < 0)
2909                 return (-ENOTSUP);
2910
2911         addr = IXGBE_VFRE(pool >= ETH_64_POOLS/2);
2912         reg = IXGBE_READ_REG(hw, addr);
2913         val = bit1 << pool;
2914
2915         if (on)
2916                 reg |= val;
2917         else
2918                 reg &= ~val;
2919
2920         IXGBE_WRITE_REG(hw, addr,reg);
2921
2922         return 0;
2923 }
2924
2925 static int
2926 ixgbe_set_pool_tx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on)
2927 {
2928         uint32_t reg,addr;
2929         uint32_t val;
2930         const uint8_t bit1 = 0x1;
2931
2932         struct ixgbe_hw *hw =
2933                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2934
2935         if (ixgbe_vmdq_mode_check(hw) < 0)
2936                 return (-ENOTSUP);
2937
2938         addr = IXGBE_VFTE(pool >= ETH_64_POOLS/2);
2939         reg = IXGBE_READ_REG(hw, addr);
2940         val = bit1 << pool;
2941
2942         if (on)
2943                 reg |= val;
2944         else
2945                 reg &= ~val;
2946
2947         IXGBE_WRITE_REG(hw, addr,reg);
2948
2949         return 0;
2950 }
2951
2952 static int
2953 ixgbe_set_pool_vlan_filter(struct rte_eth_dev *dev, uint16_t vlan,
2954                         uint64_t pool_mask, uint8_t vlan_on)
2955 {
2956         int ret = 0;
2957         uint16_t pool_idx;
2958         struct ixgbe_hw *hw =
2959                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2960
2961         if (ixgbe_vmdq_mode_check(hw) < 0)
2962                 return (-ENOTSUP);
2963         for (pool_idx = 0; pool_idx < ETH_64_POOLS; pool_idx++) {
2964                 if (pool_mask & ((uint64_t)(1ULL << pool_idx)))
2965                         ret = hw->mac.ops.set_vfta(hw,vlan,pool_idx,vlan_on);
2966                         if (ret < 0)
2967                                 return ret;
2968         }
2969
2970         return ret;
2971 }
2972
2973 static int
2974 ixgbe_mirror_rule_set(struct rte_eth_dev *dev,
2975                         struct rte_eth_vmdq_mirror_conf *mirror_conf,
2976                         uint8_t rule_id, uint8_t on)
2977 {
2978         uint32_t mr_ctl,vlvf;
2979         uint32_t mp_lsb = 0;
2980         uint32_t mv_msb = 0;
2981         uint32_t mv_lsb = 0;
2982         uint32_t mp_msb = 0;
2983         uint8_t i = 0;
2984         int reg_index = 0;
2985         uint64_t vlan_mask = 0;
2986
2987         const uint8_t pool_mask_offset = 32;
2988         const uint8_t vlan_mask_offset = 32;
2989         const uint8_t dst_pool_offset = 8;
2990         const uint8_t rule_mr_offset  = 4;
2991         const uint8_t mirror_rule_mask= 0x0F;
2992
2993         struct ixgbe_mirror_info *mr_info =
2994                         (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private));
2995         struct ixgbe_hw *hw =
2996                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2997
2998         if (ixgbe_vmdq_mode_check(hw) < 0)
2999                 return (-ENOTSUP);
3000
3001         /* Check if vlan mask is valid */
3002         if ((mirror_conf->rule_type_mask & ETH_VMDQ_VLAN_MIRROR) && (on)) {
3003                 if (mirror_conf->vlan.vlan_mask == 0)
3004                         return (-EINVAL);
3005         }
3006
3007         /* Check if vlan id is valid and find conresponding VLAN ID index in VLVF */
3008         if (mirror_conf->rule_type_mask & ETH_VMDQ_VLAN_MIRROR) {
3009                 for (i = 0;i < IXGBE_VLVF_ENTRIES; i++) {
3010                         if (mirror_conf->vlan.vlan_mask & (1ULL << i)) {
3011                                 /* search vlan id related pool vlan filter index */
3012                                 reg_index = ixgbe_find_vlvf_slot(hw,
3013                                                 mirror_conf->vlan.vlan_id[i]);
3014                                 if(reg_index < 0)
3015                                         return (-EINVAL);
3016                                 vlvf = IXGBE_READ_REG(hw, IXGBE_VLVF(reg_index));
3017                                 if ((vlvf & IXGBE_VLVF_VIEN) &&
3018                                         ((vlvf & IXGBE_VLVF_VLANID_MASK)
3019                                                 == mirror_conf->vlan.vlan_id[i]))
3020                                         vlan_mask |= (1ULL << reg_index);
3021                                 else
3022                                         return (-EINVAL);
3023                         }
3024                 }
3025
3026                 if (on) {
3027                         mv_lsb = vlan_mask & 0xFFFFFFFF;
3028                         mv_msb = vlan_mask >> vlan_mask_offset;
3029
3030                         mr_info->mr_conf[rule_id].vlan.vlan_mask =
3031                                                 mirror_conf->vlan.vlan_mask;
3032                         for(i = 0 ;i < ETH_VMDQ_MAX_VLAN_FILTERS; i++) {
3033                                 if(mirror_conf->vlan.vlan_mask & (1ULL << i))
3034                                         mr_info->mr_conf[rule_id].vlan.vlan_id[i] =
3035                                                 mirror_conf->vlan.vlan_id[i];
3036                         }
3037                 } else {
3038                         mv_lsb = 0;
3039                         mv_msb = 0;
3040                         mr_info->mr_conf[rule_id].vlan.vlan_mask = 0;
3041                         for(i = 0 ;i < ETH_VMDQ_MAX_VLAN_FILTERS; i++)
3042                                 mr_info->mr_conf[rule_id].vlan.vlan_id[i] = 0;
3043                 }
3044         }
3045
3046         /*
3047          * if enable pool mirror, write related pool mask register,if disable
3048          * pool mirror, clear PFMRVM register
3049          */
3050         if (mirror_conf->rule_type_mask & ETH_VMDQ_POOL_MIRROR) {
3051                 if (on) {
3052                         mp_lsb = mirror_conf->pool_mask & 0xFFFFFFFF;
3053                         mp_msb = mirror_conf->pool_mask >> pool_mask_offset;
3054                         mr_info->mr_conf[rule_id].pool_mask =
3055                                         mirror_conf->pool_mask;
3056
3057                 } else {
3058                         mp_lsb = 0;
3059                         mp_msb = 0;
3060                         mr_info->mr_conf[rule_id].pool_mask = 0;
3061                 }
3062         }
3063
3064         /* read  mirror control register and recalculate it */
3065         mr_ctl = IXGBE_READ_REG(hw,IXGBE_MRCTL(rule_id));
3066
3067         if (on) {
3068                 mr_ctl |= mirror_conf->rule_type_mask;
3069                 mr_ctl &= mirror_rule_mask;
3070                 mr_ctl |= mirror_conf->dst_pool << dst_pool_offset;
3071         } else
3072                 mr_ctl &= ~(mirror_conf->rule_type_mask & mirror_rule_mask);
3073
3074         mr_info->mr_conf[rule_id].rule_type_mask = (uint8_t)(mr_ctl & mirror_rule_mask);
3075         mr_info->mr_conf[rule_id].dst_pool = mirror_conf->dst_pool;
3076
3077         /* write mirrror control  register */
3078         IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl);
3079
3080         /* write pool mirrror control  register */
3081         if (mirror_conf->rule_type_mask & ETH_VMDQ_POOL_MIRROR) {
3082                 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), mp_lsb);
3083                 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset),
3084                                 mp_msb);
3085         }
3086         /* write VLAN mirrror control  register */
3087         if (mirror_conf->rule_type_mask & ETH_VMDQ_VLAN_MIRROR) {
3088                 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), mv_lsb);
3089                 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset),
3090                                 mv_msb);
3091         }
3092
3093         return 0;
3094 }
3095
3096 static int
3097 ixgbe_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t rule_id)
3098 {
3099         int mr_ctl = 0;
3100         uint32_t lsb_val = 0;
3101         uint32_t msb_val = 0;
3102         const uint8_t rule_mr_offset = 4;
3103
3104         struct ixgbe_hw *hw =
3105                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3106         struct ixgbe_mirror_info *mr_info =
3107                 (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private));
3108
3109         if (ixgbe_vmdq_mode_check(hw) < 0)
3110                 return (-ENOTSUP);
3111
3112         memset(&mr_info->mr_conf[rule_id], 0,
3113                 sizeof(struct rte_eth_vmdq_mirror_conf));
3114
3115         /* clear PFVMCTL register */
3116         IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl);
3117
3118         /* clear pool mask register */
3119         IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), lsb_val);
3120         IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset), msb_val);
3121
3122         /* clear vlan mask register */
3123         IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), lsb_val);
3124         IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset), msb_val);
3125
3126         return 0;
3127 }
3128
3129 static void
3130 ixgbevf_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
3131                      __attribute__((unused)) uint32_t index,
3132                      __attribute__((unused)) uint32_t pool)
3133 {
3134         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3135         int diag;
3136
3137         /*
3138          * On a 82599 VF, adding again the same MAC addr is not an idempotent
3139          * operation. Trap this case to avoid exhausting the [very limited]
3140          * set of PF resources used to store VF MAC addresses.
3141          */
3142         if (memcmp(hw->mac.perm_addr, mac_addr, sizeof(struct ether_addr)) == 0)
3143                 return;
3144         diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes);
3145         if (diag == 0)
3146                 return;
3147         PMD_DRV_LOG(ERR, "Unable to add MAC address - diag=%d", diag);
3148 }
3149
3150 static void
3151 ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index)
3152 {
3153         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3154         struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr;
3155         struct ether_addr *mac_addr;
3156         uint32_t i;
3157         int diag;
3158
3159         /*
3160          * The IXGBE_VF_SET_MACVLAN command of the ixgbe-pf driver does
3161          * not support the deletion of a given MAC address.
3162          * Instead, it imposes to delete all MAC addresses, then to add again
3163          * all MAC addresses with the exception of the one to be deleted.
3164          */
3165         (void) ixgbevf_set_uc_addr_vf(hw, 0, NULL);
3166
3167         /*
3168          * Add again all MAC addresses, with the exception of the deleted one
3169          * and of the permanent MAC address.
3170          */
3171         for (i = 0, mac_addr = dev->data->mac_addrs;
3172              i < hw->mac.num_rar_entries; i++, mac_addr++) {
3173                 /* Skip the deleted MAC address */
3174                 if (i == index)
3175                         continue;
3176                 /* Skip NULL MAC addresses */
3177                 if (is_zero_ether_addr(mac_addr))
3178                         continue;
3179                 /* Skip the permanent MAC address */
3180                 if (memcmp(perm_addr, mac_addr, sizeof(struct ether_addr)) == 0)
3181                         continue;
3182                 diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes);
3183                 if (diag != 0)
3184                         PMD_DRV_LOG(ERR,
3185                                     "Adding again MAC address "
3186                                     "%02x:%02x:%02x:%02x:%02x:%02x failed "
3187                                     "diag=%d",
3188                                     mac_addr->addr_bytes[0],
3189                                     mac_addr->addr_bytes[1],
3190                                     mac_addr->addr_bytes[2],
3191                                     mac_addr->addr_bytes[3],
3192                                     mac_addr->addr_bytes[4],
3193                                     mac_addr->addr_bytes[5],
3194                                     diag);
3195         }
3196 }
3197
3198 static struct rte_driver rte_ixgbe_driver = {
3199         .type = PMD_PDEV,
3200         .init = rte_ixgbe_pmd_init,
3201 };
3202
3203 static struct rte_driver rte_ixgbevf_driver = {
3204         .type = PMD_PDEV,
3205         .init = rte_ixgbevf_pmd_init,
3206 };
3207
3208 PMD_REGISTER_DRIVER(rte_ixgbe_driver);
3209 PMD_REGISTER_DRIVER(rte_ixgbevf_driver);