lib/librte_pmd_ixgbe/ixgbe_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <inttypes.h>
  42 #include <rte_byteorder.h>
  43 #include <rte_common.h>
  44 #include <rte_cycles.h>
  45
  46 #include <rte_interrupts.h>
  47 #include <rte_log.h>
  48 #include <rte_debug.h>
  49 #include <rte_pci.h>
  50 #include <rte_atomic.h>
  51 #include <rte_branch_prediction.h>
  52 #include <rte_memory.h>
  53 #include <rte_memzone.h>
  54 #include <rte_tailq.h>
  55 #include <rte_eal.h>
  56 #include <rte_alarm.h>
  57 #include <rte_ether.h>
  58 #include <rte_ethdev.h>
  59 #include <rte_atomic.h>
  60 #include <rte_malloc.h>
  61 #include <rte_random.h>
  62 #include <rte_dev.h>
  63
  64 #include "ixgbe_logs.h"
  65 #include "ixgbe/ixgbe_api.h"
  66 #include "ixgbe/ixgbe_vf.h"
  67 #include "ixgbe/ixgbe_common.h"
  68 #include "ixgbe_ethdev.h"
  69 #include "ixgbe_bypass.h"
  70
  71 /*
  72  * High threshold controlling when to start sending XOFF frames. Must be at
  73  * least 8 bytes less than receive packet buffer size. This value is in units
  74  * of 1024 bytes.
  75  */
  76 #define IXGBE_FC_HI    0x80
  77
  78 /*
  79  * Low threshold controlling when to start sending XON frames. This value is
  80  * in units of 1024 bytes.
  81  */
  82 #define IXGBE_FC_LO    0x40
  83
  84 /* Timer value included in XOFF frames. */
  85 #define IXGBE_FC_PAUSE 0x680
  86
  87 #define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */
  88 #define IXGBE_LINK_UP_CHECK_TIMEOUT   1000 /* ms */
  89 #define IXGBE_VMDQ_NUM_UC_MAC         4096 /* Maximum nb. of UC MAC addr. */
  90
  91
  92 #define IXGBEVF_PMD_NAME "rte_ixgbevf_pmd" /* PMD name */
  93
  94 #define IXGBE_QUEUE_STAT_COUNTERS (sizeof(hw_stats->qprc) / sizeof(hw_stats->qprc[0]))
  95
  96 static int eth_ixgbe_dev_init(struct eth_driver *eth_drv,
  97                 struct rte_eth_dev *eth_dev);
  98 static int  ixgbe_dev_configure(struct rte_eth_dev *dev);
  99 static int  ixgbe_dev_start(struct rte_eth_dev *dev);
 100 static void ixgbe_dev_stop(struct rte_eth_dev *dev);
 101 static void ixgbe_dev_close(struct rte_eth_dev *dev);
 102 static void ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev);
 103 static void ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev);
 104 static void ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev);
 105 static void ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev);
 106 static int ixgbe_dev_link_update(struct rte_eth_dev *dev,
 107                                 int wait_to_complete);
 108 static void ixgbe_dev_stats_get(struct rte_eth_dev *dev,
 109                                 struct rte_eth_stats *stats);
 110 static void ixgbe_dev_stats_reset(struct rte_eth_dev *dev);
 111 static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 112                                              uint16_t queue_id,
 113                                              uint8_t stat_idx,
 114                                              uint8_t is_rx);
 115 static void ixgbe_dev_info_get(struct rte_eth_dev *dev,
 116                                 struct rte_eth_dev_info *dev_info);
 117 static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev,
 118                 uint16_t vlan_id, int on);
 119 static void ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 120 static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev,
 121                 uint16_t queue, bool on);
 122 static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue,
 123                 int on);
 124 static void ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 125 static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue);
 126 static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue);
 127 static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 128 static void ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 129
 130 static int ixgbe_dev_led_on(struct rte_eth_dev *dev);
 131 static int ixgbe_dev_led_off(struct rte_eth_dev *dev);
 132 static int  ixgbe_flow_ctrl_set(struct rte_eth_dev *dev,
 133                 struct rte_eth_fc_conf *fc_conf);
 134 static int ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 135                 struct rte_eth_pfc_conf *pfc_conf);
 136 static int ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
 137                 struct rte_eth_rss_reta *reta_conf);
 138 static int ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
 139                 struct rte_eth_rss_reta *reta_conf);
 140 static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev);
 141 static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev);
 142 static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev);
 143 static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev);
 144 static void ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle,
 145                 void *param);
 146 static void ixgbe_dev_interrupt_delayed_handler(void *param);
 147 static void ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 148                 uint32_t index, uint32_t pool);
 149 static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index);
 150 static void ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config);
 151
 152 /* For Virtual Function support */
 153 static int eth_ixgbevf_dev_init(struct eth_driver *eth_drv,
 154                 struct rte_eth_dev *eth_dev);
 155 static int  ixgbevf_dev_configure(struct rte_eth_dev *dev);
 156 static int  ixgbevf_dev_start(struct rte_eth_dev *dev);
 157 static void ixgbevf_dev_stop(struct rte_eth_dev *dev);
 158 static void ixgbevf_dev_close(struct rte_eth_dev *dev);
 159 static void ixgbevf_intr_disable(struct ixgbe_hw *hw);
 160 static void ixgbevf_dev_stats_get(struct rte_eth_dev *dev,
 161                 struct rte_eth_stats *stats);
 162 static void ixgbevf_dev_stats_reset(struct rte_eth_dev *dev);
 163 static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev,
 164                 uint16_t vlan_id, int on);
 165 static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev,
 166                 uint16_t queue, int on);
 167 static void ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 168 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 169
 170 /* For Eth VMDQ APIs support */
 171 static int ixgbe_uc_hash_table_set(struct rte_eth_dev *dev, struct
 172                 ether_addr* mac_addr,uint8_t on);
 173 static int ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev,uint8_t on);
 174 static int  ixgbe_set_pool_rx_mode(struct rte_eth_dev *dev,  uint16_t pool,
 175                 uint16_t rx_mask, uint8_t on);
 176 static int ixgbe_set_pool_rx(struct rte_eth_dev *dev,uint16_t pool,uint8_t on);
 177 static int ixgbe_set_pool_tx(struct rte_eth_dev *dev,uint16_t pool,uint8_t on);
 178 static int ixgbe_set_pool_vlan_filter(struct rte_eth_dev *dev, uint16_t vlan,
 179                 uint64_t pool_mask,uint8_t vlan_on);
 180 static int ixgbe_mirror_rule_set(struct rte_eth_dev *dev,
 181                 struct rte_eth_vmdq_mirror_conf *mirror_conf,
 182                 uint8_t rule_id, uint8_t on);
 183 static int ixgbe_mirror_rule_reset(struct rte_eth_dev *dev,
 184                 uint8_t rule_id);
 185
 186 static void ixgbevf_add_mac_addr(struct rte_eth_dev *dev,
 187                                  struct ether_addr *mac_addr,
 188                                  uint32_t index, uint32_t pool);
 189 static void ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index);
 190
 191 /*
 192  * Define VF Stats MACRO for Non "cleared on read" register
 193  */
 194 #define UPDATE_VF_STAT(reg, last, cur)                          \
 195 {                                                               \
 196         u32 latest = IXGBE_READ_REG(hw, reg);                   \
 197         cur += latest - last;                                   \
 198         last = latest;                                          \
 199 }
 200
 201 #define UPDATE_VF_STAT_36BIT(lsb, msb, last, cur)                \
 202 {                                                                \
 203         u64 new_lsb = IXGBE_READ_REG(hw, lsb);                   \
 204         u64 new_msb = IXGBE_READ_REG(hw, msb);                   \
 205         u64 latest = ((new_msb << 32) | new_lsb);                \
 206         cur += (0x1000000000LL + latest - last) & 0xFFFFFFFFFLL; \
 207         last = latest;                                           \
 208 }
 209
 210 #define IXGBE_SET_HWSTRIP(h, q) do{\
 211                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 212                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 213                 (h)->bitmap[idx] |= 1 << bit;\
 214         }while(0)
 215
 216 #define IXGBE_CLEAR_HWSTRIP(h, q) do{\
 217                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 218                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 219                 (h)->bitmap[idx] &= ~(1 << bit);\
 220         }while(0)
 221
 222 #define IXGBE_GET_HWSTRIP(h, q, r) do{\
 223                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 224                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 225                 (r) = (h)->bitmap[idx] >> bit & 1;\
 226         }while(0)
 227
 228 /*
 229  * The set of PCI devices this driver supports
 230  */
 231 static struct rte_pci_id pci_id_ixgbe_map[] = {
 232
 233 #define RTE_PCI_DEV_ID_DECL_IXGBE(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 234 #include "rte_pci_dev_ids.h"
 235
 236 { .vendor_id = 0, /* sentinel */ },
 237 };
 238
 239
 240 /*
 241  * The set of PCI devices this driver supports (for 82599 VF)
 242  */
 243 static struct rte_pci_id pci_id_ixgbevf_map[] = {
 244
 245 #define RTE_PCI_DEV_ID_DECL_IXGBEVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 246 #include "rte_pci_dev_ids.h"
 247 { .vendor_id = 0, /* sentinel */ },
 248
 249 };
 250
 251 static struct eth_dev_ops ixgbe_eth_dev_ops = {
 252         .dev_configure        = ixgbe_dev_configure,
 253         .dev_start            = ixgbe_dev_start,
 254         .dev_stop             = ixgbe_dev_stop,
 255         .dev_close            = ixgbe_dev_close,
 256         .promiscuous_enable   = ixgbe_dev_promiscuous_enable,
 257         .promiscuous_disable  = ixgbe_dev_promiscuous_disable,
 258         .allmulticast_enable  = ixgbe_dev_allmulticast_enable,
 259         .allmulticast_disable = ixgbe_dev_allmulticast_disable,
 260         .link_update          = ixgbe_dev_link_update,
 261         .stats_get            = ixgbe_dev_stats_get,
 262         .stats_reset          = ixgbe_dev_stats_reset,
 263         .queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set,
 264         .dev_infos_get        = ixgbe_dev_info_get,
 265         .vlan_filter_set      = ixgbe_vlan_filter_set,
 266         .vlan_tpid_set        = ixgbe_vlan_tpid_set,
 267         .vlan_offload_set     = ixgbe_vlan_offload_set,
 268         .vlan_strip_queue_set = ixgbe_vlan_strip_queue_set,
 269         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 270         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 271         .rx_queue_count       = ixgbe_dev_rx_queue_count,
 272         .rx_descriptor_done   = ixgbe_dev_rx_descriptor_done,
 273         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 274         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 275         .dev_led_on           = ixgbe_dev_led_on,
 276         .dev_led_off          = ixgbe_dev_led_off,
 277         .flow_ctrl_set        = ixgbe_flow_ctrl_set,
 278         .priority_flow_ctrl_set = ixgbe_priority_flow_ctrl_set,
 279         .mac_addr_add         = ixgbe_add_rar,
 280         .mac_addr_remove      = ixgbe_remove_rar,
 281         .uc_hash_table_set    = ixgbe_uc_hash_table_set,
 282         .uc_all_hash_table_set  = ixgbe_uc_all_hash_table_set,
 283         .mirror_rule_set        = ixgbe_mirror_rule_set,
 284         .mirror_rule_reset      = ixgbe_mirror_rule_reset,
 285         .set_vf_rx_mode       = ixgbe_set_pool_rx_mode,
 286         .set_vf_rx            = ixgbe_set_pool_rx,
 287         .set_vf_tx            = ixgbe_set_pool_tx,
 288         .set_vf_vlan_filter   = ixgbe_set_pool_vlan_filter,
 289         .fdir_add_signature_filter    = ixgbe_fdir_add_signature_filter,
 290         .fdir_update_signature_filter = ixgbe_fdir_update_signature_filter,
 291         .fdir_remove_signature_filter = ixgbe_fdir_remove_signature_filter,
 292         .fdir_infos_get               = ixgbe_fdir_info_get,
 293         .fdir_add_perfect_filter      = ixgbe_fdir_add_perfect_filter,
 294         .fdir_update_perfect_filter   = ixgbe_fdir_update_perfect_filter,
 295         .fdir_remove_perfect_filter   = ixgbe_fdir_remove_perfect_filter,
 296         .fdir_set_masks               = ixgbe_fdir_set_masks,
 297         .reta_update          = ixgbe_dev_rss_reta_update,
 298         .reta_query           = ixgbe_dev_rss_reta_query,
 299 #ifdef RTE_NIC_BYPASS
 300         .bypass_init          = ixgbe_bypass_init,
 301         .bypass_state_set     = ixgbe_bypass_state_store,
 302         .bypass_state_show    = ixgbe_bypass_state_show,
 303         .bypass_event_set     = ixgbe_bypass_event_store,
 304         .bypass_event_show    = ixgbe_bypass_event_show,
 305         .bypass_wd_timeout_set  = ixgbe_bypass_wd_timeout_store,
 306         .bypass_wd_timeout_show = ixgbe_bypass_wd_timeout_show,
 307         .bypass_ver_show      = ixgbe_bypass_ver_show,
 308         .bypass_wd_reset      = ixgbe_bypass_wd_reset,
 309 #endif /* RTE_NIC_BYPASS */
 310         .rss_hash_update = ixgbe_dev_rss_hash_update,
 311 };
 312
 313 /*
 314  * dev_ops for virtual function, bare necessities for basic vf
 315  * operation have been implemented
 316  */
 317 static struct eth_dev_ops ixgbevf_eth_dev_ops = {
 318
 319         .dev_configure        = ixgbevf_dev_configure,
 320         .dev_start            = ixgbevf_dev_start,
 321         .dev_stop             = ixgbevf_dev_stop,
 322         .link_update          = ixgbe_dev_link_update,
 323         .stats_get            = ixgbevf_dev_stats_get,
 324         .stats_reset          = ixgbevf_dev_stats_reset,
 325         .dev_close            = ixgbevf_dev_close,
 326         .dev_infos_get        = ixgbe_dev_info_get,
 327         .vlan_filter_set      = ixgbevf_vlan_filter_set,
 328         .vlan_strip_queue_set = ixgbevf_vlan_strip_queue_set,
 329         .vlan_offload_set     = ixgbevf_vlan_offload_set,
 330         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 331         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 332         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 333         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 334         .mac_addr_add         = ixgbevf_add_mac_addr,
 335         .mac_addr_remove      = ixgbevf_remove_mac_addr,
 336 };
 337
 338 /**
 339  * Atomically reads the link status information from global
 340  * structure rte_eth_dev.
 341  *
 342  * @param dev
 343  *   - Pointer to the structure rte_eth_dev to read from.
 344  *   - Pointer to the buffer to be saved with the link status.
 345  *
 346  * @return
 347  *   - On success, zero.
 348  *   - On failure, negative value.
 349  */
 350 static inline int
 351 rte_ixgbe_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 352                                 struct rte_eth_link *link)
 353 {
 354         struct rte_eth_link *dst = link;
 355         struct rte_eth_link *src = &(dev->data->dev_link);
 356
 357         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 358                                         *(uint64_t *)src) == 0)
 359                 return -1;
 360
 361         return 0;
 362 }
 363
 364 /**
 365  * Atomically writes the link status information into global
 366  * structure rte_eth_dev.
 367  *
 368  * @param dev
 369  *   - Pointer to the structure rte_eth_dev to read from.
 370  *   - Pointer to the buffer to be saved with the link status.
 371  *
 372  * @return
 373  *   - On success, zero.
 374  *   - On failure, negative value.
 375  */
 376 static inline int
 377 rte_ixgbe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 378                                 struct rte_eth_link *link)
 379 {
 380         struct rte_eth_link *dst = &(dev->data->dev_link);
 381         struct rte_eth_link *src = link;
 382
 383         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 384                                         *(uint64_t *)src) == 0)
 385                 return -1;
 386
 387         return 0;
 388 }
 389
 390 /*
 391  * This function is the same as ixgbe_is_sfp() in ixgbe/ixgbe.h.
 392  */
 393 static inline int
 394 ixgbe_is_sfp(struct ixgbe_hw *hw)
 395 {
 396         switch (hw->phy.type) {
 397         case ixgbe_phy_sfp_avago:
 398         case ixgbe_phy_sfp_ftl:
 399         case ixgbe_phy_sfp_intel:
 400         case ixgbe_phy_sfp_unknown:
 401         case ixgbe_phy_sfp_passive_tyco:
 402         case ixgbe_phy_sfp_passive_unknown:
 403                 return 1;
 404         default:
 405                 return 0;
 406         }
 407 }
 408
 409 static inline int32_t
 410 ixgbe_pf_reset_hw(struct ixgbe_hw *hw)
 411 {
 412         uint32_t ctrl_ext;
 413         int32_t status;
 414
 415         status = ixgbe_reset_hw(hw);
 416
 417         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 418         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 419         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
 420         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 421         IXGBE_WRITE_FLUSH(hw);
 422
 423         return status;
 424 }
 425
 426 static inline void
 427 ixgbe_enable_intr(struct rte_eth_dev *dev)
 428 {
 429         struct ixgbe_interrupt *intr =
 430                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 431         struct ixgbe_hw *hw =
 432                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 433
 434         IXGBE_WRITE_REG(hw, IXGBE_EIMS, intr->mask);
 435         IXGBE_WRITE_FLUSH(hw);
 436 }
 437
 438 /*
 439  * This function is based on ixgbe_disable_intr() in ixgbe/ixgbe.h.
 440  */
 441 static void
 442 ixgbe_disable_intr(struct ixgbe_hw *hw)
 443 {
 444         PMD_INIT_FUNC_TRACE();
 445
 446         if (hw->mac.type == ixgbe_mac_82598EB) {
 447                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ~0);
 448         } else {
 449                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xFFFF0000);
 450                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), ~0);
 451                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), ~0);
 452         }
 453         IXGBE_WRITE_FLUSH(hw);
 454 }
 455
 456 /*
 457  * This function resets queue statistics mapping registers.
 458  * From Niantic datasheet, Initialization of Statistics section:
 459  * "...if software requires the queue counters, the RQSMR and TQSM registers
 460  * must be re-programmed following a device reset.
 461  */
 462 static void
 463 ixgbe_reset_qstat_mappings(struct ixgbe_hw *hw)
 464 {
 465         uint32_t i;
 466
 467         for(i = 0; i != IXGBE_NB_STAT_MAPPING_REGS; i++) {
 468                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), 0);
 469                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), 0);
 470         }
 471 }
 472
 473
 474 static int
 475 ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 476                                   uint16_t queue_id,
 477                                   uint8_t stat_idx,
 478                                   uint8_t is_rx)
 479 {
 480 #define QSM_REG_NB_BITS_PER_QMAP_FIELD 8
 481 #define NB_QMAP_FIELDS_PER_QSM_REG 4
 482 #define QMAP_FIELD_RESERVED_BITS_MASK 0x0f
 483
 484         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 485         struct ixgbe_stat_mapping_registers *stat_mappings =
 486                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(eth_dev->data->dev_private);
 487         uint32_t qsmr_mask = 0;
 488         uint32_t clearing_mask = QMAP_FIELD_RESERVED_BITS_MASK;
 489         uint32_t q_map;
 490         uint8_t n, offset;
 491
 492         if ((hw->mac.type != ixgbe_mac_82599EB) && (hw->mac.type != ixgbe_mac_X540))
 493                 return -ENOSYS;
 494
 495         PMD_INIT_LOG(INFO, "Setting port %d, %s queue_id %d to stat index %d\n",
 496                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx);
 497
 498         n = (uint8_t)(queue_id / NB_QMAP_FIELDS_PER_QSM_REG);
 499         if (n >= IXGBE_NB_STAT_MAPPING_REGS) {
 500                 PMD_INIT_LOG(ERR, "Nb of stat mapping registers exceeded\n");
 501                 return -EIO;
 502         }
 503         offset = (uint8_t)(queue_id % NB_QMAP_FIELDS_PER_QSM_REG);
 504
 505         /* Now clear any previous stat_idx set */
 506         clearing_mask <<= (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 507         if (!is_rx)
 508                 stat_mappings->tqsm[n] &= ~clearing_mask;
 509         else
 510                 stat_mappings->rqsmr[n] &= ~clearing_mask;
 511
 512         q_map = (uint32_t)stat_idx;
 513         q_map &= QMAP_FIELD_RESERVED_BITS_MASK;
 514         qsmr_mask = q_map << (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 515         if (!is_rx)
 516                 stat_mappings->tqsm[n] |= qsmr_mask;
 517         else
 518                 stat_mappings->rqsmr[n] |= qsmr_mask;
 519
 520         PMD_INIT_LOG(INFO, "Set port %d, %s queue_id %d to stat index %d\n"
 521                      "%s[%d] = 0x%08x\n",
 522                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx,
 523                      is_rx ? "RQSMR" : "TQSM",n, is_rx ? stat_mappings->rqsmr[n] : stat_mappings->tqsm[n]);
 524
 525         /* Now write the mapping in the appropriate register */
 526         if (is_rx) {
 527                 PMD_INIT_LOG(INFO, "Write 0x%x to RX IXGBE stat mapping reg:%d\n",
 528                              stat_mappings->rqsmr[n], n);
 529                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(n), stat_mappings->rqsmr[n]);
 530         }
 531         else {
 532                 PMD_INIT_LOG(INFO, "Write 0x%x to TX IXGBE stat mapping reg:%d\n",
 533                              stat_mappings->tqsm[n], n);
 534                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(n), stat_mappings->tqsm[n]);
 535         }
 536         return 0;
 537 }
 538
 539 static void
 540 ixgbe_restore_statistics_mapping(struct rte_eth_dev * dev)
 541 {
 542         struct ixgbe_stat_mapping_registers *stat_mappings =
 543                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(dev->data->dev_private);
 544         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 545         int i;
 546
 547         /* write whatever was in stat mapping table to the NIC */
 548         for (i = 0; i < IXGBE_NB_STAT_MAPPING_REGS; i++) {
 549                 /* rx */
 550                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), stat_mappings->rqsmr[i]);
 551
 552                 /* tx */
 553                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), stat_mappings->tqsm[i]);
 554         }
 555 }
 556
 557 static void
 558 ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config)
 559 {
 560         uint8_t i;
 561         struct ixgbe_dcb_tc_config *tc;
 562         uint8_t dcb_max_tc = IXGBE_DCB_MAX_TRAFFIC_CLASS;
 563
 564         dcb_config->num_tcs.pg_tcs = dcb_max_tc;
 565         dcb_config->num_tcs.pfc_tcs = dcb_max_tc;
 566         for (i = 0; i < dcb_max_tc; i++) {
 567                 tc = &dcb_config->tc_config[i];
 568                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_id = i;
 569                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
 570                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 571                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_id = i;
 572                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
 573                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 574                 tc->pfc = ixgbe_dcb_pfc_disabled;
 575         }
 576
 577         /* Initialize default user to priority mapping, UPx->TC0 */
 578         tc = &dcb_config->tc_config[0];
 579         tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
 580         tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
 581         for (i = 0; i< IXGBE_DCB_MAX_BW_GROUP; i++) {
 582                 dcb_config->bw_percentage[IXGBE_DCB_TX_CONFIG][i] = 100;
 583                 dcb_config->bw_percentage[IXGBE_DCB_RX_CONFIG][i] = 100;
 584         }
 585         dcb_config->rx_pba_cfg = ixgbe_dcb_pba_equal;
 586         dcb_config->pfc_mode_enable = false;
 587         dcb_config->vt_mode = true;
 588         dcb_config->round_robin_enable = false;
 589         /* support all DCB capabilities in 82599 */
 590         dcb_config->support.capabilities = 0xFF;
 591
 592         /*we only support 4 Tcs for X540*/
 593         if (hw->mac.type == ixgbe_mac_X540) {
 594                 dcb_config->num_tcs.pg_tcs = 4;
 595                 dcb_config->num_tcs.pfc_tcs = 4;
 596         }
 597 }
 598
 599 /*
 600  * Ensure that all locks are released before first NVM or PHY access
 601  */
 602 static void
 603 ixgbe_swfw_lock_reset(struct ixgbe_hw *hw)
 604 {
 605         uint16_t mask;
 606
 607         /*
 608          * Phy lock should not fail in this early stage. If this is the case,
 609          * it is due to an improper exit of the application.
 610          * So force the release of the faulty lock. Release of common lock
 611          * is done automatically by swfw_sync function.
 612          */
 613         mask = IXGBE_GSSR_PHY0_SM << hw->bus.func;
 614         if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) {
 615                    DEBUGOUT1("SWFW phy%d lock released", hw->bus.func);
 616         }
 617         ixgbe_release_swfw_semaphore(hw, mask);
 618
 619         /*
 620          * These ones are more tricky since they are common to all ports; but
 621          * swfw_sync retries last long enough (1s) to be almost sure that if
 622          * lock can not be taken it is due to an improper lock of the
 623          * semaphore.
 624          */
 625         mask = IXGBE_GSSR_EEP_SM | IXGBE_GSSR_MAC_CSR_SM | IXGBE_GSSR_SW_MNG_SM;
 626         if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) {
 627                    DEBUGOUT("SWFW common locks released");
 628         }
 629         ixgbe_release_swfw_semaphore(hw, mask);
 630 }
 631
 632 /*
 633  * This function is based on code in ixgbe_attach() in ixgbe/ixgbe.c.
 634  * It returns 0 on success.
 635  */
 636 static int
 637 eth_ixgbe_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 638                      struct rte_eth_dev *eth_dev)
 639 {
 640         struct rte_pci_device *pci_dev;
 641         struct ixgbe_hw *hw =
 642                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 643         struct ixgbe_vfta * shadow_vfta =
 644                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 645         struct ixgbe_hwstrip *hwstrip =
 646                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 647         struct ixgbe_dcb_config *dcb_config =
 648                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private);
 649         uint32_t ctrl_ext;
 650         uint16_t csum;
 651         int diag, i;
 652
 653         PMD_INIT_FUNC_TRACE();
 654
 655         eth_dev->dev_ops = &ixgbe_eth_dev_ops;
 656         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
 657         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
 658
 659         /* for secondary processes, we don't initialise any further as primary
 660          * has already done this work. Only check we don't need a different
 661          * RX function */
 662         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 663                 if (eth_dev->data->scattered_rx)
 664                         eth_dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
 665                 return 0;
 666         }
 667         pci_dev = eth_dev->pci_dev;
 668
 669         /* Vendor and Device ID need to be set before init of shared code */
 670         hw->device_id = pci_dev->id.device_id;
 671         hw->vendor_id = pci_dev->id.vendor_id;
 672         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 673 #ifdef RTE_LIBRTE_IXGBE_ALLOW_UNSUPPORTED_SFP
 674         hw->allow_unsupported_sfp = 1;
 675 #endif
 676
 677         /* Initialize the shared code */
 678 #ifdef RTE_NIC_BYPASS
 679         diag = ixgbe_bypass_init_shared_code(hw);
 680 #else
 681         diag = ixgbe_init_shared_code(hw);
 682 #endif /* RTE_NIC_BYPASS */
 683
 684         if (diag != IXGBE_SUCCESS) {
 685                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
 686                 return -EIO;
 687         }
 688
 689         /* pick up the PCI bus settings for reporting later */
 690         ixgbe_get_bus_info(hw);
 691
 692         /* Unlock any pending hardware semaphore */
 693         ixgbe_swfw_lock_reset(hw);
 694
 695         /* Initialize DCB configuration*/
 696         memset(dcb_config, 0, sizeof(struct ixgbe_dcb_config));
 697         ixgbe_dcb_init(hw,dcb_config);
 698         /* Get Hardware Flow Control setting */
 699         hw->fc.requested_mode = ixgbe_fc_full;
 700         hw->fc.current_mode = ixgbe_fc_full;
 701         hw->fc.pause_time = IXGBE_FC_PAUSE;
 702         for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
 703                 hw->fc.low_water[i] = IXGBE_FC_LO;
 704                 hw->fc.high_water[i] = IXGBE_FC_HI;
 705         }
 706         hw->fc.send_xon = 1;
 707
 708         /* Make sure we have a good EEPROM before we read from it */
 709         diag = ixgbe_validate_eeprom_checksum(hw, &csum);
 710         if (diag != IXGBE_SUCCESS) {
 711                 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag);
 712                 return -EIO;
 713         }
 714
 715 #ifdef RTE_NIC_BYPASS
 716         diag = ixgbe_bypass_init_hw(hw);
 717 #else
 718         diag = ixgbe_init_hw(hw);
 719 #endif /* RTE_NIC_BYPASS */
 720
 721         /*
 722          * Devices with copper phys will fail to initialise if ixgbe_init_hw()
 723          * is called too soon after the kernel driver unbinding/binding occurs.
 724          * The failure occurs in ixgbe_identify_phy_generic() for all devices,
 725          * but for non-copper devies, ixgbe_identify_sfp_module_generic() is
 726          * also called. See ixgbe_identify_phy_82599(). The reason for the
 727          * failure is not known, and only occuts when virtualisation features
 728          * are disabled in the bios. A delay of 100ms  was found to be enough by
 729          * trial-and-error, and is doubled to be safe.
 730          */
 731         if (diag && (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper)) {
 732                 rte_delay_ms(200);
 733                 diag = ixgbe_init_hw(hw);
 734         }
 735
 736         if (diag == IXGBE_ERR_EEPROM_VERSION) {
 737                 PMD_INIT_LOG(ERR, "This device is a pre-production adapter/"
 738                     "LOM.  Please be aware there may be issues associated "
 739                     "with your hardware.\n If you are experiencing problems "
 740                     "please contact your Intel or hardware representative "
 741                     "who provided you with this hardware.\n");
 742         } else if (diag == IXGBE_ERR_SFP_NOT_SUPPORTED)
 743                 PMD_INIT_LOG(ERR, "Unsupported SFP+ Module\n");
 744         if (diag) {
 745                 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag);
 746                 return -EIO;
 747         }
 748
 749         /* disable interrupt */
 750         ixgbe_disable_intr(hw);
 751
 752         /* reset mappings for queue statistics hw counters*/
 753         ixgbe_reset_qstat_mappings(hw);
 754
 755         /* Allocate memory for storing MAC addresses */
 756         eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 757                         hw->mac.num_rar_entries, 0);
 758         if (eth_dev->data->mac_addrs == NULL) {
 759                 PMD_INIT_LOG(ERR,
 760                         "Failed to allocate %u bytes needed to store "
 761                         "MAC addresses",
 762                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 763                 return -ENOMEM;
 764         }
 765         /* Copy the permanent MAC address */
 766         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 767                         &eth_dev->data->mac_addrs[0]);
 768
 769         /* Allocate memory for storing hash filter MAC addresses */
 770         eth_dev->data->hash_mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 771                         IXGBE_VMDQ_NUM_UC_MAC, 0);
 772         if (eth_dev->data->hash_mac_addrs == NULL) {
 773                 PMD_INIT_LOG(ERR,
 774                         "Failed to allocate %d bytes needed to store MAC addresses",
 775                         ETHER_ADDR_LEN * IXGBE_VMDQ_NUM_UC_MAC);
 776                 return -ENOMEM;
 777         }
 778
 779         /* initialize the vfta */
 780         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 781
 782         /* initialize the hw strip bitmap*/
 783         memset(hwstrip, 0, sizeof(*hwstrip));
 784
 785         /* initialize PF if max_vfs not zero */
 786         ixgbe_pf_host_init(eth_dev);
 787
 788         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 789         /* let hardware know driver is loaded */
 790         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 791         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 792         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
 793         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 794         IXGBE_WRITE_FLUSH(hw);
 795
 796         if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
 797                 PMD_INIT_LOG(DEBUG,
 798                              "MAC: %d, PHY: %d, SFP+: %d<n",
 799                              (int) hw->mac.type, (int) hw->phy.type,
 800                              (int) hw->phy.sfp_type);
 801         else
 802                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d\n",
 803                              (int) hw->mac.type, (int) hw->phy.type);
 804
 805         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
 806                         eth_dev->data->port_id, pci_dev->id.vendor_id,
 807                         pci_dev->id.device_id);
 808
 809         rte_intr_callback_register(&(pci_dev->intr_handle),
 810                 ixgbe_dev_interrupt_handler, (void *)eth_dev);
 811
 812         /* enable uio intr after callback register */
 813         rte_intr_enable(&(pci_dev->intr_handle));
 814
 815         /* enable support intr */
 816         ixgbe_enable_intr(eth_dev);
 817
 818         return 0;
 819 }
 820
 821
 822 /*
 823  * Negotiate mailbox API version with the PF.
 824  * After reset API version is always set to the basic one (ixgbe_mbox_api_10).
 825  * Then we try to negotiate starting with the most recent one.
 826  * If all negotiation attempts fail, then we will proceed with
 827  * the default one (ixgbe_mbox_api_10).
 828  */
 829 static void
 830 ixgbevf_negotiate_api(struct ixgbe_hw *hw)
 831 {
 832         int32_t i;
 833
 834         /* start with highest supported, proceed down */
 835         static const enum ixgbe_pfvf_api_rev sup_ver[] = {
 836                 ixgbe_mbox_api_11,
 837                 ixgbe_mbox_api_10,
 838         };
 839
 840         for (i = 0;
 841                         i != RTE_DIM(sup_ver) &&
 842                         ixgbevf_negotiate_api_version(hw, sup_ver[i]) != 0;
 843                         i++)
 844                 ;
 845 }
 846
 847 static void
 848 generate_random_mac_addr(struct ether_addr *mac_addr)
 849 {
 850         uint64_t random;
 851
 852         /* Set Organizationally Unique Identifier (OUI) prefix. */
 853         mac_addr->addr_bytes[0] = 0x00;
 854         mac_addr->addr_bytes[1] = 0x09;
 855         mac_addr->addr_bytes[2] = 0xC0;
 856         /* Force indication of locally assigned MAC address. */
 857         mac_addr->addr_bytes[0] |= ETHER_LOCAL_ADMIN_ADDR;
 858         /* Generate the last 3 bytes of the MAC address with a random number. */
 859         random = rte_rand();
 860         memcpy(&mac_addr->addr_bytes[3], &random, 3);
 861 }
 862
 863 /*
 864  * Virtual Function device init
 865  */
 866 static int
 867 eth_ixgbevf_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 868                      struct rte_eth_dev *eth_dev)
 869 {
 870         int diag;
 871         uint32_t tc, tcs;
 872         struct rte_pci_device *pci_dev;
 873         struct ixgbe_hw *hw =
 874                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 875         struct ixgbe_vfta * shadow_vfta =
 876                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 877         struct ixgbe_hwstrip *hwstrip =
 878                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 879         struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr;
 880
 881         PMD_INIT_LOG(DEBUG, "eth_ixgbevf_dev_init");
 882
 883         eth_dev->dev_ops = &ixgbevf_eth_dev_ops;
 884         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
 885         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
 886
 887         /* for secondary processes, we don't initialise any further as primary
 888          * has already done this work. Only check we don't need a different
 889          * RX function */
 890         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 891                 if (eth_dev->data->scattered_rx)
 892                         eth_dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
 893                 return 0;
 894         }
 895
 896         pci_dev = eth_dev->pci_dev;
 897
 898         hw->device_id = pci_dev->id.device_id;
 899         hw->vendor_id = pci_dev->id.vendor_id;
 900         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 901
 902         /* initialize the vfta */
 903         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 904
 905         /* initialize the hw strip bitmap*/
 906         memset(hwstrip, 0, sizeof(*hwstrip));
 907
 908         /* Initialize the shared code */
 909         diag = ixgbe_init_shared_code(hw);
 910         if (diag != IXGBE_SUCCESS) {
 911                 PMD_INIT_LOG(ERR, "Shared code init failed for ixgbevf: %d", diag);
 912                 return -EIO;
 913         }
 914
 915         /* init_mailbox_params */
 916         hw->mbx.ops.init_params(hw);
 917
 918         /* Disable the interrupts for VF */
 919         ixgbevf_intr_disable(hw);
 920
 921         hw->mac.num_rar_entries = 128; /* The MAX of the underlying PF */
 922         diag = hw->mac.ops.reset_hw(hw);
 923
 924         /*
 925          * The VF reset operation returns the IXGBE_ERR_INVALID_MAC_ADDR when
 926          * the underlying PF driver has not assigned a MAC address to the VF.
 927          * In this case, assign a random MAC address.
 928          */
 929         if ((diag != IXGBE_SUCCESS) && (diag != IXGBE_ERR_INVALID_MAC_ADDR)) {
 930                 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
 931                 return (diag);
 932         }
 933
 934         /* negotiate mailbox API version to use with the PF. */
 935         ixgbevf_negotiate_api(hw);
 936
 937         /* Get Rx/Tx queue count via mailbox, which is ready after reset_hw */
 938         ixgbevf_get_queues(hw, &tcs, &tc);
 939
 940         /* Allocate memory for storing MAC addresses */
 941         eth_dev->data->mac_addrs = rte_zmalloc("ixgbevf", ETHER_ADDR_LEN *
 942                         hw->mac.num_rar_entries, 0);
 943         if (eth_dev->data->mac_addrs == NULL) {
 944                 PMD_INIT_LOG(ERR,
 945                         "Failed to allocate %u bytes needed to store "
 946                         "MAC addresses",
 947                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 948                 return -ENOMEM;
 949         }
 950
 951         /* Generate a random MAC address, if none was assigned by PF. */
 952         if (is_zero_ether_addr(perm_addr)) {
 953                 generate_random_mac_addr(perm_addr);
 954                 diag = ixgbe_set_rar_vf(hw, 1, perm_addr->addr_bytes, 0, 1);
 955                 if (diag) {
 956                         rte_free(eth_dev->data->mac_addrs);
 957                         eth_dev->data->mac_addrs = NULL;
 958                         return diag;
 959                 }
 960                 RTE_LOG(INFO, PMD,
 961                         "\tVF MAC address not assigned by Host PF\n"
 962                         "\tAssign randomly generated MAC address "
 963                         "%02x:%02x:%02x:%02x:%02x:%02x\n",
 964                         perm_addr->addr_bytes[0],
 965                         perm_addr->addr_bytes[1],
 966                         perm_addr->addr_bytes[2],
 967                         perm_addr->addr_bytes[3],
 968                         perm_addr->addr_bytes[4],
 969                         perm_addr->addr_bytes[5]);
 970         }
 971
 972         /* Copy the permanent MAC address */
 973         ether_addr_copy(perm_addr, &eth_dev->data->mac_addrs[0]);
 974
 975         /* reset the hardware with the new settings */
 976         diag = hw->mac.ops.start_hw(hw);
 977         switch (diag) {
 978                 case  0:
 979                         break;
 980
 981                 default:
 982                         PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
 983                         return (-EIO);
 984         }
 985
 986         PMD_INIT_LOG(DEBUG, "\nport %d vendorID=0x%x deviceID=0x%x mac.type=%s\n",
 987                          eth_dev->data->port_id, pci_dev->id.vendor_id, pci_dev->id.device_id,
 988                          "ixgbe_mac_82599_vf");
 989
 990         return 0;
 991 }
 992
 993 static struct eth_driver rte_ixgbe_pmd = {
 994         {
 995                 .name = "rte_ixgbe_pmd",
 996                 .id_table = pci_id_ixgbe_map,
 997                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
 998         },
 999         .eth_dev_init = eth_ixgbe_dev_init,
1000         .dev_private_size = sizeof(struct ixgbe_adapter),
1001 };
1002
1003 /*
1004  * virtual function driver struct
1005  */
1006 static struct eth_driver rte_ixgbevf_pmd = {
1007         {
1008                 .name = "rte_ixgbevf_pmd",
1009                 .id_table = pci_id_ixgbevf_map,
1010                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
1011         },
1012         .eth_dev_init = eth_ixgbevf_dev_init,
1013         .dev_private_size = sizeof(struct ixgbe_adapter),
1014 };
1015
1016 /*
1017  * Driver initialization routine.
1018  * Invoked once at EAL init time.
1019  * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
1020  */
1021 static int
1022 rte_ixgbe_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
1023 {
1024         PMD_INIT_FUNC_TRACE();
1025
1026         rte_eth_driver_register(&rte_ixgbe_pmd);
1027         return 0;
1028 }
1029
1030 /*
1031  * VF Driver initialization routine.
1032  * Invoked one at EAL init time.
1033  * Register itself as the [Virtual Poll Mode] Driver of PCI niantic devices.
1034  */
1035 static int
1036 rte_ixgbevf_pmd_init(const char *name __rte_unused, const char *param __rte_unused)
1037 {
1038         DEBUGFUNC("rte_ixgbevf_pmd_init");
1039
1040         rte_eth_driver_register(&rte_ixgbevf_pmd);
1041         return (0);
1042 }
1043
1044 static int
1045 ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1046 {
1047         struct ixgbe_hw *hw =
1048                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1049         struct ixgbe_vfta * shadow_vfta =
1050                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1051         uint32_t vfta;
1052         uint32_t vid_idx;
1053         uint32_t vid_bit;
1054
1055         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
1056         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
1057         vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(vid_idx));
1058         if (on)
1059                 vfta |= vid_bit;
1060         else
1061                 vfta &= ~vid_bit;
1062         IXGBE_WRITE_REG(hw, IXGBE_VFTA(vid_idx), vfta);
1063
1064         /* update local VFTA copy */
1065         shadow_vfta->vfta[vid_idx] = vfta;
1066
1067         return 0;
1068 }
1069
1070 static void
1071 ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
1072 {
1073         if (on)
1074                 ixgbe_vlan_hw_strip_enable(dev, queue);
1075         else
1076                 ixgbe_vlan_hw_strip_disable(dev, queue);
1077 }
1078
1079 static void
1080 ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
1081 {
1082         struct ixgbe_hw *hw =
1083                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1084
1085         /* Only the high 16-bits is valid */
1086         IXGBE_WRITE_REG(hw, IXGBE_EXVET, tpid << 16);
1087 }
1088
1089 void
1090 ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev)
1091 {
1092         struct ixgbe_hw *hw =
1093                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1094         uint32_t vlnctrl;
1095
1096         PMD_INIT_FUNC_TRACE();
1097
1098         /* Filter Table Disable */
1099         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1100         vlnctrl &= ~IXGBE_VLNCTRL_VFE;
1101
1102         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
1103 }
1104
1105 void
1106 ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1107 {
1108         struct ixgbe_hw *hw =
1109                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1110         struct ixgbe_vfta * shadow_vfta =
1111                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1112         uint32_t vlnctrl;
1113         uint16_t i;
1114
1115         PMD_INIT_FUNC_TRACE();
1116
1117         /* Filter Table Enable */
1118         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1119         vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
1120         vlnctrl |= IXGBE_VLNCTRL_VFE;
1121
1122         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
1123
1124         /* write whatever is in local vfta copy */
1125         for (i = 0; i < IXGBE_VFTA_SIZE; i++)
1126                 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), shadow_vfta->vfta[i]);
1127 }
1128
1129 static void
1130 ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on)
1131 {
1132         struct ixgbe_hwstrip *hwstrip =
1133                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(dev->data->dev_private);
1134
1135         if(queue >= IXGBE_MAX_RX_QUEUE_NUM)
1136                 return;
1137
1138         if (on)
1139                 IXGBE_SET_HWSTRIP(hwstrip, queue);
1140         else
1141                 IXGBE_CLEAR_HWSTRIP(hwstrip, queue);
1142 }
1143
1144 static void
1145 ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue)
1146 {
1147         struct ixgbe_hw *hw =
1148                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1149         uint32_t ctrl;
1150
1151         PMD_INIT_FUNC_TRACE();
1152
1153         if (hw->mac.type == ixgbe_mac_82598EB) {
1154                 /* No queue level support */
1155                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
1156                 return;
1157         }
1158         else {
1159                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1160                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1161                 ctrl &= ~IXGBE_RXDCTL_VME;
1162                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
1163         }
1164         /* record those setting for HW strip per queue */
1165         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 0);
1166 }
1167
1168 static void
1169 ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue)
1170 {
1171         struct ixgbe_hw *hw =
1172                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1173         uint32_t ctrl;
1174
1175         PMD_INIT_FUNC_TRACE();
1176
1177         if (hw->mac.type == ixgbe_mac_82598EB) {
1178                 /* No queue level supported */
1179                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
1180                 return;
1181         }
1182         else {
1183                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1184                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1185                 ctrl |= IXGBE_RXDCTL_VME;
1186                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
1187         }
1188         /* record those setting for HW strip per queue */
1189         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 1);
1190 }
1191
1192 void
1193 ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev)
1194 {
1195         struct ixgbe_hw *hw =
1196                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1197         uint32_t ctrl;
1198         uint16_t i;
1199
1200         PMD_INIT_FUNC_TRACE();
1201
1202         if (hw->mac.type == ixgbe_mac_82598EB) {
1203                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1204                 ctrl &= ~IXGBE_VLNCTRL_VME;
1205                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1206         }
1207         else {
1208                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1209                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1210                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1211                         ctrl &= ~IXGBE_RXDCTL_VME;
1212                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1213
1214                         /* record those setting for HW strip per queue */
1215                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 0);
1216                 }
1217         }
1218 }
1219
1220 void
1221 ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev)
1222 {
1223         struct ixgbe_hw *hw =
1224                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1225         uint32_t ctrl;
1226         uint16_t i;
1227
1228         PMD_INIT_FUNC_TRACE();
1229
1230         if (hw->mac.type == ixgbe_mac_82598EB) {
1231                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1232                 ctrl |= IXGBE_VLNCTRL_VME;
1233                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1234         }
1235         else {
1236                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1237                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1238                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1239                         ctrl |= IXGBE_RXDCTL_VME;
1240                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1241
1242                         /* record those setting for HW strip per queue */
1243                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 1);
1244                 }
1245         }
1246 }
1247
1248 static void
1249 ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1250 {
1251         struct ixgbe_hw *hw =
1252                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1253         uint32_t ctrl;
1254
1255         PMD_INIT_FUNC_TRACE();
1256
1257         /* DMATXCTRL: Geric Double VLAN Disable */
1258         ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1259         ctrl &= ~IXGBE_DMATXCTL_GDV;
1260         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1261
1262         /* CTRL_EXT: Global Double VLAN Disable */
1263         ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1264         ctrl &= ~IXGBE_EXTENDED_VLAN;
1265         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1266
1267 }
1268
1269 static void
1270 ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1271 {
1272         struct ixgbe_hw *hw =
1273                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1274         uint32_t ctrl;
1275
1276         PMD_INIT_FUNC_TRACE();
1277
1278         /* DMATXCTRL: Geric Double VLAN Enable */
1279         ctrl  = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1280         ctrl |= IXGBE_DMATXCTL_GDV;
1281         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1282
1283         /* CTRL_EXT: Global Double VLAN Enable */
1284         ctrl  = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1285         ctrl |= IXGBE_EXTENDED_VLAN;
1286         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1287
1288         /*
1289          * VET EXT field in the EXVET register = 0x8100 by default
1290          * So no need to change. Same to VT field of DMATXCTL register
1291          */
1292 }
1293
1294 static void
1295 ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1296 {
1297         if(mask & ETH_VLAN_STRIP_MASK){
1298                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1299                         ixgbe_vlan_hw_strip_enable_all(dev);
1300                 else
1301                         ixgbe_vlan_hw_strip_disable_all(dev);
1302         }
1303
1304         if(mask & ETH_VLAN_FILTER_MASK){
1305                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1306                         ixgbe_vlan_hw_filter_enable(dev);
1307                 else
1308                         ixgbe_vlan_hw_filter_disable(dev);
1309         }
1310
1311         if(mask & ETH_VLAN_EXTEND_MASK){
1312                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1313                         ixgbe_vlan_hw_extend_enable(dev);
1314                 else
1315                         ixgbe_vlan_hw_extend_disable(dev);
1316         }
1317 }
1318
1319 static void
1320 ixgbe_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1321 {
1322         struct ixgbe_hw *hw =
1323                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1324         /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
1325         uint32_t vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1326         vlanctrl |= IXGBE_VLNCTRL_VFE ; /* enable vlan filters */
1327         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
1328 }
1329
1330 static int
1331 ixgbe_dev_configure(struct rte_eth_dev *dev)
1332 {
1333         struct ixgbe_interrupt *intr =
1334                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1335
1336         PMD_INIT_FUNC_TRACE();
1337
1338         /* set flag to update link status after init */
1339         intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1340
1341         return 0;
1342 }
1343
1344 /*
1345  * Configure device link speed and setup link.
1346  * It returns 0 on success.
1347  */
1348 static int
1349 ixgbe_dev_start(struct rte_eth_dev *dev)
1350 {
1351         struct ixgbe_hw *hw =
1352                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1353         int err, link_up = 0, negotiate = 0;
1354         uint32_t speed = 0;
1355         int mask = 0;
1356         int status;
1357
1358         PMD_INIT_FUNC_TRACE();
1359
1360         /* IXGBE devices don't support half duplex */
1361         if ((dev->data->dev_conf.link_duplex != ETH_LINK_AUTONEG_DUPLEX) &&
1362                         (dev->data->dev_conf.link_duplex != ETH_LINK_FULL_DUPLEX)) {
1363                 PMD_INIT_LOG(ERR, "Invalid link_duplex (%hu) for port %hhu\n",
1364                                 dev->data->dev_conf.link_duplex,
1365                                 dev->data->port_id);
1366                 return -EINVAL;
1367         }
1368
1369         /* stop adapter */
1370         hw->adapter_stopped = FALSE;
1371         ixgbe_stop_adapter(hw);
1372
1373         /* reinitialize adapter
1374          * this calls reset and start */
1375         status = ixgbe_pf_reset_hw(hw);
1376         if (status != 0)
1377                 return -1;
1378         hw->mac.ops.start_hw(hw);
1379
1380         /* configure PF module if SRIOV enabled */
1381         ixgbe_pf_host_configure(dev);
1382
1383         /* initialize transmission unit */
1384         ixgbe_dev_tx_init(dev);
1385
1386         /* This can fail when allocating mbufs for descriptor rings */
1387         err = ixgbe_dev_rx_init(dev);
1388         if (err) {
1389                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware\n");
1390                 goto error;
1391         }
1392
1393         ixgbe_dev_rxtx_start(dev);
1394
1395         if (ixgbe_is_sfp(hw) && hw->phy.multispeed_fiber) {
1396                 err = hw->mac.ops.setup_sfp(hw);
1397                 if (err)
1398                         goto error;
1399         }
1400
1401         /* Turn on the laser */
1402         ixgbe_enable_tx_laser(hw);
1403
1404         /* Skip link setup if loopback mode is enabled for 82599. */
1405         if (hw->mac.type == ixgbe_mac_82599EB &&
1406                         dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_82599_TX_RX)
1407                 goto skip_link_setup;
1408
1409         err = ixgbe_check_link(hw, &speed, &link_up, 0);
1410         if (err)
1411                 goto error;
1412         err = ixgbe_get_link_capabilities(hw, &speed, &negotiate);
1413         if (err)
1414                 goto error;
1415
1416         switch(dev->data->dev_conf.link_speed) {
1417         case ETH_LINK_SPEED_AUTONEG:
1418                 speed = (hw->mac.type != ixgbe_mac_82598EB) ?
1419                                 IXGBE_LINK_SPEED_82599_AUTONEG :
1420                                 IXGBE_LINK_SPEED_82598_AUTONEG;
1421                 break;
1422         case ETH_LINK_SPEED_100:
1423                 /*
1424                  * Invalid for 82598 but error will be detected by
1425                  * ixgbe_setup_link()
1426                  */
1427                 speed = IXGBE_LINK_SPEED_100_FULL;
1428                 break;
1429         case ETH_LINK_SPEED_1000:
1430                 speed = IXGBE_LINK_SPEED_1GB_FULL;
1431                 break;
1432         case ETH_LINK_SPEED_10000:
1433                 speed = IXGBE_LINK_SPEED_10GB_FULL;
1434                 break;
1435         default:
1436                 PMD_INIT_LOG(ERR, "Invalid link_speed (%hu) for port %hhu\n",
1437                                 dev->data->dev_conf.link_speed,
1438                                 dev->data->port_id);
1439                 goto error;
1440         }
1441
1442         err = ixgbe_setup_link(hw, speed, negotiate, link_up);
1443         if (err)
1444                 goto error;
1445
1446 skip_link_setup:
1447
1448         /* check if lsc interrupt is enabled */
1449         if (dev->data->dev_conf.intr_conf.lsc != 0)
1450                 ixgbe_dev_lsc_interrupt_setup(dev);
1451
1452         /* resume enabled intr since hw reset */
1453         ixgbe_enable_intr(dev);
1454
1455         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
1456                 ETH_VLAN_EXTEND_MASK;
1457         ixgbe_vlan_offload_set(dev, mask);
1458
1459         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
1460                 /* Enable vlan filtering for VMDq */
1461                 ixgbe_vmdq_vlan_hw_filter_enable(dev);
1462         }
1463
1464         /* Configure DCB hw */
1465         ixgbe_configure_dcb(dev);
1466
1467         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) {
1468                 err = ixgbe_fdir_configure(dev);
1469                 if (err)
1470                         goto error;
1471         }
1472
1473         ixgbe_restore_statistics_mapping(dev);
1474
1475         return (0);
1476
1477 error:
1478         PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err);
1479         ixgbe_dev_clear_queues(dev);
1480         return -EIO;
1481 }
1482
1483 /*
1484  * Stop device: disable rx and tx functions to allow for reconfiguring.
1485  */
1486 static void
1487 ixgbe_dev_stop(struct rte_eth_dev *dev)
1488 {
1489         struct rte_eth_link link;
1490         struct ixgbe_hw *hw =
1491                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1492         struct ixgbe_vf_info *vfinfo =
1493                 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private);
1494         int vf;
1495
1496         PMD_INIT_FUNC_TRACE();
1497
1498         /* disable interrupts */
1499         ixgbe_disable_intr(hw);
1500
1501         /* reset the NIC */
1502         ixgbe_pf_reset_hw(hw);
1503         hw->adapter_stopped = FALSE;
1504
1505         /* stop adapter */
1506         ixgbe_stop_adapter(hw);
1507
1508         for (vf = 0; vfinfo != NULL &&
1509                      vf < dev->pci_dev->max_vfs; vf++)
1510                 vfinfo[vf].clear_to_send = false;
1511
1512         /* Turn off the laser */
1513         ixgbe_disable_tx_laser(hw);
1514
1515         ixgbe_dev_clear_queues(dev);
1516
1517         /* Clear recorded link status */
1518         memset(&link, 0, sizeof(link));
1519         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1520 }
1521
1522 /*
1523  * Reest and stop device.
1524  */
1525 static void
1526 ixgbe_dev_close(struct rte_eth_dev *dev)
1527 {
1528         struct ixgbe_hw *hw =
1529                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1530
1531         PMD_INIT_FUNC_TRACE();
1532
1533         ixgbe_pf_reset_hw(hw);
1534
1535         ixgbe_dev_stop(dev);
1536         hw->adapter_stopped = 1;
1537
1538         ixgbe_disable_pcie_master(hw);
1539
1540         /* reprogram the RAR[0] in case user changed it. */
1541         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
1542 }
1543
1544 /*
1545  * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c
1546  */
1547 static void
1548 ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1549 {
1550         struct ixgbe_hw *hw =
1551                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1552         struct ixgbe_hw_stats *hw_stats =
1553                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1554         uint32_t bprc, lxon, lxoff, total;
1555         uint64_t total_missed_rx, total_qbrc, total_qprc;
1556         unsigned i;
1557
1558         total_missed_rx = 0;
1559         total_qbrc = 0;
1560         total_qprc = 0;
1561
1562         hw_stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
1563         hw_stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
1564         hw_stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
1565         hw_stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
1566
1567         for (i = 0; i < 8; i++) {
1568                 uint32_t mp;
1569                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
1570                 /* global total per queue */
1571                 hw_stats->mpc[i] += mp;
1572                 /* Running comprehensive total for stats display */
1573                 total_missed_rx += hw_stats->mpc[i];
1574                 if (hw->mac.type == ixgbe_mac_82598EB)
1575                         hw_stats->rnbc[i] +=
1576                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
1577                 hw_stats->pxontxc[i] +=
1578                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
1579                 hw_stats->pxonrxc[i] +=
1580                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
1581                 hw_stats->pxofftxc[i] +=
1582                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
1583                 hw_stats->pxoffrxc[i] +=
1584                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
1585                 hw_stats->pxon2offc[i] +=
1586                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
1587         }
1588         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1589                 hw_stats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
1590                 hw_stats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
1591                 hw_stats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
1592                 hw_stats->qbrc[i] +=
1593                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32);
1594                 hw_stats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
1595                 hw_stats->qbtc[i] +=
1596                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32);
1597                 hw_stats->qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
1598
1599                 total_qprc += hw_stats->qprc[i];
1600                 total_qbrc += hw_stats->qbrc[i];
1601         }
1602         hw_stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
1603         hw_stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
1604         hw_stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
1605
1606         /* Note that gprc counts missed packets */
1607         hw_stats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
1608
1609         if (hw->mac.type != ixgbe_mac_82598EB) {
1610                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
1611                 hw_stats->gorc += ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
1612                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
1613                 hw_stats->gotc += ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
1614                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
1615                 hw_stats->tor += ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
1616                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
1617                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
1618         } else {
1619                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
1620                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
1621                 /* 82598 only has a counter in the high register */
1622                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
1623                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
1624                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
1625         }
1626
1627         /*
1628          * Workaround: mprc hardware is incorrectly counting
1629          * broadcasts, so for now we subtract those.
1630          */
1631         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
1632         hw_stats->bprc += bprc;
1633         hw_stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
1634         if (hw->mac.type == ixgbe_mac_82598EB)
1635                 hw_stats->mprc -= bprc;
1636
1637         hw_stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
1638         hw_stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
1639         hw_stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
1640         hw_stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
1641         hw_stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
1642         hw_stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
1643
1644         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
1645         hw_stats->lxontxc += lxon;
1646         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
1647         hw_stats->lxofftxc += lxoff;
1648         total = lxon + lxoff;
1649
1650         hw_stats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
1651         hw_stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
1652         hw_stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
1653         hw_stats->gptc -= total;
1654         hw_stats->mptc -= total;
1655         hw_stats->ptc64 -= total;
1656         hw_stats->gotc -= total * ETHER_MIN_LEN;
1657
1658         hw_stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
1659         hw_stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
1660         hw_stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
1661         hw_stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
1662         hw_stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
1663         hw_stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
1664         hw_stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
1665         hw_stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
1666         hw_stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
1667         hw_stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
1668         hw_stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
1669         hw_stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
1670         hw_stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
1671         hw_stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
1672         hw_stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
1673         hw_stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);
1674         hw_stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
1675         hw_stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
1676         /* Only read FCOE on 82599 */
1677         if (hw->mac.type != ixgbe_mac_82598EB) {
1678                 hw_stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
1679                 hw_stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
1680                 hw_stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
1681                 hw_stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
1682                 hw_stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
1683         }
1684
1685         if (stats == NULL)
1686                 return;
1687
1688         /* Fill out the rte_eth_stats statistics structure */
1689         stats->ipackets = total_qprc;
1690         stats->ibytes = total_qbrc;
1691         stats->opackets = hw_stats->gptc;
1692         stats->obytes = hw_stats->gotc;
1693         stats->imcasts = hw_stats->mprc;
1694
1695         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1696                 stats->q_ipackets[i] = hw_stats->qprc[i];
1697                 stats->q_opackets[i] = hw_stats->qptc[i];
1698                 stats->q_ibytes[i] = hw_stats->qbrc[i];
1699                 stats->q_obytes[i] = hw_stats->qbtc[i];
1700                 stats->q_errors[i] = hw_stats->qprdc[i];
1701         }
1702
1703         /* Rx Errors */
1704         stats->ierrors = total_missed_rx + hw_stats->crcerrs +
1705                 hw_stats->rlec;
1706
1707         stats->oerrors  = 0;
1708
1709         /* XON/XOFF pause frames */
1710         stats->tx_pause_xon  = hw_stats->lxontxc;
1711         stats->rx_pause_xon  = hw_stats->lxonrxc;
1712         stats->tx_pause_xoff = hw_stats->lxofftxc;
1713         stats->rx_pause_xoff = hw_stats->lxoffrxc;
1714
1715         /* Flow Director Stats registers */
1716         hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
1717         hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
1718         stats->fdirmatch = hw_stats->fdirmatch;
1719         stats->fdirmiss = hw_stats->fdirmiss;
1720 }
1721
1722 static void
1723 ixgbe_dev_stats_reset(struct rte_eth_dev *dev)
1724 {
1725         struct ixgbe_hw_stats *stats =
1726                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1727
1728         /* HW registers are cleared on read */
1729         ixgbe_dev_stats_get(dev, NULL);
1730
1731         /* Reset software totals */
1732         memset(stats, 0, sizeof(*stats));
1733 }
1734
1735 static void
1736 ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1737 {
1738         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1739         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1740                           IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1741
1742         /* Good Rx packet, include VF loopback */
1743         UPDATE_VF_STAT(IXGBE_VFGPRC,
1744             hw_stats->last_vfgprc, hw_stats->vfgprc);
1745
1746         /* Good Rx octets, include VF loopback */
1747         UPDATE_VF_STAT_36BIT(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
1748             hw_stats->last_vfgorc, hw_stats->vfgorc);
1749
1750         /* Good Tx packet, include VF loopback */
1751         UPDATE_VF_STAT(IXGBE_VFGPTC,
1752             hw_stats->last_vfgptc, hw_stats->vfgptc);
1753
1754         /* Good Tx octets, include VF loopback */
1755         UPDATE_VF_STAT_36BIT(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
1756             hw_stats->last_vfgotc, hw_stats->vfgotc);
1757
1758         /* Rx Multicst Packet */
1759         UPDATE_VF_STAT(IXGBE_VFMPRC,
1760             hw_stats->last_vfmprc, hw_stats->vfmprc);
1761
1762         if (stats == NULL)
1763                 return;
1764
1765         memset(stats, 0, sizeof(*stats));
1766         stats->ipackets = hw_stats->vfgprc;
1767         stats->ibytes = hw_stats->vfgorc;
1768         stats->opackets = hw_stats->vfgptc;
1769         stats->obytes = hw_stats->vfgotc;
1770         stats->imcasts = hw_stats->vfmprc;
1771 }
1772
1773 static void
1774 ixgbevf_dev_stats_reset(struct rte_eth_dev *dev)
1775 {
1776         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1777                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1778
1779         /* Sync HW register to the last stats */
1780         ixgbevf_dev_stats_get(dev, NULL);
1781
1782         /* reset HW current stats*/
1783         hw_stats->vfgprc = 0;
1784         hw_stats->vfgorc = 0;
1785         hw_stats->vfgptc = 0;
1786         hw_stats->vfgotc = 0;
1787         hw_stats->vfmprc = 0;
1788
1789 }
1790
1791 static void
1792 ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1793 {
1794         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1795
1796         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
1797         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
1798         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL register */
1799         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */
1800         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
1801         dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC;
1802         dev_info->max_vfs = dev->pci_dev->max_vfs;
1803         if (hw->mac.type == ixgbe_mac_82598EB)
1804                 dev_info->max_vmdq_pools = ETH_16_POOLS;
1805         else
1806                 dev_info->max_vmdq_pools = ETH_64_POOLS;
1807         dev_info->rx_offload_capa =
1808                 DEV_RX_OFFLOAD_VLAN_STRIP |
1809                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1810                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1811                 DEV_RX_OFFLOAD_TCP_CKSUM;
1812         dev_info->tx_offload_capa =
1813                 DEV_TX_OFFLOAD_VLAN_INSERT |
1814                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1815                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1816                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1817                 DEV_TX_OFFLOAD_SCTP_CKSUM;
1818 }
1819
1820 /* return 0 means link status changed, -1 means not changed */
1821 static int
1822 ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1823 {
1824         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1825         struct rte_eth_link link, old;
1826         ixgbe_link_speed link_speed;
1827         int link_up;
1828         int diag;
1829
1830         link.link_status = 0;
1831         link.link_speed = 0;
1832         link.link_duplex = 0;
1833         memset(&old, 0, sizeof(old));
1834         rte_ixgbe_dev_atomic_read_link_status(dev, &old);
1835
1836         /* check if it needs to wait to complete, if lsc interrupt is enabled */
1837         if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
1838                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 0);
1839         else
1840                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 1);
1841         if (diag != 0) {
1842                 link.link_speed = ETH_LINK_SPEED_100;
1843                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1844                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1845                 if (link.link_status == old.link_status)
1846                         return -1;
1847                 return 0;
1848         }
1849
1850         if (link_up == 0) {
1851                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1852                 if (link.link_status == old.link_status)
1853                         return -1;
1854                 return 0;
1855         }
1856         link.link_status = 1;
1857         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1858
1859         switch (link_speed) {
1860         default:
1861         case IXGBE_LINK_SPEED_UNKNOWN:
1862                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1863                 link.link_speed = ETH_LINK_SPEED_100;
1864                 break;
1865
1866         case IXGBE_LINK_SPEED_100_FULL:
1867                 link.link_speed = ETH_LINK_SPEED_100;
1868                 break;
1869
1870         case IXGBE_LINK_SPEED_1GB_FULL:
1871                 link.link_speed = ETH_LINK_SPEED_1000;
1872                 break;
1873
1874         case IXGBE_LINK_SPEED_10GB_FULL:
1875                 link.link_speed = ETH_LINK_SPEED_10000;
1876                 break;
1877         }
1878         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1879
1880         if (link.link_status == old.link_status)
1881                 return -1;
1882
1883         return 0;
1884 }
1885
1886 static void
1887 ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev)
1888 {
1889         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1890         uint32_t fctrl;
1891
1892         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1893         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1894         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1895 }
1896
1897 static void
1898 ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev)
1899 {
1900         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1901         uint32_t fctrl;
1902
1903         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1904         fctrl &= (~IXGBE_FCTRL_UPE);
1905         if (dev->data->all_multicast == 1)
1906                 fctrl |= IXGBE_FCTRL_MPE;
1907         else
1908                 fctrl &= (~IXGBE_FCTRL_MPE);
1909         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1910 }
1911
1912 static void
1913 ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev)
1914 {
1915         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1916         uint32_t fctrl;
1917
1918         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1919         fctrl |= IXGBE_FCTRL_MPE;
1920         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1921 }
1922
1923 static void
1924 ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev)
1925 {
1926         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1927         uint32_t fctrl;
1928
1929         if (dev->data->promiscuous == 1)
1930                 return; /* must remain in all_multicast mode */
1931
1932         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1933         fctrl &= (~IXGBE_FCTRL_MPE);
1934         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1935 }
1936
1937 /**
1938  * It clears the interrupt causes and enables the interrupt.
1939  * It will be called once only during nic initialized.
1940  *
1941  * @param dev
1942  *  Pointer to struct rte_eth_dev.
1943  *
1944  * @return
1945  *  - On success, zero.
1946  *  - On failure, a negative value.
1947  */
1948 static int
1949 ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev)
1950 {
1951         struct ixgbe_interrupt *intr =
1952                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1953
1954         ixgbe_dev_link_status_print(dev);
1955         intr->mask |= IXGBE_EICR_LSC;
1956
1957         return 0;
1958 }
1959
1960 /*
1961  * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update.
1962  *
1963  * @param dev
1964  *  Pointer to struct rte_eth_dev.
1965  *
1966  * @return
1967  *  - On success, zero.
1968  *  - On failure, a negative value.
1969  */
1970 static int
1971 ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
1972 {
1973         uint32_t eicr;
1974         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1975         struct ixgbe_interrupt *intr =
1976                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1977
1978         /* clear all cause mask */
1979         ixgbe_disable_intr(hw);
1980
1981         /* read-on-clear nic registers here */
1982         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1983         PMD_DRV_LOG(INFO, "eicr %x", eicr);
1984
1985         intr->flags = 0;
1986         if (eicr & IXGBE_EICR_LSC) {
1987                 /* set flag for async link update */
1988                 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1989         }
1990
1991         if (eicr & IXGBE_EICR_MAILBOX)
1992                 intr->flags |= IXGBE_FLAG_MAILBOX;
1993
1994         return 0;
1995 }
1996
1997 /**
1998  * It gets and then prints the link status.
1999  *
2000  * @param dev
2001  *  Pointer to struct rte_eth_dev.
2002  *
2003  * @return
2004  *  - On success, zero.
2005  *  - On failure, a negative value.
2006  */
2007 static void
2008 ixgbe_dev_link_status_print(struct rte_eth_dev *dev)
2009 {
2010         struct rte_eth_link link;
2011
2012         memset(&link, 0, sizeof(link));
2013         rte_ixgbe_dev_atomic_read_link_status(dev, &link);
2014         if (link.link_status) {
2015                 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
2016                                         (int)(dev->data->port_id),
2017                                         (unsigned)link.link_speed,
2018                         link.link_duplex == ETH_LINK_FULL_DUPLEX ?
2019                                         "full-duplex" : "half-duplex");
2020         } else {
2021                 PMD_INIT_LOG(INFO, " Port %d: Link Down",
2022                                 (int)(dev->data->port_id));
2023         }
2024         PMD_INIT_LOG(INFO, "PCI Address: %04d:%02d:%02d:%d",
2025                                 dev->pci_dev->addr.domain,
2026                                 dev->pci_dev->addr.bus,
2027                                 dev->pci_dev->addr.devid,
2028                                 dev->pci_dev->addr.function);
2029 }
2030
2031 /*
2032  * It executes link_update after knowing an interrupt occurred.
2033  *
2034  * @param dev
2035  *  Pointer to struct rte_eth_dev.
2036  *
2037  * @return
2038  *  - On success, zero.
2039  *  - On failure, a negative value.
2040  */
2041 static int
2042 ixgbe_dev_interrupt_action(struct rte_eth_dev *dev)
2043 {
2044         struct ixgbe_interrupt *intr =
2045                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2046         int64_t timeout;
2047         struct rte_eth_link link;
2048         int intr_enable_delay = false;
2049
2050         PMD_DRV_LOG(DEBUG, "intr action type %d\n", intr->flags);
2051
2052         if (intr->flags & IXGBE_FLAG_MAILBOX) {
2053                 ixgbe_pf_mbx_process(dev);
2054                 intr->flags &= ~IXGBE_FLAG_MAILBOX;
2055         }
2056
2057         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
2058                 /* get the link status before link update, for predicting later */
2059                 memset(&link, 0, sizeof(link));
2060                 rte_ixgbe_dev_atomic_read_link_status(dev, &link);
2061
2062                 ixgbe_dev_link_update(dev, 0);
2063
2064                 /* likely to up */
2065                 if (!link.link_status)
2066                         /* handle it 1 sec later, wait it being stable */
2067                         timeout = IXGBE_LINK_UP_CHECK_TIMEOUT;
2068                 /* likely to down */
2069                 else
2070                         /* handle it 4 sec later, wait it being stable */
2071                         timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT;
2072
2073                 ixgbe_dev_link_status_print(dev);
2074
2075                 intr_enable_delay = true;
2076         }
2077
2078         if (intr_enable_delay) {
2079                 if (rte_eal_alarm_set(timeout * 1000,
2080                                       ixgbe_dev_interrupt_delayed_handler, (void*)dev) < 0)
2081                         PMD_DRV_LOG(ERR, "Error setting alarm");
2082         } else {
2083                 PMD_DRV_LOG(DEBUG, "enable intr immediately");
2084                 ixgbe_enable_intr(dev);
2085                 rte_intr_enable(&(dev->pci_dev->intr_handle));
2086         }
2087
2088
2089         return 0;
2090 }
2091
2092 /**
2093  * Interrupt handler which shall be registered for alarm callback for delayed
2094  * handling specific interrupt to wait for the stable nic state. As the
2095  * NIC interrupt state is not stable for ixgbe after link is just down,
2096  * it needs to wait 4 seconds to get the stable status.
2097  *
2098  * @param handle
2099  *  Pointer to interrupt handle.
2100  * @param param
2101  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2102  *
2103  * @return
2104  *  void
2105  */
2106 static void
2107 ixgbe_dev_interrupt_delayed_handler(void *param)
2108 {
2109         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2110         struct ixgbe_interrupt *intr =
2111                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2112         struct ixgbe_hw *hw =
2113                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2114         uint32_t eicr;
2115
2116         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
2117         if (eicr & IXGBE_EICR_MAILBOX)
2118                 ixgbe_pf_mbx_process(dev);
2119
2120         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
2121                 ixgbe_dev_link_update(dev, 0);
2122                 intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
2123                 ixgbe_dev_link_status_print(dev);
2124                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
2125         }
2126
2127         PMD_DRV_LOG(DEBUG, "enable intr in delayed handler S[%08x]\n", eicr);
2128         ixgbe_enable_intr(dev);
2129         rte_intr_enable(&(dev->pci_dev->intr_handle));
2130 }
2131
2132 /**
2133  * Interrupt handler triggered by NIC  for handling
2134  * specific interrupt.
2135  *
2136  * @param handle
2137  *  Pointer to interrupt handle.
2138  * @param param
2139  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2140  *
2141  * @return
2142  *  void
2143  */
2144 static void
2145 ixgbe_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
2146                                                         void *param)
2147 {
2148         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2149         ixgbe_dev_interrupt_get_status(dev);
2150         ixgbe_dev_interrupt_action(dev);
2151 }
2152
2153 static int
2154 ixgbe_dev_led_on(struct rte_eth_dev *dev)
2155 {
2156         struct ixgbe_hw *hw;
2157
2158         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2159         return (ixgbe_led_on(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
2160 }
2161
2162 static int
2163 ixgbe_dev_led_off(struct rte_eth_dev *dev)
2164 {
2165         struct ixgbe_hw *hw;
2166
2167         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2168         return (ixgbe_led_off(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
2169 }
2170
2171 static int
2172 ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2173 {
2174         struct ixgbe_hw *hw;
2175         int err;
2176         uint32_t rx_buf_size;
2177         uint32_t max_high_water;
2178         uint32_t mflcn;
2179         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
2180                 ixgbe_fc_none,
2181                 ixgbe_fc_rx_pause,
2182                 ixgbe_fc_tx_pause,
2183                 ixgbe_fc_full
2184         };
2185
2186         PMD_INIT_FUNC_TRACE();
2187
2188         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2189         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0));
2190         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
2191
2192         /*
2193          * At least reserve one Ethernet frame for watermark
2194          * high_water/low_water in kilo bytes for ixgbe
2195          */
2196         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
2197         if ((fc_conf->high_water > max_high_water) ||
2198                 (fc_conf->high_water < fc_conf->low_water)) {
2199                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
2200                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
2201                 return (-EINVAL);
2202         }
2203
2204         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[fc_conf->mode];
2205         hw->fc.pause_time     = fc_conf->pause_time;
2206         hw->fc.high_water[0]  = fc_conf->high_water;
2207         hw->fc.low_water[0]   = fc_conf->low_water;
2208         hw->fc.send_xon       = fc_conf->send_xon;
2209
2210         err = ixgbe_fc_enable(hw);
2211
2212         /* Not negotiated is not an error case */
2213         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) {
2214
2215                 /* check if we want to forward MAC frames - driver doesn't have native
2216                  * capability to do that, so we'll write the registers ourselves */
2217
2218                 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN);
2219
2220                 /* set or clear MFLCN.PMCF bit depending on configuration */
2221                 if (fc_conf->mac_ctrl_frame_fwd != 0)
2222                         mflcn |= IXGBE_MFLCN_PMCF;
2223                 else
2224                         mflcn &= ~IXGBE_MFLCN_PMCF;
2225
2226                 IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn);
2227                 IXGBE_WRITE_FLUSH(hw);
2228
2229                 return 0;
2230         }
2231
2232         PMD_INIT_LOG(ERR, "ixgbe_fc_enable = 0x%x \n", err);
2233         return -EIO;
2234 }
2235
2236 /**
2237  *  ixgbe_pfc_enable_generic - Enable flow control
2238  *  @hw: pointer to hardware structure
2239  *  @tc_num: traffic class number
2240  *  Enable flow control according to the current settings.
2241  */
2242 static int
2243 ixgbe_dcb_pfc_enable_generic(struct ixgbe_hw *hw,uint8_t tc_num)
2244 {
2245         int ret_val = 0;
2246         uint32_t mflcn_reg, fccfg_reg;
2247         uint32_t reg;
2248         uint32_t fcrtl, fcrth;
2249         uint8_t i;
2250         uint8_t nb_rx_en;
2251
2252         /* Validate the water mark configuration */
2253         if (!hw->fc.pause_time) {
2254                 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2255                 goto out;
2256         }
2257
2258         /* Low water mark of zero causes XOFF floods */
2259         if (hw->fc.current_mode & ixgbe_fc_tx_pause) {
2260                  /* High/Low water can not be 0 */
2261                 if( (!hw->fc.high_water[tc_num])|| (!hw->fc.low_water[tc_num])) {
2262                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
2263                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2264                         goto out;
2265                 }
2266
2267                 if(hw->fc.low_water[tc_num] >= hw->fc.high_water[tc_num]) {
2268                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
2269                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2270                         goto out;
2271                 }
2272         }
2273         /* Negotiate the fc mode to use */
2274         ixgbe_fc_autoneg(hw);
2275
2276         /* Disable any previous flow control settings */
2277         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
2278         mflcn_reg &= ~(IXGBE_MFLCN_RPFCE_SHIFT | IXGBE_MFLCN_RFCE|IXGBE_MFLCN_RPFCE);
2279
2280         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
2281         fccfg_reg &= ~(IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY);
2282
2283         switch (hw->fc.current_mode) {
2284         case ixgbe_fc_none:
2285                 /*
2286                  * If the count of enabled RX Priority Flow control >1,
2287                  * and the TX pause can not be disabled
2288                  */
2289                 nb_rx_en = 0;
2290                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
2291                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
2292                         if (reg & IXGBE_FCRTH_FCEN)
2293                                 nb_rx_en++;
2294                 }
2295                 if (nb_rx_en > 1)
2296                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2297                 break;
2298         case ixgbe_fc_rx_pause:
2299                 /*
2300                  * Rx Flow control is enabled and Tx Flow control is
2301                  * disabled by software override. Since there really
2302                  * isn't a way to advertise that we are capable of RX
2303                  * Pause ONLY, we will advertise that we support both
2304                  * symmetric and asymmetric Rx PAUSE.  Later, we will
2305                  * disable the adapter's ability to send PAUSE frames.
2306                  */
2307                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
2308                 /*
2309                  * If the count of enabled RX Priority Flow control >1,
2310                  * and the TX pause can not be disabled
2311                  */
2312                 nb_rx_en = 0;
2313                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
2314                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
2315                         if (reg & IXGBE_FCRTH_FCEN)
2316                                 nb_rx_en++;
2317                 }
2318                 if (nb_rx_en > 1)
2319                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2320                 break;
2321         case ixgbe_fc_tx_pause:
2322                 /*
2323                  * Tx Flow control is enabled, and Rx Flow control is
2324                  * disabled by software override.
2325                  */
2326                 fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2327                 break;
2328         case ixgbe_fc_full:
2329                 /* Flow control (both Rx and Tx) is enabled by SW override. */
2330                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
2331                 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
2332                 break;
2333         default:
2334                 DEBUGOUT("Flow control param set incorrectly\n");
2335                 ret_val = IXGBE_ERR_CONFIG;
2336                 goto out;
2337                 break;
2338         }
2339
2340         /* Set 802.3x based flow control settings. */
2341         mflcn_reg |= IXGBE_MFLCN_DPF;
2342         IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg);
2343         IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg);
2344
2345         /* Set up and enable Rx high/low water mark thresholds, enable XON. */
2346         if ((hw->fc.current_mode & ixgbe_fc_tx_pause) &&
2347                 hw->fc.high_water[tc_num]) {
2348                 fcrtl = (hw->fc.low_water[tc_num] << 10) | IXGBE_FCRTL_XONE;
2349                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), fcrtl);
2350                 fcrth = (hw->fc.high_water[tc_num] << 10) | IXGBE_FCRTH_FCEN;
2351         } else {
2352                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), 0);
2353                 /*
2354                  * In order to prevent Tx hangs when the internal Tx
2355                  * switch is enabled we must set the high water mark
2356                  * to the maximum FCRTH value.  This allows the Tx
2357                  * switch to function even under heavy Rx workloads.
2358                  */
2359                 fcrth = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)) - 32;
2360         }
2361         IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(tc_num), fcrth);
2362
2363         /* Configure pause time (2 TCs per register) */
2364         reg = hw->fc.pause_time * 0x00010001;
2365         for (i = 0; i < (IXGBE_DCB_MAX_TRAFFIC_CLASS / 2); i++)
2366                 IXGBE_WRITE_REG(hw, IXGBE_FCTTV(i), reg);
2367
2368         /* Configure flow control refresh threshold value */
2369         IXGBE_WRITE_REG(hw, IXGBE_FCRTV, hw->fc.pause_time / 2);
2370
2371 out:
2372         return ret_val;
2373 }
2374
2375 static int
2376 ixgbe_dcb_pfc_enable(struct rte_eth_dev *dev,uint8_t tc_num)
2377 {
2378         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2379         int32_t ret_val = IXGBE_NOT_IMPLEMENTED;
2380
2381         if(hw->mac.type != ixgbe_mac_82598EB) {
2382                 ret_val = ixgbe_dcb_pfc_enable_generic(hw,tc_num);
2383         }
2384         return ret_val;
2385 }
2386
2387 static int
2388 ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf)
2389 {
2390         int err;
2391         uint32_t rx_buf_size;
2392         uint32_t max_high_water;
2393         uint8_t tc_num;
2394         uint8_t  map[IXGBE_DCB_MAX_USER_PRIORITY] = { 0 };
2395         struct ixgbe_hw *hw =
2396                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2397         struct ixgbe_dcb_config *dcb_config =
2398                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
2399
2400         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
2401                 ixgbe_fc_none,
2402                 ixgbe_fc_rx_pause,
2403                 ixgbe_fc_tx_pause,
2404                 ixgbe_fc_full
2405         };
2406
2407         PMD_INIT_FUNC_TRACE();
2408
2409         ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
2410         tc_num = map[pfc_conf->priority];
2411         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num));
2412         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
2413         /*
2414          * At least reserve one Ethernet frame for watermark
2415          * high_water/low_water in kilo bytes for ixgbe
2416          */
2417         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
2418         if ((pfc_conf->fc.high_water > max_high_water) ||
2419                 (pfc_conf->fc.high_water <= pfc_conf->fc.low_water)) {
2420                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
2421                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
2422                 return (-EINVAL);
2423         }
2424
2425         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[pfc_conf->fc.mode];
2426         hw->fc.pause_time = pfc_conf->fc.pause_time;
2427         hw->fc.send_xon = pfc_conf->fc.send_xon;
2428         hw->fc.low_water[tc_num] =  pfc_conf->fc.low_water;
2429         hw->fc.high_water[tc_num] = pfc_conf->fc.high_water;
2430
2431         err = ixgbe_dcb_pfc_enable(dev,tc_num);
2432
2433         /* Not negotiated is not an error case */
2434         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED))
2435                 return 0;
2436
2437         PMD_INIT_LOG(ERR, "ixgbe_dcb_pfc_enable = 0x%x \n", err);
2438         return -EIO;
2439 }
2440
2441 static int
2442 ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
2443                                 struct rte_eth_rss_reta *reta_conf)
2444 {
2445         uint8_t i,j,mask;
2446         uint32_t reta;
2447         struct ixgbe_hw *hw =
2448                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2449
2450         PMD_INIT_FUNC_TRACE();
2451         /*
2452         * Update Redirection Table RETA[n],n=0...31,The redirection table has
2453         * 128-entries in 32 registers
2454          */
2455         for(i = 0; i < ETH_RSS_RETA_NUM_ENTRIES; i += 4) {
2456                 if (i < ETH_RSS_RETA_NUM_ENTRIES/2)
2457                         mask = (uint8_t)((reta_conf->mask_lo >> i) & 0xF);
2458                 else
2459                         mask = (uint8_t)((reta_conf->mask_hi >>
2460                                 (i - ETH_RSS_RETA_NUM_ENTRIES/2)) & 0xF);
2461                 if (mask != 0) {
2462                         reta = 0;
2463                         if (mask != 0xF)
2464                                 reta = IXGBE_READ_REG(hw,IXGBE_RETA(i >> 2));
2465
2466                         for (j = 0; j < 4; j++) {
2467                                 if (mask & (0x1 << j)) {
2468                                         if (mask != 0xF)
2469                                                 reta &= ~(0xFF << 8 * j);
2470                                         reta |= reta_conf->reta[i + j] << 8*j;
2471                                 }
2472                         }
2473                         IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2),reta);
2474                 }
2475         }
2476
2477         return 0;
2478 }
2479
2480 static int
2481 ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
2482                                 struct rte_eth_rss_reta *reta_conf)
2483 {
2484         uint8_t i,j,mask;
2485         uint32_t reta;
2486         struct ixgbe_hw *hw =
2487                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2488
2489         PMD_INIT_FUNC_TRACE();
2490         /*
2491          * Read Redirection Table RETA[n],n=0...31,The redirection table has
2492          * 128-entries in 32 registers
2493          */
2494         for(i = 0; i < ETH_RSS_RETA_NUM_ENTRIES; i += 4) {
2495                 if (i < ETH_RSS_RETA_NUM_ENTRIES/2)
2496                         mask = (uint8_t)((reta_conf->mask_lo >> i) & 0xF);
2497                 else
2498                         mask = (uint8_t)((reta_conf->mask_hi >>
2499                                 (i - ETH_RSS_RETA_NUM_ENTRIES/2)) & 0xF);
2500
2501                 if (mask != 0) {
2502                         reta = IXGBE_READ_REG(hw,IXGBE_RETA(i >> 2));
2503                         for (j = 0; j < 4; j++) {
2504                                 if (mask & (0x1 << j))
2505                                         reta_conf->reta[i + j] =
2506                                                 (uint8_t)((reta >> 8 * j) & 0xFF);
2507                         }
2508                 }
2509         }
2510
2511         return 0;
2512 }
2513
2514 static void
2515 ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
2516                                 uint32_t index, uint32_t pool)
2517 {
2518         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2519         uint32_t enable_addr = 1;
2520
2521         ixgbe_set_rar(hw, index, mac_addr->addr_bytes, pool, enable_addr);
2522 }
2523
2524 static void
2525 ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
2526 {
2527         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2528
2529         ixgbe_clear_rar(hw, index);
2530 }
2531
2532 /*
2533  * Virtual Function operations
2534  */
2535 static void
2536 ixgbevf_intr_disable(struct ixgbe_hw *hw)
2537 {
2538         PMD_INIT_LOG(DEBUG, "ixgbevf_intr_disable");
2539
2540         /* Clear interrupt mask to stop from interrupts being generated */
2541         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, IXGBE_VF_IRQ_CLEAR_MASK);
2542
2543         IXGBE_WRITE_FLUSH(hw);
2544 }
2545
2546 static int
2547 ixgbevf_dev_configure(struct rte_eth_dev *dev)
2548 {
2549         struct rte_eth_conf* conf = &dev->data->dev_conf;
2550
2551         PMD_INIT_LOG(DEBUG, "\nConfigured Virtual Function port id: %d\n",
2552                 dev->data->port_id);
2553
2554         /*
2555          * VF has no ability to enable/disable HW CRC
2556          * Keep the persistent behavior the same as Host PF
2557          */
2558 #ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC
2559         if (!conf->rxmode.hw_strip_crc) {
2560                 PMD_INIT_LOG(INFO, "VF can't disable HW CRC Strip\n");
2561                 conf->rxmode.hw_strip_crc = 1;
2562         }
2563 #else
2564         if (conf->rxmode.hw_strip_crc) {
2565                 PMD_INIT_LOG(INFO, "VF can't enable HW CRC Strip\n");
2566                 conf->rxmode.hw_strip_crc = 0;
2567         }
2568 #endif
2569
2570         return 0;
2571 }
2572
2573 static int
2574 ixgbevf_dev_start(struct rte_eth_dev *dev)
2575 {
2576         struct ixgbe_hw *hw =
2577                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2578         int err, mask = 0;
2579
2580         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_start");
2581
2582         hw->mac.ops.reset_hw(hw);
2583
2584         /* negotiate mailbox API version to use with the PF. */
2585         ixgbevf_negotiate_api(hw);
2586
2587         ixgbevf_dev_tx_init(dev);
2588
2589         /* This can fail when allocating mbufs for descriptor rings */
2590         err = ixgbevf_dev_rx_init(dev);
2591         if (err) {
2592                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware (%d)\n", err);
2593                 ixgbe_dev_clear_queues(dev);
2594                 return err;
2595         }
2596
2597         /* Set vfta */
2598         ixgbevf_set_vfta_all(dev,1);
2599
2600         /* Set HW strip */
2601         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
2602                 ETH_VLAN_EXTEND_MASK;
2603         ixgbevf_vlan_offload_set(dev, mask);
2604
2605         ixgbevf_dev_rxtx_start(dev);
2606
2607         return 0;
2608 }
2609
2610 static void
2611 ixgbevf_dev_stop(struct rte_eth_dev *dev)
2612 {
2613         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2614
2615         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_stop");
2616
2617         hw->adapter_stopped = TRUE;
2618         ixgbe_stop_adapter(hw);
2619
2620         /*
2621           * Clear what we set, but we still keep shadow_vfta to
2622           * restore after device starts
2623           */
2624         ixgbevf_set_vfta_all(dev,0);
2625
2626         ixgbe_dev_clear_queues(dev);
2627 }
2628
2629 static void
2630 ixgbevf_dev_close(struct rte_eth_dev *dev)
2631 {
2632         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2633
2634         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_close");
2635
2636         ixgbe_reset_hw(hw);
2637
2638         ixgbevf_dev_stop(dev);
2639
2640         /* reprogram the RAR[0] in case user changed it. */
2641         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
2642 }
2643
2644 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on)
2645 {
2646         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2647         struct ixgbe_vfta * shadow_vfta =
2648                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2649         int i = 0, j = 0, vfta = 0, mask = 1;
2650
2651         for (i = 0; i < IXGBE_VFTA_SIZE; i++){
2652                 vfta = shadow_vfta->vfta[i];
2653                 if(vfta){
2654                         mask = 1;
2655                         for (j = 0; j < 32; j++){
2656                                 if(vfta & mask)
2657                                         ixgbe_set_vfta(hw, (i<<5)+j, 0, on);
2658                                 mask<<=1;
2659                         }
2660                 }
2661         }
2662
2663 }
2664
2665 static int
2666 ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2667 {
2668         struct ixgbe_hw *hw =
2669                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2670         struct ixgbe_vfta * shadow_vfta =
2671                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2672         uint32_t vid_idx = 0;
2673         uint32_t vid_bit = 0;
2674         int ret = 0;
2675
2676         PMD_INIT_FUNC_TRACE();
2677
2678         /* vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf */
2679         ret = ixgbe_set_vfta(hw, vlan_id, 0, !!on);
2680         if(ret){
2681                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
2682                 return ret;
2683         }
2684         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
2685         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
2686
2687         /* Save what we set and retore it after device reset */
2688         if (on)
2689                 shadow_vfta->vfta[vid_idx] |= vid_bit;
2690         else
2691                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
2692
2693         return 0;
2694 }
2695
2696 static void
2697 ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
2698 {
2699         struct ixgbe_hw *hw =
2700                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2701         uint32_t ctrl;
2702
2703         PMD_INIT_FUNC_TRACE();
2704
2705         if(queue >= hw->mac.max_rx_queues)
2706                 return;
2707
2708         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
2709         if(on)
2710                 ctrl |= IXGBE_RXDCTL_VME;
2711         else
2712                 ctrl &= ~IXGBE_RXDCTL_VME;
2713         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
2714
2715         ixgbe_vlan_hw_strip_bitmap_set( dev, queue, on);
2716 }
2717
2718 static void
2719 ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2720 {
2721         struct ixgbe_hw *hw =
2722                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2723         uint16_t i;
2724         int on = 0;
2725
2726         /* VF function only support hw strip feature, others are not support */
2727         if(mask & ETH_VLAN_STRIP_MASK){
2728                 on = !!(dev->data->dev_conf.rxmode.hw_vlan_strip);
2729
2730                 for(i=0; i < hw->mac.max_rx_queues; i++)
2731                         ixgbevf_vlan_strip_queue_set(dev,i,on);
2732         }
2733 }
2734
2735 static int
2736 ixgbe_vmdq_mode_check(struct ixgbe_hw *hw)
2737 {
2738         uint32_t reg_val;
2739
2740         /* we only need to do this if VMDq is enabled */
2741         reg_val = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
2742         if (!(reg_val & IXGBE_VT_CTL_VT_ENABLE)) {
2743                 PMD_INIT_LOG(ERR, "VMDq must be enabled for this setting\n");
2744                 return (-1);
2745         }
2746
2747         return 0;
2748 }
2749
2750 static uint32_t
2751 ixgbe_uta_vector(struct ixgbe_hw *hw, struct ether_addr* uc_addr)
2752 {
2753         uint32_t vector = 0;
2754         switch (hw->mac.mc_filter_type) {
2755         case 0:   /* use bits [47:36] of the address */
2756                 vector = ((uc_addr->addr_bytes[4] >> 4) |
2757                         (((uint16_t)uc_addr->addr_bytes[5]) << 4));
2758                 break;
2759         case 1:   /* use bits [46:35] of the address */
2760                 vector = ((uc_addr->addr_bytes[4] >> 3) |
2761                         (((uint16_t)uc_addr->addr_bytes[5]) << 5));
2762                 break;
2763         case 2:   /* use bits [45:34] of the address */
2764                 vector = ((uc_addr->addr_bytes[4] >> 2) |
2765                         (((uint16_t)uc_addr->addr_bytes[5]) << 6));
2766                 break;
2767         case 3:   /* use bits [43:32] of the address */
2768                 vector = ((uc_addr->addr_bytes[4]) |
2769                         (((uint16_t)uc_addr->addr_bytes[5]) << 8));
2770                 break;
2771         default:  /* Invalid mc_filter_type */
2772                 break;
2773         }
2774
2775         /* vector can only be 12-bits or boundary will be exceeded */
2776         vector &= 0xFFF;
2777         return vector;
2778 }
2779
2780 static int
2781 ixgbe_uc_hash_table_set(struct rte_eth_dev *dev,struct ether_addr* mac_addr,
2782                                uint8_t on)
2783 {
2784         uint32_t vector;
2785         uint32_t uta_idx;
2786         uint32_t reg_val;
2787         uint32_t uta_shift;
2788         uint32_t rc;
2789         const uint32_t ixgbe_uta_idx_mask = 0x7F;
2790         const uint32_t ixgbe_uta_bit_shift = 5;
2791         const uint32_t ixgbe_uta_bit_mask = (0x1 << ixgbe_uta_bit_shift) - 1;
2792         const uint32_t bit1 = 0x1;
2793
2794         struct ixgbe_hw *hw =
2795                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2796         struct ixgbe_uta_info *uta_info =
2797                 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private);
2798
2799         /* The UTA table only exists on 82599 hardware and newer */
2800         if (hw->mac.type < ixgbe_mac_82599EB)
2801                 return (-ENOTSUP);
2802
2803         vector = ixgbe_uta_vector(hw,mac_addr);
2804         uta_idx = (vector >> ixgbe_uta_bit_shift) & ixgbe_uta_idx_mask;
2805         uta_shift = vector & ixgbe_uta_bit_mask;
2806
2807         rc = ((uta_info->uta_shadow[uta_idx] >> uta_shift & bit1) != 0);
2808         if(rc == on)
2809                 return 0;
2810
2811         reg_val = IXGBE_READ_REG(hw, IXGBE_UTA(uta_idx));
2812         if (on) {
2813                 uta_info->uta_in_use++;
2814                 reg_val |= (bit1 << uta_shift);
2815                 uta_info->uta_shadow[uta_idx] |= (bit1 << uta_shift);
2816         } else {
2817                 uta_info->uta_in_use--;
2818                 reg_val &= ~(bit1 << uta_shift);
2819                 uta_info->uta_shadow[uta_idx] &= ~(bit1 << uta_shift);
2820         }
2821
2822         IXGBE_WRITE_REG(hw, IXGBE_UTA(uta_idx), reg_val);
2823
2824         if (uta_info->uta_in_use > 0)
2825                 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL,
2826                                 IXGBE_MCSTCTRL_MFE | hw->mac.mc_filter_type);
2827         else
2828                 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL,hw->mac.mc_filter_type);
2829
2830         return 0;
2831 }
2832
2833 static int
2834 ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on)
2835 {
2836         int i;
2837         struct ixgbe_hw *hw =
2838                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2839         struct ixgbe_uta_info *uta_info =
2840                 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private);
2841
2842         /* The UTA table only exists on 82599 hardware and newer */
2843         if (hw->mac.type < ixgbe_mac_82599EB)
2844                 return (-ENOTSUP);
2845
2846         if(on) {
2847                 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
2848                         uta_info->uta_shadow[i] = ~0;
2849                         IXGBE_WRITE_REG(hw, IXGBE_UTA(i), ~0);
2850                 }
2851         } else {
2852                 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
2853                         uta_info->uta_shadow[i] = 0;
2854                         IXGBE_WRITE_REG(hw, IXGBE_UTA(i), 0);
2855                 }
2856         }
2857         return 0;
2858
2859 }
2860 static int
2861 ixgbe_set_pool_rx_mode(struct rte_eth_dev *dev, uint16_t pool,
2862                                uint16_t rx_mask, uint8_t on)
2863 {
2864         int val = 0;
2865
2866         struct ixgbe_hw *hw =
2867                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2868         uint32_t vmolr = IXGBE_READ_REG(hw, IXGBE_VMOLR(pool));
2869
2870         if (hw->mac.type == ixgbe_mac_82598EB) {
2871                 PMD_INIT_LOG(ERR, "setting VF receive mode set should be done"
2872                         " on 82599 hardware and newer\n");
2873                 return (-ENOTSUP);
2874         }
2875         if (ixgbe_vmdq_mode_check(hw) < 0)
2876                 return (-ENOTSUP);
2877
2878         if (rx_mask & ETH_VMDQ_ACCEPT_UNTAG )
2879                 val |= IXGBE_VMOLR_AUPE;
2880         if (rx_mask & ETH_VMDQ_ACCEPT_HASH_MC )
2881                 val |= IXGBE_VMOLR_ROMPE;
2882         if (rx_mask & ETH_VMDQ_ACCEPT_HASH_UC)
2883                 val |= IXGBE_VMOLR_ROPE;
2884         if (rx_mask & ETH_VMDQ_ACCEPT_BROADCAST)
2885                 val |= IXGBE_VMOLR_BAM;
2886         if (rx_mask & ETH_VMDQ_ACCEPT_MULTICAST)
2887                 val |= IXGBE_VMOLR_MPE;
2888
2889         if (on)
2890                 vmolr |= val;
2891         else
2892                 vmolr &= ~val;
2893
2894         IXGBE_WRITE_REG(hw, IXGBE_VMOLR(pool), vmolr);
2895
2896         return 0;
2897 }
2898
2899 static int
2900 ixgbe_set_pool_rx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on)
2901 {
2902         uint32_t reg,addr;
2903         uint32_t val;
2904         const uint8_t bit1 = 0x1;
2905
2906         struct ixgbe_hw *hw =
2907                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2908
2909         if (ixgbe_vmdq_mode_check(hw) < 0)
2910                 return (-ENOTSUP);
2911
2912         addr = IXGBE_VFRE(pool >= ETH_64_POOLS/2);
2913         reg = IXGBE_READ_REG(hw, addr);
2914         val = bit1 << pool;
2915
2916         if (on)
2917                 reg |= val;
2918         else
2919                 reg &= ~val;
2920
2921         IXGBE_WRITE_REG(hw, addr,reg);
2922
2923         return 0;
2924 }
2925
2926 static int
2927 ixgbe_set_pool_tx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on)
2928 {
2929         uint32_t reg,addr;
2930         uint32_t val;
2931         const uint8_t bit1 = 0x1;
2932
2933         struct ixgbe_hw *hw =
2934                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2935
2936         if (ixgbe_vmdq_mode_check(hw) < 0)
2937                 return (-ENOTSUP);
2938
2939         addr = IXGBE_VFTE(pool >= ETH_64_POOLS/2);
2940         reg = IXGBE_READ_REG(hw, addr);
2941         val = bit1 << pool;
2942
2943         if (on)
2944                 reg |= val;
2945         else
2946                 reg &= ~val;
2947
2948         IXGBE_WRITE_REG(hw, addr,reg);
2949
2950         return 0;
2951 }
2952
2953 static int
2954 ixgbe_set_pool_vlan_filter(struct rte_eth_dev *dev, uint16_t vlan,
2955                         uint64_t pool_mask, uint8_t vlan_on)
2956 {
2957         int ret = 0;
2958         uint16_t pool_idx;
2959         struct ixgbe_hw *hw =
2960                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2961
2962         if (ixgbe_vmdq_mode_check(hw) < 0)
2963                 return (-ENOTSUP);
2964         for (pool_idx = 0; pool_idx < ETH_64_POOLS; pool_idx++) {
2965                 if (pool_mask & ((uint64_t)(1ULL << pool_idx)))
2966                         ret = hw->mac.ops.set_vfta(hw,vlan,pool_idx,vlan_on);
2967                         if (ret < 0)
2968                                 return ret;
2969         }
2970
2971         return ret;
2972 }
2973
2974 static int
2975 ixgbe_mirror_rule_set(struct rte_eth_dev *dev,
2976                         struct rte_eth_vmdq_mirror_conf *mirror_conf,
2977                         uint8_t rule_id, uint8_t on)
2978 {
2979         uint32_t mr_ctl,vlvf;
2980         uint32_t mp_lsb = 0;
2981         uint32_t mv_msb = 0;
2982         uint32_t mv_lsb = 0;
2983         uint32_t mp_msb = 0;
2984         uint8_t i = 0;
2985         int reg_index = 0;
2986         uint64_t vlan_mask = 0;
2987
2988         const uint8_t pool_mask_offset = 32;
2989         const uint8_t vlan_mask_offset = 32;
2990         const uint8_t dst_pool_offset = 8;
2991         const uint8_t rule_mr_offset  = 4;
2992         const uint8_t mirror_rule_mask= 0x0F;
2993
2994         struct ixgbe_mirror_info *mr_info =
2995                         (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private));
2996         struct ixgbe_hw *hw =
2997                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2998
2999         if (ixgbe_vmdq_mode_check(hw) < 0)
3000                 return (-ENOTSUP);
3001
3002         /* Check if vlan mask is valid */
3003         if ((mirror_conf->rule_type_mask & ETH_VMDQ_VLAN_MIRROR) && (on)) {
3004                 if (mirror_conf->vlan.vlan_mask == 0)
3005                         return (-EINVAL);
3006         }
3007
3008         /* Check if vlan id is valid and find conresponding VLAN ID index in VLVF */
3009         if (mirror_conf->rule_type_mask & ETH_VMDQ_VLAN_MIRROR) {
3010                 for (i = 0;i < IXGBE_VLVF_ENTRIES; i++) {
3011                         if (mirror_conf->vlan.vlan_mask & (1ULL << i)) {
3012                                 /* search vlan id related pool vlan filter index */
3013                                 reg_index = ixgbe_find_vlvf_slot(hw,
3014                                                 mirror_conf->vlan.vlan_id[i]);
3015                                 if(reg_index < 0)
3016                                         return (-EINVAL);
3017                                 vlvf = IXGBE_READ_REG(hw, IXGBE_VLVF(reg_index));
3018                                 if ((vlvf & IXGBE_VLVF_VIEN) &&
3019                                         ((vlvf & IXGBE_VLVF_VLANID_MASK)
3020                                                 == mirror_conf->vlan.vlan_id[i]))
3021                                         vlan_mask |= (1ULL << reg_index);
3022                                 else
3023                                         return (-EINVAL);
3024                         }
3025                 }
3026
3027                 if (on) {
3028                         mv_lsb = vlan_mask & 0xFFFFFFFF;
3029                         mv_msb = vlan_mask >> vlan_mask_offset;
3030
3031                         mr_info->mr_conf[rule_id].vlan.vlan_mask =
3032                                                 mirror_conf->vlan.vlan_mask;
3033                         for(i = 0 ;i < ETH_VMDQ_MAX_VLAN_FILTERS; i++) {
3034                                 if(mirror_conf->vlan.vlan_mask & (1ULL << i))
3035                                         mr_info->mr_conf[rule_id].vlan.vlan_id[i] =
3036                                                 mirror_conf->vlan.vlan_id[i];
3037                         }
3038                 } else {
3039                         mv_lsb = 0;
3040                         mv_msb = 0;
3041                         mr_info->mr_conf[rule_id].vlan.vlan_mask = 0;
3042                         for(i = 0 ;i < ETH_VMDQ_MAX_VLAN_FILTERS; i++)
3043                                 mr_info->mr_conf[rule_id].vlan.vlan_id[i] = 0;
3044                 }
3045         }
3046
3047         /*
3048          * if enable pool mirror, write related pool mask register,if disable
3049          * pool mirror, clear PFMRVM register
3050          */
3051         if (mirror_conf->rule_type_mask & ETH_VMDQ_POOL_MIRROR) {
3052                 if (on) {
3053                         mp_lsb = mirror_conf->pool_mask & 0xFFFFFFFF;
3054                         mp_msb = mirror_conf->pool_mask >> pool_mask_offset;
3055                         mr_info->mr_conf[rule_id].pool_mask =
3056                                         mirror_conf->pool_mask;
3057
3058                 } else {
3059                         mp_lsb = 0;
3060                         mp_msb = 0;
3061                         mr_info->mr_conf[rule_id].pool_mask = 0;
3062                 }
3063         }
3064
3065         /* read  mirror control register and recalculate it */
3066         mr_ctl = IXGBE_READ_REG(hw,IXGBE_MRCTL(rule_id));
3067
3068         if (on) {
3069                 mr_ctl |= mirror_conf->rule_type_mask;
3070                 mr_ctl &= mirror_rule_mask;
3071                 mr_ctl |= mirror_conf->dst_pool << dst_pool_offset;
3072         } else
3073                 mr_ctl &= ~(mirror_conf->rule_type_mask & mirror_rule_mask);
3074
3075         mr_info->mr_conf[rule_id].rule_type_mask = (uint8_t)(mr_ctl & mirror_rule_mask);
3076         mr_info->mr_conf[rule_id].dst_pool = mirror_conf->dst_pool;
3077
3078         /* write mirrror control  register */
3079         IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl);
3080
3081         /* write pool mirrror control  register */
3082         if (mirror_conf->rule_type_mask & ETH_VMDQ_POOL_MIRROR) {
3083                 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), mp_lsb);
3084                 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset),
3085                                 mp_msb);
3086         }
3087         /* write VLAN mirrror control  register */
3088         if (mirror_conf->rule_type_mask & ETH_VMDQ_VLAN_MIRROR) {
3089                 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), mv_lsb);
3090                 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset),
3091                                 mv_msb);
3092         }
3093
3094         return 0;
3095 }
3096
3097 static int
3098 ixgbe_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t rule_id)
3099 {
3100         int mr_ctl = 0;
3101         uint32_t lsb_val = 0;
3102         uint32_t msb_val = 0;
3103         const uint8_t rule_mr_offset = 4;
3104
3105         struct ixgbe_hw *hw =
3106                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3107         struct ixgbe_mirror_info *mr_info =
3108                 (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private));
3109
3110         if (ixgbe_vmdq_mode_check(hw) < 0)
3111                 return (-ENOTSUP);
3112
3113         memset(&mr_info->mr_conf[rule_id], 0,
3114                 sizeof(struct rte_eth_vmdq_mirror_conf));
3115
3116         /* clear PFVMCTL register */
3117         IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl);
3118
3119         /* clear pool mask register */
3120         IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), lsb_val);
3121         IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset), msb_val);
3122
3123         /* clear vlan mask register */
3124         IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), lsb_val);
3125         IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset), msb_val);
3126
3127         return 0;
3128 }
3129
3130 static void
3131 ixgbevf_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
3132                      __attribute__((unused)) uint32_t index,
3133                      __attribute__((unused)) uint32_t pool)
3134 {
3135         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3136         int diag;
3137
3138         /*
3139          * On a 82599 VF, adding again the same MAC addr is not an idempotent
3140          * operation. Trap this case to avoid exhausting the [very limited]
3141          * set of PF resources used to store VF MAC addresses.
3142          */
3143         if (memcmp(hw->mac.perm_addr, mac_addr, sizeof(struct ether_addr)) == 0)
3144                 return;
3145         diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes);
3146         if (diag == 0)
3147                 return;
3148         PMD_DRV_LOG(ERR, "Unable to add MAC address - diag=%d", diag);
3149 }
3150
3151 static void
3152 ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index)
3153 {
3154         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3155         struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr;
3156         struct ether_addr *mac_addr;
3157         uint32_t i;
3158         int diag;
3159
3160         /*
3161          * The IXGBE_VF_SET_MACVLAN command of the ixgbe-pf driver does
3162          * not support the deletion of a given MAC address.
3163          * Instead, it imposes to delete all MAC addresses, then to add again
3164          * all MAC addresses with the exception of the one to be deleted.
3165          */
3166         (void) ixgbevf_set_uc_addr_vf(hw, 0, NULL);
3167
3168         /*
3169          * Add again all MAC addresses, with the exception of the deleted one
3170          * and of the permanent MAC address.
3171          */
3172         for (i = 0, mac_addr = dev->data->mac_addrs;
3173              i < hw->mac.num_rar_entries; i++, mac_addr++) {
3174                 /* Skip the deleted MAC address */
3175                 if (i == index)
3176                         continue;
3177                 /* Skip NULL MAC addresses */
3178                 if (is_zero_ether_addr(mac_addr))
3179                         continue;
3180                 /* Skip the permanent MAC address */
3181                 if (memcmp(perm_addr, mac_addr, sizeof(struct ether_addr)) == 0)
3182                         continue;
3183                 diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes);
3184                 if (diag != 0)
3185                         PMD_DRV_LOG(ERR,
3186                                     "Adding again MAC address "
3187                                     "%02x:%02x:%02x:%02x:%02x:%02x failed "
3188                                     "diag=%d",
3189                                     mac_addr->addr_bytes[0],
3190                                     mac_addr->addr_bytes[1],
3191                                     mac_addr->addr_bytes[2],
3192                                     mac_addr->addr_bytes[3],
3193                                     mac_addr->addr_bytes[4],
3194                                     mac_addr->addr_bytes[5],
3195                                     diag);
3196         }
3197 }
3198
3199 static struct rte_driver rte_ixgbe_driver = {
3200         .type = PMD_PDEV,
3201         .init = rte_ixgbe_pmd_init,
3202 };
3203
3204 static struct rte_driver rte_ixgbevf_driver = {
3205         .type = PMD_PDEV,
3206         .init = rte_ixgbevf_pmd_init,
3207 };
3208
3209 PMD_REGISTER_DRIVER(rte_ixgbe_driver);
3210 PMD_REGISTER_DRIVER(rte_ixgbevf_driver);