lib/librte_pmd_ixgbe/ixgbe_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2013 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  *
  33  */
  34
  35 #include <sys/queue.h>
  36 #include <stdio.h>
  37 #include <errno.h>
  38 #include <stdint.h>
  39 #include <string.h>
  40 #include <unistd.h>
  41 #include <stdarg.h>
  42 #include <inttypes.h>
  43 #include <rte_byteorder.h>
  44 #include <rte_common.h>
  45 #include <rte_cycles.h>
  46
  47 #include <rte_interrupts.h>
  48 #include <rte_log.h>
  49 #include <rte_debug.h>
  50 #include <rte_pci.h>
  51 #include <rte_atomic.h>
  52 #include <rte_branch_prediction.h>
  53 #include <rte_memory.h>
  54 #include <rte_memzone.h>
  55 #include <rte_tailq.h>
  56 #include <rte_eal.h>
  57 #include <rte_alarm.h>
  58 #include <rte_ether.h>
  59 #include <rte_ethdev.h>
  60 #include <rte_atomic.h>
  61 #include <rte_malloc.h>
  62
  63 #include "ixgbe_logs.h"
  64 #include "ixgbe/ixgbe_api.h"
  65 #include "ixgbe/ixgbe_vf.h"
  66 #include "ixgbe/ixgbe_common.h"
  67 #include "ixgbe_ethdev.h"
  68
  69 /*
  70  * High threshold controlling when to start sending XOFF frames. Must be at
  71  * least 8 bytes less than receive packet buffer size. This value is in units
  72  * of 1024 bytes.
  73  */
  74 #define IXGBE_FC_HI    0x80
  75
  76 /*
  77  * Low threshold controlling when to start sending XON frames. This value is
  78  * in units of 1024 bytes.
  79  */
  80 #define IXGBE_FC_LO    0x40
  81
  82 /* Timer value included in XOFF frames. */
  83 #define IXGBE_FC_PAUSE 0x680
  84
  85 #define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */
  86 #define IXGBE_LINK_UP_CHECK_TIMEOUT   1000 /* ms */
  87
  88 #define IXGBE_QUEUE_STAT_COUNTERS (sizeof(hw_stats->qprc) / sizeof(hw_stats->qprc[0]))
  89
  90 static int eth_ixgbe_dev_init(struct eth_driver *eth_drv,
  91                 struct rte_eth_dev *eth_dev);
  92 static int  ixgbe_dev_configure(struct rte_eth_dev *dev);
  93 static int  ixgbe_dev_start(struct rte_eth_dev *dev);
  94 static void ixgbe_dev_stop(struct rte_eth_dev *dev);
  95 static void ixgbe_dev_close(struct rte_eth_dev *dev);
  96 static void ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev);
  97 static void ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev);
  98 static void ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev);
  99 static void ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev);
 100 static int ixgbe_dev_link_update(struct rte_eth_dev *dev,
 101                                 int wait_to_complete);
 102 static void ixgbe_dev_stats_get(struct rte_eth_dev *dev,
 103                                 struct rte_eth_stats *stats);
 104 static void ixgbe_dev_stats_reset(struct rte_eth_dev *dev);
 105 static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 106                                              uint16_t queue_id,
 107                                              uint8_t stat_idx,
 108                                              uint8_t is_rx);
 109 static void ixgbe_dev_info_get(struct rte_eth_dev *dev,
 110                                 struct rte_eth_dev_info *dev_info);
 111 static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev,
 112                 uint16_t vlan_id, int on);
 113 static void ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 114 static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev,
 115                 uint16_t queue, bool on);
 116 static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue,
 117                 int on);
 118 static void ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 119 static void ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev);
 120 static void ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev);
 121 static void ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev);
 122 static void ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev);
 123 static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue);
 124 static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue);
 125 static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 126 static void ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 127
 128 static int ixgbe_dev_led_on(struct rte_eth_dev *dev);
 129 static int ixgbe_dev_led_off(struct rte_eth_dev *dev);
 130 static int  ixgbe_flow_ctrl_set(struct rte_eth_dev *dev,
 131                 struct rte_eth_fc_conf *fc_conf);
 132 static int ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 133                 struct rte_eth_pfc_conf *pfc_conf);
 134 static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev);
 135 static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev);
 136 static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev);
 137 static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev);
 138 static void ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle,
 139                 void *param);
 140 static void ixgbe_dev_interrupt_delayed_handler(void *param);
 141 static void ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 142                 uint32_t index, uint32_t pool);
 143 static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index);
 144 static void ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config);
 145
 146 /* For Virtual Function support */
 147 static int eth_ixgbevf_dev_init(struct eth_driver *eth_drv,
 148                 struct rte_eth_dev *eth_dev);
 149 static int  ixgbevf_dev_configure(struct rte_eth_dev *dev);
 150 static int  ixgbevf_dev_start(struct rte_eth_dev *dev);
 151 static void ixgbevf_dev_stop(struct rte_eth_dev *dev);
 152 static void ixgbevf_dev_close(struct rte_eth_dev *dev);
 153 static void ixgbevf_intr_disable(struct ixgbe_hw *hw);
 154 static void ixgbevf_dev_stats_get(struct rte_eth_dev *dev,
 155                 struct rte_eth_stats *stats);
 156 static void ixgbevf_dev_stats_reset(struct rte_eth_dev *dev);
 157 static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev,
 158                 uint16_t vlan_id, int on);
 159 static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev,
 160                 uint16_t queue, int on);
 161 static void ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 162 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 163
 164
 165 /*
 166  * Define VF Stats MACRO for Non "cleared on read" register
 167  */
 168 #define UPDATE_VF_STAT(reg, last, cur)                          \
 169 {                                                               \
 170         u32 latest = IXGBE_READ_REG(hw, reg);                   \
 171         cur += latest - last;                                   \
 172         last = latest;                                          \
 173 }
 174
 175 #define UPDATE_VF_STAT_36BIT(lsb, msb, last, cur)                \
 176 {                                                                \
 177         u64 new_lsb = IXGBE_READ_REG(hw, lsb);                   \
 178         u64 new_msb = IXGBE_READ_REG(hw, msb);                   \
 179         u64 latest = ((new_msb << 32) | new_lsb);                \
 180         cur += (0x1000000000LL + latest - last) & 0xFFFFFFFFFLL; \
 181         last = latest;                                           \
 182 }
 183
 184 #define IXGBE_SET_HWSTRIP(h, q) do{\
 185                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 186                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 187                 (h)->bitmap[idx] |= 1 << bit;\
 188         }while(0)
 189
 190 #define IXGBE_CLEAR_HWSTRIP(h, q) do{\
 191                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 192                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 193                 (h)->bitmap[idx] &= ~(1 << bit);\
 194         }while(0)
 195
 196 #define IXGBE_GET_HWSTRIP(h, q, r) do{\
 197                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 198                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 199                 (r) = (h)->bitmap[idx] >> bit & 1;\
 200         }while(0)
 201
 202 /*
 203  * The set of PCI devices this driver supports
 204  */
 205 static struct rte_pci_id pci_id_ixgbe_map[] = {
 206
 207 #define RTE_PCI_DEV_ID_DECL_IXGBE(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 208 #include "rte_pci_dev_ids.h"
 209
 210 { .vendor_id = 0, /* sentinel */ },
 211 };
 212
 213
 214 /*
 215  * The set of PCI devices this driver supports (for 82599 VF)
 216  */
 217 static struct rte_pci_id pci_id_ixgbevf_map[] = {
 218
 219 #define RTE_PCI_DEV_ID_DECL_IXGBEVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 220 #include "rte_pci_dev_ids.h"
 221 { .vendor_id = 0, /* sentinel */ },
 222
 223 };
 224
 225 static struct eth_dev_ops ixgbe_eth_dev_ops = {
 226         .dev_configure        = ixgbe_dev_configure,
 227         .dev_start            = ixgbe_dev_start,
 228         .dev_stop             = ixgbe_dev_stop,
 229         .dev_close            = ixgbe_dev_close,
 230         .promiscuous_enable   = ixgbe_dev_promiscuous_enable,
 231         .promiscuous_disable  = ixgbe_dev_promiscuous_disable,
 232         .allmulticast_enable  = ixgbe_dev_allmulticast_enable,
 233         .allmulticast_disable = ixgbe_dev_allmulticast_disable,
 234         .link_update          = ixgbe_dev_link_update,
 235         .stats_get            = ixgbe_dev_stats_get,
 236         .stats_reset          = ixgbe_dev_stats_reset,
 237         .queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set,
 238         .dev_infos_get        = ixgbe_dev_info_get,
 239         .vlan_filter_set      = ixgbe_vlan_filter_set,
 240         .vlan_tpid_set        = ixgbe_vlan_tpid_set,
 241         .vlan_offload_set     = ixgbe_vlan_offload_set,
 242         .vlan_strip_queue_set = ixgbe_vlan_strip_queue_set,
 243         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 244         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 245         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 246         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 247         .dev_led_on           = ixgbe_dev_led_on,
 248         .dev_led_off          = ixgbe_dev_led_off,
 249         .flow_ctrl_set        = ixgbe_flow_ctrl_set,
 250         .priority_flow_ctrl_set = ixgbe_priority_flow_ctrl_set,
 251         .mac_addr_add         = ixgbe_add_rar,
 252         .mac_addr_remove      = ixgbe_remove_rar,
 253         .fdir_add_signature_filter    = ixgbe_fdir_add_signature_filter,
 254         .fdir_update_signature_filter = ixgbe_fdir_update_signature_filter,
 255         .fdir_remove_signature_filter = ixgbe_fdir_remove_signature_filter,
 256         .fdir_infos_get               = ixgbe_fdir_info_get,
 257         .fdir_add_perfect_filter      = ixgbe_fdir_add_perfect_filter,
 258         .fdir_update_perfect_filter   = ixgbe_fdir_update_perfect_filter,
 259         .fdir_remove_perfect_filter   = ixgbe_fdir_remove_perfect_filter,
 260         .fdir_set_masks               = ixgbe_fdir_set_masks,
 261 };
 262
 263 /*
 264  * dev_ops for virtual function, bare necessities for basic vf
 265  * operation have been implemented
 266  */
 267 static struct eth_dev_ops ixgbevf_eth_dev_ops = {
 268
 269         .dev_configure        = ixgbevf_dev_configure,
 270         .dev_start            = ixgbevf_dev_start,
 271         .dev_stop             = ixgbevf_dev_stop,
 272         .link_update          = ixgbe_dev_link_update,
 273         .stats_get            = ixgbevf_dev_stats_get,
 274         .stats_reset          = ixgbevf_dev_stats_reset,
 275         .dev_close            = ixgbevf_dev_close,
 276         .dev_infos_get        = ixgbe_dev_info_get,
 277         .vlan_filter_set      = ixgbevf_vlan_filter_set,
 278         .vlan_strip_queue_set = ixgbevf_vlan_strip_queue_set,
 279         .vlan_offload_set     = ixgbevf_vlan_offload_set,
 280         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 281         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 282         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 283         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 284 };
 285
 286 /**
 287  * Atomically reads the link status information from global
 288  * structure rte_eth_dev.
 289  *
 290  * @param dev
 291  *   - Pointer to the structure rte_eth_dev to read from.
 292  *   - Pointer to the buffer to be saved with the link status.
 293  *
 294  * @return
 295  *   - On success, zero.
 296  *   - On failure, negative value.
 297  */
 298 static inline int
 299 rte_ixgbe_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 300                                 struct rte_eth_link *link)
 301 {
 302         struct rte_eth_link *dst = link;
 303         struct rte_eth_link *src = &(dev->data->dev_link);
 304
 305         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 306                                         *(uint64_t *)src) == 0)
 307                 return -1;
 308
 309         return 0;
 310 }
 311
 312 /**
 313  * Atomically writes the link status information into global
 314  * structure rte_eth_dev.
 315  *
 316  * @param dev
 317  *   - Pointer to the structure rte_eth_dev to read from.
 318  *   - Pointer to the buffer to be saved with the link status.
 319  *
 320  * @return
 321  *   - On success, zero.
 322  *   - On failure, negative value.
 323  */
 324 static inline int
 325 rte_ixgbe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 326                                 struct rte_eth_link *link)
 327 {
 328         struct rte_eth_link *dst = &(dev->data->dev_link);
 329         struct rte_eth_link *src = link;
 330
 331         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 332                                         *(uint64_t *)src) == 0)
 333                 return -1;
 334
 335         return 0;
 336 }
 337
 338 /*
 339  * This function is the same as ixgbe_is_sfp() in ixgbe/ixgbe.h.
 340  */
 341 static inline int
 342 ixgbe_is_sfp(struct ixgbe_hw *hw)
 343 {
 344         switch (hw->phy.type) {
 345         case ixgbe_phy_sfp_avago:
 346         case ixgbe_phy_sfp_ftl:
 347         case ixgbe_phy_sfp_intel:
 348         case ixgbe_phy_sfp_unknown:
 349         case ixgbe_phy_sfp_passive_tyco:
 350         case ixgbe_phy_sfp_passive_unknown:
 351                 return 1;
 352         default:
 353                 return 0;
 354         }
 355 }
 356
 357 static inline void
 358 ixgbe_enable_intr(struct rte_eth_dev *dev)
 359 {
 360         struct ixgbe_interrupt *intr =
 361                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 362         struct ixgbe_hw *hw =
 363                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 364
 365         IXGBE_WRITE_REG(hw, IXGBE_EIMS, intr->mask);
 366         IXGBE_WRITE_FLUSH(hw);
 367 }
 368
 369 /*
 370  * This function is based on ixgbe_disable_intr() in ixgbe/ixgbe.h.
 371  */
 372 static void
 373 ixgbe_disable_intr(struct ixgbe_hw *hw)
 374 {
 375         PMD_INIT_FUNC_TRACE();
 376
 377         if (hw->mac.type == ixgbe_mac_82598EB) {
 378                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ~0);
 379         } else {
 380                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xFFFF0000);
 381                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), ~0);
 382                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), ~0);
 383         }
 384         IXGBE_WRITE_FLUSH(hw);
 385 }
 386
 387 /*
 388  * This function resets queue statistics mapping registers.
 389  * From Niantic datasheet, Initialization of Statistics section:
 390  * "...if software requires the queue counters, the RQSMR and TQSM registers
 391  * must be re-programmed following a device reset.
 392  */
 393 static void
 394 ixgbe_reset_qstat_mappings(struct ixgbe_hw *hw)
 395 {
 396         uint32_t i;
 397
 398         for(i = 0; i != IXGBE_NB_STAT_MAPPING_REGS; i++) {
 399                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), 0);
 400                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), 0);
 401         }
 402 }
 403
 404
 405 static int
 406 ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 407                                   uint16_t queue_id,
 408                                   uint8_t stat_idx,
 409                                   uint8_t is_rx)
 410 {
 411 #define QSM_REG_NB_BITS_PER_QMAP_FIELD 8
 412 #define NB_QMAP_FIELDS_PER_QSM_REG 4
 413 #define QMAP_FIELD_RESERVED_BITS_MASK 0x0f
 414
 415         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 416         struct ixgbe_stat_mapping_registers *stat_mappings =
 417                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(eth_dev->data->dev_private);
 418         uint32_t qsmr_mask = 0;
 419         uint32_t clearing_mask = QMAP_FIELD_RESERVED_BITS_MASK;
 420         uint32_t q_map;
 421         uint8_t n, offset;
 422
 423         if ((hw->mac.type != ixgbe_mac_82599EB) && (hw->mac.type != ixgbe_mac_X540))
 424                 return -ENOSYS;
 425
 426         PMD_INIT_LOG(INFO, "Setting port %d, %s queue_id %d to stat index %d\n",
 427                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx);
 428
 429         n = (uint8_t)(queue_id / NB_QMAP_FIELDS_PER_QSM_REG);
 430         if (n >= IXGBE_NB_STAT_MAPPING_REGS) {
 431                 PMD_INIT_LOG(ERR, "Nb of stat mapping registers exceeded\n");
 432                 return -EIO;
 433         }
 434         offset = (uint8_t)(queue_id % NB_QMAP_FIELDS_PER_QSM_REG);
 435
 436         /* Now clear any previous stat_idx set */
 437         clearing_mask <<= (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 438         if (!is_rx)
 439                 stat_mappings->tqsm[n] &= ~clearing_mask;
 440         else
 441                 stat_mappings->rqsmr[n] &= ~clearing_mask;
 442
 443         q_map = (uint32_t)stat_idx;
 444         q_map &= QMAP_FIELD_RESERVED_BITS_MASK;
 445         qsmr_mask = q_map << (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 446         if (!is_rx)
 447                 stat_mappings->tqsm[n] |= qsmr_mask;
 448         else
 449                 stat_mappings->rqsmr[n] |= qsmr_mask;
 450
 451         PMD_INIT_LOG(INFO, "Set port %d, %s queue_id %d to stat index %d\n"
 452                      "%s[%d] = 0x%08x\n",
 453                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx,
 454                      is_rx ? "RQSMR" : "TQSM",n, is_rx ? stat_mappings->rqsmr[n] : stat_mappings->tqsm[n]);
 455
 456         /* Now write the mapping in the appropriate register */
 457         if (is_rx) {
 458                 PMD_INIT_LOG(INFO, "Write 0x%x to RX IXGBE stat mapping reg:%d\n",
 459                              stat_mappings->rqsmr[n], n);
 460                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(n), stat_mappings->rqsmr[n]);
 461         }
 462         else {
 463                 PMD_INIT_LOG(INFO, "Write 0x%x to TX IXGBE stat mapping reg:%d\n",
 464                              stat_mappings->tqsm[n], n);
 465                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(n), stat_mappings->tqsm[n]);
 466         }
 467         return 0;
 468 }
 469
 470 static void
 471 ixgbe_restore_statistics_mapping(struct rte_eth_dev * dev)
 472 {
 473         struct ixgbe_stat_mapping_registers *stat_mappings =
 474                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(dev->data->dev_private);
 475         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 476         int i;
 477
 478         /* write whatever was in stat mapping table to the NIC */
 479         for (i = 0; i < IXGBE_NB_STAT_MAPPING_REGS; i++) {
 480                 /* rx */
 481                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), stat_mappings->rqsmr[i]);
 482
 483                 /* tx */
 484                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), stat_mappings->tqsm[i]);
 485         }
 486 }
 487
 488 static void
 489 ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config)
 490 {
 491         uint8_t i;
 492         struct ixgbe_dcb_tc_config *tc;
 493         uint8_t dcb_max_tc = IXGBE_DCB_MAX_TRAFFIC_CLASS;
 494
 495         dcb_config->num_tcs.pg_tcs = dcb_max_tc;
 496         dcb_config->num_tcs.pfc_tcs = dcb_max_tc;
 497         for (i = 0; i < dcb_max_tc; i++) {
 498                 tc = &dcb_config->tc_config[i];
 499                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_id = i;
 500                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
 501                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 502                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_id = i;
 503                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
 504                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 505                 tc->pfc = ixgbe_dcb_pfc_disabled;
 506         }
 507
 508         /* Initialize default user to priority mapping, UPx->TC0 */
 509         tc = &dcb_config->tc_config[0];
 510         tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
 511         tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
 512         for (i = 0; i< IXGBE_DCB_MAX_BW_GROUP; i++) {
 513                 dcb_config->bw_percentage[IXGBE_DCB_TX_CONFIG][i] = 100;
 514                 dcb_config->bw_percentage[IXGBE_DCB_RX_CONFIG][i] = 100;
 515         }
 516         dcb_config->rx_pba_cfg = ixgbe_dcb_pba_equal;
 517         dcb_config->pfc_mode_enable = false;
 518         dcb_config->vt_mode = true;
 519         dcb_config->round_robin_enable = false;
 520         /* support all DCB capabilities in 82599 */
 521         dcb_config->support.capabilities = 0xFF;
 522
 523         /*we only support 4 Tcs for X540*/
 524         if (hw->mac.type == ixgbe_mac_X540) {
 525                 dcb_config->num_tcs.pg_tcs = 4;
 526                 dcb_config->num_tcs.pfc_tcs = 4;
 527         }
 528 }
 529
 530 /*
 531  * This function is based on code in ixgbe_attach() in ixgbe/ixgbe.c.
 532  * It returns 0 on success.
 533  */
 534 static int
 535 eth_ixgbe_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 536                      struct rte_eth_dev *eth_dev)
 537 {
 538         struct rte_pci_device *pci_dev;
 539         struct ixgbe_hw *hw =
 540                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 541         struct ixgbe_vfta * shadow_vfta =
 542                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 543         struct ixgbe_hwstrip *hwstrip =
 544                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 545         struct ixgbe_dcb_config *dcb_config =
 546                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private);
 547         uint32_t ctrl_ext;
 548         uint16_t csum;
 549         int diag, i;
 550
 551         PMD_INIT_FUNC_TRACE();
 552
 553         eth_dev->dev_ops = &ixgbe_eth_dev_ops;
 554         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
 555         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
 556
 557         /* for secondary processes, we don't initialise any further as primary
 558          * has already done this work. Only check we don't need a different
 559          * RX function */
 560         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 561                 if (eth_dev->data->scattered_rx)
 562                         eth_dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
 563                 return 0;
 564         }
 565         pci_dev = eth_dev->pci_dev;
 566
 567         /* Vendor and Device ID need to be set before init of shared code */
 568         hw->device_id = pci_dev->id.device_id;
 569         hw->vendor_id = pci_dev->id.vendor_id;
 570         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 571
 572         /* Initialize the shared code */
 573         diag = ixgbe_init_shared_code(hw);
 574         if (diag != IXGBE_SUCCESS) {
 575                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
 576                 return -EIO;
 577         }
 578
 579         /* Initialize DCB configuration*/
 580         memset(dcb_config, 0, sizeof(struct ixgbe_dcb_config));
 581         ixgbe_dcb_init(hw,dcb_config);
 582         /* Get Hardware Flow Control setting */
 583         hw->fc.requested_mode = ixgbe_fc_full;
 584         hw->fc.current_mode = ixgbe_fc_full;
 585         hw->fc.pause_time = IXGBE_FC_PAUSE;
 586         for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
 587                 hw->fc.low_water[i] = IXGBE_FC_LO;
 588                 hw->fc.high_water[i] = IXGBE_FC_HI;
 589         }
 590         hw->fc.send_xon = 1;
 591
 592         /* Make sure we have a good EEPROM before we read from it */
 593         diag = ixgbe_validate_eeprom_checksum(hw, &csum);
 594         if (diag != IXGBE_SUCCESS) {
 595                 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag);
 596                 return -EIO;
 597         }
 598
 599         diag = ixgbe_init_hw(hw);
 600
 601         /*
 602          * Devices with copper phys will fail to initialise if ixgbe_init_hw()
 603          * is called too soon after the kernel driver unbinding/binding occurs.
 604          * The failure occurs in ixgbe_identify_phy_generic() for all devices,
 605          * but for non-copper devies, ixgbe_identify_sfp_module_generic() is
 606          * also called. See ixgbe_identify_phy_82599(). The reason for the
 607          * failure is not known, and only occuts when virtualisation features
 608          * are disabled in the bios. A delay of 100ms  was found to be enough by
 609          * trial-and-error, and is doubled to be safe.
 610          */
 611         if (diag && (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper)) {
 612                 rte_delay_ms(200);
 613                 diag = ixgbe_init_hw(hw);
 614         }
 615
 616         if (diag == IXGBE_ERR_EEPROM_VERSION) {
 617                 PMD_INIT_LOG(ERR, "This device is a pre-production adapter/"
 618                     "LOM.  Please be aware there may be issues associated "
 619                     "with your hardware.\n If you are experiencing problems "
 620                     "please contact your Intel or hardware representative "
 621                     "who provided you with this hardware.\n");
 622         } else if (diag == IXGBE_ERR_SFP_NOT_SUPPORTED)
 623                 PMD_INIT_LOG(ERR, "Unsupported SFP+ Module\n");
 624         if (diag) {
 625                 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag);
 626                 return -EIO;
 627         }
 628
 629         /* disable interrupt */
 630         ixgbe_disable_intr(hw);
 631
 632         /* pick up the PCI bus settings for reporting later */
 633         ixgbe_get_bus_info(hw);
 634
 635         /* reset mappings for queue statistics hw counters*/
 636         ixgbe_reset_qstat_mappings(hw);
 637
 638         /* Allocate memory for storing MAC addresses */
 639         eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 640                         hw->mac.num_rar_entries, 0);
 641         if (eth_dev->data->mac_addrs == NULL) {
 642                 PMD_INIT_LOG(ERR,
 643                         "Failed to allocate %d bytes needed to store MAC addresses",
 644                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 645                 return -ENOMEM;
 646         }
 647         /* Copy the permanent MAC address */
 648         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 649                         &eth_dev->data->mac_addrs[0]);
 650
 651         /* initialize the vfta */
 652         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 653
 654         /* initialize the hw strip bitmap*/
 655         memset(hwstrip, 0, sizeof(*hwstrip));
 656
 657         /* let hardware know driver is loaded */
 658         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 659         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 660         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 661
 662         if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
 663                 PMD_INIT_LOG(DEBUG,
 664                              "MAC: %d, PHY: %d, SFP+: %d<n",
 665                              (int) hw->mac.type, (int) hw->phy.type,
 666                              (int) hw->phy.sfp_type);
 667         else
 668                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d\n",
 669                              (int) hw->mac.type, (int) hw->phy.type);
 670
 671         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
 672                         eth_dev->data->port_id, pci_dev->id.vendor_id,
 673                         pci_dev->id.device_id);
 674
 675         rte_intr_callback_register(&(pci_dev->intr_handle),
 676                 ixgbe_dev_interrupt_handler, (void *)eth_dev);
 677
 678         /* enable uio intr after callback register */
 679         rte_intr_enable(&(pci_dev->intr_handle));
 680
 681         /* enable support intr */
 682         ixgbe_enable_intr(eth_dev);
 683
 684         return 0;
 685 }
 686
 687 /*
 688  * Virtual Function device init
 689  */
 690 static int
 691 eth_ixgbevf_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 692                      struct rte_eth_dev *eth_dev)
 693 {
 694         struct rte_pci_device *pci_dev;
 695         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 696         int diag;
 697         struct ixgbe_vfta * shadow_vfta =
 698                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 699         struct ixgbe_hwstrip *hwstrip =
 700                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 701
 702         PMD_INIT_LOG(DEBUG, "eth_ixgbevf_dev_init");
 703
 704         eth_dev->dev_ops = &ixgbevf_eth_dev_ops;
 705         pci_dev = eth_dev->pci_dev;
 706
 707         hw->device_id = pci_dev->id.device_id;
 708         hw->vendor_id = pci_dev->id.vendor_id;
 709         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 710
 711         /* initialize the vfta */
 712         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 713
 714         /* initialize the hw strip bitmap*/
 715         memset(hwstrip, 0, sizeof(*hwstrip));
 716
 717         /* Initialize the shared code */
 718         diag = ixgbe_init_shared_code(hw);
 719         if (diag != IXGBE_SUCCESS) {
 720                 PMD_INIT_LOG(ERR, "Shared code init failed for ixgbevf: %d", diag);
 721                 return -EIO;
 722         }
 723
 724         /* init_mailbox_params */
 725         hw->mbx.ops.init_params(hw);
 726
 727         /* Disable the interrupts for VF */
 728         ixgbevf_intr_disable(hw);
 729
 730         hw->mac.num_rar_entries = hw->mac.max_rx_queues;
 731         diag = hw->mac.ops.reset_hw(hw);
 732         if (diag != IXGBE_SUCCESS) {
 733                 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
 734                 return (diag);
 735         }
 736
 737         /* Allocate memory for storing MAC addresses */
 738         eth_dev->data->mac_addrs = rte_zmalloc("ixgbevf", ETHER_ADDR_LEN *
 739                         hw->mac.num_rar_entries, 0);
 740         if (eth_dev->data->mac_addrs == NULL) {
 741                 PMD_INIT_LOG(ERR,
 742                         "Failed to allocate %d bytes needed to store MAC addresses",
 743                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 744                 return -ENOMEM;
 745         }
 746         /* Copy the permanent MAC address */
 747         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 748                         &eth_dev->data->mac_addrs[0]);
 749
 750         /* reset the hardware with the new settings */
 751         diag = hw->mac.ops.start_hw(hw);
 752         switch (diag) {
 753                 case  0:
 754                         break;
 755
 756                 default:
 757                         PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
 758                         return (-EIO);
 759         }
 760
 761         PMD_INIT_LOG(DEBUG, "\nport %d vendorID=0x%x deviceID=0x%x mac.type=%s\n",
 762                          eth_dev->data->port_id, pci_dev->id.vendor_id, pci_dev->id.device_id,
 763                          "ixgbe_mac_82599_vf");
 764
 765         return 0;
 766 }
 767
 768 static struct eth_driver rte_ixgbe_pmd = {
 769         {
 770                 .name = "rte_ixgbe_pmd",
 771                 .id_table = pci_id_ixgbe_map,
 772                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
 773         },
 774         .eth_dev_init = eth_ixgbe_dev_init,
 775         .dev_private_size = sizeof(struct ixgbe_adapter),
 776 };
 777
 778 /*
 779  * virtual function driver struct
 780  */
 781 static struct eth_driver rte_ixgbevf_pmd = {
 782         {
 783                 .name = "rte_ixgbevf_pmd",
 784                 .id_table = pci_id_ixgbevf_map,
 785                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
 786         },
 787         .eth_dev_init = eth_ixgbevf_dev_init,
 788         .dev_private_size = sizeof(struct ixgbe_adapter),
 789 };
 790
 791 /*
 792  * Driver initialization routine.
 793  * Invoked once at EAL init time.
 794  * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
 795  */
 796 int
 797 rte_ixgbe_pmd_init(void)
 798 {
 799         PMD_INIT_FUNC_TRACE();
 800
 801         rte_eth_driver_register(&rte_ixgbe_pmd);
 802         return 0;
 803 }
 804
 805 /*
 806  * VF Driver initialization routine.
 807  * Invoked one at EAL init time.
 808  * Register itself as the [Virtual Poll Mode] Driver of PCI niantic devices.
 809  */
 810 int
 811 rte_ixgbevf_pmd_init(void)
 812 {
 813         DEBUGFUNC("rte_ixgbevf_pmd_init");
 814
 815         rte_eth_driver_register(&rte_ixgbevf_pmd);
 816         return (0);
 817 }
 818
 819 static int
 820 ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 821 {
 822         struct ixgbe_hw *hw =
 823                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 824         struct ixgbe_vfta * shadow_vfta =
 825                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
 826         uint32_t vfta;
 827         uint32_t vid_idx;
 828         uint32_t vid_bit;
 829
 830         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
 831         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
 832         vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(vid_idx));
 833         if (on)
 834                 vfta |= vid_bit;
 835         else
 836                 vfta &= ~vid_bit;
 837         IXGBE_WRITE_REG(hw, IXGBE_VFTA(vid_idx), vfta);
 838
 839         /* update local VFTA copy */
 840         shadow_vfta->vfta[vid_idx] = vfta;
 841
 842         return 0;
 843 }
 844
 845 static void
 846 ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
 847 {
 848         if (on)
 849                 ixgbe_vlan_hw_strip_enable(dev, queue);
 850         else
 851                 ixgbe_vlan_hw_strip_disable(dev, queue);
 852 }
 853
 854 static void
 855 ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
 856 {
 857         struct ixgbe_hw *hw =
 858                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 859
 860         /* Only the high 16-bits is valid */
 861         IXGBE_WRITE_REG(hw, IXGBE_EXVET, tpid << 16);
 862 }
 863
 864 static void
 865 ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev)
 866 {
 867         struct ixgbe_hw *hw =
 868                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 869         uint32_t vlnctrl;
 870
 871         PMD_INIT_FUNC_TRACE();
 872
 873         /* Filter Table Disable */
 874         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 875         vlnctrl &= ~IXGBE_VLNCTRL_VFE;
 876
 877         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
 878 }
 879
 880 static void
 881 ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev)
 882 {
 883         struct ixgbe_hw *hw =
 884                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 885         struct ixgbe_vfta * shadow_vfta =
 886                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
 887         uint32_t vlnctrl;
 888         uint16_t i;
 889
 890         PMD_INIT_FUNC_TRACE();
 891
 892         /* Filter Table Enable */
 893         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 894         vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
 895         vlnctrl |= IXGBE_VLNCTRL_VFE;
 896
 897         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
 898
 899         /* write whatever is in local vfta copy */
 900         for (i = 0; i < IXGBE_VFTA_SIZE; i++)
 901                 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), shadow_vfta->vfta[i]);
 902 }
 903
 904 static void
 905 ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on)
 906 {
 907         struct ixgbe_hwstrip *hwstrip =
 908                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(dev->data->dev_private);
 909
 910         if(queue >= IXGBE_MAX_RX_QUEUE_NUM)
 911                 return;
 912
 913         if (on)
 914                 IXGBE_SET_HWSTRIP(hwstrip, queue);
 915         else
 916                 IXGBE_CLEAR_HWSTRIP(hwstrip, queue);
 917 }
 918
 919 static void
 920 ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue)
 921 {
 922         struct ixgbe_hw *hw =
 923                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 924         uint32_t ctrl;
 925
 926         PMD_INIT_FUNC_TRACE();
 927
 928         if (hw->mac.type == ixgbe_mac_82598EB) {
 929                 /* No queue level support */
 930                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
 931                 return;
 932         }
 933         else {
 934                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 935                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
 936                 ctrl &= ~IXGBE_RXDCTL_VME;
 937                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
 938         }
 939         /* record those setting for HW strip per queue */
 940         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 0);
 941 }
 942
 943 static void
 944 ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue)
 945 {
 946         struct ixgbe_hw *hw =
 947                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 948         uint32_t ctrl;
 949
 950         PMD_INIT_FUNC_TRACE();
 951
 952         if (hw->mac.type == ixgbe_mac_82598EB) {
 953                 /* No queue level supported */
 954                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
 955                 return;
 956         }
 957         else {
 958                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 959                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
 960                 ctrl |= IXGBE_RXDCTL_VME;
 961                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
 962         }
 963         /* record those setting for HW strip per queue */
 964         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 1);
 965 }
 966
 967 static void
 968 ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev)
 969 {
 970         struct ixgbe_hw *hw =
 971                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 972         uint32_t ctrl;
 973         uint16_t i;
 974
 975         PMD_INIT_FUNC_TRACE();
 976
 977         if (hw->mac.type == ixgbe_mac_82598EB) {
 978                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 979                 ctrl &= ~IXGBE_VLNCTRL_VME;
 980                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
 981         }
 982         else {
 983                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 984                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
 985                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
 986                         ctrl &= ~IXGBE_RXDCTL_VME;
 987                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
 988
 989                         /* record those setting for HW strip per queue */
 990                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 0);
 991                 }
 992         }
 993 }
 994
 995 static void
 996 ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev)
 997 {
 998         struct ixgbe_hw *hw =
 999                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1000         uint32_t ctrl;
1001         uint16_t i;
1002
1003         PMD_INIT_FUNC_TRACE();
1004
1005         if (hw->mac.type == ixgbe_mac_82598EB) {
1006                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1007                 ctrl |= IXGBE_VLNCTRL_VME;
1008                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1009         }
1010         else {
1011                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1012                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1013                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1014                         ctrl |= IXGBE_RXDCTL_VME;
1015                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1016
1017                         /* record those setting for HW strip per queue */
1018                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 1);
1019                 }
1020         }
1021 }
1022
1023 static void
1024 ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1025 {
1026         struct ixgbe_hw *hw =
1027                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1028         uint32_t ctrl;
1029
1030         PMD_INIT_FUNC_TRACE();
1031
1032         /* DMATXCTRL: Geric Double VLAN Disable */
1033         ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1034         ctrl &= ~IXGBE_DMATXCTL_GDV;
1035         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1036
1037         /* CTRL_EXT: Global Double VLAN Disable */
1038         ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1039         ctrl &= ~IXGBE_EXTENDED_VLAN;
1040         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1041
1042 }
1043
1044 static void
1045 ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1046 {
1047         struct ixgbe_hw *hw =
1048                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1049         uint32_t ctrl;
1050
1051         PMD_INIT_FUNC_TRACE();
1052
1053         /* DMATXCTRL: Geric Double VLAN Enable */
1054         ctrl  = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1055         ctrl |= IXGBE_DMATXCTL_GDV;
1056         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1057
1058         /* CTRL_EXT: Global Double VLAN Enable */
1059         ctrl  = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1060         ctrl |= IXGBE_EXTENDED_VLAN;
1061         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1062
1063         /*
1064          * VET EXT field in the EXVET register = 0x8100 by default
1065          * So no need to change. Same to VT field of DMATXCTL register
1066          */
1067 }
1068
1069 static void
1070 ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1071 {
1072         if(mask & ETH_VLAN_STRIP_MASK){
1073                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1074                         ixgbe_vlan_hw_strip_enable_all(dev);
1075                 else
1076                         ixgbe_vlan_hw_strip_disable_all(dev);
1077         }
1078
1079         if(mask & ETH_VLAN_FILTER_MASK){
1080                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1081                         ixgbe_vlan_hw_filter_enable(dev);
1082                 else
1083                         ixgbe_vlan_hw_filter_disable(dev);
1084         }
1085
1086         if(mask & ETH_VLAN_EXTEND_MASK){
1087                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1088                         ixgbe_vlan_hw_extend_enable(dev);
1089                 else
1090                         ixgbe_vlan_hw_extend_disable(dev);
1091         }
1092 }
1093
1094 static int
1095 ixgbe_dev_configure(struct rte_eth_dev *dev)
1096 {
1097         struct ixgbe_interrupt *intr =
1098                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1099
1100         PMD_INIT_FUNC_TRACE();
1101
1102         /* set flag to update link status after init */
1103         intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1104
1105         return 0;
1106 }
1107
1108 /*
1109  * Configure device link speed and setup link.
1110  * It returns 0 on success.
1111  */
1112 static int
1113 ixgbe_dev_start(struct rte_eth_dev *dev)
1114 {
1115         struct ixgbe_hw *hw =
1116                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1117         int err, link_up = 0, negotiate = 0;
1118         uint32_t speed = 0;
1119         int mask = 0;
1120
1121         PMD_INIT_FUNC_TRACE();
1122
1123         /* IXGBE devices don't support half duplex */
1124         if ((dev->data->dev_conf.link_duplex != ETH_LINK_AUTONEG_DUPLEX) &&
1125                         (dev->data->dev_conf.link_duplex != ETH_LINK_FULL_DUPLEX)) {
1126                 PMD_INIT_LOG(ERR, "Invalid link_duplex (%u) for port %u\n",
1127                                 dev->data->dev_conf.link_duplex,
1128                                 dev->data->port_id);
1129                 return -EINVAL;
1130         }
1131
1132         /* stop adapter */
1133         hw->adapter_stopped = FALSE;
1134         ixgbe_stop_adapter(hw);
1135
1136         /* reinitialize adapter
1137          * this calls reset and start */
1138         ixgbe_init_hw(hw);
1139         hw->mac.ops.start_hw(hw);
1140
1141         /* initialize transmission unit */
1142         ixgbe_dev_tx_init(dev);
1143
1144         /* This can fail when allocating mbufs for descriptor rings */
1145         err = ixgbe_dev_rx_init(dev);
1146         if (err) {
1147                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware\n");
1148                 goto error;
1149         }
1150
1151         ixgbe_dev_rxtx_start(dev);
1152
1153         if (ixgbe_is_sfp(hw) && hw->phy.multispeed_fiber) {
1154                 err = hw->mac.ops.setup_sfp(hw);
1155                 if (err)
1156                         goto error;
1157         }
1158
1159         /* Turn on the laser */
1160         if (hw->phy.multispeed_fiber)
1161                 ixgbe_enable_tx_laser(hw);
1162
1163         err = ixgbe_check_link(hw, &speed, &link_up, 0);
1164         if (err)
1165                 goto error;
1166         err = ixgbe_get_link_capabilities(hw, &speed, &negotiate);
1167         if (err)
1168                 goto error;
1169
1170         switch(dev->data->dev_conf.link_speed) {
1171         case ETH_LINK_SPEED_AUTONEG:
1172                 speed = (hw->mac.type != ixgbe_mac_82598EB) ?
1173                                 IXGBE_LINK_SPEED_82599_AUTONEG :
1174                                 IXGBE_LINK_SPEED_82598_AUTONEG;
1175                 break;
1176         case ETH_LINK_SPEED_100:
1177                 /*
1178                  * Invalid for 82598 but error will be detected by
1179                  * ixgbe_setup_link()
1180                  */
1181                 speed = IXGBE_LINK_SPEED_100_FULL;
1182                 break;
1183         case ETH_LINK_SPEED_1000:
1184                 speed = IXGBE_LINK_SPEED_1GB_FULL;
1185                 break;
1186         case ETH_LINK_SPEED_10000:
1187                 speed = IXGBE_LINK_SPEED_10GB_FULL;
1188                 break;
1189         default:
1190                 PMD_INIT_LOG(ERR, "Invalid link_speed (%u) for port %u\n",
1191                                 dev->data->dev_conf.link_speed, dev->data->port_id);
1192                 goto error;
1193         }
1194
1195         err = ixgbe_setup_link(hw, speed, negotiate, link_up);
1196         if (err)
1197                 goto error;
1198
1199         /* check if lsc interrupt is enabled */
1200         if (dev->data->dev_conf.intr_conf.lsc != 0)
1201                 ixgbe_dev_lsc_interrupt_setup(dev);
1202
1203         /* resume enabled intr since hw reset */
1204         ixgbe_enable_intr(dev);
1205
1206         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
1207                 ETH_VLAN_EXTEND_MASK;
1208         ixgbe_vlan_offload_set(dev, mask);
1209
1210         /* Configure DCB hw */
1211         ixgbe_configure_dcb(dev);
1212
1213         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) {
1214                 err = ixgbe_fdir_configure(dev);
1215                 if (err)
1216                         goto error;
1217         }
1218
1219         ixgbe_restore_statistics_mapping(dev);
1220
1221         return (0);
1222
1223 error:
1224         PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err);
1225         ixgbe_dev_clear_queues(dev);
1226         return -EIO;
1227 }
1228
1229 /*
1230  * Stop device: disable rx and tx functions to allow for reconfiguring.
1231  */
1232 static void
1233 ixgbe_dev_stop(struct rte_eth_dev *dev)
1234 {
1235         struct rte_eth_link link;
1236         struct ixgbe_hw *hw =
1237                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1238
1239         PMD_INIT_FUNC_TRACE();
1240
1241         /* disable interrupts */
1242         ixgbe_disable_intr(hw);
1243
1244         /* reset the NIC */
1245         ixgbe_reset_hw(hw);
1246         hw->adapter_stopped = FALSE;
1247
1248         /* stop adapter */
1249         ixgbe_stop_adapter(hw);
1250
1251         /* Turn off the laser */
1252         if (hw->phy.multispeed_fiber)
1253                 ixgbe_disable_tx_laser(hw);
1254
1255         ixgbe_dev_clear_queues(dev);
1256
1257         /* Clear recorded link status */
1258         memset(&link, 0, sizeof(link));
1259         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1260 }
1261
1262 /*
1263  * Reest and stop device.
1264  */
1265 static void
1266 ixgbe_dev_close(struct rte_eth_dev *dev)
1267 {
1268         struct ixgbe_hw *hw =
1269                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1270
1271         PMD_INIT_FUNC_TRACE();
1272
1273         ixgbe_reset_hw(hw);
1274
1275
1276         ixgbe_dev_stop(dev);
1277         hw->adapter_stopped = 1;
1278
1279         ixgbe_disable_pcie_master(hw);
1280
1281         /* reprogram the RAR[0] in case user changed it. */
1282         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
1283 }
1284
1285 /*
1286  * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c
1287  */
1288 static void
1289 ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1290 {
1291         struct ixgbe_hw *hw =
1292                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1293         struct ixgbe_hw_stats *hw_stats =
1294                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1295         uint32_t bprc, lxon, lxoff, total;
1296         uint64_t total_missed_rx, total_qbrc, total_qprc;
1297         unsigned i;
1298
1299         total_missed_rx = 0;
1300         total_qbrc = 0;
1301         total_qprc = 0;
1302
1303         hw_stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
1304         hw_stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
1305         hw_stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
1306         hw_stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
1307
1308         for (i = 0; i < 8; i++) {
1309                 uint32_t mp;
1310                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
1311                 /* global total per queue */
1312                 hw_stats->mpc[i] += mp;
1313                 /* Running comprehensive total for stats display */
1314                 total_missed_rx += hw_stats->mpc[i];
1315                 if (hw->mac.type == ixgbe_mac_82598EB)
1316                         hw_stats->rnbc[i] +=
1317                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
1318                 hw_stats->pxontxc[i] +=
1319                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
1320                 hw_stats->pxonrxc[i] +=
1321                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
1322                 hw_stats->pxofftxc[i] +=
1323                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
1324                 hw_stats->pxoffrxc[i] +=
1325                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
1326                 hw_stats->pxon2offc[i] +=
1327                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
1328         }
1329         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1330                 hw_stats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
1331                 hw_stats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
1332                 hw_stats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
1333                 hw_stats->qbrc[i] +=
1334                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32);
1335                 hw_stats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
1336                 hw_stats->qbtc[i] +=
1337                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32);
1338                 hw_stats->qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
1339
1340                 total_qprc += hw_stats->qprc[i];
1341                 total_qbrc += hw_stats->qbrc[i];
1342         }
1343         hw_stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
1344         hw_stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
1345         hw_stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
1346
1347         /* Note that gprc counts missed packets */
1348         hw_stats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
1349
1350         if (hw->mac.type != ixgbe_mac_82598EB) {
1351                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
1352                 hw_stats->gorc += ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
1353                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
1354                 hw_stats->gotc += ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
1355                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
1356                 hw_stats->tor += ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
1357                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
1358                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
1359         } else {
1360                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
1361                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
1362                 /* 82598 only has a counter in the high register */
1363                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
1364                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
1365                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
1366         }
1367
1368         /*
1369          * Workaround: mprc hardware is incorrectly counting
1370          * broadcasts, so for now we subtract those.
1371          */
1372         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
1373         hw_stats->bprc += bprc;
1374         hw_stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
1375         if (hw->mac.type == ixgbe_mac_82598EB)
1376                 hw_stats->mprc -= bprc;
1377
1378         hw_stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
1379         hw_stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
1380         hw_stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
1381         hw_stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
1382         hw_stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
1383         hw_stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
1384
1385         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
1386         hw_stats->lxontxc += lxon;
1387         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
1388         hw_stats->lxofftxc += lxoff;
1389         total = lxon + lxoff;
1390
1391         hw_stats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
1392         hw_stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
1393         hw_stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
1394         hw_stats->gptc -= total;
1395         hw_stats->mptc -= total;
1396         hw_stats->ptc64 -= total;
1397         hw_stats->gotc -= total * ETHER_MIN_LEN;
1398
1399         hw_stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
1400         hw_stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
1401         hw_stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
1402         hw_stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
1403         hw_stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
1404         hw_stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
1405         hw_stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
1406         hw_stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
1407         hw_stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
1408         hw_stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
1409         hw_stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
1410         hw_stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
1411         hw_stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
1412         hw_stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
1413         hw_stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
1414         hw_stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);
1415         hw_stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
1416         hw_stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
1417         /* Only read FCOE on 82599 */
1418         if (hw->mac.type != ixgbe_mac_82598EB) {
1419                 hw_stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
1420                 hw_stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
1421                 hw_stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
1422                 hw_stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
1423                 hw_stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
1424         }
1425
1426         if (stats == NULL)
1427                 return;
1428
1429         /* Fill out the rte_eth_stats statistics structure */
1430         stats->ipackets = total_qprc;
1431         stats->ibytes = total_qbrc;
1432         stats->opackets = hw_stats->gptc;
1433         stats->obytes = hw_stats->gotc;
1434         stats->imcasts = hw_stats->mprc;
1435
1436         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1437                 stats->q_ipackets[i] = hw_stats->qprc[i];
1438                 stats->q_opackets[i] = hw_stats->qptc[i];
1439                 stats->q_ibytes[i] = hw_stats->qbrc[i];
1440                 stats->q_obytes[i] = hw_stats->qbtc[i];
1441                 stats->q_errors[i] = hw_stats->qprdc[i];
1442         }
1443
1444         /* Rx Errors */
1445         stats->ierrors = total_missed_rx + hw_stats->crcerrs +
1446                 hw_stats->rlec;
1447
1448         stats->oerrors  = 0;
1449
1450         /* Flow Director Stats registers */
1451         hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
1452         hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
1453         stats->fdirmatch = hw_stats->fdirmatch;
1454         stats->fdirmiss = hw_stats->fdirmiss;
1455 }
1456
1457 static void
1458 ixgbe_dev_stats_reset(struct rte_eth_dev *dev)
1459 {
1460         struct ixgbe_hw_stats *stats =
1461                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1462
1463         /* HW registers are cleared on read */
1464         ixgbe_dev_stats_get(dev, NULL);
1465
1466         /* Reset software totals */
1467         memset(stats, 0, sizeof(*stats));
1468 }
1469
1470 static void
1471 ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1472 {
1473         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1474         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1475                           IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1476
1477         /* Good Rx packet, include VF loopback */
1478         UPDATE_VF_STAT(IXGBE_VFGPRC,
1479             hw_stats->last_vfgprc, hw_stats->vfgprc);
1480
1481         /* Good Rx octets, include VF loopback */
1482         UPDATE_VF_STAT_36BIT(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
1483             hw_stats->last_vfgorc, hw_stats->vfgorc);
1484
1485         /* Good Tx packet, include VF loopback */
1486         UPDATE_VF_STAT(IXGBE_VFGPTC,
1487             hw_stats->last_vfgptc, hw_stats->vfgptc);
1488
1489         /* Good Tx octets, include VF loopback */
1490         UPDATE_VF_STAT_36BIT(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
1491             hw_stats->last_vfgotc, hw_stats->vfgotc);
1492
1493         /* Rx Multicst Packet */
1494         UPDATE_VF_STAT(IXGBE_VFMPRC,
1495             hw_stats->last_vfmprc, hw_stats->vfmprc);
1496
1497         if (stats == NULL)
1498                 return;
1499
1500         memset(stats, 0, sizeof(*stats));
1501         stats->ipackets = hw_stats->vfgprc;
1502         stats->ibytes = hw_stats->vfgorc;
1503         stats->opackets = hw_stats->vfgptc;
1504         stats->obytes = hw_stats->vfgotc;
1505         stats->imcasts = hw_stats->vfmprc;
1506 }
1507
1508 static void
1509 ixgbevf_dev_stats_reset(struct rte_eth_dev *dev)
1510 {
1511         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1512                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1513
1514         /* Sync HW register to the last stats */
1515         ixgbevf_dev_stats_get(dev, NULL);
1516
1517         /* reset HW current stats*/
1518         hw_stats->vfgprc = 0;
1519         hw_stats->vfgorc = 0;
1520         hw_stats->vfgptc = 0;
1521         hw_stats->vfgotc = 0;
1522         hw_stats->vfmprc = 0;
1523
1524 }
1525
1526 static void
1527 ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1528 {
1529         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1530
1531         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
1532         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
1533         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL register */
1534         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */
1535         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
1536 }
1537
1538 /* return 0 means link status changed, -1 means not changed */
1539 static int
1540 ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1541 {
1542         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1543         struct rte_eth_link link, old;
1544         ixgbe_link_speed link_speed;
1545         int link_up;
1546         int diag;
1547
1548         link.link_status = 0;
1549         link.link_speed = 0;
1550         link.link_duplex = 0;
1551         memset(&old, 0, sizeof(old));
1552         rte_ixgbe_dev_atomic_read_link_status(dev, &old);
1553
1554         /* check if it needs to wait to complete, if lsc interrupt is enabled */
1555         if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
1556                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 0);
1557         else
1558                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 1);
1559         if (diag != 0) {
1560                 link.link_speed = ETH_LINK_SPEED_100;
1561                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1562                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1563                 if (link.link_status == old.link_status)
1564                         return -1;
1565                 return 0;
1566         }
1567
1568         if (link_up == 0) {
1569                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1570                 if (link.link_status == old.link_status)
1571                         return -1;
1572                 return 0;
1573         }
1574         link.link_status = 1;
1575         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1576
1577         switch (link_speed) {
1578         default:
1579         case IXGBE_LINK_SPEED_UNKNOWN:
1580                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1581                 link.link_speed = ETH_LINK_SPEED_100;
1582                 break;
1583
1584         case IXGBE_LINK_SPEED_100_FULL:
1585                 link.link_speed = ETH_LINK_SPEED_100;
1586                 break;
1587
1588         case IXGBE_LINK_SPEED_1GB_FULL:
1589                 link.link_speed = ETH_LINK_SPEED_1000;
1590                 break;
1591
1592         case IXGBE_LINK_SPEED_10GB_FULL:
1593                 link.link_speed = ETH_LINK_SPEED_10000;
1594                 break;
1595         }
1596         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1597
1598         if (link.link_status == old.link_status)
1599                 return -1;
1600
1601         return 0;
1602 }
1603
1604 static void
1605 ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev)
1606 {
1607         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1608         uint32_t fctrl;
1609
1610         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1611         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1612         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1613 }
1614
1615 static void
1616 ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev)
1617 {
1618         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1619         uint32_t fctrl;
1620
1621         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1622         fctrl &= (~IXGBE_FCTRL_UPE);
1623         if (dev->data->all_multicast == 1)
1624                 fctrl |= IXGBE_FCTRL_MPE;
1625         else
1626                 fctrl &= (~IXGBE_FCTRL_MPE);
1627         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1628 }
1629
1630 static void
1631 ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev)
1632 {
1633         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1634         uint32_t fctrl;
1635
1636         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1637         fctrl |= IXGBE_FCTRL_MPE;
1638         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1639 }
1640
1641 static void
1642 ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev)
1643 {
1644         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1645         uint32_t fctrl;
1646
1647         if (dev->data->promiscuous == 1)
1648                 return; /* must remain in all_multicast mode */
1649
1650         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1651         fctrl &= (~IXGBE_FCTRL_MPE);
1652         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1653 }
1654
1655 /**
1656  * It clears the interrupt causes and enables the interrupt.
1657  * It will be called once only during nic initialized.
1658  *
1659  * @param dev
1660  *  Pointer to struct rte_eth_dev.
1661  *
1662  * @return
1663  *  - On success, zero.
1664  *  - On failure, a negative value.
1665  */
1666 static int
1667 ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev)
1668 {
1669         struct ixgbe_interrupt *intr =
1670                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1671
1672         ixgbe_dev_link_status_print(dev);
1673         intr->mask |= IXGBE_EICR_LSC;
1674
1675         return 0;
1676 }
1677
1678 /*
1679  * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update.
1680  *
1681  * @param dev
1682  *  Pointer to struct rte_eth_dev.
1683  *
1684  * @return
1685  *  - On success, zero.
1686  *  - On failure, a negative value.
1687  */
1688 static int
1689 ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
1690 {
1691         uint32_t eicr;
1692         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1693         struct ixgbe_interrupt *intr =
1694                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1695
1696         /* clear all cause mask */
1697         ixgbe_disable_intr(hw);
1698
1699         /* read-on-clear nic registers here */
1700         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1701         PMD_DRV_LOG(INFO, "eicr %x", eicr);
1702
1703         intr->flags = 0;
1704         if (eicr & IXGBE_EICR_LSC) {
1705                 /* set flag for async link update */
1706                 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1707         }
1708
1709         return 0;
1710 }
1711
1712 /**
1713  * It gets and then prints the link status.
1714  *
1715  * @param dev
1716  *  Pointer to struct rte_eth_dev.
1717  *
1718  * @return
1719  *  - On success, zero.
1720  *  - On failure, a negative value.
1721  */
1722 static void
1723 ixgbe_dev_link_status_print(struct rte_eth_dev *dev)
1724 {
1725         struct rte_eth_link link;
1726
1727         memset(&link, 0, sizeof(link));
1728         rte_ixgbe_dev_atomic_read_link_status(dev, &link);
1729         if (link.link_status) {
1730                 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
1731                                         (int)(dev->data->port_id),
1732                                         (unsigned)link.link_speed,
1733                         link.link_duplex == ETH_LINK_FULL_DUPLEX ?
1734                                         "full-duplex" : "half-duplex");
1735         } else {
1736                 PMD_INIT_LOG(INFO, " Port %d: Link Down",
1737                                 (int)(dev->data->port_id));
1738         }
1739         PMD_INIT_LOG(INFO, "PCI Address: %04d:%02d:%02d:%d",
1740                                 dev->pci_dev->addr.domain,
1741                                 dev->pci_dev->addr.bus,
1742                                 dev->pci_dev->addr.devid,
1743                                 dev->pci_dev->addr.function);
1744 }
1745
1746 /*
1747  * It executes link_update after knowing an interrupt occured.
1748  *
1749  * @param dev
1750  *  Pointer to struct rte_eth_dev.
1751  *
1752  * @return
1753  *  - On success, zero.
1754  *  - On failure, a negative value.
1755  */
1756 static int
1757 ixgbe_dev_interrupt_action(struct rte_eth_dev *dev)
1758 {
1759         struct ixgbe_interrupt *intr =
1760                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1761         int64_t timeout;
1762         struct rte_eth_link link;
1763         int intr_enable_delay = false;
1764
1765         PMD_DRV_LOG(DEBUG, "intr action type %d\n", intr->flags);
1766
1767         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
1768                 /* get the link status before link update, for predicting later */
1769                 memset(&link, 0, sizeof(link));
1770                 rte_ixgbe_dev_atomic_read_link_status(dev, &link);
1771
1772                 ixgbe_dev_link_update(dev, 0);
1773
1774                 /* likely to up */
1775                 if (!link.link_status)
1776                         /* handle it 1 sec later, wait it being stable */
1777                         timeout = IXGBE_LINK_UP_CHECK_TIMEOUT;
1778                 /* likely to down */
1779                 else
1780                         /* handle it 4 sec later, wait it being stable */
1781                         timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT;
1782
1783                 ixgbe_dev_link_status_print(dev);
1784
1785                 intr_enable_delay = true;
1786         }
1787
1788         if (intr_enable_delay) {
1789                 if (rte_eal_alarm_set(timeout * 1000,
1790                                       ixgbe_dev_interrupt_delayed_handler, (void*)dev) < 0)
1791                         PMD_DRV_LOG(ERR, "Error setting alarm");
1792         } else {
1793                 PMD_DRV_LOG(DEBUG, "enable intr immediately");
1794                 ixgbe_enable_intr(dev);
1795                 rte_intr_enable(&(dev->pci_dev->intr_handle));
1796         }
1797
1798
1799         return 0;
1800 }
1801
1802 /**
1803  * Interrupt handler which shall be registered for alarm callback for delayed
1804  * handling specific interrupt to wait for the stable nic state. As the
1805  * NIC interrupt state is not stable for ixgbe after link is just down,
1806  * it needs to wait 4 seconds to get the stable status.
1807  *
1808  * @param handle
1809  *  Pointer to interrupt handle.
1810  * @param param
1811  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1812  *
1813  * @return
1814  *  void
1815  */
1816 static void
1817 ixgbe_dev_interrupt_delayed_handler(void *param)
1818 {
1819         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1820         struct ixgbe_interrupt *intr =
1821                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1822
1823         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
1824                 ixgbe_dev_link_update(dev, 0);
1825                 intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
1826                 ixgbe_dev_link_status_print(dev);
1827                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
1828         }
1829
1830         PMD_DRV_LOG(DEBUG, "enable intr in delayed handler\n");
1831         ixgbe_enable_intr(dev);
1832         rte_intr_enable(&(dev->pci_dev->intr_handle));
1833 }
1834
1835 /**
1836  * Interrupt handler triggered by NIC  for handling
1837  * specific interrupt.
1838  *
1839  * @param handle
1840  *  Pointer to interrupt handle.
1841  * @param param
1842  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1843  *
1844  * @return
1845  *  void
1846  */
1847 static void
1848 ixgbe_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
1849                                                         void *param)
1850 {
1851         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1852         ixgbe_dev_interrupt_get_status(dev);
1853         ixgbe_dev_interrupt_action(dev);
1854 }
1855
1856 static int
1857 ixgbe_dev_led_on(struct rte_eth_dev *dev)
1858 {
1859         struct ixgbe_hw *hw;
1860
1861         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1862         return (ixgbe_led_on(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
1863 }
1864
1865 static int
1866 ixgbe_dev_led_off(struct rte_eth_dev *dev)
1867 {
1868         struct ixgbe_hw *hw;
1869
1870         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1871         return (ixgbe_led_off(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
1872 }
1873
1874 static int
1875 ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1876 {
1877         struct ixgbe_hw *hw;
1878         int err;
1879         uint32_t rx_buf_size;
1880         uint32_t max_high_water;
1881         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
1882                 ixgbe_fc_none,
1883                 ixgbe_fc_rx_pause,
1884                 ixgbe_fc_tx_pause,
1885                 ixgbe_fc_full
1886         };
1887
1888         PMD_INIT_FUNC_TRACE();
1889
1890         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1891         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0));
1892         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
1893
1894         /*
1895          * At least reserve one Ethernet frame for watermark
1896          * high_water/low_water in kilo bytes for ixgbe
1897          */
1898         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
1899         if ((fc_conf->high_water > max_high_water) ||
1900                 (fc_conf->high_water < fc_conf->low_water)) {
1901                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
1902                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
1903                 return (-EINVAL);
1904         }
1905
1906         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[fc_conf->mode];
1907         hw->fc.pause_time     = fc_conf->pause_time;
1908         hw->fc.high_water[0]  = fc_conf->high_water;
1909         hw->fc.low_water[0]   = fc_conf->low_water;
1910         hw->fc.send_xon       = fc_conf->send_xon;
1911
1912         err = ixgbe_fc_enable(hw);
1913         /* Not negotiated is not an error case */
1914         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) {
1915                 return 0;
1916         }
1917
1918         PMD_INIT_LOG(ERR, "ixgbe_fc_enable = 0x%x \n", err);
1919         return -EIO;
1920 }
1921
1922 /**
1923  *  ixgbe_pfc_enable_generic - Enable flow control
1924  *  @hw: pointer to hardware structure
1925  *  @tc_num: traffic class number
1926  *  Enable flow control according to the current settings.
1927  */
1928 static int
1929 ixgbe_dcb_pfc_enable_generic(struct ixgbe_hw *hw,uint8_t tc_num)
1930 {
1931         int ret_val = 0;
1932         uint32_t mflcn_reg, fccfg_reg;
1933         uint32_t reg;
1934         uint32_t fcrtl, fcrth;
1935         uint8_t i;
1936         uint8_t nb_rx_en;
1937
1938         /* Validate the water mark configuration */
1939         if (!hw->fc.pause_time) {
1940                 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
1941                 goto out;
1942         }
1943
1944         /* Low water mark of zero causes XOFF floods */
1945         if (hw->fc.current_mode & ixgbe_fc_tx_pause) {
1946                  /* High/Low water can not be 0 */
1947                 if( (!hw->fc.high_water[tc_num])|| (!hw->fc.low_water[tc_num])) {
1948                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
1949                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
1950                         goto out;
1951                 }
1952
1953                 if(hw->fc.low_water[tc_num] >= hw->fc.high_water[tc_num]) {
1954                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
1955                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
1956                         goto out;
1957                 }
1958         }
1959         /* Negotiate the fc mode to use */
1960         ixgbe_fc_autoneg(hw);
1961
1962         /* Disable any previous flow control settings */
1963         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
1964         mflcn_reg &= ~(IXGBE_MFLCN_RPFCE_SHIFT | IXGBE_MFLCN_RFCE|IXGBE_MFLCN_RPFCE);
1965
1966         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
1967         fccfg_reg &= ~(IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY);
1968
1969         switch (hw->fc.current_mode) {
1970         case ixgbe_fc_none:
1971                 /*
1972                  * If the count of enabled RX Priority Flow control >1,
1973                  * and the TX pause can not be disabled
1974                  */
1975                 nb_rx_en = 0;
1976                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
1977                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
1978                         if (reg & IXGBE_FCRTH_FCEN)
1979                                 nb_rx_en++;
1980                 }
1981                 if (nb_rx_en > 1)
1982                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
1983                 break;
1984         case ixgbe_fc_rx_pause:
1985                 /*
1986                  * Rx Flow control is enabled and Tx Flow control is
1987                  * disabled by software override. Since there really
1988                  * isn't a way to advertise that we are capable of RX
1989                  * Pause ONLY, we will advertise that we support both
1990                  * symmetric and asymmetric Rx PAUSE.  Later, we will
1991                  * disable the adapter's ability to send PAUSE frames.
1992                  */
1993                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
1994                 /*
1995                  * If the count of enabled RX Priority Flow control >1,
1996                  * and the TX pause can not be disabled
1997                  */
1998                 nb_rx_en = 0;
1999                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
2000                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
2001                         if (reg & IXGBE_FCRTH_FCEN)
2002                                 nb_rx_en++;
2003                 }
2004                 if (nb_rx_en > 1)
2005                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2006                 break;
2007         case ixgbe_fc_tx_pause:
2008                 /*
2009                  * Tx Flow control is enabled, and Rx Flow control is
2010                  * disabled by software override.
2011                  */
2012                 fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2013                 break;
2014         case ixgbe_fc_full:
2015                 /* Flow control (both Rx and Tx) is enabled by SW override. */
2016                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
2017                 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
2018                 break;
2019         default:
2020                 DEBUGOUT("Flow control param set incorrectly\n");
2021                 ret_val = IXGBE_ERR_CONFIG;
2022                 goto out;
2023                 break;
2024         }
2025
2026         /* Set 802.3x based flow control settings. */
2027         mflcn_reg |= IXGBE_MFLCN_DPF;
2028         IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg);
2029         IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg);
2030
2031         /* Set up and enable Rx high/low water mark thresholds, enable XON. */
2032         if ((hw->fc.current_mode & ixgbe_fc_tx_pause) &&
2033                 hw->fc.high_water[tc_num]) {
2034                 fcrtl = (hw->fc.low_water[tc_num] << 10) | IXGBE_FCRTL_XONE;
2035                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), fcrtl);
2036                 fcrth = (hw->fc.high_water[tc_num] << 10) | IXGBE_FCRTH_FCEN;
2037         } else {
2038                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), 0);
2039                 /*
2040                  * In order to prevent Tx hangs when the internal Tx
2041                  * switch is enabled we must set the high water mark
2042                  * to the maximum FCRTH value.  This allows the Tx
2043                  * switch to function even under heavy Rx workloads.
2044                  */
2045                 fcrth = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)) - 32;
2046         }
2047         IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(tc_num), fcrth);
2048
2049         /* Configure pause time (2 TCs per register) */
2050         reg = hw->fc.pause_time * 0x00010001;
2051         for (i = 0; i < (IXGBE_DCB_MAX_TRAFFIC_CLASS / 2); i++)
2052                 IXGBE_WRITE_REG(hw, IXGBE_FCTTV(i), reg);
2053
2054         /* Configure flow control refresh threshold value */
2055         IXGBE_WRITE_REG(hw, IXGBE_FCRTV, hw->fc.pause_time / 2);
2056
2057 out:
2058         return ret_val;
2059 }
2060
2061 static int
2062 ixgbe_dcb_pfc_enable(struct rte_eth_dev *dev,uint8_t tc_num)
2063 {
2064         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2065         int32_t ret_val = IXGBE_NOT_IMPLEMENTED;
2066
2067         if(hw->mac.type != ixgbe_mac_82598EB) {
2068                 ret_val = ixgbe_dcb_pfc_enable_generic(hw,tc_num);
2069         }
2070         return ret_val;
2071 }
2072
2073 static int
2074 ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf)
2075 {
2076         int err;
2077         uint32_t rx_buf_size;
2078         uint32_t max_high_water;
2079         uint8_t tc_num;
2080         uint8_t  map[IXGBE_DCB_MAX_USER_PRIORITY] = { 0 };
2081         struct ixgbe_hw *hw =
2082                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2083         struct ixgbe_dcb_config *dcb_config =
2084                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
2085
2086         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
2087                 ixgbe_fc_none,
2088                 ixgbe_fc_rx_pause,
2089                 ixgbe_fc_tx_pause,
2090                 ixgbe_fc_full
2091         };
2092
2093         PMD_INIT_FUNC_TRACE();
2094
2095         ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
2096         tc_num = map[pfc_conf->priority];
2097         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num));
2098         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
2099         /*
2100          * At least reserve one Ethernet frame for watermark
2101          * high_water/low_water in kilo bytes for ixgbe
2102          */
2103         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
2104         if ((pfc_conf->fc.high_water > max_high_water) ||
2105                 (pfc_conf->fc.high_water <= pfc_conf->fc.low_water)) {
2106                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
2107                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
2108                 return (-EINVAL);
2109         }
2110
2111         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[pfc_conf->fc.mode];
2112         hw->fc.pause_time = pfc_conf->fc.pause_time;
2113         hw->fc.send_xon = pfc_conf->fc.send_xon;
2114         hw->fc.low_water[tc_num] =  pfc_conf->fc.low_water;
2115         hw->fc.high_water[tc_num] = pfc_conf->fc.high_water;
2116
2117         err = ixgbe_dcb_pfc_enable(dev,tc_num);
2118
2119         /* Not negotiated is not an error case */
2120         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED))
2121                 return 0;
2122
2123         PMD_INIT_LOG(ERR, "ixgbe_dcb_pfc_enable = 0x%x \n", err);
2124         return -EIO;
2125 }
2126
2127 static void
2128 ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
2129                                 uint32_t index, uint32_t pool)
2130 {
2131         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2132         uint32_t enable_addr = 1;
2133
2134         ixgbe_set_rar(hw, index, mac_addr->addr_bytes, pool, enable_addr);
2135 }
2136
2137 static void
2138 ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
2139 {
2140         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2141
2142         ixgbe_clear_rar(hw, index);
2143 }
2144
2145 /*
2146  * Virtual Function operations
2147  */
2148 static void
2149 ixgbevf_intr_disable(struct ixgbe_hw *hw)
2150 {
2151         PMD_INIT_LOG(DEBUG, "ixgbevf_intr_disable");
2152
2153         /* Clear interrupt mask to stop from interrupts being generated */
2154         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, IXGBE_VF_IRQ_CLEAR_MASK);
2155
2156         IXGBE_WRITE_FLUSH(hw);
2157 }
2158
2159 static int
2160 ixgbevf_dev_configure(struct rte_eth_dev *dev)
2161 {
2162         struct rte_eth_conf* conf = &dev->data->dev_conf;
2163
2164         PMD_INIT_LOG(DEBUG, "\nConfigured Virtual Function port id: %d\n",
2165                 dev->data->port_id);
2166
2167         /*
2168          * VF has no ability to enable/disable HW CRC
2169          * Keep the persistent behavior the same as Host PF
2170          */
2171 #ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC
2172         if (!conf->rxmode.hw_strip_crc) {
2173                 PMD_INIT_LOG(INFO, "VF can't disable HW CRC Strip\n");
2174                 conf->rxmode.hw_strip_crc = 1;
2175         }
2176 #else
2177         if (conf->rxmode.hw_strip_crc) {
2178                 PMD_INIT_LOG(INFO, "VF can't enable HW CRC Strip\n");
2179                 conf->rxmode.hw_strip_crc = 0;
2180         }
2181 #endif
2182
2183         return 0;
2184 }
2185
2186 static int
2187 ixgbevf_dev_start(struct rte_eth_dev *dev)
2188 {
2189         struct ixgbe_hw *hw =
2190                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2191         int err, mask = 0;
2192
2193         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_start");
2194
2195         hw->mac.ops.reset_hw(hw);
2196
2197         ixgbevf_dev_tx_init(dev);
2198
2199         /* This can fail when allocating mbufs for descriptor rings */
2200         err = ixgbevf_dev_rx_init(dev);
2201         if (err) {
2202                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware (%d)\n", err);
2203                 ixgbe_dev_clear_queues(dev);
2204                 return err;
2205         }
2206
2207         /* Set vfta */
2208         ixgbevf_set_vfta_all(dev,1);
2209
2210         /* Set HW strip */
2211         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
2212                 ETH_VLAN_EXTEND_MASK;
2213         ixgbevf_vlan_offload_set(dev, mask);
2214
2215         ixgbevf_dev_rxtx_start(dev);
2216
2217         return 0;
2218 }
2219
2220 static void
2221 ixgbevf_dev_stop(struct rte_eth_dev *dev)
2222 {
2223         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2224
2225         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_stop");
2226
2227         hw->adapter_stopped = TRUE;
2228         ixgbe_stop_adapter(hw);
2229
2230         /*
2231           * Clear what we set, but we still keep shadow_vfta to
2232           * restore after device starts
2233           */
2234         ixgbevf_set_vfta_all(dev,0);
2235
2236         ixgbe_dev_clear_queues(dev);
2237 }
2238
2239 static void
2240 ixgbevf_dev_close(struct rte_eth_dev *dev)
2241 {
2242         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2243
2244         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_close");
2245
2246         ixgbe_reset_hw(hw);
2247
2248         ixgbevf_dev_stop(dev);
2249
2250         /* reprogram the RAR[0] in case user changed it. */
2251         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
2252 }
2253
2254 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on)
2255 {
2256         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2257         struct ixgbe_vfta * shadow_vfta =
2258                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2259         int i = 0, j = 0, vfta = 0, mask = 1;
2260
2261         for (i = 0; i < IXGBE_VFTA_SIZE; i++){
2262                 vfta = shadow_vfta->vfta[i];
2263                 if(vfta){
2264                         mask = 1;
2265                         for (j = 0; j < 32; j++){
2266                                 if(vfta & mask)
2267                                         ixgbe_set_vfta(hw, (i<<5)+j, 0, on);
2268                                 mask<<=1;
2269                         }
2270                 }
2271         }
2272
2273 }
2274
2275 static int
2276 ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2277 {
2278         struct ixgbe_hw *hw =
2279                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2280         struct ixgbe_vfta * shadow_vfta =
2281                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2282         uint32_t vid_idx = 0;
2283         uint32_t vid_bit = 0;
2284         int ret = 0;
2285
2286         PMD_INIT_FUNC_TRACE();
2287
2288         /* vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf */
2289         ret = ixgbe_set_vfta(hw, vlan_id, 0, !!on);
2290         if(ret){
2291                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
2292                 return ret;
2293         }
2294         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
2295         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
2296
2297         /* Save what we set and retore it after device reset */
2298         if (on)
2299                 shadow_vfta->vfta[vid_idx] |= vid_bit;
2300         else
2301                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
2302
2303         return 0;
2304 }
2305
2306 static void
2307 ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
2308 {
2309         struct ixgbe_hw *hw =
2310                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2311         uint32_t ctrl;
2312
2313         PMD_INIT_FUNC_TRACE();
2314
2315         if(queue >= hw->mac.max_rx_queues)
2316                 return;
2317
2318         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
2319         if(on)
2320                 ctrl |= IXGBE_RXDCTL_VME;
2321         else
2322                 ctrl &= ~IXGBE_RXDCTL_VME;
2323         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
2324
2325         ixgbe_vlan_hw_strip_bitmap_set( dev, queue, on);
2326 }
2327
2328 static void
2329 ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2330 {
2331         struct ixgbe_hw *hw =
2332                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2333         uint16_t i;
2334         int on = 0;
2335
2336         /* VF function only support hw strip feature, others are not support */
2337         if(mask & ETH_VLAN_STRIP_MASK){
2338                 on = !!(dev->data->dev_conf.rxmode.hw_vlan_strip);
2339
2340                 for(i=0; i < hw->mac.max_rx_queues; i++)
2341                         ixgbevf_vlan_strip_queue_set(dev,i,on);
2342         }
2343 }