lib/librte_pmd_ixgbe/ixgbe_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2013 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  *
  33  */
  34
  35 #include <sys/queue.h>
  36 #include <stdio.h>
  37 #include <errno.h>
  38 #include <stdint.h>
  39 #include <string.h>
  40 #include <unistd.h>
  41 #include <stdarg.h>
  42 #include <inttypes.h>
  43 #include <rte_byteorder.h>
  44 #include <rte_common.h>
  45 #include <rte_cycles.h>
  46
  47 #include <rte_interrupts.h>
  48 #include <rte_log.h>
  49 #include <rte_debug.h>
  50 #include <rte_pci.h>
  51 #include <rte_atomic.h>
  52 #include <rte_branch_prediction.h>
  53 #include <rte_memory.h>
  54 #include <rte_memzone.h>
  55 #include <rte_tailq.h>
  56 #include <rte_eal.h>
  57 #include <rte_alarm.h>
  58 #include <rte_ether.h>
  59 #include <rte_ethdev.h>
  60 #include <rte_atomic.h>
  61 #include <rte_malloc.h>
  62
  63 #include "ixgbe_logs.h"
  64 #include "ixgbe/ixgbe_api.h"
  65 #include "ixgbe/ixgbe_vf.h"
  66 #include "ixgbe/ixgbe_common.h"
  67 #include "ixgbe_ethdev.h"
  68
  69 /*
  70  * High threshold controlling when to start sending XOFF frames. Must be at
  71  * least 8 bytes less than receive packet buffer size. This value is in units
  72  * of 1024 bytes.
  73  */
  74 #define IXGBE_FC_HI    0x80
  75
  76 /*
  77  * Low threshold controlling when to start sending XON frames. This value is
  78  * in units of 1024 bytes.
  79  */
  80 #define IXGBE_FC_LO    0x40
  81
  82 /* Timer value included in XOFF frames. */
  83 #define IXGBE_FC_PAUSE 0x680
  84
  85 #define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */
  86 #define IXGBE_LINK_UP_CHECK_TIMEOUT   1000 /* ms */
  87
  88 #define IXGBE_QUEUE_STAT_COUNTERS (sizeof(hw_stats->qprc) / sizeof(hw_stats->qprc[0]))
  89
  90 static int eth_ixgbe_dev_init(struct eth_driver *eth_drv,
  91                 struct rte_eth_dev *eth_dev);
  92 static int  ixgbe_dev_configure(struct rte_eth_dev *dev);
  93 static int  ixgbe_dev_start(struct rte_eth_dev *dev);
  94 static void ixgbe_dev_stop(struct rte_eth_dev *dev);
  95 static void ixgbe_dev_close(struct rte_eth_dev *dev);
  96 static void ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev);
  97 static void ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev);
  98 static void ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev);
  99 static void ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev);
 100 static int ixgbe_dev_link_update(struct rte_eth_dev *dev,
 101                                 int wait_to_complete);
 102 static void ixgbe_dev_stats_get(struct rte_eth_dev *dev,
 103                                 struct rte_eth_stats *stats);
 104 static void ixgbe_dev_stats_reset(struct rte_eth_dev *dev);
 105 static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 106                                              uint16_t queue_id,
 107                                              uint8_t stat_idx,
 108                                              uint8_t is_rx);
 109 static void ixgbe_dev_info_get(struct rte_eth_dev *dev,
 110                                 struct rte_eth_dev_info *dev_info);
 111 static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev,
 112                 uint16_t vlan_id, int on);
 113 static void ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 114 static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev,
 115                 uint16_t queue, bool on);
 116 static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue,
 117                 int on);
 118 static void ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 119 static void ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev);
 120 static void ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev);
 121 static void ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev);
 122 static void ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev);
 123 static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue);
 124 static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue);
 125 static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 126 static void ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 127
 128 static int ixgbe_dev_led_on(struct rte_eth_dev *dev);
 129 static int ixgbe_dev_led_off(struct rte_eth_dev *dev);
 130 static int  ixgbe_flow_ctrl_set(struct rte_eth_dev *dev,
 131                 struct rte_eth_fc_conf *fc_conf);
 132 static int ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 133                 struct rte_eth_pfc_conf *pfc_conf);
 134 static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev);
 135 static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev);
 136 static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev);
 137 static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev);
 138 static void ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle,
 139                 void *param);
 140 static void ixgbe_dev_interrupt_delayed_handler(void *param);
 141 static void ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 142                 uint32_t index, uint32_t pool);
 143 static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index);
 144 static void ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config);
 145
 146 /* For Virtual Function support */
 147 static int eth_ixgbevf_dev_init(struct eth_driver *eth_drv,
 148                 struct rte_eth_dev *eth_dev);
 149 static int  ixgbevf_dev_configure(struct rte_eth_dev *dev);
 150 static int  ixgbevf_dev_start(struct rte_eth_dev *dev);
 151 static void ixgbevf_dev_stop(struct rte_eth_dev *dev);
 152 static void ixgbevf_dev_close(struct rte_eth_dev *dev);
 153 static void ixgbevf_intr_disable(struct ixgbe_hw *hw);
 154 static void ixgbevf_dev_stats_get(struct rte_eth_dev *dev,
 155                 struct rte_eth_stats *stats);
 156 static void ixgbevf_dev_stats_reset(struct rte_eth_dev *dev);
 157 static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev,
 158                 uint16_t vlan_id, int on);
 159 static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev,
 160                 uint16_t queue, int on);
 161 static void ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 162 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 163
 164
 165 /*
 166  * Define VF Stats MACRO for Non "cleared on read" register
 167  */
 168 #define UPDATE_VF_STAT(reg, last, cur)                          \
 169 {                                                               \
 170         u32 latest = IXGBE_READ_REG(hw, reg);                   \
 171         cur += latest - last;                                   \
 172         last = latest;                                          \
 173 }
 174
 175 #define UPDATE_VF_STAT_36BIT(lsb, msb, last, cur)                \
 176 {                                                                \
 177         u64 new_lsb = IXGBE_READ_REG(hw, lsb);                   \
 178         u64 new_msb = IXGBE_READ_REG(hw, msb);                   \
 179         u64 latest = ((new_msb << 32) | new_lsb);                \
 180         cur += (0x1000000000LL + latest - last) & 0xFFFFFFFFFLL; \
 181         last = latest;                                           \
 182 }
 183
 184 #define IXGBE_SET_HWSTRIP(h, q) do{\
 185                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 186                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 187                 (h)->bitmap[idx] |= 1 << bit;\
 188         }while(0)
 189
 190 #define IXGBE_CLEAR_HWSTRIP(h, q) do{\
 191                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 192                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 193                 (h)->bitmap[idx] &= ~(1 << bit);\
 194         }while(0)
 195
 196 #define IXGBE_GET_HWSTRIP(h, q, r) do{\
 197                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 198                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 199                 (r) = (h)->bitmap[idx] >> bit & 1;\
 200         }while(0)
 201
 202 /*
 203  * The set of PCI devices this driver supports
 204  */
 205 static struct rte_pci_id pci_id_ixgbe_map[] = {
 206
 207 #define RTE_PCI_DEV_ID_DECL_IXGBE(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 208 #include "rte_pci_dev_ids.h"
 209
 210 { .vendor_id = 0, /* sentinel */ },
 211 };
 212
 213
 214 /*
 215  * The set of PCI devices this driver supports (for 82599 VF)
 216  */
 217 static struct rte_pci_id pci_id_ixgbevf_map[] = {
 218
 219 #define RTE_PCI_DEV_ID_DECL_IXGBEVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 220 #include "rte_pci_dev_ids.h"
 221 { .vendor_id = 0, /* sentinel */ },
 222
 223 };
 224
 225 static struct eth_dev_ops ixgbe_eth_dev_ops = {
 226         .dev_configure        = ixgbe_dev_configure,
 227         .dev_start            = ixgbe_dev_start,
 228         .dev_stop             = ixgbe_dev_stop,
 229         .dev_close            = ixgbe_dev_close,
 230         .promiscuous_enable   = ixgbe_dev_promiscuous_enable,
 231         .promiscuous_disable  = ixgbe_dev_promiscuous_disable,
 232         .allmulticast_enable  = ixgbe_dev_allmulticast_enable,
 233         .allmulticast_disable = ixgbe_dev_allmulticast_disable,
 234         .link_update          = ixgbe_dev_link_update,
 235         .stats_get            = ixgbe_dev_stats_get,
 236         .stats_reset          = ixgbe_dev_stats_reset,
 237         .queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set,
 238         .dev_infos_get        = ixgbe_dev_info_get,
 239         .vlan_filter_set      = ixgbe_vlan_filter_set,
 240         .vlan_tpid_set        = ixgbe_vlan_tpid_set,
 241         .vlan_offload_set     = ixgbe_vlan_offload_set,
 242         .vlan_strip_queue_set = ixgbe_vlan_strip_queue_set,
 243         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 244         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 245         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 246         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 247         .dev_led_on           = ixgbe_dev_led_on,
 248         .dev_led_off          = ixgbe_dev_led_off,
 249         .flow_ctrl_set        = ixgbe_flow_ctrl_set,
 250         .priority_flow_ctrl_set = ixgbe_priority_flow_ctrl_set,
 251         .mac_addr_add         = ixgbe_add_rar,
 252         .mac_addr_remove      = ixgbe_remove_rar,
 253         .fdir_add_signature_filter    = ixgbe_fdir_add_signature_filter,
 254         .fdir_update_signature_filter = ixgbe_fdir_update_signature_filter,
 255         .fdir_remove_signature_filter = ixgbe_fdir_remove_signature_filter,
 256         .fdir_infos_get               = ixgbe_fdir_info_get,
 257         .fdir_add_perfect_filter      = ixgbe_fdir_add_perfect_filter,
 258         .fdir_update_perfect_filter   = ixgbe_fdir_update_perfect_filter,
 259         .fdir_remove_perfect_filter   = ixgbe_fdir_remove_perfect_filter,
 260         .fdir_set_masks               = ixgbe_fdir_set_masks,
 261 };
 262
 263 /*
 264  * dev_ops for virtual function, bare necessities for basic vf
 265  * operation have been implemented
 266  */
 267 static struct eth_dev_ops ixgbevf_eth_dev_ops = {
 268
 269         .dev_configure        = ixgbevf_dev_configure,
 270         .dev_start            = ixgbevf_dev_start,
 271         .dev_stop             = ixgbevf_dev_stop,
 272         .link_update          = ixgbe_dev_link_update,
 273         .stats_get            = ixgbevf_dev_stats_get,
 274         .stats_reset          = ixgbevf_dev_stats_reset,
 275         .dev_close            = ixgbevf_dev_close,
 276         .dev_infos_get        = ixgbe_dev_info_get,
 277         .vlan_filter_set      = ixgbevf_vlan_filter_set,
 278         .vlan_strip_queue_set = ixgbevf_vlan_strip_queue_set,
 279         .vlan_offload_set     = ixgbevf_vlan_offload_set,
 280         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 281         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 282         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 283         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 284 };
 285
 286 /**
 287  * Atomically reads the link status information from global
 288  * structure rte_eth_dev.
 289  *
 290  * @param dev
 291  *   - Pointer to the structure rte_eth_dev to read from.
 292  *   - Pointer to the buffer to be saved with the link status.
 293  *
 294  * @return
 295  *   - On success, zero.
 296  *   - On failure, negative value.
 297  */
 298 static inline int
 299 rte_ixgbe_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 300                                 struct rte_eth_link *link)
 301 {
 302         struct rte_eth_link *dst = link;
 303         struct rte_eth_link *src = &(dev->data->dev_link);
 304
 305         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 306                                         *(uint64_t *)src) == 0)
 307                 return -1;
 308
 309         return 0;
 310 }
 311
 312 /**
 313  * Atomically writes the link status information into global
 314  * structure rte_eth_dev.
 315  *
 316  * @param dev
 317  *   - Pointer to the structure rte_eth_dev to read from.
 318  *   - Pointer to the buffer to be saved with the link status.
 319  *
 320  * @return
 321  *   - On success, zero.
 322  *   - On failure, negative value.
 323  */
 324 static inline int
 325 rte_ixgbe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 326                                 struct rte_eth_link *link)
 327 {
 328         struct rte_eth_link *dst = &(dev->data->dev_link);
 329         struct rte_eth_link *src = link;
 330
 331         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 332                                         *(uint64_t *)src) == 0)
 333                 return -1;
 334
 335         return 0;
 336 }
 337
 338 /*
 339  * This function is the same as ixgbe_is_sfp() in ixgbe/ixgbe.h.
 340  */
 341 static inline int
 342 ixgbe_is_sfp(struct ixgbe_hw *hw)
 343 {
 344         switch (hw->phy.type) {
 345         case ixgbe_phy_sfp_avago:
 346         case ixgbe_phy_sfp_ftl:
 347         case ixgbe_phy_sfp_intel:
 348         case ixgbe_phy_sfp_unknown:
 349         case ixgbe_phy_sfp_passive_tyco:
 350         case ixgbe_phy_sfp_passive_unknown:
 351                 return 1;
 352         default:
 353                 return 0;
 354         }
 355 }
 356
 357 static inline void
 358 ixgbe_enable_intr(struct rte_eth_dev *dev)
 359 {
 360         struct ixgbe_interrupt *intr =
 361                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 362         struct ixgbe_hw *hw =
 363                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 364
 365         IXGBE_WRITE_REG(hw, IXGBE_EIMS, intr->mask);
 366         IXGBE_WRITE_FLUSH(hw);
 367 }
 368
 369 /*
 370  * This function is based on ixgbe_disable_intr() in ixgbe/ixgbe.h.
 371  */
 372 static void
 373 ixgbe_disable_intr(struct ixgbe_hw *hw)
 374 {
 375         PMD_INIT_FUNC_TRACE();
 376
 377         if (hw->mac.type == ixgbe_mac_82598EB) {
 378                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ~0);
 379         } else {
 380                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xFFFF0000);
 381                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), ~0);
 382                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), ~0);
 383         }
 384         IXGBE_WRITE_FLUSH(hw);
 385 }
 386
 387 /*
 388  * This function resets queue statistics mapping registers.
 389  * From Niantic datasheet, Initialization of Statistics section:
 390  * "...if software requires the queue counters, the RQSMR and TQSM registers
 391  * must be re-programmed following a device reset.
 392  */
 393 static void
 394 ixgbe_reset_qstat_mappings(struct ixgbe_hw *hw)
 395 {
 396         uint32_t i;
 397
 398         for(i = 0; i != IXGBE_NB_STAT_MAPPING_REGS; i++) {
 399                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), 0);
 400                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), 0);
 401         }
 402 }
 403
 404
 405 static int
 406 ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 407                                   uint16_t queue_id,
 408                                   uint8_t stat_idx,
 409                                   uint8_t is_rx)
 410 {
 411 #define QSM_REG_NB_BITS_PER_QMAP_FIELD 8
 412 #define NB_QMAP_FIELDS_PER_QSM_REG 4
 413 #define QMAP_FIELD_RESERVED_BITS_MASK 0x0f
 414
 415         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 416         struct ixgbe_stat_mapping_registers *stat_mappings =
 417                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(eth_dev->data->dev_private);
 418         uint32_t qsmr_mask = 0;
 419         uint32_t clearing_mask = QMAP_FIELD_RESERVED_BITS_MASK;
 420         uint32_t q_map;
 421         uint8_t n, offset;
 422
 423         if ((hw->mac.type != ixgbe_mac_82599EB) && (hw->mac.type != ixgbe_mac_X540))
 424                 return -ENOSYS;
 425
 426         PMD_INIT_LOG(INFO, "Setting port %d, %s queue_id %d to stat index %d\n",
 427                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx);
 428
 429         n = (uint8_t)(queue_id / NB_QMAP_FIELDS_PER_QSM_REG);
 430         if (n >= IXGBE_NB_STAT_MAPPING_REGS) {
 431                 PMD_INIT_LOG(ERR, "Nb of stat mapping registers exceeded\n");
 432                 return -EIO;
 433         }
 434         offset = (uint8_t)(queue_id % NB_QMAP_FIELDS_PER_QSM_REG);
 435
 436         /* Now clear any previous stat_idx set */
 437         clearing_mask <<= (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 438         if (!is_rx)
 439                 stat_mappings->tqsm[n] &= ~clearing_mask;
 440         else
 441                 stat_mappings->rqsmr[n] &= ~clearing_mask;
 442
 443         q_map = (uint32_t)stat_idx;
 444         q_map &= QMAP_FIELD_RESERVED_BITS_MASK;
 445         qsmr_mask = q_map << (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 446         if (!is_rx)
 447                 stat_mappings->tqsm[n] |= qsmr_mask;
 448         else
 449                 stat_mappings->rqsmr[n] |= qsmr_mask;
 450
 451         PMD_INIT_LOG(INFO, "Set port %d, %s queue_id %d to stat index %d\n"
 452                      "%s[%d] = 0x%08x\n",
 453                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx,
 454                      is_rx ? "RQSMR" : "TQSM",n, is_rx ? stat_mappings->rqsmr[n] : stat_mappings->tqsm[n]);
 455
 456         /* Now write the mapping in the appropriate register */
 457         if (is_rx) {
 458                 PMD_INIT_LOG(INFO, "Write 0x%x to RX IXGBE stat mapping reg:%d\n",
 459                              stat_mappings->rqsmr[n], n);
 460                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(n), stat_mappings->rqsmr[n]);
 461         }
 462         else {
 463                 PMD_INIT_LOG(INFO, "Write 0x%x to TX IXGBE stat mapping reg:%d\n",
 464                              stat_mappings->tqsm[n], n);
 465                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(n), stat_mappings->tqsm[n]);
 466         }
 467         return 0;
 468 }
 469
 470 static void
 471 ixgbe_restore_statistics_mapping(struct rte_eth_dev * dev)
 472 {
 473         struct ixgbe_stat_mapping_registers *stat_mappings =
 474                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(dev->data->dev_private);
 475         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 476         int i;
 477
 478         /* write whatever was in stat mapping table to the NIC */
 479         for (i = 0; i < IXGBE_NB_STAT_MAPPING_REGS; i++) {
 480                 /* rx */
 481                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), stat_mappings->rqsmr[i]);
 482
 483                 /* tx */
 484                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), stat_mappings->tqsm[i]);
 485         }
 486 }
 487
 488 static void
 489 ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config)
 490 {
 491         uint8_t i;
 492         struct ixgbe_dcb_tc_config *tc;
 493         uint8_t dcb_max_tc = IXGBE_DCB_MAX_TRAFFIC_CLASS;
 494
 495         dcb_config->num_tcs.pg_tcs = dcb_max_tc;
 496         dcb_config->num_tcs.pfc_tcs = dcb_max_tc;
 497         for (i = 0; i < dcb_max_tc; i++) {
 498                 tc = &dcb_config->tc_config[i];
 499                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_id = i;
 500                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
 501                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 502                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_id = i;
 503                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
 504                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 505                 tc->pfc = ixgbe_dcb_pfc_disabled;
 506         }
 507
 508         /* Initialize default user to priority mapping, UPx->TC0 */
 509         tc = &dcb_config->tc_config[0];
 510         tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
 511         tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
 512         for (i = 0; i< IXGBE_DCB_MAX_BW_GROUP; i++) {
 513                 dcb_config->bw_percentage[IXGBE_DCB_TX_CONFIG][i] = 100;
 514                 dcb_config->bw_percentage[IXGBE_DCB_RX_CONFIG][i] = 100;
 515         }
 516         dcb_config->rx_pba_cfg = ixgbe_dcb_pba_equal;
 517         dcb_config->pfc_mode_enable = false;
 518         dcb_config->vt_mode = true;
 519         dcb_config->round_robin_enable = false;
 520         /* support all DCB capabilities in 82599 */
 521         dcb_config->support.capabilities = 0xFF;
 522
 523         /*we only support 4 Tcs for X540*/
 524         if (hw->mac.type == ixgbe_mac_X540) {
 525                 dcb_config->num_tcs.pg_tcs = 4;
 526                 dcb_config->num_tcs.pfc_tcs = 4;
 527         }
 528 }
 529
 530 /*
 531  * This function is based on code in ixgbe_attach() in ixgbe/ixgbe.c.
 532  * It returns 0 on success.
 533  */
 534 static int
 535 eth_ixgbe_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 536                      struct rte_eth_dev *eth_dev)
 537 {
 538         struct rte_pci_device *pci_dev;
 539         struct ixgbe_hw *hw =
 540                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 541         struct ixgbe_vfta * shadow_vfta =
 542                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 543         struct ixgbe_hwstrip *hwstrip =
 544                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 545         struct ixgbe_dcb_config *dcb_config =
 546                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private);
 547         uint32_t ctrl_ext;
 548         uint16_t csum;
 549         int diag, i;
 550
 551         PMD_INIT_FUNC_TRACE();
 552
 553         eth_dev->dev_ops = &ixgbe_eth_dev_ops;
 554         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
 555         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
 556
 557         /* for secondary processes, we don't initialise any further as primary
 558          * has already done this work. Only check we don't need a different
 559          * RX function */
 560         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 561                 if (eth_dev->data->scattered_rx)
 562                         eth_dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
 563                 return 0;
 564         }
 565         pci_dev = eth_dev->pci_dev;
 566
 567         /* Vendor and Device ID need to be set before init of shared code */
 568         hw->device_id = pci_dev->id.device_id;
 569         hw->vendor_id = pci_dev->id.vendor_id;
 570         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 571
 572         /* Initialize the shared code */
 573         diag = ixgbe_init_shared_code(hw);
 574         if (diag != IXGBE_SUCCESS) {
 575                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
 576                 return -EIO;
 577         }
 578
 579         /* Initialize DCB configuration*/
 580         memset(dcb_config, 0, sizeof(struct ixgbe_dcb_config));
 581         ixgbe_dcb_init(hw,dcb_config);
 582         /* Get Hardware Flow Control setting */
 583         hw->fc.requested_mode = ixgbe_fc_full;
 584         hw->fc.current_mode = ixgbe_fc_full;
 585         hw->fc.pause_time = IXGBE_FC_PAUSE;
 586         for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
 587                 hw->fc.low_water[i] = IXGBE_FC_LO;
 588                 hw->fc.high_water[i] = IXGBE_FC_HI;
 589         }
 590         hw->fc.send_xon = 1;
 591
 592         /* Make sure we have a good EEPROM before we read from it */
 593         diag = ixgbe_validate_eeprom_checksum(hw, &csum);
 594         if (diag != IXGBE_SUCCESS) {
 595                 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag);
 596                 return -EIO;
 597         }
 598
 599         diag = ixgbe_init_hw(hw);
 600
 601         /*
 602          * Devices with copper phys will fail to initialise if ixgbe_init_hw()
 603          * is called too soon after the kernel driver unbinding/binding occurs.
 604          * The failure occurs in ixgbe_identify_phy_generic() for all devices,
 605          * but for non-copper devies, ixgbe_identify_sfp_module_generic() is
 606          * also called. See ixgbe_identify_phy_82599(). The reason for the
 607          * failure is not known, and only occuts when virtualisation features
 608          * are disabled in the bios. A delay of 100ms  was found to be enough by
 609          * trial-and-error, and is doubled to be safe.
 610          */
 611         if (diag && (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper)) {
 612                 rte_delay_ms(200);
 613                 diag = ixgbe_init_hw(hw);
 614         }
 615
 616         if (diag == IXGBE_ERR_EEPROM_VERSION) {
 617                 PMD_INIT_LOG(ERR, "This device is a pre-production adapter/"
 618                     "LOM.  Please be aware there may be issues associated "
 619                     "with your hardware.\n If you are experiencing problems "
 620                     "please contact your Intel or hardware representative "
 621                     "who provided you with this hardware.\n");
 622         } else if (diag == IXGBE_ERR_SFP_NOT_SUPPORTED)
 623                 PMD_INIT_LOG(ERR, "Unsupported SFP+ Module\n");
 624         if (diag) {
 625                 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag);
 626                 return -EIO;
 627         }
 628
 629         /* disable interrupt */
 630         ixgbe_disable_intr(hw);
 631
 632         /* pick up the PCI bus settings for reporting later */
 633         ixgbe_get_bus_info(hw);
 634
 635         /* reset mappings for queue statistics hw counters*/
 636         ixgbe_reset_qstat_mappings(hw);
 637
 638         /* Allocate memory for storing MAC addresses */
 639         eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 640                         hw->mac.num_rar_entries, 0);
 641         if (eth_dev->data->mac_addrs == NULL) {
 642                 PMD_INIT_LOG(ERR,
 643                         "Failed to allocate %d bytes needed to store MAC addresses",
 644                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 645                 return -ENOMEM;
 646         }
 647         /* Copy the permanent MAC address */
 648         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 649                         &eth_dev->data->mac_addrs[0]);
 650
 651         /* initialize the vfta */
 652         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 653
 654         /* initialize the hw strip bitmap*/
 655         memset(hwstrip, 0, sizeof(*hwstrip));
 656
 657         /* let hardware know driver is loaded */
 658         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 659         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 660         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 661
 662         if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
 663                 PMD_INIT_LOG(DEBUG,
 664                              "MAC: %d, PHY: %d, SFP+: %d<n",
 665                              (int) hw->mac.type, (int) hw->phy.type,
 666                              (int) hw->phy.sfp_type);
 667         else
 668                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d\n",
 669                              (int) hw->mac.type, (int) hw->phy.type);
 670
 671         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
 672                         eth_dev->data->port_id, pci_dev->id.vendor_id,
 673                         pci_dev->id.device_id);
 674
 675         rte_intr_callback_register(&(pci_dev->intr_handle),
 676                 ixgbe_dev_interrupt_handler, (void *)eth_dev);
 677
 678         /* enable uio intr after callback register */
 679         rte_intr_enable(&(pci_dev->intr_handle));
 680
 681         /* enable support intr */
 682         ixgbe_enable_intr(eth_dev);
 683
 684         return 0;
 685 }
 686
 687 /*
 688  * Virtual Function device init
 689  */
 690 static int
 691 eth_ixgbevf_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 692                      struct rte_eth_dev *eth_dev)
 693 {
 694         struct rte_pci_device *pci_dev;
 695         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 696         int diag;
 697         struct ixgbe_vfta * shadow_vfta =
 698                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 699         struct ixgbe_hwstrip *hwstrip =
 700                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 701
 702         PMD_INIT_LOG(DEBUG, "eth_ixgbevf_dev_init");
 703
 704         eth_dev->dev_ops = &ixgbevf_eth_dev_ops;
 705         pci_dev = eth_dev->pci_dev;
 706
 707         hw->device_id = pci_dev->id.device_id;
 708         hw->vendor_id = pci_dev->id.vendor_id;
 709         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 710
 711         /* initialize the vfta */
 712         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 713
 714         /* initialize the hw strip bitmap*/
 715         memset(hwstrip, 0, sizeof(*hwstrip));
 716
 717         /* Initialize the shared code */
 718         diag = ixgbe_init_shared_code(hw);
 719         if (diag != IXGBE_SUCCESS) {
 720                 PMD_INIT_LOG(ERR, "Shared code init failed for ixgbevf: %d", diag);
 721                 return -EIO;
 722         }
 723
 724         /* init_mailbox_params */
 725         hw->mbx.ops.init_params(hw);
 726
 727         /* Disable the interrupts for VF */
 728         ixgbevf_intr_disable(hw);
 729
 730         hw->mac.num_rar_entries = hw->mac.max_rx_queues;
 731         diag = hw->mac.ops.reset_hw(hw);
 732         if (diag != IXGBE_SUCCESS) {
 733                 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
 734                 return (diag);
 735         }
 736
 737         /* Allocate memory for storing MAC addresses */
 738         eth_dev->data->mac_addrs = rte_zmalloc("ixgbevf", ETHER_ADDR_LEN *
 739                         hw->mac.num_rar_entries, 0);
 740         if (eth_dev->data->mac_addrs == NULL) {
 741                 PMD_INIT_LOG(ERR,
 742                         "Failed to allocate %d bytes needed to store MAC addresses",
 743                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 744                 return -ENOMEM;
 745         }
 746         /* Copy the permanent MAC address */
 747         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 748                         &eth_dev->data->mac_addrs[0]);
 749
 750         /* reset the hardware with the new settings */
 751         diag = hw->mac.ops.start_hw(hw);
 752         switch (diag) {
 753                 case  0:
 754                         break;
 755
 756                 default:
 757                         PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
 758                         return (-EIO);
 759         }
 760
 761         PMD_INIT_LOG(DEBUG, "\nport %d vendorID=0x%x deviceID=0x%x mac.type=%s\n",
 762                          eth_dev->data->port_id, pci_dev->id.vendor_id, pci_dev->id.device_id,
 763                          "ixgbe_mac_82599_vf");
 764
 765         return 0;
 766 }
 767
 768 static struct eth_driver rte_ixgbe_pmd = {
 769         {
 770                 .name = "rte_ixgbe_pmd",
 771                 .id_table = pci_id_ixgbe_map,
 772 #ifdef RTE_EAL_UNBIND_PORTS
 773                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
 774 #endif
 775         },
 776         .eth_dev_init = eth_ixgbe_dev_init,
 777         .dev_private_size = sizeof(struct ixgbe_adapter),
 778 };
 779
 780 /*
 781  * virtual function driver struct
 782  */
 783 static struct eth_driver rte_ixgbevf_pmd = {
 784         {
 785                 .name = "rte_ixgbevf_pmd",
 786                 .id_table = pci_id_ixgbevf_map,
 787 #ifdef RTE_EAL_UNBIND_PORTS
 788                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
 789 #endif
 790         },
 791         .eth_dev_init = eth_ixgbevf_dev_init,
 792         .dev_private_size = sizeof(struct ixgbe_adapter),
 793 };
 794
 795 /*
 796  * Driver initialization routine.
 797  * Invoked once at EAL init time.
 798  * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
 799  */
 800 int
 801 rte_ixgbe_pmd_init(void)
 802 {
 803         PMD_INIT_FUNC_TRACE();
 804
 805         rte_eth_driver_register(&rte_ixgbe_pmd);
 806         return 0;
 807 }
 808
 809 /*
 810  * VF Driver initialization routine.
 811  * Invoked one at EAL init time.
 812  * Register itself as the [Virtual Poll Mode] Driver of PCI niantic devices.
 813  */
 814 int
 815 rte_ixgbevf_pmd_init(void)
 816 {
 817         DEBUGFUNC("rte_ixgbevf_pmd_init");
 818
 819         rte_eth_driver_register(&rte_ixgbevf_pmd);
 820         return (0);
 821 }
 822
 823 static int
 824 ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 825 {
 826         struct ixgbe_hw *hw =
 827                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 828         struct ixgbe_vfta * shadow_vfta =
 829                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
 830         uint32_t vfta;
 831         uint32_t vid_idx;
 832         uint32_t vid_bit;
 833
 834         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
 835         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
 836         vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(vid_idx));
 837         if (on)
 838                 vfta |= vid_bit;
 839         else
 840                 vfta &= ~vid_bit;
 841         IXGBE_WRITE_REG(hw, IXGBE_VFTA(vid_idx), vfta);
 842
 843         /* update local VFTA copy */
 844         shadow_vfta->vfta[vid_idx] = vfta;
 845
 846         return 0;
 847 }
 848
 849 static void
 850 ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
 851 {
 852         if (on)
 853                 ixgbe_vlan_hw_strip_enable(dev, queue);
 854         else
 855                 ixgbe_vlan_hw_strip_disable(dev, queue);
 856 }
 857
 858 static void
 859 ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
 860 {
 861         struct ixgbe_hw *hw =
 862                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 863
 864         /* Only the high 16-bits is valid */
 865         IXGBE_WRITE_REG(hw, IXGBE_EXVET, tpid << 16);
 866 }
 867
 868 static void
 869 ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev)
 870 {
 871         struct ixgbe_hw *hw =
 872                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 873         uint32_t vlnctrl;
 874
 875         PMD_INIT_FUNC_TRACE();
 876
 877         /* Filter Table Disable */
 878         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 879         vlnctrl &= ~IXGBE_VLNCTRL_VFE;
 880
 881         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
 882 }
 883
 884 static void
 885 ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev)
 886 {
 887         struct ixgbe_hw *hw =
 888                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 889         struct ixgbe_vfta * shadow_vfta =
 890                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
 891         uint32_t vlnctrl;
 892         uint16_t i;
 893
 894         PMD_INIT_FUNC_TRACE();
 895
 896         /* Filter Table Enable */
 897         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 898         vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
 899         vlnctrl |= IXGBE_VLNCTRL_VFE;
 900
 901         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
 902
 903         /* write whatever is in local vfta copy */
 904         for (i = 0; i < IXGBE_VFTA_SIZE; i++)
 905                 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), shadow_vfta->vfta[i]);
 906 }
 907
 908 static void
 909 ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on)
 910 {
 911         struct ixgbe_hwstrip *hwstrip =
 912                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(dev->data->dev_private);
 913
 914         if(queue >= IXGBE_MAX_RX_QUEUE_NUM)
 915                 return;
 916
 917         if (on)
 918                 IXGBE_SET_HWSTRIP(hwstrip, queue);
 919         else
 920                 IXGBE_CLEAR_HWSTRIP(hwstrip, queue);
 921 }
 922
 923 static void
 924 ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue)
 925 {
 926         struct ixgbe_hw *hw =
 927                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 928         uint32_t ctrl;
 929
 930         PMD_INIT_FUNC_TRACE();
 931
 932         if (hw->mac.type == ixgbe_mac_82598EB) {
 933                 /* No queue level support */
 934                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
 935                 return;
 936         }
 937         else {
 938                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 939                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
 940                 ctrl &= ~IXGBE_RXDCTL_VME;
 941                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
 942         }
 943         /* record those setting for HW strip per queue */
 944         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 0);
 945 }
 946
 947 static void
 948 ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue)
 949 {
 950         struct ixgbe_hw *hw =
 951                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 952         uint32_t ctrl;
 953
 954         PMD_INIT_FUNC_TRACE();
 955
 956         if (hw->mac.type == ixgbe_mac_82598EB) {
 957                 /* No queue level supported */
 958                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
 959                 return;
 960         }
 961         else {
 962                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 963                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
 964                 ctrl |= IXGBE_RXDCTL_VME;
 965                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
 966         }
 967         /* record those setting for HW strip per queue */
 968         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 1);
 969 }
 970
 971 static void
 972 ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev)
 973 {
 974         struct ixgbe_hw *hw =
 975                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 976         uint32_t ctrl;
 977         uint16_t i;
 978
 979         PMD_INIT_FUNC_TRACE();
 980
 981         if (hw->mac.type == ixgbe_mac_82598EB) {
 982                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 983                 ctrl &= ~IXGBE_VLNCTRL_VME;
 984                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
 985         }
 986         else {
 987                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 988                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
 989                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
 990                         ctrl &= ~IXGBE_RXDCTL_VME;
 991                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
 992
 993                         /* record those setting for HW strip per queue */
 994                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 0);
 995                 }
 996         }
 997 }
 998
 999 static void
1000 ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev)
1001 {
1002         struct ixgbe_hw *hw =
1003                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1004         uint32_t ctrl;
1005         uint16_t i;
1006
1007         PMD_INIT_FUNC_TRACE();
1008
1009         if (hw->mac.type == ixgbe_mac_82598EB) {
1010                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1011                 ctrl |= IXGBE_VLNCTRL_VME;
1012                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1013         }
1014         else {
1015                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1016                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1017                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1018                         ctrl |= IXGBE_RXDCTL_VME;
1019                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1020
1021                         /* record those setting for HW strip per queue */
1022                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 1);
1023                 }
1024         }
1025 }
1026
1027 static void
1028 ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1029 {
1030         struct ixgbe_hw *hw =
1031                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1032         uint32_t ctrl;
1033
1034         PMD_INIT_FUNC_TRACE();
1035
1036         /* DMATXCTRL: Geric Double VLAN Disable */
1037         ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1038         ctrl &= ~IXGBE_DMATXCTL_GDV;
1039         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1040
1041         /* CTRL_EXT: Global Double VLAN Disable */
1042         ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1043         ctrl &= ~IXGBE_EXTENDED_VLAN;
1044         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1045
1046 }
1047
1048 static void
1049 ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1050 {
1051         struct ixgbe_hw *hw =
1052                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1053         uint32_t ctrl;
1054
1055         PMD_INIT_FUNC_TRACE();
1056
1057         /* DMATXCTRL: Geric Double VLAN Enable */
1058         ctrl  = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1059         ctrl |= IXGBE_DMATXCTL_GDV;
1060         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1061
1062         /* CTRL_EXT: Global Double VLAN Enable */
1063         ctrl  = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1064         ctrl |= IXGBE_EXTENDED_VLAN;
1065         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1066
1067         /*
1068          * VET EXT field in the EXVET register = 0x8100 by default
1069          * So no need to change. Same to VT field of DMATXCTL register
1070          */
1071 }
1072
1073 static void
1074 ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1075 {
1076         if(mask & ETH_VLAN_STRIP_MASK){
1077                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1078                         ixgbe_vlan_hw_strip_enable_all(dev);
1079                 else
1080                         ixgbe_vlan_hw_strip_disable_all(dev);
1081         }
1082
1083         if(mask & ETH_VLAN_FILTER_MASK){
1084                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1085                         ixgbe_vlan_hw_filter_enable(dev);
1086                 else
1087                         ixgbe_vlan_hw_filter_disable(dev);
1088         }
1089
1090         if(mask & ETH_VLAN_EXTEND_MASK){
1091                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1092                         ixgbe_vlan_hw_extend_enable(dev);
1093                 else
1094                         ixgbe_vlan_hw_extend_disable(dev);
1095         }
1096 }
1097
1098 static int
1099 ixgbe_dev_configure(struct rte_eth_dev *dev)
1100 {
1101         struct ixgbe_interrupt *intr =
1102                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1103
1104         PMD_INIT_FUNC_TRACE();
1105
1106         /* set flag to update link status after init */
1107         intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1108
1109         return 0;
1110 }
1111
1112 /*
1113  * Configure device link speed and setup link.
1114  * It returns 0 on success.
1115  */
1116 static int
1117 ixgbe_dev_start(struct rte_eth_dev *dev)
1118 {
1119         struct ixgbe_hw *hw =
1120                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1121         int err, link_up = 0, negotiate = 0;
1122         uint32_t speed = 0;
1123         int mask = 0;
1124
1125         PMD_INIT_FUNC_TRACE();
1126
1127         /* IXGBE devices don't support half duplex */
1128         if ((dev->data->dev_conf.link_duplex != ETH_LINK_AUTONEG_DUPLEX) &&
1129                         (dev->data->dev_conf.link_duplex != ETH_LINK_FULL_DUPLEX)) {
1130                 PMD_INIT_LOG(ERR, "Invalid link_duplex (%u) for port %u\n",
1131                                 dev->data->dev_conf.link_duplex,
1132                                 dev->data->port_id);
1133                 return -EINVAL;
1134         }
1135
1136         /* stop adapter */
1137         hw->adapter_stopped = FALSE;
1138         ixgbe_stop_adapter(hw);
1139
1140         /* reinitialize adapter
1141          * this calls reset and start */
1142         ixgbe_init_hw(hw);
1143         hw->mac.ops.start_hw(hw);
1144
1145         /* initialize transmission unit */
1146         ixgbe_dev_tx_init(dev);
1147
1148         /* This can fail when allocating mbufs for descriptor rings */
1149         err = ixgbe_dev_rx_init(dev);
1150         if (err) {
1151                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware\n");
1152                 goto error;
1153         }
1154
1155         ixgbe_dev_rxtx_start(dev);
1156
1157         if (ixgbe_is_sfp(hw) && hw->phy.multispeed_fiber) {
1158                 err = hw->mac.ops.setup_sfp(hw);
1159                 if (err)
1160                         goto error;
1161         }
1162
1163         /* Turn on the laser */
1164         if (hw->phy.multispeed_fiber)
1165                 ixgbe_enable_tx_laser(hw);
1166
1167         err = ixgbe_check_link(hw, &speed, &link_up, 0);
1168         if (err)
1169                 goto error;
1170         err = ixgbe_get_link_capabilities(hw, &speed, &negotiate);
1171         if (err)
1172                 goto error;
1173
1174         switch(dev->data->dev_conf.link_speed) {
1175         case ETH_LINK_SPEED_AUTONEG:
1176                 speed = (hw->mac.type != ixgbe_mac_82598EB) ?
1177                                 IXGBE_LINK_SPEED_82599_AUTONEG :
1178                                 IXGBE_LINK_SPEED_82598_AUTONEG;
1179                 break;
1180         case ETH_LINK_SPEED_100:
1181                 /*
1182                  * Invalid for 82598 but error will be detected by
1183                  * ixgbe_setup_link()
1184                  */
1185                 speed = IXGBE_LINK_SPEED_100_FULL;
1186                 break;
1187         case ETH_LINK_SPEED_1000:
1188                 speed = IXGBE_LINK_SPEED_1GB_FULL;
1189                 break;
1190         case ETH_LINK_SPEED_10000:
1191                 speed = IXGBE_LINK_SPEED_10GB_FULL;
1192                 break;
1193         default:
1194                 PMD_INIT_LOG(ERR, "Invalid link_speed (%u) for port %u\n",
1195                                 dev->data->dev_conf.link_speed, dev->data->port_id);
1196                 goto error;
1197         }
1198
1199         err = ixgbe_setup_link(hw, speed, negotiate, link_up);
1200         if (err)
1201                 goto error;
1202
1203         /* check if lsc interrupt is enabled */
1204         if (dev->data->dev_conf.intr_conf.lsc != 0)
1205                 ixgbe_dev_lsc_interrupt_setup(dev);
1206
1207         /* resume enabled intr since hw reset */
1208         ixgbe_enable_intr(dev);
1209
1210         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
1211                 ETH_VLAN_EXTEND_MASK;
1212         ixgbe_vlan_offload_set(dev, mask);
1213
1214         /* Configure DCB hw */
1215         ixgbe_configure_dcb(dev);
1216
1217         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) {
1218                 err = ixgbe_fdir_configure(dev);
1219                 if (err)
1220                         goto error;
1221         }
1222
1223         ixgbe_restore_statistics_mapping(dev);
1224
1225         return (0);
1226
1227 error:
1228         PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err);
1229         ixgbe_dev_clear_queues(dev);
1230         return -EIO;
1231 }
1232
1233 /*
1234  * Stop device: disable rx and tx functions to allow for reconfiguring.
1235  */
1236 static void
1237 ixgbe_dev_stop(struct rte_eth_dev *dev)
1238 {
1239         struct rte_eth_link link;
1240         struct ixgbe_hw *hw =
1241                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1242
1243         PMD_INIT_FUNC_TRACE();
1244
1245         /* disable interrupts */
1246         ixgbe_disable_intr(hw);
1247
1248         /* reset the NIC */
1249         ixgbe_reset_hw(hw);
1250         hw->adapter_stopped = FALSE;
1251
1252         /* stop adapter */
1253         ixgbe_stop_adapter(hw);
1254
1255         /* Turn off the laser */
1256         if (hw->phy.multispeed_fiber)
1257                 ixgbe_disable_tx_laser(hw);
1258
1259         ixgbe_dev_clear_queues(dev);
1260
1261         /* Clear recorded link status */
1262         memset(&link, 0, sizeof(link));
1263         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1264 }
1265
1266 /*
1267  * Reest and stop device.
1268  */
1269 static void
1270 ixgbe_dev_close(struct rte_eth_dev *dev)
1271 {
1272         struct ixgbe_hw *hw =
1273                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1274
1275         PMD_INIT_FUNC_TRACE();
1276
1277         ixgbe_reset_hw(hw);
1278
1279
1280         ixgbe_dev_stop(dev);
1281         hw->adapter_stopped = 1;
1282
1283         ixgbe_disable_pcie_master(hw);
1284
1285         /* reprogram the RAR[0] in case user changed it. */
1286         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
1287 }
1288
1289 /*
1290  * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c
1291  */
1292 static void
1293 ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1294 {
1295         struct ixgbe_hw *hw =
1296                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1297         struct ixgbe_hw_stats *hw_stats =
1298                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1299         uint32_t bprc, lxon, lxoff, total;
1300         uint64_t total_missed_rx, total_qbrc, total_qprc;
1301         unsigned i;
1302
1303         total_missed_rx = 0;
1304         total_qbrc = 0;
1305         total_qprc = 0;
1306
1307         hw_stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
1308         hw_stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
1309         hw_stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
1310         hw_stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
1311
1312         for (i = 0; i < 8; i++) {
1313                 uint32_t mp;
1314                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
1315                 /* global total per queue */
1316                 hw_stats->mpc[i] += mp;
1317                 /* Running comprehensive total for stats display */
1318                 total_missed_rx += hw_stats->mpc[i];
1319                 if (hw->mac.type == ixgbe_mac_82598EB)
1320                         hw_stats->rnbc[i] +=
1321                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
1322                 hw_stats->pxontxc[i] +=
1323                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
1324                 hw_stats->pxonrxc[i] +=
1325                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
1326                 hw_stats->pxofftxc[i] +=
1327                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
1328                 hw_stats->pxoffrxc[i] +=
1329                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
1330                 hw_stats->pxon2offc[i] +=
1331                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
1332         }
1333         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1334                 hw_stats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
1335                 hw_stats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
1336                 hw_stats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
1337                 hw_stats->qbrc[i] +=
1338                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32);
1339                 hw_stats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
1340                 hw_stats->qbtc[i] +=
1341                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32);
1342                 hw_stats->qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
1343
1344                 total_qprc += hw_stats->qprc[i];
1345                 total_qbrc += hw_stats->qbrc[i];
1346         }
1347         hw_stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
1348         hw_stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
1349         hw_stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
1350
1351         /* Note that gprc counts missed packets */
1352         hw_stats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
1353
1354         if (hw->mac.type != ixgbe_mac_82598EB) {
1355                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
1356                 hw_stats->gorc += ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
1357                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
1358                 hw_stats->gotc += ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
1359                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
1360                 hw_stats->tor += ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
1361                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
1362                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
1363         } else {
1364                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
1365                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
1366                 /* 82598 only has a counter in the high register */
1367                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
1368                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
1369                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
1370         }
1371
1372         /*
1373          * Workaround: mprc hardware is incorrectly counting
1374          * broadcasts, so for now we subtract those.
1375          */
1376         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
1377         hw_stats->bprc += bprc;
1378         hw_stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
1379         if (hw->mac.type == ixgbe_mac_82598EB)
1380                 hw_stats->mprc -= bprc;
1381
1382         hw_stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
1383         hw_stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
1384         hw_stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
1385         hw_stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
1386         hw_stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
1387         hw_stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
1388
1389         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
1390         hw_stats->lxontxc += lxon;
1391         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
1392         hw_stats->lxofftxc += lxoff;
1393         total = lxon + lxoff;
1394
1395         hw_stats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
1396         hw_stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
1397         hw_stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
1398         hw_stats->gptc -= total;
1399         hw_stats->mptc -= total;
1400         hw_stats->ptc64 -= total;
1401         hw_stats->gotc -= total * ETHER_MIN_LEN;
1402
1403         hw_stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
1404         hw_stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
1405         hw_stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
1406         hw_stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
1407         hw_stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
1408         hw_stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
1409         hw_stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
1410         hw_stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
1411         hw_stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
1412         hw_stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
1413         hw_stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
1414         hw_stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
1415         hw_stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
1416         hw_stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
1417         hw_stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
1418         hw_stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);
1419         hw_stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
1420         hw_stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
1421         /* Only read FCOE on 82599 */
1422         if (hw->mac.type != ixgbe_mac_82598EB) {
1423                 hw_stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
1424                 hw_stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
1425                 hw_stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
1426                 hw_stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
1427                 hw_stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
1428         }
1429
1430         if (stats == NULL)
1431                 return;
1432
1433         /* Fill out the rte_eth_stats statistics structure */
1434         stats->ipackets = total_qprc;
1435         stats->ibytes = total_qbrc;
1436         stats->opackets = hw_stats->gptc;
1437         stats->obytes = hw_stats->gotc;
1438         stats->imcasts = hw_stats->mprc;
1439
1440         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1441                 stats->q_ipackets[i] = hw_stats->qprc[i];
1442                 stats->q_opackets[i] = hw_stats->qptc[i];
1443                 stats->q_ibytes[i] = hw_stats->qbrc[i];
1444                 stats->q_obytes[i] = hw_stats->qbtc[i];
1445                 stats->q_errors[i] = hw_stats->qprdc[i];
1446         }
1447
1448         /* Rx Errors */
1449         stats->ierrors = total_missed_rx + hw_stats->crcerrs +
1450                 hw_stats->rlec;
1451
1452         stats->oerrors  = 0;
1453
1454         /* Flow Director Stats registers */
1455         hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
1456         hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
1457         stats->fdirmatch = hw_stats->fdirmatch;
1458         stats->fdirmiss = hw_stats->fdirmiss;
1459 }
1460
1461 static void
1462 ixgbe_dev_stats_reset(struct rte_eth_dev *dev)
1463 {
1464         struct ixgbe_hw_stats *stats =
1465                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1466
1467         /* HW registers are cleared on read */
1468         ixgbe_dev_stats_get(dev, NULL);
1469
1470         /* Reset software totals */
1471         memset(stats, 0, sizeof(*stats));
1472 }
1473
1474 static void
1475 ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1476 {
1477         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1478         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1479                           IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1480
1481         /* Good Rx packet, include VF loopback */
1482         UPDATE_VF_STAT(IXGBE_VFGPRC,
1483             hw_stats->last_vfgprc, hw_stats->vfgprc);
1484
1485         /* Good Rx octets, include VF loopback */
1486         UPDATE_VF_STAT_36BIT(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
1487             hw_stats->last_vfgorc, hw_stats->vfgorc);
1488
1489         /* Good Tx packet, include VF loopback */
1490         UPDATE_VF_STAT(IXGBE_VFGPTC,
1491             hw_stats->last_vfgptc, hw_stats->vfgptc);
1492
1493         /* Good Tx octets, include VF loopback */
1494         UPDATE_VF_STAT_36BIT(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
1495             hw_stats->last_vfgotc, hw_stats->vfgotc);
1496
1497         /* Rx Multicst Packet */
1498         UPDATE_VF_STAT(IXGBE_VFMPRC,
1499             hw_stats->last_vfmprc, hw_stats->vfmprc);
1500
1501         if (stats == NULL)
1502                 return;
1503
1504         memset(stats, 0, sizeof(*stats));
1505         stats->ipackets = hw_stats->vfgprc;
1506         stats->ibytes = hw_stats->vfgorc;
1507         stats->opackets = hw_stats->vfgptc;
1508         stats->obytes = hw_stats->vfgotc;
1509         stats->imcasts = hw_stats->vfmprc;
1510 }
1511
1512 static void
1513 ixgbevf_dev_stats_reset(struct rte_eth_dev *dev)
1514 {
1515         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1516                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1517
1518         /* Sync HW register to the last stats */
1519         ixgbevf_dev_stats_get(dev, NULL);
1520
1521         /* reset HW current stats*/
1522         hw_stats->vfgprc = 0;
1523         hw_stats->vfgorc = 0;
1524         hw_stats->vfgptc = 0;
1525         hw_stats->vfgotc = 0;
1526         hw_stats->vfmprc = 0;
1527
1528 }
1529
1530 static void
1531 ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1532 {
1533         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1534
1535         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
1536         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
1537         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL register */
1538         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */
1539         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
1540 }
1541
1542 /* return 0 means link status changed, -1 means not changed */
1543 static int
1544 ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1545 {
1546         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1547         struct rte_eth_link link, old;
1548         ixgbe_link_speed link_speed;
1549         int link_up;
1550         int diag;
1551
1552         link.link_status = 0;
1553         link.link_speed = 0;
1554         link.link_duplex = 0;
1555         memset(&old, 0, sizeof(old));
1556         rte_ixgbe_dev_atomic_read_link_status(dev, &old);
1557
1558         /* check if it needs to wait to complete, if lsc interrupt is enabled */
1559         if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
1560                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 0);
1561         else
1562                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 1);
1563         if (diag != 0) {
1564                 link.link_speed = ETH_LINK_SPEED_100;
1565                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1566                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1567                 if (link.link_status == old.link_status)
1568                         return -1;
1569                 return 0;
1570         }
1571
1572         if (link_up == 0) {
1573                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1574                 if (link.link_status == old.link_status)
1575                         return -1;
1576                 return 0;
1577         }
1578         link.link_status = 1;
1579         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1580
1581         switch (link_speed) {
1582         default:
1583         case IXGBE_LINK_SPEED_UNKNOWN:
1584                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1585                 link.link_speed = ETH_LINK_SPEED_100;
1586                 break;
1587
1588         case IXGBE_LINK_SPEED_100_FULL:
1589                 link.link_speed = ETH_LINK_SPEED_100;
1590                 break;
1591
1592         case IXGBE_LINK_SPEED_1GB_FULL:
1593                 link.link_speed = ETH_LINK_SPEED_1000;
1594                 break;
1595
1596         case IXGBE_LINK_SPEED_10GB_FULL:
1597                 link.link_speed = ETH_LINK_SPEED_10000;
1598                 break;
1599         }
1600         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1601
1602         if (link.link_status == old.link_status)
1603                 return -1;
1604
1605         return 0;
1606 }
1607
1608 static void
1609 ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev)
1610 {
1611         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1612         uint32_t fctrl;
1613
1614         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1615         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1616         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1617 }
1618
1619 static void
1620 ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev)
1621 {
1622         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1623         uint32_t fctrl;
1624
1625         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1626         fctrl &= (~IXGBE_FCTRL_UPE);
1627         if (dev->data->all_multicast == 1)
1628                 fctrl |= IXGBE_FCTRL_MPE;
1629         else
1630                 fctrl &= (~IXGBE_FCTRL_MPE);
1631         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1632 }
1633
1634 static void
1635 ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev)
1636 {
1637         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1638         uint32_t fctrl;
1639
1640         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1641         fctrl |= IXGBE_FCTRL_MPE;
1642         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1643 }
1644
1645 static void
1646 ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev)
1647 {
1648         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1649         uint32_t fctrl;
1650
1651         if (dev->data->promiscuous == 1)
1652                 return; /* must remain in all_multicast mode */
1653
1654         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1655         fctrl &= (~IXGBE_FCTRL_MPE);
1656         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1657 }
1658
1659 /**
1660  * It clears the interrupt causes and enables the interrupt.
1661  * It will be called once only during nic initialized.
1662  *
1663  * @param dev
1664  *  Pointer to struct rte_eth_dev.
1665  *
1666  * @return
1667  *  - On success, zero.
1668  *  - On failure, a negative value.
1669  */
1670 static int
1671 ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev)
1672 {
1673         struct ixgbe_interrupt *intr =
1674                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1675
1676         ixgbe_dev_link_status_print(dev);
1677         intr->mask |= IXGBE_EICR_LSC;
1678
1679         return 0;
1680 }
1681
1682 /*
1683  * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update.
1684  *
1685  * @param dev
1686  *  Pointer to struct rte_eth_dev.
1687  *
1688  * @return
1689  *  - On success, zero.
1690  *  - On failure, a negative value.
1691  */
1692 static int
1693 ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
1694 {
1695         uint32_t eicr;
1696         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1697         struct ixgbe_interrupt *intr =
1698                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1699
1700         /* clear all cause mask */
1701         ixgbe_disable_intr(hw);
1702
1703         /* read-on-clear nic registers here */
1704         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1705         PMD_DRV_LOG(INFO, "eicr %x", eicr);
1706
1707         intr->flags = 0;
1708         if (eicr & IXGBE_EICR_LSC) {
1709                 /* set flag for async link update */
1710                 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1711         }
1712
1713         return 0;
1714 }
1715
1716 /**
1717  * It gets and then prints the link status.
1718  *
1719  * @param dev
1720  *  Pointer to struct rte_eth_dev.
1721  *
1722  * @return
1723  *  - On success, zero.
1724  *  - On failure, a negative value.
1725  */
1726 static void
1727 ixgbe_dev_link_status_print(struct rte_eth_dev *dev)
1728 {
1729         struct rte_eth_link link;
1730
1731         memset(&link, 0, sizeof(link));
1732         rte_ixgbe_dev_atomic_read_link_status(dev, &link);
1733         if (link.link_status) {
1734                 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
1735                                         (int)(dev->data->port_id),
1736                                         (unsigned)link.link_speed,
1737                         link.link_duplex == ETH_LINK_FULL_DUPLEX ?
1738                                         "full-duplex" : "half-duplex");
1739         } else {
1740                 PMD_INIT_LOG(INFO, " Port %d: Link Down",
1741                                 (int)(dev->data->port_id));
1742         }
1743         PMD_INIT_LOG(INFO, "PCI Address: %04d:%02d:%02d:%d",
1744                                 dev->pci_dev->addr.domain,
1745                                 dev->pci_dev->addr.bus,
1746                                 dev->pci_dev->addr.devid,
1747                                 dev->pci_dev->addr.function);
1748 }
1749
1750 /*
1751  * It executes link_update after knowing an interrupt occured.
1752  *
1753  * @param dev
1754  *  Pointer to struct rte_eth_dev.
1755  *
1756  * @return
1757  *  - On success, zero.
1758  *  - On failure, a negative value.
1759  */
1760 static int
1761 ixgbe_dev_interrupt_action(struct rte_eth_dev *dev)
1762 {
1763         struct ixgbe_interrupt *intr =
1764                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1765         int64_t timeout;
1766         struct rte_eth_link link;
1767         int intr_enable_delay = false;
1768
1769         PMD_DRV_LOG(DEBUG, "intr action type %d\n", intr->flags);
1770
1771         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
1772                 /* get the link status before link update, for predicting later */
1773                 memset(&link, 0, sizeof(link));
1774                 rte_ixgbe_dev_atomic_read_link_status(dev, &link);
1775
1776                 ixgbe_dev_link_update(dev, 0);
1777
1778                 /* likely to up */
1779                 if (!link.link_status)
1780                         /* handle it 1 sec later, wait it being stable */
1781                         timeout = IXGBE_LINK_UP_CHECK_TIMEOUT;
1782                 /* likely to down */
1783                 else
1784                         /* handle it 4 sec later, wait it being stable */
1785                         timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT;
1786
1787                 ixgbe_dev_link_status_print(dev);
1788
1789                 intr_enable_delay = true;
1790         }
1791
1792         if (intr_enable_delay) {
1793                 if (rte_eal_alarm_set(timeout * 1000,
1794                                       ixgbe_dev_interrupt_delayed_handler, (void*)dev) < 0)
1795                         PMD_DRV_LOG(ERR, "Error setting alarm");
1796         } else {
1797                 PMD_DRV_LOG(DEBUG, "enable intr immediately");
1798                 ixgbe_enable_intr(dev);
1799                 rte_intr_enable(&(dev->pci_dev->intr_handle));
1800         }
1801
1802
1803         return 0;
1804 }
1805
1806 /**
1807  * Interrupt handler which shall be registered for alarm callback for delayed
1808  * handling specific interrupt to wait for the stable nic state. As the
1809  * NIC interrupt state is not stable for ixgbe after link is just down,
1810  * it needs to wait 4 seconds to get the stable status.
1811  *
1812  * @param handle
1813  *  Pointer to interrupt handle.
1814  * @param param
1815  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1816  *
1817  * @return
1818  *  void
1819  */
1820 static void
1821 ixgbe_dev_interrupt_delayed_handler(void *param)
1822 {
1823         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1824         struct ixgbe_interrupt *intr =
1825                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1826
1827         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
1828                 ixgbe_dev_link_update(dev, 0);
1829                 intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
1830                 ixgbe_dev_link_status_print(dev);
1831                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
1832         }
1833
1834         PMD_DRV_LOG(DEBUG, "enable intr in delayed handler\n");
1835         ixgbe_enable_intr(dev);
1836         rte_intr_enable(&(dev->pci_dev->intr_handle));
1837 }
1838
1839 /**
1840  * Interrupt handler triggered by NIC  for handling
1841  * specific interrupt.
1842  *
1843  * @param handle
1844  *  Pointer to interrupt handle.
1845  * @param param
1846  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1847  *
1848  * @return
1849  *  void
1850  */
1851 static void
1852 ixgbe_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
1853                                                         void *param)
1854 {
1855         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1856         ixgbe_dev_interrupt_get_status(dev);
1857         ixgbe_dev_interrupt_action(dev);
1858 }
1859
1860 static int
1861 ixgbe_dev_led_on(struct rte_eth_dev *dev)
1862 {
1863         struct ixgbe_hw *hw;
1864
1865         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1866         return (ixgbe_led_on(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
1867 }
1868
1869 static int
1870 ixgbe_dev_led_off(struct rte_eth_dev *dev)
1871 {
1872         struct ixgbe_hw *hw;
1873
1874         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1875         return (ixgbe_led_off(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
1876 }
1877
1878 static int
1879 ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1880 {
1881         struct ixgbe_hw *hw;
1882         int err;
1883         uint32_t rx_buf_size;
1884         uint32_t max_high_water;
1885         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
1886                 ixgbe_fc_none,
1887                 ixgbe_fc_rx_pause,
1888                 ixgbe_fc_tx_pause,
1889                 ixgbe_fc_full
1890         };
1891
1892         PMD_INIT_FUNC_TRACE();
1893
1894         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1895         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0));
1896         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
1897
1898         /*
1899          * At least reserve one Ethernet frame for watermark
1900          * high_water/low_water in kilo bytes for ixgbe
1901          */
1902         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
1903         if ((fc_conf->high_water > max_high_water) ||
1904                 (fc_conf->high_water < fc_conf->low_water)) {
1905                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
1906                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
1907                 return (-EINVAL);
1908         }
1909
1910         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[fc_conf->mode];
1911         hw->fc.pause_time     = fc_conf->pause_time;
1912         hw->fc.high_water[0]  = fc_conf->high_water;
1913         hw->fc.low_water[0]   = fc_conf->low_water;
1914         hw->fc.send_xon       = fc_conf->send_xon;
1915
1916         err = ixgbe_fc_enable(hw);
1917         /* Not negotiated is not an error case */
1918         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) {
1919                 return 0;
1920         }
1921
1922         PMD_INIT_LOG(ERR, "ixgbe_fc_enable = 0x%x \n", err);
1923         return -EIO;
1924 }
1925
1926 /**
1927  *  ixgbe_pfc_enable_generic - Enable flow control
1928  *  @hw: pointer to hardware structure
1929  *  @tc_num: traffic class number
1930  *  Enable flow control according to the current settings.
1931  */
1932 static int
1933 ixgbe_dcb_pfc_enable_generic(struct ixgbe_hw *hw,uint8_t tc_num)
1934 {
1935         int ret_val = 0;
1936         uint32_t mflcn_reg, fccfg_reg;
1937         uint32_t reg;
1938         uint32_t fcrtl, fcrth;
1939         uint8_t i;
1940         uint8_t nb_rx_en;
1941
1942         /* Validate the water mark configuration */
1943         if (!hw->fc.pause_time) {
1944                 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
1945                 goto out;
1946         }
1947
1948         /* Low water mark of zero causes XOFF floods */
1949         if (hw->fc.current_mode & ixgbe_fc_tx_pause) {
1950                  /* High/Low water can not be 0 */
1951                 if( (!hw->fc.high_water[tc_num])|| (!hw->fc.low_water[tc_num])) {
1952                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
1953                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
1954                         goto out;
1955                 }
1956
1957                 if(hw->fc.low_water[tc_num] >= hw->fc.high_water[tc_num]) {
1958                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
1959                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
1960                         goto out;
1961                 }
1962         }
1963         /* Negotiate the fc mode to use */
1964         ixgbe_fc_autoneg(hw);
1965
1966         /* Disable any previous flow control settings */
1967         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
1968         mflcn_reg &= ~(IXGBE_MFLCN_RPFCE_SHIFT | IXGBE_MFLCN_RFCE|IXGBE_MFLCN_RPFCE);
1969
1970         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
1971         fccfg_reg &= ~(IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY);
1972
1973         switch (hw->fc.current_mode) {
1974         case ixgbe_fc_none:
1975                 /*
1976                  * If the count of enabled RX Priority Flow control >1,
1977                  * and the TX pause can not be disabled
1978                  */
1979                 nb_rx_en = 0;
1980                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
1981                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
1982                         if (reg & IXGBE_FCRTH_FCEN)
1983                                 nb_rx_en++;
1984                 }
1985                 if (nb_rx_en > 1)
1986                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
1987                 break;
1988         case ixgbe_fc_rx_pause:
1989                 /*
1990                  * Rx Flow control is enabled and Tx Flow control is
1991                  * disabled by software override. Since there really
1992                  * isn't a way to advertise that we are capable of RX
1993                  * Pause ONLY, we will advertise that we support both
1994                  * symmetric and asymmetric Rx PAUSE.  Later, we will
1995                  * disable the adapter's ability to send PAUSE frames.
1996                  */
1997                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
1998                 /*
1999                  * If the count of enabled RX Priority Flow control >1,
2000                  * and the TX pause can not be disabled
2001                  */
2002                 nb_rx_en = 0;
2003                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
2004                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
2005                         if (reg & IXGBE_FCRTH_FCEN)
2006                                 nb_rx_en++;
2007                 }
2008                 if (nb_rx_en > 1)
2009                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2010                 break;
2011         case ixgbe_fc_tx_pause:
2012                 /*
2013                  * Tx Flow control is enabled, and Rx Flow control is
2014                  * disabled by software override.
2015                  */
2016                 fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2017                 break;
2018         case ixgbe_fc_full:
2019                 /* Flow control (both Rx and Tx) is enabled by SW override. */
2020                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
2021                 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
2022                 break;
2023         default:
2024                 DEBUGOUT("Flow control param set incorrectly\n");
2025                 ret_val = IXGBE_ERR_CONFIG;
2026                 goto out;
2027                 break;
2028         }
2029
2030         /* Set 802.3x based flow control settings. */
2031         mflcn_reg |= IXGBE_MFLCN_DPF;
2032         IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg);
2033         IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg);
2034
2035         /* Set up and enable Rx high/low water mark thresholds, enable XON. */
2036         if ((hw->fc.current_mode & ixgbe_fc_tx_pause) &&
2037                 hw->fc.high_water[tc_num]) {
2038                 fcrtl = (hw->fc.low_water[tc_num] << 10) | IXGBE_FCRTL_XONE;
2039                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), fcrtl);
2040                 fcrth = (hw->fc.high_water[tc_num] << 10) | IXGBE_FCRTH_FCEN;
2041         } else {
2042                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), 0);
2043                 /*
2044                  * In order to prevent Tx hangs when the internal Tx
2045                  * switch is enabled we must set the high water mark
2046                  * to the maximum FCRTH value.  This allows the Tx
2047                  * switch to function even under heavy Rx workloads.
2048                  */
2049                 fcrth = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)) - 32;
2050         }
2051         IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(tc_num), fcrth);
2052
2053         /* Configure pause time (2 TCs per register) */
2054         reg = hw->fc.pause_time * 0x00010001;
2055         for (i = 0; i < (IXGBE_DCB_MAX_TRAFFIC_CLASS / 2); i++)
2056                 IXGBE_WRITE_REG(hw, IXGBE_FCTTV(i), reg);
2057
2058         /* Configure flow control refresh threshold value */
2059         IXGBE_WRITE_REG(hw, IXGBE_FCRTV, hw->fc.pause_time / 2);
2060
2061 out:
2062         return ret_val;
2063 }
2064
2065 static int
2066 ixgbe_dcb_pfc_enable(struct rte_eth_dev *dev,uint8_t tc_num)
2067 {
2068         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2069         int32_t ret_val = IXGBE_NOT_IMPLEMENTED;
2070
2071         if(hw->mac.type != ixgbe_mac_82598EB) {
2072                 ret_val = ixgbe_dcb_pfc_enable_generic(hw,tc_num);
2073         }
2074         return ret_val;
2075 }
2076
2077 static int
2078 ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf)
2079 {
2080         int err;
2081         uint32_t rx_buf_size;
2082         uint32_t max_high_water;
2083         uint8_t tc_num;
2084         uint8_t  map[IXGBE_DCB_MAX_USER_PRIORITY] = { 0 };
2085         struct ixgbe_hw *hw =
2086                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2087         struct ixgbe_dcb_config *dcb_config =
2088                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
2089
2090         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
2091                 ixgbe_fc_none,
2092                 ixgbe_fc_rx_pause,
2093                 ixgbe_fc_tx_pause,
2094                 ixgbe_fc_full
2095         };
2096
2097         PMD_INIT_FUNC_TRACE();
2098
2099         ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
2100         tc_num = map[pfc_conf->priority];
2101         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num));
2102         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
2103         /*
2104          * At least reserve one Ethernet frame for watermark
2105          * high_water/low_water in kilo bytes for ixgbe
2106          */
2107         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
2108         if ((pfc_conf->fc.high_water > max_high_water) ||
2109                 (pfc_conf->fc.high_water <= pfc_conf->fc.low_water)) {
2110                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
2111                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
2112                 return (-EINVAL);
2113         }
2114
2115         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[pfc_conf->fc.mode];
2116         hw->fc.pause_time = pfc_conf->fc.pause_time;
2117         hw->fc.send_xon = pfc_conf->fc.send_xon;
2118         hw->fc.low_water[tc_num] =  pfc_conf->fc.low_water;
2119         hw->fc.high_water[tc_num] = pfc_conf->fc.high_water;
2120
2121         err = ixgbe_dcb_pfc_enable(dev,tc_num);
2122
2123         /* Not negotiated is not an error case */
2124         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED))
2125                 return 0;
2126
2127         PMD_INIT_LOG(ERR, "ixgbe_dcb_pfc_enable = 0x%x \n", err);
2128         return -EIO;
2129 }
2130
2131 static void
2132 ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
2133                                 uint32_t index, uint32_t pool)
2134 {
2135         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2136         uint32_t enable_addr = 1;
2137
2138         ixgbe_set_rar(hw, index, mac_addr->addr_bytes, pool, enable_addr);
2139 }
2140
2141 static void
2142 ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
2143 {
2144         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2145
2146         ixgbe_clear_rar(hw, index);
2147 }
2148
2149 /*
2150  * Virtual Function operations
2151  */
2152 static void
2153 ixgbevf_intr_disable(struct ixgbe_hw *hw)
2154 {
2155         PMD_INIT_LOG(DEBUG, "ixgbevf_intr_disable");
2156
2157         /* Clear interrupt mask to stop from interrupts being generated */
2158         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, IXGBE_VF_IRQ_CLEAR_MASK);
2159
2160         IXGBE_WRITE_FLUSH(hw);
2161 }
2162
2163 static int
2164 ixgbevf_dev_configure(struct rte_eth_dev *dev)
2165 {
2166         struct rte_eth_conf* conf = &dev->data->dev_conf;
2167
2168         PMD_INIT_LOG(DEBUG, "\nConfigured Virtual Function port id: %d\n",
2169                 dev->data->port_id);
2170
2171         /*
2172          * VF has no ability to enable/disable HW CRC
2173          * Keep the persistent behavior the same as Host PF
2174          */
2175 #ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC
2176         if (!conf->rxmode.hw_strip_crc) {
2177                 PMD_INIT_LOG(INFO, "VF can't disable HW CRC Strip\n");
2178                 conf->rxmode.hw_strip_crc = 1;
2179         }
2180 #else
2181         if (conf->rxmode.hw_strip_crc) {
2182                 PMD_INIT_LOG(INFO, "VF can't enable HW CRC Strip\n");
2183                 conf->rxmode.hw_strip_crc = 0;
2184         }
2185 #endif
2186
2187         return 0;
2188 }
2189
2190 static int
2191 ixgbevf_dev_start(struct rte_eth_dev *dev)
2192 {
2193         struct ixgbe_hw *hw =
2194                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2195         int err, mask = 0;
2196
2197         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_start");
2198
2199         hw->mac.ops.reset_hw(hw);
2200
2201         ixgbevf_dev_tx_init(dev);
2202
2203         /* This can fail when allocating mbufs for descriptor rings */
2204         err = ixgbevf_dev_rx_init(dev);
2205         if (err) {
2206                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware (%d)\n", err);
2207                 ixgbe_dev_clear_queues(dev);
2208                 return err;
2209         }
2210
2211         /* Set vfta */
2212         ixgbevf_set_vfta_all(dev,1);
2213
2214         /* Set HW strip */
2215         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
2216                 ETH_VLAN_EXTEND_MASK;
2217         ixgbevf_vlan_offload_set(dev, mask);
2218
2219         ixgbevf_dev_rxtx_start(dev);
2220
2221         return 0;
2222 }
2223
2224 static void
2225 ixgbevf_dev_stop(struct rte_eth_dev *dev)
2226 {
2227         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2228
2229         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_stop");
2230
2231         hw->adapter_stopped = TRUE;
2232         ixgbe_stop_adapter(hw);
2233
2234         /*
2235           * Clear what we set, but we still keep shadow_vfta to
2236           * restore after device starts
2237           */
2238         ixgbevf_set_vfta_all(dev,0);
2239
2240         ixgbe_dev_clear_queues(dev);
2241 }
2242
2243 static void
2244 ixgbevf_dev_close(struct rte_eth_dev *dev)
2245 {
2246         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2247
2248         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_close");
2249
2250         ixgbe_reset_hw(hw);
2251
2252         ixgbevf_dev_stop(dev);
2253
2254         /* reprogram the RAR[0] in case user changed it. */
2255         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
2256 }
2257
2258 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on)
2259 {
2260         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2261         struct ixgbe_vfta * shadow_vfta =
2262                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2263         int i = 0, j = 0, vfta = 0, mask = 1;
2264
2265         for (i = 0; i < IXGBE_VFTA_SIZE; i++){
2266                 vfta = shadow_vfta->vfta[i];
2267                 if(vfta){
2268                         mask = 1;
2269                         for (j = 0; j < 32; j++){
2270                                 if(vfta & mask)
2271                                         ixgbe_set_vfta(hw, (i<<5)+j, 0, on);
2272                                 mask<<=1;
2273                         }
2274                 }
2275         }
2276
2277 }
2278
2279 static int
2280 ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2281 {
2282         struct ixgbe_hw *hw =
2283                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2284         struct ixgbe_vfta * shadow_vfta =
2285                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2286         uint32_t vid_idx = 0;
2287         uint32_t vid_bit = 0;
2288         int ret = 0;
2289
2290         PMD_INIT_FUNC_TRACE();
2291
2292         /* vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf */
2293         ret = ixgbe_set_vfta(hw, vlan_id, 0, !!on);
2294         if(ret){
2295                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
2296                 return ret;
2297         }
2298         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
2299         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
2300
2301         /* Save what we set and retore it after device reset */
2302         if (on)
2303                 shadow_vfta->vfta[vid_idx] |= vid_bit;
2304         else
2305                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
2306
2307         return 0;
2308 }
2309
2310 static void
2311 ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
2312 {
2313         struct ixgbe_hw *hw =
2314                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2315         uint32_t ctrl;
2316
2317         PMD_INIT_FUNC_TRACE();
2318
2319         if(queue >= hw->mac.max_rx_queues)
2320                 return;
2321
2322         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
2323         if(on)
2324                 ctrl |= IXGBE_RXDCTL_VME;
2325         else
2326                 ctrl &= ~IXGBE_RXDCTL_VME;
2327         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
2328
2329         ixgbe_vlan_hw_strip_bitmap_set( dev, queue, on);
2330 }
2331
2332 static void
2333 ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2334 {
2335         struct ixgbe_hw *hw =
2336                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2337         uint16_t i;
2338         int on = 0;
2339
2340         /* VF function only support hw strip feature, others are not support */
2341         if(mask & ETH_VLAN_STRIP_MASK){
2342                 on = !!(dev->data->dev_conf.rxmode.hw_vlan_strip);
2343
2344                 for(i=0; i < hw->mac.max_rx_queues; i++)
2345                         ixgbevf_vlan_strip_queue_set(dev,i,on);
2346         }
2347 }