lib/librte_pmd_ixgbe/ixgbe_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2013 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  *
  33  */
  34
  35 #include <sys/queue.h>
  36 #include <stdio.h>
  37 #include <errno.h>
  38 #include <stdint.h>
  39 #include <string.h>
  40 #include <unistd.h>
  41 #include <stdarg.h>
  42 #include <inttypes.h>
  43 #include <rte_byteorder.h>
  44 #include <rte_common.h>
  45 #include <rte_cycles.h>
  46
  47 #include <rte_interrupts.h>
  48 #include <rte_log.h>
  49 #include <rte_debug.h>
  50 #include <rte_pci.h>
  51 #include <rte_atomic.h>
  52 #include <rte_branch_prediction.h>
  53 #include <rte_memory.h>
  54 #include <rte_memzone.h>
  55 #include <rte_tailq.h>
  56 #include <rte_eal.h>
  57 #include <rte_alarm.h>
  58 #include <rte_ether.h>
  59 #include <rte_ethdev.h>
  60 #include <rte_atomic.h>
  61 #include <rte_malloc.h>
  62
  63 #include "ixgbe_logs.h"
  64 #include "ixgbe/ixgbe_api.h"
  65 #include "ixgbe/ixgbe_vf.h"
  66 #include "ixgbe/ixgbe_common.h"
  67 #include "ixgbe_ethdev.h"
  68
  69 /*
  70  * High threshold controlling when to start sending XOFF frames. Must be at
  71  * least 8 bytes less than receive packet buffer size. This value is in units
  72  * of 1024 bytes.
  73  */
  74 #define IXGBE_FC_HI    0x80
  75
  76 /*
  77  * Low threshold controlling when to start sending XON frames. This value is
  78  * in units of 1024 bytes.
  79  */
  80 #define IXGBE_FC_LO    0x40
  81
  82 /* Timer value included in XOFF frames. */
  83 #define IXGBE_FC_PAUSE 0x680
  84
  85 #define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */
  86 #define IXGBE_LINK_UP_CHECK_TIMEOUT   1000 /* ms */
  87
  88 #define IXGBE_QUEUE_STAT_COUNTERS (sizeof(hw_stats->qprc) / sizeof(hw_stats->qprc[0]))
  89
  90 static int eth_ixgbe_dev_init(struct eth_driver *eth_drv,
  91                 struct rte_eth_dev *eth_dev);
  92 static int  ixgbe_dev_configure(struct rte_eth_dev *dev);
  93 static int  ixgbe_dev_start(struct rte_eth_dev *dev);
  94 static void ixgbe_dev_stop(struct rte_eth_dev *dev);
  95 static void ixgbe_dev_close(struct rte_eth_dev *dev);
  96 static void ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev);
  97 static void ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev);
  98 static void ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev);
  99 static void ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev);
 100 static int ixgbe_dev_link_update(struct rte_eth_dev *dev,
 101                                 int wait_to_complete);
 102 static void ixgbe_dev_stats_get(struct rte_eth_dev *dev,
 103                                 struct rte_eth_stats *stats);
 104 static void ixgbe_dev_stats_reset(struct rte_eth_dev *dev);
 105 static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 106                                              uint16_t queue_id,
 107                                              uint8_t stat_idx,
 108                                              uint8_t is_rx);
 109 static void ixgbe_dev_info_get(struct rte_eth_dev *dev,
 110                                 struct rte_eth_dev_info *dev_info);
 111 static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev,
 112                 uint16_t vlan_id, int on);
 113 static void ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 114 static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev,
 115                 uint16_t queue, bool on);
 116 static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue,
 117                 int on);
 118 static void ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 119 static void ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev);
 120 static void ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev);
 121 static void ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev);
 122 static void ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev);
 123 static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue);
 124 static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue);
 125 static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 126 static void ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 127
 128 static int ixgbe_dev_led_on(struct rte_eth_dev *dev);
 129 static int ixgbe_dev_led_off(struct rte_eth_dev *dev);
 130 static int  ixgbe_flow_ctrl_set(struct rte_eth_dev *dev,
 131                 struct rte_eth_fc_conf *fc_conf);
 132 static int ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 133                 struct rte_eth_pfc_conf *pfc_conf);
 134 static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev);
 135 static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev);
 136 static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev);
 137 static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev);
 138 static void ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle,
 139                 void *param);
 140 static void ixgbe_dev_interrupt_delayed_handler(void *param);
 141 static void ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 142                 uint32_t index, uint32_t pool);
 143 static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index);
 144 static void ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config);
 145
 146 /* For Virtual Function support */
 147 static int eth_ixgbevf_dev_init(struct eth_driver *eth_drv,
 148                 struct rte_eth_dev *eth_dev);
 149 static int  ixgbevf_dev_configure(struct rte_eth_dev *dev);
 150 static int  ixgbevf_dev_start(struct rte_eth_dev *dev);
 151 static void ixgbevf_dev_stop(struct rte_eth_dev *dev);
 152 static void ixgbevf_dev_close(struct rte_eth_dev *dev);
 153 static void ixgbevf_intr_disable(struct ixgbe_hw *hw);
 154 static void ixgbevf_dev_stats_get(struct rte_eth_dev *dev,
 155                 struct rte_eth_stats *stats);
 156 static void ixgbevf_dev_stats_reset(struct rte_eth_dev *dev);
 157 static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev,
 158                 uint16_t vlan_id, int on);
 159 static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev,
 160                 uint16_t queue, int on);
 161 static void ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 162 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 163
 164
 165 /*
 166  * Define VF Stats MACRO for Non "cleared on read" register
 167  */
 168 #define UPDATE_VF_STAT(reg, last, cur)                          \
 169 {                                                               \
 170         u32 latest = IXGBE_READ_REG(hw, reg);                   \
 171         cur += latest - last;                                   \
 172         last = latest;                                          \
 173 }
 174
 175 #define UPDATE_VF_STAT_36BIT(lsb, msb, last, cur)                \
 176 {                                                                \
 177         u64 new_lsb = IXGBE_READ_REG(hw, lsb);                   \
 178         u64 new_msb = IXGBE_READ_REG(hw, msb);                   \
 179         u64 latest = ((new_msb << 32) | new_lsb);                \
 180         cur += (0x1000000000LL + latest - last) & 0xFFFFFFFFFLL; \
 181         last = latest;                                           \
 182 }
 183
 184 #define IXGBE_SET_HWSTRIP(h, q) do{\
 185                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 186                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 187                 (h)->bitmap[idx] |= 1 << bit;\
 188         }while(0)
 189
 190 #define IXGBE_CLEAR_HWSTRIP(h, q) do{\
 191                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 192                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 193                 (h)->bitmap[idx] &= ~(1 << bit);\
 194         }while(0)
 195
 196 #define IXGBE_GET_HWSTRIP(h, q, r) do{\
 197                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 198                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 199                 (r) = (h)->bitmap[idx] >> bit & 1;\
 200         }while(0)
 201
 202 /*
 203  * The set of PCI devices this driver supports
 204  */
 205 static struct rte_pci_id pci_id_ixgbe_map[] = {
 206
 207 #define RTE_PCI_DEV_ID_DECL_IXGBE(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 208 #include "rte_pci_dev_ids.h"
 209
 210 { .vendor_id = 0, /* sentinel */ },
 211 };
 212
 213
 214 /*
 215  * The set of PCI devices this driver supports (for 82599 VF)
 216  */
 217 static struct rte_pci_id pci_id_ixgbevf_map[] = {
 218
 219 #define RTE_PCI_DEV_ID_DECL_IXGBEVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 220 #include "rte_pci_dev_ids.h"
 221 { .vendor_id = 0, /* sentinel */ },
 222
 223 };
 224
 225 static struct eth_dev_ops ixgbe_eth_dev_ops = {
 226         .dev_configure        = ixgbe_dev_configure,
 227         .dev_start            = ixgbe_dev_start,
 228         .dev_stop             = ixgbe_dev_stop,
 229         .dev_close            = ixgbe_dev_close,
 230         .promiscuous_enable   = ixgbe_dev_promiscuous_enable,
 231         .promiscuous_disable  = ixgbe_dev_promiscuous_disable,
 232         .allmulticast_enable  = ixgbe_dev_allmulticast_enable,
 233         .allmulticast_disable = ixgbe_dev_allmulticast_disable,
 234         .link_update          = ixgbe_dev_link_update,
 235         .stats_get            = ixgbe_dev_stats_get,
 236         .stats_reset          = ixgbe_dev_stats_reset,
 237         .queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set,
 238         .dev_infos_get        = ixgbe_dev_info_get,
 239         .vlan_filter_set      = ixgbe_vlan_filter_set,
 240         .vlan_tpid_set        = ixgbe_vlan_tpid_set,
 241         .vlan_offload_set     = ixgbe_vlan_offload_set,
 242         .vlan_strip_queue_set = ixgbe_vlan_strip_queue_set,
 243         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 244         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 245         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 246         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 247         .dev_led_on           = ixgbe_dev_led_on,
 248         .dev_led_off          = ixgbe_dev_led_off,
 249         .flow_ctrl_set        = ixgbe_flow_ctrl_set,
 250         .priority_flow_ctrl_set = ixgbe_priority_flow_ctrl_set,
 251         .mac_addr_add         = ixgbe_add_rar,
 252         .mac_addr_remove      = ixgbe_remove_rar,
 253         .fdir_add_signature_filter    = ixgbe_fdir_add_signature_filter,
 254         .fdir_update_signature_filter = ixgbe_fdir_update_signature_filter,
 255         .fdir_remove_signature_filter = ixgbe_fdir_remove_signature_filter,
 256         .fdir_infos_get               = ixgbe_fdir_info_get,
 257         .fdir_add_perfect_filter      = ixgbe_fdir_add_perfect_filter,
 258         .fdir_update_perfect_filter   = ixgbe_fdir_update_perfect_filter,
 259         .fdir_remove_perfect_filter   = ixgbe_fdir_remove_perfect_filter,
 260         .fdir_set_masks               = ixgbe_fdir_set_masks,
 261 };
 262
 263 /*
 264  * dev_ops for virtual function, bare necessities for basic vf
 265  * operation have been implemented
 266  */
 267 static struct eth_dev_ops ixgbevf_eth_dev_ops = {
 268
 269         .dev_configure        = ixgbevf_dev_configure,
 270         .dev_start            = ixgbevf_dev_start,
 271         .dev_stop             = ixgbevf_dev_stop,
 272         .link_update          = ixgbe_dev_link_update,
 273         .stats_get            = ixgbevf_dev_stats_get,
 274         .stats_reset          = ixgbevf_dev_stats_reset,
 275         .dev_close            = ixgbevf_dev_close,
 276         .dev_infos_get        = ixgbe_dev_info_get,
 277         .vlan_filter_set      = ixgbevf_vlan_filter_set,
 278         .vlan_strip_queue_set = ixgbevf_vlan_strip_queue_set,
 279         .vlan_offload_set     = ixgbevf_vlan_offload_set,
 280         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 281         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 282         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 283         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 284 };
 285
 286 /**
 287  * Atomically reads the link status information from global
 288  * structure rte_eth_dev.
 289  *
 290  * @param dev
 291  *   - Pointer to the structure rte_eth_dev to read from.
 292  *   - Pointer to the buffer to be saved with the link status.
 293  *
 294  * @return
 295  *   - On success, zero.
 296  *   - On failure, negative value.
 297  */
 298 static inline int
 299 rte_ixgbe_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 300                                 struct rte_eth_link *link)
 301 {
 302         struct rte_eth_link *dst = link;
 303         struct rte_eth_link *src = &(dev->data->dev_link);
 304
 305         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 306                                         *(uint64_t *)src) == 0)
 307                 return -1;
 308
 309         return 0;
 310 }
 311
 312 /**
 313  * Atomically writes the link status information into global
 314  * structure rte_eth_dev.
 315  *
 316  * @param dev
 317  *   - Pointer to the structure rte_eth_dev to read from.
 318  *   - Pointer to the buffer to be saved with the link status.
 319  *
 320  * @return
 321  *   - On success, zero.
 322  *   - On failure, negative value.
 323  */
 324 static inline int
 325 rte_ixgbe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 326                                 struct rte_eth_link *link)
 327 {
 328         struct rte_eth_link *dst = &(dev->data->dev_link);
 329         struct rte_eth_link *src = link;
 330
 331         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 332                                         *(uint64_t *)src) == 0)
 333                 return -1;
 334
 335         return 0;
 336 }
 337
 338 /*
 339  * This function is the same as ixgbe_is_sfp() in ixgbe/ixgbe.h.
 340  */
 341 static inline int
 342 ixgbe_is_sfp(struct ixgbe_hw *hw)
 343 {
 344         switch (hw->phy.type) {
 345         case ixgbe_phy_sfp_avago:
 346         case ixgbe_phy_sfp_ftl:
 347         case ixgbe_phy_sfp_intel:
 348         case ixgbe_phy_sfp_unknown:
 349         case ixgbe_phy_sfp_passive_tyco:
 350         case ixgbe_phy_sfp_passive_unknown:
 351                 return 1;
 352         default:
 353                 return 0;
 354         }
 355 }
 356
 357 static inline void
 358 ixgbe_enable_intr(struct rte_eth_dev *dev)
 359 {
 360         struct ixgbe_interrupt *intr =
 361                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 362         struct ixgbe_hw *hw =
 363                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 364
 365         IXGBE_WRITE_REG(hw, IXGBE_EIMS, intr->mask);
 366         IXGBE_WRITE_FLUSH(hw);
 367 }
 368
 369 /*
 370  * This function is based on ixgbe_disable_intr() in ixgbe/ixgbe.h.
 371  */
 372 static void
 373 ixgbe_disable_intr(struct ixgbe_hw *hw)
 374 {
 375         PMD_INIT_FUNC_TRACE();
 376
 377         if (hw->mac.type == ixgbe_mac_82598EB) {
 378                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ~0);
 379         } else {
 380                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xFFFF0000);
 381                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), ~0);
 382                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), ~0);
 383         }
 384         IXGBE_WRITE_FLUSH(hw);
 385 }
 386
 387 /*
 388  * This function resets queue statistics mapping registers.
 389  * From Niantic datasheet, Initialization of Statistics section:
 390  * "...if software requires the queue counters, the RQSMR and TQSM registers
 391  * must be re-programmed following a device reset.
 392  */
 393 static void
 394 ixgbe_reset_qstat_mappings(struct ixgbe_hw *hw)
 395 {
 396         uint32_t i;
 397
 398         for(i = 0; i != IXGBE_NB_STAT_MAPPING_REGS; i++) {
 399                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), 0);
 400                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), 0);
 401         }
 402 }
 403
 404
 405 static int
 406 ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 407                                   uint16_t queue_id,
 408                                   uint8_t stat_idx,
 409                                   uint8_t is_rx)
 410 {
 411 #define QSM_REG_NB_BITS_PER_QMAP_FIELD 8
 412 #define NB_QMAP_FIELDS_PER_QSM_REG 4
 413 #define QMAP_FIELD_RESERVED_BITS_MASK 0x0f
 414
 415         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 416         struct ixgbe_stat_mapping_registers *stat_mappings =
 417                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(eth_dev->data->dev_private);
 418         uint32_t qsmr_mask = 0;
 419         uint32_t clearing_mask = QMAP_FIELD_RESERVED_BITS_MASK;
 420         uint32_t q_map;
 421         uint8_t n, offset;
 422
 423         if ((hw->mac.type != ixgbe_mac_82599EB) && (hw->mac.type != ixgbe_mac_X540))
 424                 return -ENOSYS;
 425
 426         PMD_INIT_LOG(INFO, "Setting port %d, %s queue_id %d to stat index %d\n",
 427                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx);
 428
 429         n = (uint8_t)(queue_id / NB_QMAP_FIELDS_PER_QSM_REG);
 430         if (n >= IXGBE_NB_STAT_MAPPING_REGS) {
 431                 PMD_INIT_LOG(ERR, "Nb of stat mapping registers exceeded\n");
 432                 return -EIO;
 433         }
 434         offset = (uint8_t)(queue_id % NB_QMAP_FIELDS_PER_QSM_REG);
 435
 436         /* Now clear any previous stat_idx set */
 437         clearing_mask <<= (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 438         if (!is_rx)
 439                 stat_mappings->tqsm[n] &= ~clearing_mask;
 440         else
 441                 stat_mappings->rqsmr[n] &= ~clearing_mask;
 442
 443         q_map = (uint32_t)stat_idx;
 444         q_map &= QMAP_FIELD_RESERVED_BITS_MASK;
 445         qsmr_mask = q_map << (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 446         if (!is_rx)
 447                 stat_mappings->tqsm[n] |= qsmr_mask;
 448         else
 449                 stat_mappings->rqsmr[n] |= qsmr_mask;
 450
 451         PMD_INIT_LOG(INFO, "Set port %d, %s queue_id %d to stat index %d\n"
 452                      "%s[%d] = 0x%08x\n",
 453                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx,
 454                      is_rx ? "RQSMR" : "TQSM",n, is_rx ? stat_mappings->rqsmr[n] : stat_mappings->tqsm[n]);
 455
 456         /* Now write the mapping in the appropriate register */
 457         if (is_rx) {
 458                 PMD_INIT_LOG(INFO, "Write 0x%x to RX IXGBE stat mapping reg:%d\n",
 459                              stat_mappings->rqsmr[n], n);
 460                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(n), stat_mappings->rqsmr[n]);
 461         }
 462         else {
 463                 PMD_INIT_LOG(INFO, "Write 0x%x to TX IXGBE stat mapping reg:%d\n",
 464                              stat_mappings->tqsm[n], n);
 465                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(n), stat_mappings->tqsm[n]);
 466         }
 467         return 0;
 468 }
 469
 470 static void
 471 ixgbe_restore_statistics_mapping(struct rte_eth_dev * dev)
 472 {
 473         struct ixgbe_stat_mapping_registers *stat_mappings =
 474                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(dev->data->dev_private);
 475         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 476         int i;
 477
 478         /* write whatever was in stat mapping table to the NIC */
 479         for (i = 0; i < IXGBE_NB_STAT_MAPPING_REGS; i++) {
 480                 /* rx */
 481                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), stat_mappings->rqsmr[i]);
 482
 483                 /* tx */
 484                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), stat_mappings->tqsm[i]);
 485         }
 486 }
 487
 488 static void
 489 ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config)
 490 {
 491         uint8_t i;
 492         struct ixgbe_dcb_tc_config *tc;
 493         uint8_t dcb_max_tc = IXGBE_DCB_MAX_TRAFFIC_CLASS;
 494
 495         dcb_config->num_tcs.pg_tcs = dcb_max_tc;
 496         dcb_config->num_tcs.pfc_tcs = dcb_max_tc;
 497         for (i = 0; i < dcb_max_tc; i++) {
 498                 tc = &dcb_config->tc_config[i];
 499                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_id = i;
 500                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
 501                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 502                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_id = i;
 503                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
 504                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 505                 tc->pfc = ixgbe_dcb_pfc_disabled;
 506         }
 507
 508         /* Initialize default user to priority mapping, UPx->TC0 */
 509         tc = &dcb_config->tc_config[0];
 510         tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
 511         tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
 512         for (i = 0; i< IXGBE_DCB_MAX_BW_GROUP; i++) {
 513                 dcb_config->bw_percentage[IXGBE_DCB_TX_CONFIG][i] = 100;
 514                 dcb_config->bw_percentage[IXGBE_DCB_RX_CONFIG][i] = 100;
 515         }
 516         dcb_config->rx_pba_cfg = ixgbe_dcb_pba_equal;
 517         dcb_config->pfc_mode_enable = false;
 518         dcb_config->vt_mode = true;
 519         dcb_config->round_robin_enable = false;
 520         /* support all DCB capabilities in 82599 */
 521         dcb_config->support.capabilities = 0xFF;
 522
 523         /*we only support 4 Tcs for X540*/
 524         if (hw->mac.type == ixgbe_mac_X540) {
 525                 dcb_config->num_tcs.pg_tcs = 4;
 526                 dcb_config->num_tcs.pfc_tcs = 4;
 527         }
 528 }
 529
 530 /*
 531  * This function is based on code in ixgbe_attach() in ixgbe/ixgbe.c.
 532  * It returns 0 on success.
 533  */
 534 static int
 535 eth_ixgbe_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 536                      struct rte_eth_dev *eth_dev)
 537 {
 538         struct rte_pci_device *pci_dev;
 539         struct ixgbe_hw *hw =
 540                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 541         struct ixgbe_vfta * shadow_vfta =
 542                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 543         struct ixgbe_hwstrip *hwstrip =
 544                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 545         struct ixgbe_dcb_config *dcb_config =
 546                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private);
 547         uint32_t ctrl_ext;
 548         uint16_t csum;
 549         int diag, i;
 550
 551         PMD_INIT_FUNC_TRACE();
 552
 553         eth_dev->dev_ops = &ixgbe_eth_dev_ops;
 554         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
 555         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
 556
 557         /* for secondary processes, we don't initialise any further as primary
 558          * has already done this work. Only check we don't need a different
 559          * RX function */
 560         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 561                 if (eth_dev->data->scattered_rx)
 562                         eth_dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
 563                 return 0;
 564         }
 565         pci_dev = eth_dev->pci_dev;
 566
 567         /* Vendor and Device ID need to be set before init of shared code */
 568         hw->device_id = pci_dev->id.device_id;
 569         hw->vendor_id = pci_dev->id.vendor_id;
 570         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 571 #ifdef RTE_LIBRTE_IXGBE_ALLOW_UNSUPPORTED_SFP
 572         hw->allow_unsupported_sfp = 1;
 573 #endif
 574
 575         /* Initialize the shared code */
 576         diag = ixgbe_init_shared_code(hw);
 577         if (diag != IXGBE_SUCCESS) {
 578                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
 579                 return -EIO;
 580         }
 581
 582         /* Initialize DCB configuration*/
 583         memset(dcb_config, 0, sizeof(struct ixgbe_dcb_config));
 584         ixgbe_dcb_init(hw,dcb_config);
 585         /* Get Hardware Flow Control setting */
 586         hw->fc.requested_mode = ixgbe_fc_full;
 587         hw->fc.current_mode = ixgbe_fc_full;
 588         hw->fc.pause_time = IXGBE_FC_PAUSE;
 589         for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
 590                 hw->fc.low_water[i] = IXGBE_FC_LO;
 591                 hw->fc.high_water[i] = IXGBE_FC_HI;
 592         }
 593         hw->fc.send_xon = 1;
 594
 595         /* Make sure we have a good EEPROM before we read from it */
 596         diag = ixgbe_validate_eeprom_checksum(hw, &csum);
 597         if (diag != IXGBE_SUCCESS) {
 598                 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag);
 599                 return -EIO;
 600         }
 601
 602         diag = ixgbe_init_hw(hw);
 603
 604         /*
 605          * Devices with copper phys will fail to initialise if ixgbe_init_hw()
 606          * is called too soon after the kernel driver unbinding/binding occurs.
 607          * The failure occurs in ixgbe_identify_phy_generic() for all devices,
 608          * but for non-copper devies, ixgbe_identify_sfp_module_generic() is
 609          * also called. See ixgbe_identify_phy_82599(). The reason for the
 610          * failure is not known, and only occuts when virtualisation features
 611          * are disabled in the bios. A delay of 100ms  was found to be enough by
 612          * trial-and-error, and is doubled to be safe.
 613          */
 614         if (diag && (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper)) {
 615                 rte_delay_ms(200);
 616                 diag = ixgbe_init_hw(hw);
 617         }
 618
 619         if (diag == IXGBE_ERR_EEPROM_VERSION) {
 620                 PMD_INIT_LOG(ERR, "This device is a pre-production adapter/"
 621                     "LOM.  Please be aware there may be issues associated "
 622                     "with your hardware.\n If you are experiencing problems "
 623                     "please contact your Intel or hardware representative "
 624                     "who provided you with this hardware.\n");
 625         } else if (diag == IXGBE_ERR_SFP_NOT_SUPPORTED)
 626                 PMD_INIT_LOG(ERR, "Unsupported SFP+ Module\n");
 627         if (diag) {
 628                 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag);
 629                 return -EIO;
 630         }
 631
 632         /* disable interrupt */
 633         ixgbe_disable_intr(hw);
 634
 635         /* pick up the PCI bus settings for reporting later */
 636         ixgbe_get_bus_info(hw);
 637
 638         /* reset mappings for queue statistics hw counters*/
 639         ixgbe_reset_qstat_mappings(hw);
 640
 641         /* Allocate memory for storing MAC addresses */
 642         eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 643                         hw->mac.num_rar_entries, 0);
 644         if (eth_dev->data->mac_addrs == NULL) {
 645                 PMD_INIT_LOG(ERR,
 646                         "Failed to allocate %d bytes needed to store MAC addresses",
 647                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 648                 return -ENOMEM;
 649         }
 650         /* Copy the permanent MAC address */
 651         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 652                         &eth_dev->data->mac_addrs[0]);
 653
 654         /* initialize the vfta */
 655         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 656
 657         /* initialize the hw strip bitmap*/
 658         memset(hwstrip, 0, sizeof(*hwstrip));
 659
 660         /* let hardware know driver is loaded */
 661         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 662         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 663         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 664
 665         if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
 666                 PMD_INIT_LOG(DEBUG,
 667                              "MAC: %d, PHY: %d, SFP+: %d<n",
 668                              (int) hw->mac.type, (int) hw->phy.type,
 669                              (int) hw->phy.sfp_type);
 670         else
 671                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d\n",
 672                              (int) hw->mac.type, (int) hw->phy.type);
 673
 674         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
 675                         eth_dev->data->port_id, pci_dev->id.vendor_id,
 676                         pci_dev->id.device_id);
 677
 678         rte_intr_callback_register(&(pci_dev->intr_handle),
 679                 ixgbe_dev_interrupt_handler, (void *)eth_dev);
 680
 681         /* enable uio intr after callback register */
 682         rte_intr_enable(&(pci_dev->intr_handle));
 683
 684         /* enable support intr */
 685         ixgbe_enable_intr(eth_dev);
 686
 687         return 0;
 688 }
 689
 690 /*
 691  * Virtual Function device init
 692  */
 693 static int
 694 eth_ixgbevf_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 695                      struct rte_eth_dev *eth_dev)
 696 {
 697         struct rte_pci_device *pci_dev;
 698         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 699         int diag;
 700         struct ixgbe_vfta * shadow_vfta =
 701                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 702         struct ixgbe_hwstrip *hwstrip =
 703                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 704
 705         PMD_INIT_LOG(DEBUG, "eth_ixgbevf_dev_init");
 706
 707         eth_dev->dev_ops = &ixgbevf_eth_dev_ops;
 708         pci_dev = eth_dev->pci_dev;
 709
 710         hw->device_id = pci_dev->id.device_id;
 711         hw->vendor_id = pci_dev->id.vendor_id;
 712         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 713
 714         /* initialize the vfta */
 715         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 716
 717         /* initialize the hw strip bitmap*/
 718         memset(hwstrip, 0, sizeof(*hwstrip));
 719
 720         /* Initialize the shared code */
 721         diag = ixgbe_init_shared_code(hw);
 722         if (diag != IXGBE_SUCCESS) {
 723                 PMD_INIT_LOG(ERR, "Shared code init failed for ixgbevf: %d", diag);
 724                 return -EIO;
 725         }
 726
 727         /* init_mailbox_params */
 728         hw->mbx.ops.init_params(hw);
 729
 730         /* Disable the interrupts for VF */
 731         ixgbevf_intr_disable(hw);
 732
 733         hw->mac.num_rar_entries = hw->mac.max_rx_queues;
 734         diag = hw->mac.ops.reset_hw(hw);
 735         if (diag != IXGBE_SUCCESS) {
 736                 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
 737                 return (diag);
 738         }
 739
 740         /* Allocate memory for storing MAC addresses */
 741         eth_dev->data->mac_addrs = rte_zmalloc("ixgbevf", ETHER_ADDR_LEN *
 742                         hw->mac.num_rar_entries, 0);
 743         if (eth_dev->data->mac_addrs == NULL) {
 744                 PMD_INIT_LOG(ERR,
 745                         "Failed to allocate %d bytes needed to store MAC addresses",
 746                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 747                 return -ENOMEM;
 748         }
 749         /* Copy the permanent MAC address */
 750         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 751                         &eth_dev->data->mac_addrs[0]);
 752
 753         /* reset the hardware with the new settings */
 754         diag = hw->mac.ops.start_hw(hw);
 755         switch (diag) {
 756                 case  0:
 757                         break;
 758
 759                 default:
 760                         PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
 761                         return (-EIO);
 762         }
 763
 764         PMD_INIT_LOG(DEBUG, "\nport %d vendorID=0x%x deviceID=0x%x mac.type=%s\n",
 765                          eth_dev->data->port_id, pci_dev->id.vendor_id, pci_dev->id.device_id,
 766                          "ixgbe_mac_82599_vf");
 767
 768         return 0;
 769 }
 770
 771 static struct eth_driver rte_ixgbe_pmd = {
 772         {
 773                 .name = "rte_ixgbe_pmd",
 774                 .id_table = pci_id_ixgbe_map,
 775 #ifdef RTE_EAL_UNBIND_PORTS
 776                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
 777 #endif
 778         },
 779         .eth_dev_init = eth_ixgbe_dev_init,
 780         .dev_private_size = sizeof(struct ixgbe_adapter),
 781 };
 782
 783 /*
 784  * virtual function driver struct
 785  */
 786 static struct eth_driver rte_ixgbevf_pmd = {
 787         {
 788                 .name = "rte_ixgbevf_pmd",
 789                 .id_table = pci_id_ixgbevf_map,
 790 #ifdef RTE_EAL_UNBIND_PORTS
 791                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
 792 #endif
 793         },
 794         .eth_dev_init = eth_ixgbevf_dev_init,
 795         .dev_private_size = sizeof(struct ixgbe_adapter),
 796 };
 797
 798 /*
 799  * Driver initialization routine.
 800  * Invoked once at EAL init time.
 801  * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
 802  */
 803 int
 804 rte_ixgbe_pmd_init(void)
 805 {
 806         PMD_INIT_FUNC_TRACE();
 807
 808         rte_eth_driver_register(&rte_ixgbe_pmd);
 809         return 0;
 810 }
 811
 812 /*
 813  * VF Driver initialization routine.
 814  * Invoked one at EAL init time.
 815  * Register itself as the [Virtual Poll Mode] Driver of PCI niantic devices.
 816  */
 817 int
 818 rte_ixgbevf_pmd_init(void)
 819 {
 820         DEBUGFUNC("rte_ixgbevf_pmd_init");
 821
 822         rte_eth_driver_register(&rte_ixgbevf_pmd);
 823         return (0);
 824 }
 825
 826 static int
 827 ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 828 {
 829         struct ixgbe_hw *hw =
 830                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 831         struct ixgbe_vfta * shadow_vfta =
 832                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
 833         uint32_t vfta;
 834         uint32_t vid_idx;
 835         uint32_t vid_bit;
 836
 837         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
 838         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
 839         vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(vid_idx));
 840         if (on)
 841                 vfta |= vid_bit;
 842         else
 843                 vfta &= ~vid_bit;
 844         IXGBE_WRITE_REG(hw, IXGBE_VFTA(vid_idx), vfta);
 845
 846         /* update local VFTA copy */
 847         shadow_vfta->vfta[vid_idx] = vfta;
 848
 849         return 0;
 850 }
 851
 852 static void
 853 ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
 854 {
 855         if (on)
 856                 ixgbe_vlan_hw_strip_enable(dev, queue);
 857         else
 858                 ixgbe_vlan_hw_strip_disable(dev, queue);
 859 }
 860
 861 static void
 862 ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
 863 {
 864         struct ixgbe_hw *hw =
 865                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 866
 867         /* Only the high 16-bits is valid */
 868         IXGBE_WRITE_REG(hw, IXGBE_EXVET, tpid << 16);
 869 }
 870
 871 static void
 872 ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev)
 873 {
 874         struct ixgbe_hw *hw =
 875                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 876         uint32_t vlnctrl;
 877
 878         PMD_INIT_FUNC_TRACE();
 879
 880         /* Filter Table Disable */
 881         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 882         vlnctrl &= ~IXGBE_VLNCTRL_VFE;
 883
 884         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
 885 }
 886
 887 static void
 888 ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev)
 889 {
 890         struct ixgbe_hw *hw =
 891                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 892         struct ixgbe_vfta * shadow_vfta =
 893                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
 894         uint32_t vlnctrl;
 895         uint16_t i;
 896
 897         PMD_INIT_FUNC_TRACE();
 898
 899         /* Filter Table Enable */
 900         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 901         vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
 902         vlnctrl |= IXGBE_VLNCTRL_VFE;
 903
 904         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
 905
 906         /* write whatever is in local vfta copy */
 907         for (i = 0; i < IXGBE_VFTA_SIZE; i++)
 908                 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), shadow_vfta->vfta[i]);
 909 }
 910
 911 static void
 912 ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on)
 913 {
 914         struct ixgbe_hwstrip *hwstrip =
 915                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(dev->data->dev_private);
 916
 917         if(queue >= IXGBE_MAX_RX_QUEUE_NUM)
 918                 return;
 919
 920         if (on)
 921                 IXGBE_SET_HWSTRIP(hwstrip, queue);
 922         else
 923                 IXGBE_CLEAR_HWSTRIP(hwstrip, queue);
 924 }
 925
 926 static void
 927 ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue)
 928 {
 929         struct ixgbe_hw *hw =
 930                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 931         uint32_t ctrl;
 932
 933         PMD_INIT_FUNC_TRACE();
 934
 935         if (hw->mac.type == ixgbe_mac_82598EB) {
 936                 /* No queue level support */
 937                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
 938                 return;
 939         }
 940         else {
 941                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 942                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
 943                 ctrl &= ~IXGBE_RXDCTL_VME;
 944                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
 945         }
 946         /* record those setting for HW strip per queue */
 947         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 0);
 948 }
 949
 950 static void
 951 ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue)
 952 {
 953         struct ixgbe_hw *hw =
 954                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 955         uint32_t ctrl;
 956
 957         PMD_INIT_FUNC_TRACE();
 958
 959         if (hw->mac.type == ixgbe_mac_82598EB) {
 960                 /* No queue level supported */
 961                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
 962                 return;
 963         }
 964         else {
 965                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 966                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
 967                 ctrl |= IXGBE_RXDCTL_VME;
 968                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
 969         }
 970         /* record those setting for HW strip per queue */
 971         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 1);
 972 }
 973
 974 static void
 975 ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev)
 976 {
 977         struct ixgbe_hw *hw =
 978                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 979         uint32_t ctrl;
 980         uint16_t i;
 981
 982         PMD_INIT_FUNC_TRACE();
 983
 984         if (hw->mac.type == ixgbe_mac_82598EB) {
 985                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 986                 ctrl &= ~IXGBE_VLNCTRL_VME;
 987                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
 988         }
 989         else {
 990                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 991                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
 992                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
 993                         ctrl &= ~IXGBE_RXDCTL_VME;
 994                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
 995
 996                         /* record those setting for HW strip per queue */
 997                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 0);
 998                 }
 999         }
1000 }
1001
1002 static void
1003 ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev)
1004 {
1005         struct ixgbe_hw *hw =
1006                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1007         uint32_t ctrl;
1008         uint16_t i;
1009
1010         PMD_INIT_FUNC_TRACE();
1011
1012         if (hw->mac.type == ixgbe_mac_82598EB) {
1013                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1014                 ctrl |= IXGBE_VLNCTRL_VME;
1015                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1016         }
1017         else {
1018                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1019                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1020                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1021                         ctrl |= IXGBE_RXDCTL_VME;
1022                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1023
1024                         /* record those setting for HW strip per queue */
1025                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 1);
1026                 }
1027         }
1028 }
1029
1030 static void
1031 ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1032 {
1033         struct ixgbe_hw *hw =
1034                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1035         uint32_t ctrl;
1036
1037         PMD_INIT_FUNC_TRACE();
1038
1039         /* DMATXCTRL: Geric Double VLAN Disable */
1040         ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1041         ctrl &= ~IXGBE_DMATXCTL_GDV;
1042         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1043
1044         /* CTRL_EXT: Global Double VLAN Disable */
1045         ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1046         ctrl &= ~IXGBE_EXTENDED_VLAN;
1047         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1048
1049 }
1050
1051 static void
1052 ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1053 {
1054         struct ixgbe_hw *hw =
1055                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1056         uint32_t ctrl;
1057
1058         PMD_INIT_FUNC_TRACE();
1059
1060         /* DMATXCTRL: Geric Double VLAN Enable */
1061         ctrl  = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1062         ctrl |= IXGBE_DMATXCTL_GDV;
1063         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1064
1065         /* CTRL_EXT: Global Double VLAN Enable */
1066         ctrl  = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1067         ctrl |= IXGBE_EXTENDED_VLAN;
1068         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1069
1070         /*
1071          * VET EXT field in the EXVET register = 0x8100 by default
1072          * So no need to change. Same to VT field of DMATXCTL register
1073          */
1074 }
1075
1076 static void
1077 ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1078 {
1079         if(mask & ETH_VLAN_STRIP_MASK){
1080                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1081                         ixgbe_vlan_hw_strip_enable_all(dev);
1082                 else
1083                         ixgbe_vlan_hw_strip_disable_all(dev);
1084         }
1085
1086         if(mask & ETH_VLAN_FILTER_MASK){
1087                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1088                         ixgbe_vlan_hw_filter_enable(dev);
1089                 else
1090                         ixgbe_vlan_hw_filter_disable(dev);
1091         }
1092
1093         if(mask & ETH_VLAN_EXTEND_MASK){
1094                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1095                         ixgbe_vlan_hw_extend_enable(dev);
1096                 else
1097                         ixgbe_vlan_hw_extend_disable(dev);
1098         }
1099 }
1100
1101 static int
1102 ixgbe_dev_configure(struct rte_eth_dev *dev)
1103 {
1104         struct ixgbe_interrupt *intr =
1105                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1106
1107         PMD_INIT_FUNC_TRACE();
1108
1109         /* set flag to update link status after init */
1110         intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1111
1112         return 0;
1113 }
1114
1115 /*
1116  * Configure device link speed and setup link.
1117  * It returns 0 on success.
1118  */
1119 static int
1120 ixgbe_dev_start(struct rte_eth_dev *dev)
1121 {
1122         struct ixgbe_hw *hw =
1123                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1124         int err, link_up = 0, negotiate = 0;
1125         uint32_t speed = 0;
1126         int mask = 0;
1127
1128         PMD_INIT_FUNC_TRACE();
1129
1130         /* IXGBE devices don't support half duplex */
1131         if ((dev->data->dev_conf.link_duplex != ETH_LINK_AUTONEG_DUPLEX) &&
1132                         (dev->data->dev_conf.link_duplex != ETH_LINK_FULL_DUPLEX)) {
1133                 PMD_INIT_LOG(ERR, "Invalid link_duplex (%u) for port %u\n",
1134                                 dev->data->dev_conf.link_duplex,
1135                                 dev->data->port_id);
1136                 return -EINVAL;
1137         }
1138
1139         /* stop adapter */
1140         hw->adapter_stopped = FALSE;
1141         ixgbe_stop_adapter(hw);
1142
1143         /* reinitialize adapter
1144          * this calls reset and start */
1145         ixgbe_init_hw(hw);
1146         hw->mac.ops.start_hw(hw);
1147
1148         /* initialize transmission unit */
1149         ixgbe_dev_tx_init(dev);
1150
1151         /* This can fail when allocating mbufs for descriptor rings */
1152         err = ixgbe_dev_rx_init(dev);
1153         if (err) {
1154                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware\n");
1155                 goto error;
1156         }
1157
1158         ixgbe_dev_rxtx_start(dev);
1159
1160         if (ixgbe_is_sfp(hw) && hw->phy.multispeed_fiber) {
1161                 err = hw->mac.ops.setup_sfp(hw);
1162                 if (err)
1163                         goto error;
1164         }
1165
1166         /* Turn on the laser */
1167         if (hw->phy.multispeed_fiber)
1168                 ixgbe_enable_tx_laser(hw);
1169
1170         err = ixgbe_check_link(hw, &speed, &link_up, 0);
1171         if (err)
1172                 goto error;
1173         err = ixgbe_get_link_capabilities(hw, &speed, &negotiate);
1174         if (err)
1175                 goto error;
1176
1177         switch(dev->data->dev_conf.link_speed) {
1178         case ETH_LINK_SPEED_AUTONEG:
1179                 speed = (hw->mac.type != ixgbe_mac_82598EB) ?
1180                                 IXGBE_LINK_SPEED_82599_AUTONEG :
1181                                 IXGBE_LINK_SPEED_82598_AUTONEG;
1182                 break;
1183         case ETH_LINK_SPEED_100:
1184                 /*
1185                  * Invalid for 82598 but error will be detected by
1186                  * ixgbe_setup_link()
1187                  */
1188                 speed = IXGBE_LINK_SPEED_100_FULL;
1189                 break;
1190         case ETH_LINK_SPEED_1000:
1191                 speed = IXGBE_LINK_SPEED_1GB_FULL;
1192                 break;
1193         case ETH_LINK_SPEED_10000:
1194                 speed = IXGBE_LINK_SPEED_10GB_FULL;
1195                 break;
1196         default:
1197                 PMD_INIT_LOG(ERR, "Invalid link_speed (%u) for port %u\n",
1198                                 dev->data->dev_conf.link_speed, dev->data->port_id);
1199                 goto error;
1200         }
1201
1202         err = ixgbe_setup_link(hw, speed, negotiate, link_up);
1203         if (err)
1204                 goto error;
1205
1206         /* check if lsc interrupt is enabled */
1207         if (dev->data->dev_conf.intr_conf.lsc != 0)
1208                 ixgbe_dev_lsc_interrupt_setup(dev);
1209
1210         /* resume enabled intr since hw reset */
1211         ixgbe_enable_intr(dev);
1212
1213         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
1214                 ETH_VLAN_EXTEND_MASK;
1215         ixgbe_vlan_offload_set(dev, mask);
1216
1217         /* Configure DCB hw */
1218         ixgbe_configure_dcb(dev);
1219
1220         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) {
1221                 err = ixgbe_fdir_configure(dev);
1222                 if (err)
1223                         goto error;
1224         }
1225
1226         ixgbe_restore_statistics_mapping(dev);
1227
1228         return (0);
1229
1230 error:
1231         PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err);
1232         ixgbe_dev_clear_queues(dev);
1233         return -EIO;
1234 }
1235
1236 /*
1237  * Stop device: disable rx and tx functions to allow for reconfiguring.
1238  */
1239 static void
1240 ixgbe_dev_stop(struct rte_eth_dev *dev)
1241 {
1242         struct rte_eth_link link;
1243         struct ixgbe_hw *hw =
1244                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1245
1246         PMD_INIT_FUNC_TRACE();
1247
1248         /* disable interrupts */
1249         ixgbe_disable_intr(hw);
1250
1251         /* reset the NIC */
1252         ixgbe_reset_hw(hw);
1253         hw->adapter_stopped = FALSE;
1254
1255         /* stop adapter */
1256         ixgbe_stop_adapter(hw);
1257
1258         /* Turn off the laser */
1259         if (hw->phy.multispeed_fiber)
1260                 ixgbe_disable_tx_laser(hw);
1261
1262         ixgbe_dev_clear_queues(dev);
1263
1264         /* Clear recorded link status */
1265         memset(&link, 0, sizeof(link));
1266         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1267 }
1268
1269 /*
1270  * Reest and stop device.
1271  */
1272 static void
1273 ixgbe_dev_close(struct rte_eth_dev *dev)
1274 {
1275         struct ixgbe_hw *hw =
1276                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1277
1278         PMD_INIT_FUNC_TRACE();
1279
1280         ixgbe_reset_hw(hw);
1281
1282
1283         ixgbe_dev_stop(dev);
1284         hw->adapter_stopped = 1;
1285
1286         ixgbe_disable_pcie_master(hw);
1287
1288         /* reprogram the RAR[0] in case user changed it. */
1289         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
1290 }
1291
1292 /*
1293  * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c
1294  */
1295 static void
1296 ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1297 {
1298         struct ixgbe_hw *hw =
1299                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1300         struct ixgbe_hw_stats *hw_stats =
1301                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1302         uint32_t bprc, lxon, lxoff, total;
1303         uint64_t total_missed_rx, total_qbrc, total_qprc;
1304         unsigned i;
1305
1306         total_missed_rx = 0;
1307         total_qbrc = 0;
1308         total_qprc = 0;
1309
1310         hw_stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
1311         hw_stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
1312         hw_stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
1313         hw_stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
1314
1315         for (i = 0; i < 8; i++) {
1316                 uint32_t mp;
1317                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
1318                 /* global total per queue */
1319                 hw_stats->mpc[i] += mp;
1320                 /* Running comprehensive total for stats display */
1321                 total_missed_rx += hw_stats->mpc[i];
1322                 if (hw->mac.type == ixgbe_mac_82598EB)
1323                         hw_stats->rnbc[i] +=
1324                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
1325                 hw_stats->pxontxc[i] +=
1326                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
1327                 hw_stats->pxonrxc[i] +=
1328                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
1329                 hw_stats->pxofftxc[i] +=
1330                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
1331                 hw_stats->pxoffrxc[i] +=
1332                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
1333                 hw_stats->pxon2offc[i] +=
1334                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
1335         }
1336         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1337                 hw_stats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
1338                 hw_stats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
1339                 hw_stats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
1340                 hw_stats->qbrc[i] +=
1341                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32);
1342                 hw_stats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
1343                 hw_stats->qbtc[i] +=
1344                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32);
1345                 hw_stats->qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
1346
1347                 total_qprc += hw_stats->qprc[i];
1348                 total_qbrc += hw_stats->qbrc[i];
1349         }
1350         hw_stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
1351         hw_stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
1352         hw_stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
1353
1354         /* Note that gprc counts missed packets */
1355         hw_stats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
1356
1357         if (hw->mac.type != ixgbe_mac_82598EB) {
1358                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
1359                 hw_stats->gorc += ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
1360                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
1361                 hw_stats->gotc += ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
1362                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
1363                 hw_stats->tor += ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
1364                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
1365                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
1366         } else {
1367                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
1368                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
1369                 /* 82598 only has a counter in the high register */
1370                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
1371                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
1372                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
1373         }
1374
1375         /*
1376          * Workaround: mprc hardware is incorrectly counting
1377          * broadcasts, so for now we subtract those.
1378          */
1379         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
1380         hw_stats->bprc += bprc;
1381         hw_stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
1382         if (hw->mac.type == ixgbe_mac_82598EB)
1383                 hw_stats->mprc -= bprc;
1384
1385         hw_stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
1386         hw_stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
1387         hw_stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
1388         hw_stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
1389         hw_stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
1390         hw_stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
1391
1392         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
1393         hw_stats->lxontxc += lxon;
1394         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
1395         hw_stats->lxofftxc += lxoff;
1396         total = lxon + lxoff;
1397
1398         hw_stats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
1399         hw_stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
1400         hw_stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
1401         hw_stats->gptc -= total;
1402         hw_stats->mptc -= total;
1403         hw_stats->ptc64 -= total;
1404         hw_stats->gotc -= total * ETHER_MIN_LEN;
1405
1406         hw_stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
1407         hw_stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
1408         hw_stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
1409         hw_stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
1410         hw_stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
1411         hw_stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
1412         hw_stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
1413         hw_stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
1414         hw_stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
1415         hw_stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
1416         hw_stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
1417         hw_stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
1418         hw_stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
1419         hw_stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
1420         hw_stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
1421         hw_stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);
1422         hw_stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
1423         hw_stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
1424         /* Only read FCOE on 82599 */
1425         if (hw->mac.type != ixgbe_mac_82598EB) {
1426                 hw_stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
1427                 hw_stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
1428                 hw_stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
1429                 hw_stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
1430                 hw_stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
1431         }
1432
1433         if (stats == NULL)
1434                 return;
1435
1436         /* Fill out the rte_eth_stats statistics structure */
1437         stats->ipackets = total_qprc;
1438         stats->ibytes = total_qbrc;
1439         stats->opackets = hw_stats->gptc;
1440         stats->obytes = hw_stats->gotc;
1441         stats->imcasts = hw_stats->mprc;
1442
1443         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1444                 stats->q_ipackets[i] = hw_stats->qprc[i];
1445                 stats->q_opackets[i] = hw_stats->qptc[i];
1446                 stats->q_ibytes[i] = hw_stats->qbrc[i];
1447                 stats->q_obytes[i] = hw_stats->qbtc[i];
1448                 stats->q_errors[i] = hw_stats->qprdc[i];
1449         }
1450
1451         /* Rx Errors */
1452         stats->ierrors = total_missed_rx + hw_stats->crcerrs +
1453                 hw_stats->rlec;
1454
1455         stats->oerrors  = 0;
1456
1457         /* Flow Director Stats registers */
1458         hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
1459         hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
1460         stats->fdirmatch = hw_stats->fdirmatch;
1461         stats->fdirmiss = hw_stats->fdirmiss;
1462 }
1463
1464 static void
1465 ixgbe_dev_stats_reset(struct rte_eth_dev *dev)
1466 {
1467         struct ixgbe_hw_stats *stats =
1468                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1469
1470         /* HW registers are cleared on read */
1471         ixgbe_dev_stats_get(dev, NULL);
1472
1473         /* Reset software totals */
1474         memset(stats, 0, sizeof(*stats));
1475 }
1476
1477 static void
1478 ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1479 {
1480         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1481         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1482                           IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1483
1484         /* Good Rx packet, include VF loopback */
1485         UPDATE_VF_STAT(IXGBE_VFGPRC,
1486             hw_stats->last_vfgprc, hw_stats->vfgprc);
1487
1488         /* Good Rx octets, include VF loopback */
1489         UPDATE_VF_STAT_36BIT(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
1490             hw_stats->last_vfgorc, hw_stats->vfgorc);
1491
1492         /* Good Tx packet, include VF loopback */
1493         UPDATE_VF_STAT(IXGBE_VFGPTC,
1494             hw_stats->last_vfgptc, hw_stats->vfgptc);
1495
1496         /* Good Tx octets, include VF loopback */
1497         UPDATE_VF_STAT_36BIT(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
1498             hw_stats->last_vfgotc, hw_stats->vfgotc);
1499
1500         /* Rx Multicst Packet */
1501         UPDATE_VF_STAT(IXGBE_VFMPRC,
1502             hw_stats->last_vfmprc, hw_stats->vfmprc);
1503
1504         if (stats == NULL)
1505                 return;
1506
1507         memset(stats, 0, sizeof(*stats));
1508         stats->ipackets = hw_stats->vfgprc;
1509         stats->ibytes = hw_stats->vfgorc;
1510         stats->opackets = hw_stats->vfgptc;
1511         stats->obytes = hw_stats->vfgotc;
1512         stats->imcasts = hw_stats->vfmprc;
1513 }
1514
1515 static void
1516 ixgbevf_dev_stats_reset(struct rte_eth_dev *dev)
1517 {
1518         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1519                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1520
1521         /* Sync HW register to the last stats */
1522         ixgbevf_dev_stats_get(dev, NULL);
1523
1524         /* reset HW current stats*/
1525         hw_stats->vfgprc = 0;
1526         hw_stats->vfgorc = 0;
1527         hw_stats->vfgptc = 0;
1528         hw_stats->vfgotc = 0;
1529         hw_stats->vfmprc = 0;
1530
1531 }
1532
1533 static void
1534 ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1535 {
1536         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1537
1538         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
1539         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
1540         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL register */
1541         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */
1542         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
1543 }
1544
1545 /* return 0 means link status changed, -1 means not changed */
1546 static int
1547 ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1548 {
1549         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1550         struct rte_eth_link link, old;
1551         ixgbe_link_speed link_speed;
1552         int link_up;
1553         int diag;
1554
1555         link.link_status = 0;
1556         link.link_speed = 0;
1557         link.link_duplex = 0;
1558         memset(&old, 0, sizeof(old));
1559         rte_ixgbe_dev_atomic_read_link_status(dev, &old);
1560
1561         /* check if it needs to wait to complete, if lsc interrupt is enabled */
1562         if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
1563                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 0);
1564         else
1565                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 1);
1566         if (diag != 0) {
1567                 link.link_speed = ETH_LINK_SPEED_100;
1568                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1569                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1570                 if (link.link_status == old.link_status)
1571                         return -1;
1572                 return 0;
1573         }
1574
1575         if (link_up == 0) {
1576                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1577                 if (link.link_status == old.link_status)
1578                         return -1;
1579                 return 0;
1580         }
1581         link.link_status = 1;
1582         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1583
1584         switch (link_speed) {
1585         default:
1586         case IXGBE_LINK_SPEED_UNKNOWN:
1587                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1588                 link.link_speed = ETH_LINK_SPEED_100;
1589                 break;
1590
1591         case IXGBE_LINK_SPEED_100_FULL:
1592                 link.link_speed = ETH_LINK_SPEED_100;
1593                 break;
1594
1595         case IXGBE_LINK_SPEED_1GB_FULL:
1596                 link.link_speed = ETH_LINK_SPEED_1000;
1597                 break;
1598
1599         case IXGBE_LINK_SPEED_10GB_FULL:
1600                 link.link_speed = ETH_LINK_SPEED_10000;
1601                 break;
1602         }
1603         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1604
1605         if (link.link_status == old.link_status)
1606                 return -1;
1607
1608         return 0;
1609 }
1610
1611 static void
1612 ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev)
1613 {
1614         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1615         uint32_t fctrl;
1616
1617         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1618         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1619         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1620 }
1621
1622 static void
1623 ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev)
1624 {
1625         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1626         uint32_t fctrl;
1627
1628         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1629         fctrl &= (~IXGBE_FCTRL_UPE);
1630         if (dev->data->all_multicast == 1)
1631                 fctrl |= IXGBE_FCTRL_MPE;
1632         else
1633                 fctrl &= (~IXGBE_FCTRL_MPE);
1634         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1635 }
1636
1637 static void
1638 ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev)
1639 {
1640         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1641         uint32_t fctrl;
1642
1643         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1644         fctrl |= IXGBE_FCTRL_MPE;
1645         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1646 }
1647
1648 static void
1649 ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev)
1650 {
1651         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1652         uint32_t fctrl;
1653
1654         if (dev->data->promiscuous == 1)
1655                 return; /* must remain in all_multicast mode */
1656
1657         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1658         fctrl &= (~IXGBE_FCTRL_MPE);
1659         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1660 }
1661
1662 /**
1663  * It clears the interrupt causes and enables the interrupt.
1664  * It will be called once only during nic initialized.
1665  *
1666  * @param dev
1667  *  Pointer to struct rte_eth_dev.
1668  *
1669  * @return
1670  *  - On success, zero.
1671  *  - On failure, a negative value.
1672  */
1673 static int
1674 ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev)
1675 {
1676         struct ixgbe_interrupt *intr =
1677                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1678
1679         ixgbe_dev_link_status_print(dev);
1680         intr->mask |= IXGBE_EICR_LSC;
1681
1682         return 0;
1683 }
1684
1685 /*
1686  * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update.
1687  *
1688  * @param dev
1689  *  Pointer to struct rte_eth_dev.
1690  *
1691  * @return
1692  *  - On success, zero.
1693  *  - On failure, a negative value.
1694  */
1695 static int
1696 ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
1697 {
1698         uint32_t eicr;
1699         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1700         struct ixgbe_interrupt *intr =
1701                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1702
1703         /* clear all cause mask */
1704         ixgbe_disable_intr(hw);
1705
1706         /* read-on-clear nic registers here */
1707         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1708         PMD_DRV_LOG(INFO, "eicr %x", eicr);
1709
1710         intr->flags = 0;
1711         if (eicr & IXGBE_EICR_LSC) {
1712                 /* set flag for async link update */
1713                 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1714         }
1715
1716         return 0;
1717 }
1718
1719 /**
1720  * It gets and then prints the link status.
1721  *
1722  * @param dev
1723  *  Pointer to struct rte_eth_dev.
1724  *
1725  * @return
1726  *  - On success, zero.
1727  *  - On failure, a negative value.
1728  */
1729 static void
1730 ixgbe_dev_link_status_print(struct rte_eth_dev *dev)
1731 {
1732         struct rte_eth_link link;
1733
1734         memset(&link, 0, sizeof(link));
1735         rte_ixgbe_dev_atomic_read_link_status(dev, &link);
1736         if (link.link_status) {
1737                 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
1738                                         (int)(dev->data->port_id),
1739                                         (unsigned)link.link_speed,
1740                         link.link_duplex == ETH_LINK_FULL_DUPLEX ?
1741                                         "full-duplex" : "half-duplex");
1742         } else {
1743                 PMD_INIT_LOG(INFO, " Port %d: Link Down",
1744                                 (int)(dev->data->port_id));
1745         }
1746         PMD_INIT_LOG(INFO, "PCI Address: %04d:%02d:%02d:%d",
1747                                 dev->pci_dev->addr.domain,
1748                                 dev->pci_dev->addr.bus,
1749                                 dev->pci_dev->addr.devid,
1750                                 dev->pci_dev->addr.function);
1751 }
1752
1753 /*
1754  * It executes link_update after knowing an interrupt occured.
1755  *
1756  * @param dev
1757  *  Pointer to struct rte_eth_dev.
1758  *
1759  * @return
1760  *  - On success, zero.
1761  *  - On failure, a negative value.
1762  */
1763 static int
1764 ixgbe_dev_interrupt_action(struct rte_eth_dev *dev)
1765 {
1766         struct ixgbe_interrupt *intr =
1767                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1768         int64_t timeout;
1769         struct rte_eth_link link;
1770         int intr_enable_delay = false;
1771
1772         PMD_DRV_LOG(DEBUG, "intr action type %d\n", intr->flags);
1773
1774         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
1775                 /* get the link status before link update, for predicting later */
1776                 memset(&link, 0, sizeof(link));
1777                 rte_ixgbe_dev_atomic_read_link_status(dev, &link);
1778
1779                 ixgbe_dev_link_update(dev, 0);
1780
1781                 /* likely to up */
1782                 if (!link.link_status)
1783                         /* handle it 1 sec later, wait it being stable */
1784                         timeout = IXGBE_LINK_UP_CHECK_TIMEOUT;
1785                 /* likely to down */
1786                 else
1787                         /* handle it 4 sec later, wait it being stable */
1788                         timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT;
1789
1790                 ixgbe_dev_link_status_print(dev);
1791
1792                 intr_enable_delay = true;
1793         }
1794
1795         if (intr_enable_delay) {
1796                 if (rte_eal_alarm_set(timeout * 1000,
1797                                       ixgbe_dev_interrupt_delayed_handler, (void*)dev) < 0)
1798                         PMD_DRV_LOG(ERR, "Error setting alarm");
1799         } else {
1800                 PMD_DRV_LOG(DEBUG, "enable intr immediately");
1801                 ixgbe_enable_intr(dev);
1802                 rte_intr_enable(&(dev->pci_dev->intr_handle));
1803         }
1804
1805
1806         return 0;
1807 }
1808
1809 /**
1810  * Interrupt handler which shall be registered for alarm callback for delayed
1811  * handling specific interrupt to wait for the stable nic state. As the
1812  * NIC interrupt state is not stable for ixgbe after link is just down,
1813  * it needs to wait 4 seconds to get the stable status.
1814  *
1815  * @param handle
1816  *  Pointer to interrupt handle.
1817  * @param param
1818  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1819  *
1820  * @return
1821  *  void
1822  */
1823 static void
1824 ixgbe_dev_interrupt_delayed_handler(void *param)
1825 {
1826         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1827         struct ixgbe_interrupt *intr =
1828                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1829
1830         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
1831                 ixgbe_dev_link_update(dev, 0);
1832                 intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
1833                 ixgbe_dev_link_status_print(dev);
1834                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
1835         }
1836
1837         PMD_DRV_LOG(DEBUG, "enable intr in delayed handler\n");
1838         ixgbe_enable_intr(dev);
1839         rte_intr_enable(&(dev->pci_dev->intr_handle));
1840 }
1841
1842 /**
1843  * Interrupt handler triggered by NIC  for handling
1844  * specific interrupt.
1845  *
1846  * @param handle
1847  *  Pointer to interrupt handle.
1848  * @param param
1849  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1850  *
1851  * @return
1852  *  void
1853  */
1854 static void
1855 ixgbe_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
1856                                                         void *param)
1857 {
1858         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1859         ixgbe_dev_interrupt_get_status(dev);
1860         ixgbe_dev_interrupt_action(dev);
1861 }
1862
1863 static int
1864 ixgbe_dev_led_on(struct rte_eth_dev *dev)
1865 {
1866         struct ixgbe_hw *hw;
1867
1868         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1869         return (ixgbe_led_on(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
1870 }
1871
1872 static int
1873 ixgbe_dev_led_off(struct rte_eth_dev *dev)
1874 {
1875         struct ixgbe_hw *hw;
1876
1877         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1878         return (ixgbe_led_off(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
1879 }
1880
1881 static int
1882 ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1883 {
1884         struct ixgbe_hw *hw;
1885         int err;
1886         uint32_t rx_buf_size;
1887         uint32_t max_high_water;
1888         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
1889                 ixgbe_fc_none,
1890                 ixgbe_fc_rx_pause,
1891                 ixgbe_fc_tx_pause,
1892                 ixgbe_fc_full
1893         };
1894
1895         PMD_INIT_FUNC_TRACE();
1896
1897         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1898         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0));
1899         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
1900
1901         /*
1902          * At least reserve one Ethernet frame for watermark
1903          * high_water/low_water in kilo bytes for ixgbe
1904          */
1905         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
1906         if ((fc_conf->high_water > max_high_water) ||
1907                 (fc_conf->high_water < fc_conf->low_water)) {
1908                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
1909                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
1910                 return (-EINVAL);
1911         }
1912
1913         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[fc_conf->mode];
1914         hw->fc.pause_time     = fc_conf->pause_time;
1915         hw->fc.high_water[0]  = fc_conf->high_water;
1916         hw->fc.low_water[0]   = fc_conf->low_water;
1917         hw->fc.send_xon       = fc_conf->send_xon;
1918
1919         err = ixgbe_fc_enable(hw);
1920         /* Not negotiated is not an error case */
1921         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) {
1922                 return 0;
1923         }
1924
1925         PMD_INIT_LOG(ERR, "ixgbe_fc_enable = 0x%x \n", err);
1926         return -EIO;
1927 }
1928
1929 /**
1930  *  ixgbe_pfc_enable_generic - Enable flow control
1931  *  @hw: pointer to hardware structure
1932  *  @tc_num: traffic class number
1933  *  Enable flow control according to the current settings.
1934  */
1935 static int
1936 ixgbe_dcb_pfc_enable_generic(struct ixgbe_hw *hw,uint8_t tc_num)
1937 {
1938         int ret_val = 0;
1939         uint32_t mflcn_reg, fccfg_reg;
1940         uint32_t reg;
1941         uint32_t fcrtl, fcrth;
1942         uint8_t i;
1943         uint8_t nb_rx_en;
1944
1945         /* Validate the water mark configuration */
1946         if (!hw->fc.pause_time) {
1947                 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
1948                 goto out;
1949         }
1950
1951         /* Low water mark of zero causes XOFF floods */
1952         if (hw->fc.current_mode & ixgbe_fc_tx_pause) {
1953                  /* High/Low water can not be 0 */
1954                 if( (!hw->fc.high_water[tc_num])|| (!hw->fc.low_water[tc_num])) {
1955                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
1956                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
1957                         goto out;
1958                 }
1959
1960                 if(hw->fc.low_water[tc_num] >= hw->fc.high_water[tc_num]) {
1961                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
1962                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
1963                         goto out;
1964                 }
1965         }
1966         /* Negotiate the fc mode to use */
1967         ixgbe_fc_autoneg(hw);
1968
1969         /* Disable any previous flow control settings */
1970         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
1971         mflcn_reg &= ~(IXGBE_MFLCN_RPFCE_SHIFT | IXGBE_MFLCN_RFCE|IXGBE_MFLCN_RPFCE);
1972
1973         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
1974         fccfg_reg &= ~(IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY);
1975
1976         switch (hw->fc.current_mode) {
1977         case ixgbe_fc_none:
1978                 /*
1979                  * If the count of enabled RX Priority Flow control >1,
1980                  * and the TX pause can not be disabled
1981                  */
1982                 nb_rx_en = 0;
1983                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
1984                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
1985                         if (reg & IXGBE_FCRTH_FCEN)
1986                                 nb_rx_en++;
1987                 }
1988                 if (nb_rx_en > 1)
1989                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
1990                 break;
1991         case ixgbe_fc_rx_pause:
1992                 /*
1993                  * Rx Flow control is enabled and Tx Flow control is
1994                  * disabled by software override. Since there really
1995                  * isn't a way to advertise that we are capable of RX
1996                  * Pause ONLY, we will advertise that we support both
1997                  * symmetric and asymmetric Rx PAUSE.  Later, we will
1998                  * disable the adapter's ability to send PAUSE frames.
1999                  */
2000                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
2001                 /*
2002                  * If the count of enabled RX Priority Flow control >1,
2003                  * and the TX pause can not be disabled
2004                  */
2005                 nb_rx_en = 0;
2006                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
2007                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
2008                         if (reg & IXGBE_FCRTH_FCEN)
2009                                 nb_rx_en++;
2010                 }
2011                 if (nb_rx_en > 1)
2012                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2013                 break;
2014         case ixgbe_fc_tx_pause:
2015                 /*
2016                  * Tx Flow control is enabled, and Rx Flow control is
2017                  * disabled by software override.
2018                  */
2019                 fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2020                 break;
2021         case ixgbe_fc_full:
2022                 /* Flow control (both Rx and Tx) is enabled by SW override. */
2023                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
2024                 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
2025                 break;
2026         default:
2027                 DEBUGOUT("Flow control param set incorrectly\n");
2028                 ret_val = IXGBE_ERR_CONFIG;
2029                 goto out;
2030                 break;
2031         }
2032
2033         /* Set 802.3x based flow control settings. */
2034         mflcn_reg |= IXGBE_MFLCN_DPF;
2035         IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg);
2036         IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg);
2037
2038         /* Set up and enable Rx high/low water mark thresholds, enable XON. */
2039         if ((hw->fc.current_mode & ixgbe_fc_tx_pause) &&
2040                 hw->fc.high_water[tc_num]) {
2041                 fcrtl = (hw->fc.low_water[tc_num] << 10) | IXGBE_FCRTL_XONE;
2042                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), fcrtl);
2043                 fcrth = (hw->fc.high_water[tc_num] << 10) | IXGBE_FCRTH_FCEN;
2044         } else {
2045                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), 0);
2046                 /*
2047                  * In order to prevent Tx hangs when the internal Tx
2048                  * switch is enabled we must set the high water mark
2049                  * to the maximum FCRTH value.  This allows the Tx
2050                  * switch to function even under heavy Rx workloads.
2051                  */
2052                 fcrth = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)) - 32;
2053         }
2054         IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(tc_num), fcrth);
2055
2056         /* Configure pause time (2 TCs per register) */
2057         reg = hw->fc.pause_time * 0x00010001;
2058         for (i = 0; i < (IXGBE_DCB_MAX_TRAFFIC_CLASS / 2); i++)
2059                 IXGBE_WRITE_REG(hw, IXGBE_FCTTV(i), reg);
2060
2061         /* Configure flow control refresh threshold value */
2062         IXGBE_WRITE_REG(hw, IXGBE_FCRTV, hw->fc.pause_time / 2);
2063
2064 out:
2065         return ret_val;
2066 }
2067
2068 static int
2069 ixgbe_dcb_pfc_enable(struct rte_eth_dev *dev,uint8_t tc_num)
2070 {
2071         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2072         int32_t ret_val = IXGBE_NOT_IMPLEMENTED;
2073
2074         if(hw->mac.type != ixgbe_mac_82598EB) {
2075                 ret_val = ixgbe_dcb_pfc_enable_generic(hw,tc_num);
2076         }
2077         return ret_val;
2078 }
2079
2080 static int
2081 ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf)
2082 {
2083         int err;
2084         uint32_t rx_buf_size;
2085         uint32_t max_high_water;
2086         uint8_t tc_num;
2087         uint8_t  map[IXGBE_DCB_MAX_USER_PRIORITY] = { 0 };
2088         struct ixgbe_hw *hw =
2089                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2090         struct ixgbe_dcb_config *dcb_config =
2091                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
2092
2093         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
2094                 ixgbe_fc_none,
2095                 ixgbe_fc_rx_pause,
2096                 ixgbe_fc_tx_pause,
2097                 ixgbe_fc_full
2098         };
2099
2100         PMD_INIT_FUNC_TRACE();
2101
2102         ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
2103         tc_num = map[pfc_conf->priority];
2104         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num));
2105         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
2106         /*
2107          * At least reserve one Ethernet frame for watermark
2108          * high_water/low_water in kilo bytes for ixgbe
2109          */
2110         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
2111         if ((pfc_conf->fc.high_water > max_high_water) ||
2112                 (pfc_conf->fc.high_water <= pfc_conf->fc.low_water)) {
2113                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
2114                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
2115                 return (-EINVAL);
2116         }
2117
2118         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[pfc_conf->fc.mode];
2119         hw->fc.pause_time = pfc_conf->fc.pause_time;
2120         hw->fc.send_xon = pfc_conf->fc.send_xon;
2121         hw->fc.low_water[tc_num] =  pfc_conf->fc.low_water;
2122         hw->fc.high_water[tc_num] = pfc_conf->fc.high_water;
2123
2124         err = ixgbe_dcb_pfc_enable(dev,tc_num);
2125
2126         /* Not negotiated is not an error case */
2127         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED))
2128                 return 0;
2129
2130         PMD_INIT_LOG(ERR, "ixgbe_dcb_pfc_enable = 0x%x \n", err);
2131         return -EIO;
2132 }
2133
2134 static void
2135 ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
2136                                 uint32_t index, uint32_t pool)
2137 {
2138         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2139         uint32_t enable_addr = 1;
2140
2141         ixgbe_set_rar(hw, index, mac_addr->addr_bytes, pool, enable_addr);
2142 }
2143
2144 static void
2145 ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
2146 {
2147         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2148
2149         ixgbe_clear_rar(hw, index);
2150 }
2151
2152 /*
2153  * Virtual Function operations
2154  */
2155 static void
2156 ixgbevf_intr_disable(struct ixgbe_hw *hw)
2157 {
2158         PMD_INIT_LOG(DEBUG, "ixgbevf_intr_disable");
2159
2160         /* Clear interrupt mask to stop from interrupts being generated */
2161         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, IXGBE_VF_IRQ_CLEAR_MASK);
2162
2163         IXGBE_WRITE_FLUSH(hw);
2164 }
2165
2166 static int
2167 ixgbevf_dev_configure(struct rte_eth_dev *dev)
2168 {
2169         struct rte_eth_conf* conf = &dev->data->dev_conf;
2170
2171         PMD_INIT_LOG(DEBUG, "\nConfigured Virtual Function port id: %d\n",
2172                 dev->data->port_id);
2173
2174         /*
2175          * VF has no ability to enable/disable HW CRC
2176          * Keep the persistent behavior the same as Host PF
2177          */
2178 #ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC
2179         if (!conf->rxmode.hw_strip_crc) {
2180                 PMD_INIT_LOG(INFO, "VF can't disable HW CRC Strip\n");
2181                 conf->rxmode.hw_strip_crc = 1;
2182         }
2183 #else
2184         if (conf->rxmode.hw_strip_crc) {
2185                 PMD_INIT_LOG(INFO, "VF can't enable HW CRC Strip\n");
2186                 conf->rxmode.hw_strip_crc = 0;
2187         }
2188 #endif
2189
2190         return 0;
2191 }
2192
2193 static int
2194 ixgbevf_dev_start(struct rte_eth_dev *dev)
2195 {
2196         struct ixgbe_hw *hw =
2197                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2198         int err, mask = 0;
2199
2200         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_start");
2201
2202         hw->mac.ops.reset_hw(hw);
2203
2204         ixgbevf_dev_tx_init(dev);
2205
2206         /* This can fail when allocating mbufs for descriptor rings */
2207         err = ixgbevf_dev_rx_init(dev);
2208         if (err) {
2209                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware (%d)\n", err);
2210                 ixgbe_dev_clear_queues(dev);
2211                 return err;
2212         }
2213
2214         /* Set vfta */
2215         ixgbevf_set_vfta_all(dev,1);
2216
2217         /* Set HW strip */
2218         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
2219                 ETH_VLAN_EXTEND_MASK;
2220         ixgbevf_vlan_offload_set(dev, mask);
2221
2222         ixgbevf_dev_rxtx_start(dev);
2223
2224         return 0;
2225 }
2226
2227 static void
2228 ixgbevf_dev_stop(struct rte_eth_dev *dev)
2229 {
2230         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2231
2232         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_stop");
2233
2234         hw->adapter_stopped = TRUE;
2235         ixgbe_stop_adapter(hw);
2236
2237         /*
2238           * Clear what we set, but we still keep shadow_vfta to
2239           * restore after device starts
2240           */
2241         ixgbevf_set_vfta_all(dev,0);
2242
2243         ixgbe_dev_clear_queues(dev);
2244 }
2245
2246 static void
2247 ixgbevf_dev_close(struct rte_eth_dev *dev)
2248 {
2249         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2250
2251         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_close");
2252
2253         ixgbe_reset_hw(hw);
2254
2255         ixgbevf_dev_stop(dev);
2256
2257         /* reprogram the RAR[0] in case user changed it. */
2258         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
2259 }
2260
2261 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on)
2262 {
2263         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2264         struct ixgbe_vfta * shadow_vfta =
2265                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2266         int i = 0, j = 0, vfta = 0, mask = 1;
2267
2268         for (i = 0; i < IXGBE_VFTA_SIZE; i++){
2269                 vfta = shadow_vfta->vfta[i];
2270                 if(vfta){
2271                         mask = 1;
2272                         for (j = 0; j < 32; j++){
2273                                 if(vfta & mask)
2274                                         ixgbe_set_vfta(hw, (i<<5)+j, 0, on);
2275                                 mask<<=1;
2276                         }
2277                 }
2278         }
2279
2280 }
2281
2282 static int
2283 ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2284 {
2285         struct ixgbe_hw *hw =
2286                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2287         struct ixgbe_vfta * shadow_vfta =
2288                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2289         uint32_t vid_idx = 0;
2290         uint32_t vid_bit = 0;
2291         int ret = 0;
2292
2293         PMD_INIT_FUNC_TRACE();
2294
2295         /* vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf */
2296         ret = ixgbe_set_vfta(hw, vlan_id, 0, !!on);
2297         if(ret){
2298                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
2299                 return ret;
2300         }
2301         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
2302         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
2303
2304         /* Save what we set and retore it after device reset */
2305         if (on)
2306                 shadow_vfta->vfta[vid_idx] |= vid_bit;
2307         else
2308                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
2309
2310         return 0;
2311 }
2312
2313 static void
2314 ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
2315 {
2316         struct ixgbe_hw *hw =
2317                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2318         uint32_t ctrl;
2319
2320         PMD_INIT_FUNC_TRACE();
2321
2322         if(queue >= hw->mac.max_rx_queues)
2323                 return;
2324
2325         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
2326         if(on)
2327                 ctrl |= IXGBE_RXDCTL_VME;
2328         else
2329                 ctrl &= ~IXGBE_RXDCTL_VME;
2330         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
2331
2332         ixgbe_vlan_hw_strip_bitmap_set( dev, queue, on);
2333 }
2334
2335 static void
2336 ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2337 {
2338         struct ixgbe_hw *hw =
2339                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2340         uint16_t i;
2341         int on = 0;
2342
2343         /* VF function only support hw strip feature, others are not support */
2344         if(mask & ETH_VLAN_STRIP_MASK){
2345                 on = !!(dev->data->dev_conf.rxmode.hw_vlan_strip);
2346
2347                 for(i=0; i < hw->mac.max_rx_queues; i++)
2348                         ixgbevf_vlan_strip_queue_set(dev,i,on);
2349         }
2350 }