lib/librte_pmd_ixgbe/ixgbe_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2012 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  *
  33  */
  34
  35 #include <sys/queue.h>
  36 #include <stdio.h>
  37 #include <errno.h>
  38 #include <stdint.h>
  39 #include <string.h>
  40 #include <unistd.h>
  41 #include <stdarg.h>
  42 #include <inttypes.h>
  43 #include <rte_byteorder.h>
  44 #include <rte_common.h>
  45 #include <rte_cycles.h>
  46
  47 #include <rte_interrupts.h>
  48 #include <rte_log.h>
  49 #include <rte_debug.h>
  50 #include <rte_pci.h>
  51 #include <rte_atomic.h>
  52 #include <rte_branch_prediction.h>
  53 #include <rte_memory.h>
  54 #include <rte_memzone.h>
  55 #include <rte_tailq.h>
  56 #include <rte_eal.h>
  57 #include <rte_alarm.h>
  58 #include <rte_ether.h>
  59 #include <rte_ethdev.h>
  60 #include <rte_atomic.h>
  61 #include <rte_malloc.h>
  62
  63 #include "ixgbe_logs.h"
  64 #include "ixgbe/ixgbe_api.h"
  65 #include "ixgbe/ixgbe_vf.h"
  66 #include "ixgbe/ixgbe_common.h"
  67 #include "ixgbe_ethdev.h"
  68
  69 /*
  70  * High threshold controlling when to start sending XOFF frames. Must be at
  71  * least 8 bytes less than receive packet buffer size. This value is in units
  72  * of 1024 bytes.
  73  */
  74 #define IXGBE_FC_HI    0x80
  75
  76 /*
  77  * Low threshold controlling when to start sending XON frames. This value is
  78  * in units of 1024 bytes.
  79  */
  80 #define IXGBE_FC_LO    0x40
  81
  82 /* Timer value included in XOFF frames. */
  83 #define IXGBE_FC_PAUSE 0x680
  84
  85 #define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */
  86 #define IXGBE_LINK_UP_CHECK_TIMEOUT   1000 /* ms */
  87
  88 #define IXGBE_QUEUE_STAT_COUNTERS (sizeof(hw_stats->qprc) / sizeof(hw_stats->qprc[0]))
  89
  90 static int eth_ixgbe_dev_init(struct eth_driver *eth_drv,
  91                 struct rte_eth_dev *eth_dev);
  92 static int  ixgbe_dev_configure(struct rte_eth_dev *dev);
  93 static int  ixgbe_dev_start(struct rte_eth_dev *dev);
  94 static void ixgbe_dev_stop(struct rte_eth_dev *dev);
  95 static void ixgbe_dev_close(struct rte_eth_dev *dev);
  96 static void ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev);
  97 static void ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev);
  98 static void ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev);
  99 static void ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev);
 100 static int ixgbe_dev_link_update(struct rte_eth_dev *dev,
 101                                 int wait_to_complete);
 102 static void ixgbe_dev_stats_get(struct rte_eth_dev *dev,
 103                                 struct rte_eth_stats *stats);
 104 static void ixgbe_dev_stats_reset(struct rte_eth_dev *dev);
 105 static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 106                                              uint16_t queue_id,
 107                                              uint8_t stat_idx,
 108                                              uint8_t is_rx);
 109 static void ixgbe_dev_info_get(struct rte_eth_dev *dev,
 110                                 struct rte_eth_dev_info *dev_info);
 111 static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev,
 112                 uint16_t vlan_id, int on);
 113 static void ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 114 static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev,
 115                 uint16_t queue, bool on);
 116 static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue,
 117                                   int on);
 118 static void ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 119 static void ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev);
 120 static void ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev);
 121 static void ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev);
 122 static void ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev);
 123 static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue);
 124 static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue);
 125 static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 126 static void ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 127
 128 static int ixgbe_dev_led_on(struct rte_eth_dev *dev);
 129 static int ixgbe_dev_led_off(struct rte_eth_dev *dev);
 130 static int  ixgbe_flow_ctrl_set(struct rte_eth_dev *dev,
 131                                 struct rte_eth_fc_conf *fc_conf);
 132 static int ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 133                 struct rte_eth_pfc_conf *pfc_conf);
 134 static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev);
 135 static int ixgbe_dev_interrupt_setup(struct rte_eth_dev *dev);
 136 static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev);
 137 static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev);
 138 static void ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle,
 139                                                         void *param);
 140 static void ixgbe_dev_interrupt_delayed_handler(void *param);
 141 static void ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 142                                 uint32_t index, uint32_t pool);
 143 static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index);
 144 static void ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config);
 145
 146 /* For Virtual Function support */
 147 static int eth_ixgbevf_dev_init(struct eth_driver *eth_drv,
 148                 struct rte_eth_dev *eth_dev);
 149 static int  ixgbevf_dev_configure(struct rte_eth_dev *dev);
 150 static int  ixgbevf_dev_start(struct rte_eth_dev *dev);
 151 static void ixgbevf_dev_stop(struct rte_eth_dev *dev);
 152 static void ixgbevf_intr_disable(struct ixgbe_hw *hw);
 153 static void ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats);
 154 static void ixgbevf_dev_stats_reset(struct rte_eth_dev *dev);
 155 static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev,
 156                 uint16_t vlan_id, int on);
 157 static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev,
 158                 uint16_t queue, int on);
 159 static void ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 160 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 161
 162
 163 /*
 164  *  * Define VF Stats MACRO for Non "cleared on read" register
 165  *   */
 166 #define UPDATE_VF_STAT(reg, last, cur)                          \
 167 {                                                               \
 168         u32 latest = IXGBE_READ_REG(hw, reg);                   \
 169         cur += latest - last;                                   \
 170         last = latest;                                          \
 171 }
 172
 173 #define UPDATE_VF_STAT_36BIT(lsb, msb, last, cur)                \
 174 {                                                                \
 175         u64 new_lsb = IXGBE_READ_REG(hw, lsb);                   \
 176         u64 new_msb = IXGBE_READ_REG(hw, msb);                   \
 177         u64 latest = ((new_msb << 32) | new_lsb);                \
 178         cur += (0x1000000000LL + latest - last) & 0xFFFFFFFFFLL; \
 179         last = latest;                                           \
 180 }
 181
 182 #define IXGBE_SET_HWSTRIP(h, q) do{\
 183                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 184                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 185                 (h)->bitmap[idx] |= 1 << bit;\
 186         }while(0)
 187
 188 #define IXGBE_CLEAR_HWSTRIP(h, q) do{\
 189                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 190                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 191                 (h)->bitmap[idx] &= ~(1 << bit);\
 192         }while(0)
 193
 194 #define IXGBE_GET_HWSTRIP(h, q, r) do{\
 195                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 196                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 197                 (r) = (h)->bitmap[idx] >> bit & 1;\
 198         }while(0)
 199
 200 /*
 201  * The set of PCI devices this driver supports
 202  */
 203 static struct rte_pci_id pci_id_ixgbe_map[] = {
 204
 205 #undef RTE_LIBRTE_IGB_PMD
 206 #define RTE_PCI_DEV_ID_DECL(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 207 #include "rte_pci_dev_ids.h"
 208
 209 { .vendor_id = 0, /* sentinel */ },
 210 };
 211
 212
 213 /*
 214  * The set of PCI devices this driver supports (for 82599 VF)
 215  */
 216 static struct rte_pci_id pci_id_ixgbevf_map[] = {
 217 {
 218         .vendor_id = PCI_VENDOR_ID_INTEL,
 219         .device_id = IXGBE_DEV_ID_82599_VF,
 220         .subsystem_vendor_id = PCI_ANY_ID,
 221         .subsystem_device_id = PCI_ANY_ID,
 222 },
 223 { .vendor_id = 0, /* sentinel */ },
 224 };
 225
 226 static struct eth_dev_ops ixgbe_eth_dev_ops = {
 227         .dev_configure        = ixgbe_dev_configure,
 228         .dev_start            = ixgbe_dev_start,
 229         .dev_stop             = ixgbe_dev_stop,
 230         .dev_close            = ixgbe_dev_close,
 231         .promiscuous_enable   = ixgbe_dev_promiscuous_enable,
 232         .promiscuous_disable  = ixgbe_dev_promiscuous_disable,
 233         .allmulticast_enable  = ixgbe_dev_allmulticast_enable,
 234         .allmulticast_disable = ixgbe_dev_allmulticast_disable,
 235         .link_update          = ixgbe_dev_link_update,
 236         .stats_get            = ixgbe_dev_stats_get,
 237         .stats_reset          = ixgbe_dev_stats_reset,
 238         .queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set,
 239         .dev_infos_get        = ixgbe_dev_info_get,
 240         .vlan_filter_set      = ixgbe_vlan_filter_set,
 241         .vlan_tpid_set        = ixgbe_vlan_tpid_set,
 242         .vlan_offload_set     = ixgbe_vlan_offload_set,
 243         .vlan_strip_queue_set = ixgbe_vlan_strip_queue_set,
 244         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 245         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 246         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 247         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 248         .dev_led_on           = ixgbe_dev_led_on,
 249         .dev_led_off          = ixgbe_dev_led_off,
 250         .flow_ctrl_set        = ixgbe_flow_ctrl_set,
 251         .priority_flow_ctrl_set = ixgbe_priority_flow_ctrl_set,
 252         .mac_addr_add         = ixgbe_add_rar,
 253         .mac_addr_remove      = ixgbe_remove_rar,
 254         .fdir_add_signature_filter    = ixgbe_fdir_add_signature_filter,
 255         .fdir_update_signature_filter = ixgbe_fdir_update_signature_filter,
 256         .fdir_remove_signature_filter = ixgbe_fdir_remove_signature_filter,
 257         .fdir_infos_get               = ixgbe_fdir_info_get,
 258         .fdir_add_perfect_filter      = ixgbe_fdir_add_perfect_filter,
 259         .fdir_update_perfect_filter   = ixgbe_fdir_update_perfect_filter,
 260         .fdir_remove_perfect_filter   = ixgbe_fdir_remove_perfect_filter,
 261         .fdir_set_masks               = ixgbe_fdir_set_masks,
 262 };
 263
 264 /*
 265  * dev_ops for virtual function, bare necessities for basic vf
 266  * operation have been implemented
 267  */
 268 static struct eth_dev_ops ixgbevf_eth_dev_ops = {
 269
 270         .dev_configure        = ixgbevf_dev_configure,
 271         .dev_start            = ixgbevf_dev_start,
 272         .dev_stop             = ixgbevf_dev_stop,
 273         .link_update          = ixgbe_dev_link_update,
 274         .stats_get            = ixgbevf_dev_stats_get,
 275         .stats_reset          = ixgbevf_dev_stats_reset,
 276         .dev_close            = ixgbevf_dev_stop,
 277
 278         .dev_infos_get        = ixgbe_dev_info_get,
 279         .vlan_filter_set      = ixgbevf_vlan_filter_set,
 280         .vlan_strip_queue_set = ixgbevf_vlan_strip_queue_set,
 281         .vlan_offload_set     = ixgbevf_vlan_offload_set,
 282         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 283         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 284         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 285         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 286 };
 287
 288 /**
 289  * Atomically reads the link status information from global
 290  * structure rte_eth_dev.
 291  *
 292  * @param dev
 293  *   - Pointer to the structure rte_eth_dev to read from.
 294  *   - Pointer to the buffer to be saved with the link status.
 295  *
 296  * @return
 297  *   - On success, zero.
 298  *   - On failure, negative value.
 299  */
 300 static inline int
 301 rte_ixgbe_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 302                                 struct rte_eth_link *link)
 303 {
 304         struct rte_eth_link *dst = link;
 305         struct rte_eth_link *src = &(dev->data->dev_link);
 306
 307         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 308                                         *(uint64_t *)src) == 0)
 309                 return -1;
 310
 311         return 0;
 312 }
 313
 314 /**
 315  * Atomically writes the link status information into global
 316  * structure rte_eth_dev.
 317  *
 318  * @param dev
 319  *   - Pointer to the structure rte_eth_dev to read from.
 320  *   - Pointer to the buffer to be saved with the link status.
 321  *
 322  * @return
 323  *   - On success, zero.
 324  *   - On failure, negative value.
 325  */
 326 static inline int
 327 rte_ixgbe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 328                                 struct rte_eth_link *link)
 329 {
 330         struct rte_eth_link *dst = &(dev->data->dev_link);
 331         struct rte_eth_link *src = link;
 332
 333         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 334                                         *(uint64_t *)src) == 0)
 335                 return -1;
 336
 337         return 0;
 338 }
 339
 340 /*
 341  * This function is the same as ixgbe_is_sfp() in ixgbe/ixgbe.h.
 342  */
 343 static inline int
 344 ixgbe_is_sfp(struct ixgbe_hw *hw)
 345 {
 346         switch (hw->phy.type) {
 347         case ixgbe_phy_sfp_avago:
 348         case ixgbe_phy_sfp_ftl:
 349         case ixgbe_phy_sfp_intel:
 350         case ixgbe_phy_sfp_unknown:
 351         case ixgbe_phy_sfp_passive_tyco:
 352         case ixgbe_phy_sfp_passive_unknown:
 353                 return 1;
 354         default:
 355                 return 0;
 356         }
 357 }
 358
 359 /*
 360  * This function is based on ixgbe_disable_intr() in ixgbe/ixgbe.h.
 361  */
 362 static void
 363 ixgbe_disable_intr(struct ixgbe_hw *hw)
 364 {
 365         PMD_INIT_FUNC_TRACE();
 366
 367         if (hw->mac.type == ixgbe_mac_82598EB) {
 368                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ~0);
 369         } else {
 370                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xFFFF0000);
 371                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), ~0);
 372                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), ~0);
 373         }
 374         IXGBE_WRITE_FLUSH(hw);
 375 }
 376
 377 /*
 378  * This function resets queue statistics mapping registers.
 379  * From Niantic datasheet, Initialization of Statistics section:
 380  * "...if software requires the queue counters, the RQSMR and TQSM registers
 381  * must be re-programmed following a device reset.
 382  */
 383 static void
 384 ixgbe_reset_qstat_mappings(struct ixgbe_hw *hw)
 385 {
 386         uint32_t i;
 387
 388         for(i = 0; i != IXGBE_NB_STAT_MAPPING_REGS; i++) {
 389                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), 0);
 390                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), 0);
 391         }
 392 }
 393
 394
 395 static int
 396 ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 397                                   uint16_t queue_id,
 398                                   uint8_t stat_idx,
 399                                   uint8_t is_rx)
 400 {
 401 #define QSM_REG_NB_BITS_PER_QMAP_FIELD 8
 402 #define NB_QMAP_FIELDS_PER_QSM_REG 4
 403 #define QMAP_FIELD_RESERVED_BITS_MASK 0x0f
 404
 405         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 406         struct ixgbe_stat_mapping_registers *stat_mappings =
 407                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(eth_dev->data->dev_private);
 408         uint32_t qsmr_mask = 0;
 409         uint32_t clearing_mask = QMAP_FIELD_RESERVED_BITS_MASK;
 410         uint32_t q_map;
 411         uint8_t n, offset;
 412
 413         if ((hw->mac.type != ixgbe_mac_82599EB) && (hw->mac.type != ixgbe_mac_X540))
 414                 return -ENOSYS;
 415
 416         PMD_INIT_LOG(INFO, "Setting port %d, %s queue_id %d to stat index %d\n",
 417                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx);
 418
 419         n = queue_id / NB_QMAP_FIELDS_PER_QSM_REG;
 420         if (n >= IXGBE_NB_STAT_MAPPING_REGS) {
 421                 PMD_INIT_LOG(ERR, "Nb of stat mapping registers exceeded\n");
 422                 return -EIO;
 423         }
 424         offset = queue_id % NB_QMAP_FIELDS_PER_QSM_REG;
 425
 426         /* Now clear any previous stat_idx set */
 427         clearing_mask <<= (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 428         if (!is_rx)
 429                 stat_mappings->tqsm[n] &= ~clearing_mask;
 430         else
 431                 stat_mappings->rqsmr[n] &= ~clearing_mask;
 432
 433         q_map = (uint32_t)stat_idx;
 434         q_map &= QMAP_FIELD_RESERVED_BITS_MASK;
 435         qsmr_mask = q_map << (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 436         if (!is_rx)
 437                 stat_mappings->tqsm[n] |= qsmr_mask;
 438         else
 439                 stat_mappings->rqsmr[n] |= qsmr_mask;
 440
 441         PMD_INIT_LOG(INFO, "Set port %d, %s queue_id %d to stat index %d\n"
 442                      "%s[%d] = 0x%08x\n",
 443                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx,
 444                      is_rx ? "RQSMR" : "TQSM",n, is_rx ? stat_mappings->rqsmr[n] : stat_mappings->tqsm[n]);
 445
 446         /* Now write the mapping in the appropriate register */
 447         if (is_rx) {
 448                 PMD_INIT_LOG(INFO, "Write 0x%x to RX IXGBE stat mapping reg:%d\n",
 449                              stat_mappings->rqsmr[n], n);
 450                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(n), stat_mappings->rqsmr[n]);
 451         }
 452         else {
 453                 PMD_INIT_LOG(INFO, "Write 0x%x to TX IXGBE stat mapping reg:%d\n",
 454                              stat_mappings->tqsm[n], n);
 455                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(n), stat_mappings->tqsm[n]);
 456         }
 457         return 0;
 458 }
 459
 460 static void
 461 ixgbe_restore_statistics_mapping(struct rte_eth_dev * dev)
 462 {
 463         struct ixgbe_stat_mapping_registers *stat_mappings =
 464                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(dev->data->dev_private);
 465         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 466         int i;
 467
 468         /* write whatever was in stat mapping table to the NIC */
 469         for (i = 0; i < IXGBE_NB_STAT_MAPPING_REGS; i++) {
 470                 /* rx */
 471                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), stat_mappings->rqsmr[i]);
 472
 473                 /* tx */
 474                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), stat_mappings->tqsm[i]);
 475         }
 476 }
 477
 478 static void
 479 ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config)
 480 {
 481         uint8_t i;
 482         struct ixgbe_dcb_tc_config *tc;
 483         int dcb_max_tc = IXGBE_DCB_MAX_TRAFFIC_CLASS;
 484
 485         dcb_config->num_tcs.pg_tcs = dcb_max_tc;
 486         dcb_config->num_tcs.pfc_tcs = dcb_max_tc;
 487         for (i = 0; i < dcb_max_tc; i++) {
 488                 tc = &dcb_config->tc_config[i];
 489                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_id = i;
 490                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = 100/dcb_max_tc + (i & 1);
 491                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_id = i;
 492                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent = 100/dcb_max_tc + (i & 1);
 493                 tc->pfc = ixgbe_dcb_pfc_disabled;
 494         }
 495
 496         /* Initialize default user to priority mapping, UPx->TC0 */
 497         tc = &dcb_config->tc_config[0];
 498         tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
 499         tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
 500         for (i = 0; i< IXGBE_DCB_MAX_BW_GROUP; i++) {
 501                 dcb_config->bw_percentage[IXGBE_DCB_TX_CONFIG][i] = 100;
 502                 dcb_config->bw_percentage[IXGBE_DCB_RX_CONFIG][i] = 100;
 503         }
 504         dcb_config->rx_pba_cfg = ixgbe_dcb_pba_equal;
 505         dcb_config->pfc_mode_enable = false;
 506         dcb_config->vt_mode = true;
 507         dcb_config->round_robin_enable = false;
 508         /* support all DCB capabilities in 82599 */
 509         dcb_config->support.capabilities = 0xFF;
 510
 511         /*we only support 4 Tcs for X540*/
 512         if (hw->mac.type == ixgbe_mac_X540) {
 513                 dcb_config->num_tcs.pg_tcs = 4;
 514                 dcb_config->num_tcs.pfc_tcs = 4;
 515         }
 516 }
 517
 518 /*
 519  * This function is based on code in ixgbe_attach() in ixgbe/ixgbe.c.
 520  * It returns 0 on success.
 521  */
 522 static int
 523 eth_ixgbe_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 524                      struct rte_eth_dev *eth_dev)
 525 {
 526         struct rte_pci_device *pci_dev;
 527         struct ixgbe_hw *hw =
 528                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 529         struct ixgbe_vfta * shadow_vfta =
 530                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 531         struct ixgbe_hwstrip *hwstrip =
 532                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 533         struct ixgbe_dcb_config *dcb_config =
 534                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private);
 535         uint32_t ctrl_ext;
 536         uint16_t csum;
 537         int diag, i;
 538
 539         PMD_INIT_FUNC_TRACE();
 540
 541         eth_dev->dev_ops = &ixgbe_eth_dev_ops;
 542         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
 543         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
 544
 545         /* for secondary processes, we don't initialise any further as primary
 546          * has already done this work. Only check we don't need a different
 547          * RX function */
 548         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 549                 if (eth_dev->data->scattered_rx)
 550                         eth_dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
 551                 return 0;
 552         }
 553         pci_dev = eth_dev->pci_dev;
 554
 555         /* Vendor and Device ID need to be set before init of shared code */
 556         hw->device_id = pci_dev->id.device_id;
 557         hw->vendor_id = pci_dev->id.vendor_id;
 558         hw->hw_addr = (void *)pci_dev->mem_resource.addr;
 559
 560         /* Initialize the shared code */
 561         diag = ixgbe_init_shared_code(hw);
 562         if (diag != IXGBE_SUCCESS) {
 563                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
 564                 return -EIO;
 565         }
 566
 567         /* Initialize DCB configuration*/
 568         memset(dcb_config, 0, sizeof(struct ixgbe_dcb_config));
 569         ixgbe_dcb_init(hw,dcb_config);
 570         /* Get Hardware Flow Control setting */
 571         hw->fc.requested_mode = ixgbe_fc_full;
 572         hw->fc.current_mode = ixgbe_fc_full;
 573         hw->fc.pause_time = IXGBE_FC_PAUSE;
 574         for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
 575                 hw->fc.low_water[i] = IXGBE_FC_LO;
 576                 hw->fc.high_water[i] = IXGBE_FC_HI;
 577         }
 578         hw->fc.send_xon = 1;
 579
 580         ixgbe_disable_intr(hw);
 581
 582         /* Make sure we have a good EEPROM before we read from it */
 583         diag = ixgbe_validate_eeprom_checksum(hw, &csum);
 584         if (diag != IXGBE_SUCCESS) {
 585                 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag);
 586                 return -EIO;
 587         }
 588
 589         diag = ixgbe_init_hw(hw);
 590
 591         /*
 592          * Devices with copper phys will fail to initialise if ixgbe_init_hw()
 593          * is called too soon after the kernel driver unbinding/binding occurs.
 594          * The failure occurs in ixgbe_identify_phy_generic() for all devices,
 595          * but for non-copper devies, ixgbe_identify_sfp_module_generic() is
 596          * also called. See ixgbe_identify_phy_82599(). The reason for the
 597          * failure is not known, and only occuts when virtualisation features
 598          * are disabled in the bios. A delay of 100ms  was found to be enough by
 599          * trial-and-error, and is doubled to be safe.
 600          */
 601         if (diag && (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper)) {
 602                 rte_delay_ms(200);
 603                 diag = ixgbe_init_hw(hw);
 604         }
 605
 606         if (diag == IXGBE_ERR_EEPROM_VERSION) {
 607                 PMD_INIT_LOG(ERR, "This device is a pre-production adapter/"
 608                     "LOM.  Please be aware there may be issues associated "
 609                     "with your hardware.\n If you are experiencing problems "
 610                     "please contact your Intel or hardware representative "
 611                     "who provided you with this hardware.\n");
 612         } else if (diag == IXGBE_ERR_SFP_NOT_SUPPORTED)
 613                 PMD_INIT_LOG(ERR, "Unsupported SFP+ Module\n");
 614         if (diag) {
 615                 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag);
 616                 return -EIO;
 617         }
 618
 619         /* pick up the PCI bus settings for reporting later */
 620         ixgbe_get_bus_info(hw);
 621
 622         /* reset mappings for queue statistics hw counters*/
 623         ixgbe_reset_qstat_mappings(hw);
 624
 625         /* Allocate memory for storing MAC addresses */
 626         eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 627                         hw->mac.num_rar_entries, 0);
 628         if (eth_dev->data->mac_addrs == NULL) {
 629                 PMD_INIT_LOG(ERR,
 630                         "Failed to allocate %d bytes needed to store MAC addresses",
 631                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 632                 return -ENOMEM;
 633         }
 634         /* Copy the permanent MAC address */
 635         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 636                         &eth_dev->data->mac_addrs[0]);
 637
 638         /* initialize the vfta */
 639         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 640
 641         /* initialize the hw strip bitmap*/
 642         memset(hwstrip, 0, sizeof(*hwstrip));
 643
 644         /* let hardware know driver is loaded */
 645         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 646         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 647         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 648
 649         if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
 650                 PMD_INIT_LOG(DEBUG,
 651                              "MAC: %d, PHY: %d, SFP+: %d<n",
 652                              (int) hw->mac.type, (int) hw->phy.type,
 653                              (int) hw->phy.sfp_type);
 654         else
 655                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d\n",
 656                              (int) hw->mac.type, (int) hw->phy.type);
 657
 658         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
 659                         eth_dev->data->port_id, pci_dev->id.vendor_id,
 660                         pci_dev->id.device_id);
 661
 662         rte_intr_callback_register(&(pci_dev->intr_handle),
 663                 ixgbe_dev_interrupt_handler, (void *)eth_dev);
 664
 665         return 0;
 666 }
 667
 668 /*
 669  * Virtual Function device init
 670  */
 671 static int
 672 eth_ixgbevf_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 673                      struct rte_eth_dev *eth_dev)
 674 {
 675         struct rte_pci_device *pci_dev;
 676         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 677         int diag;
 678         struct ixgbe_vfta * shadow_vfta =
 679                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 680         struct ixgbe_hwstrip *hwstrip =
 681                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 682
 683         PMD_INIT_LOG(DEBUG, "eth_ixgbevf_dev_init");
 684
 685         eth_dev->dev_ops = &ixgbevf_eth_dev_ops;
 686         pci_dev = eth_dev->pci_dev;
 687
 688         hw->device_id = pci_dev->id.device_id;
 689         hw->vendor_id = pci_dev->id.vendor_id;
 690         hw->hw_addr = (void *)pci_dev->mem_resource.addr;
 691
 692         /* initialize the vfta */
 693         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 694
 695         /* initialize the hw strip bitmap*/
 696         memset(hwstrip, 0, sizeof(*hwstrip));
 697
 698         /* Initialize the shared code */
 699         diag = ixgbe_init_shared_code(hw);
 700         if (diag != IXGBE_SUCCESS) {
 701                 PMD_INIT_LOG(ERR, "Shared code init failed for ixgbevf: %d", diag);
 702                 return -EIO;
 703         }
 704
 705         /* init_mailbox_params */
 706         hw->mbx.ops.init_params(hw);
 707
 708         /* Disable the interrupts for VF */
 709         ixgbevf_intr_disable(hw);
 710
 711         hw->mac.num_rar_entries = hw->mac.max_rx_queues;
 712         diag = hw->mac.ops.reset_hw(hw);
 713
 714         /* Allocate memory for storing MAC addresses */
 715         eth_dev->data->mac_addrs = rte_zmalloc("ixgbevf", ETHER_ADDR_LEN *
 716                         hw->mac.num_rar_entries, 0);
 717         if (eth_dev->data->mac_addrs == NULL) {
 718                 PMD_INIT_LOG(ERR,
 719                         "Failed to allocate %d bytes needed to store MAC addresses",
 720                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 721                 return -ENOMEM;
 722         }
 723         /* Copy the permanent MAC address */
 724         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 725                         &eth_dev->data->mac_addrs[0]);
 726
 727         /* reset the hardware with the new settings */
 728         diag = hw->mac.ops.start_hw(hw);
 729         switch (diag) {
 730                 case  0:
 731                         break;
 732
 733                 default:
 734                         PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
 735                         return (diag);
 736         }
 737
 738         PMD_INIT_LOG(DEBUG, "\nport %d vendorID=0x%x deviceID=0x%x mac.type=%s\n",
 739                          eth_dev->data->port_id, pci_dev->id.vendor_id, pci_dev->id.device_id,
 740                          "ixgbe_mac_82599_vf");
 741
 742         return 0;
 743 }
 744
 745 static struct eth_driver rte_ixgbe_pmd = {
 746         {
 747                 .name = "rte_ixgbe_pmd",
 748                 .id_table = pci_id_ixgbe_map,
 749                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
 750         },
 751         .eth_dev_init = eth_ixgbe_dev_init,
 752         .dev_private_size = sizeof(struct ixgbe_adapter),
 753 };
 754
 755 /*
 756  * virtual function driver struct
 757  */
 758 static struct eth_driver rte_ixgbevf_pmd = {
 759         {
 760                 .name = "rte_ixgbevf_pmd",
 761                 .id_table = pci_id_ixgbevf_map,
 762                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
 763         },
 764         .eth_dev_init = eth_ixgbevf_dev_init,
 765         .dev_private_size = sizeof(struct ixgbe_adapter),
 766 };
 767
 768 /*
 769  * Driver initialization routine.
 770  * Invoked once at EAL init time.
 771  * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
 772  */
 773 int
 774 rte_ixgbe_pmd_init(void)
 775 {
 776         PMD_INIT_FUNC_TRACE();
 777
 778         rte_eth_driver_register(&rte_ixgbe_pmd);
 779         return 0;
 780 }
 781
 782 /*
 783  * VF Driver initialization routine.
 784  * Invoked one at EAL init time.
 785  * Register itself as the [Virtual Poll Mode] Driver of PCI niantic devices.
 786  */
 787 int
 788 rte_ixgbevf_pmd_init(void)
 789 {
 790         DEBUGFUNC("rte_ixgbevf_pmd_init");
 791
 792         rte_eth_driver_register(&rte_ixgbevf_pmd);
 793         return (0);
 794 }
 795
 796 static int
 797 ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 798 {
 799         struct ixgbe_hw *hw =
 800                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 801         struct ixgbe_vfta * shadow_vfta =
 802                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
 803         uint32_t vfta;
 804         uint32_t vid_idx;
 805         uint32_t vid_bit;
 806
 807         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
 808         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
 809         vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(vid_idx));
 810         if (on)
 811                 vfta |= vid_bit;
 812         else
 813                 vfta &= ~vid_bit;
 814         IXGBE_WRITE_REG(hw, IXGBE_VFTA(vid_idx), vfta);
 815
 816         /* update local VFTA copy */
 817         shadow_vfta->vfta[vid_idx] = vfta;
 818
 819         return 0;
 820 }
 821
 822 static void
 823 ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
 824 {
 825         if (on)
 826                 ixgbe_vlan_hw_strip_enable(dev, queue);
 827         else
 828                 ixgbe_vlan_hw_strip_disable(dev, queue);
 829 }
 830
 831 static void
 832 ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
 833 {
 834         struct ixgbe_hw *hw =
 835                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 836
 837         /* Only the high 16-bits is valid */
 838         IXGBE_WRITE_REG(hw, IXGBE_EXVET, tpid << 16);
 839 }
 840
 841 static void
 842 ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev)
 843 {
 844         struct ixgbe_hw *hw =
 845                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 846         uint32_t vlnctrl;
 847
 848         PMD_INIT_FUNC_TRACE();
 849
 850         /* Filter Table Disable */
 851         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 852         vlnctrl &= ~IXGBE_VLNCTRL_VFE;
 853
 854         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
 855 }
 856
 857 static void
 858 ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev)
 859 {
 860         struct ixgbe_hw *hw =
 861                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 862         struct ixgbe_vfta * shadow_vfta =
 863                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
 864         uint32_t vlnctrl;
 865         uint16_t i;
 866
 867         PMD_INIT_FUNC_TRACE();
 868
 869         /* Filter Table Enable */
 870         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 871         vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
 872         vlnctrl |= IXGBE_VLNCTRL_VFE;
 873
 874         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
 875
 876         /* write whatever is in local vfta copy */
 877         for (i = 0; i < IXGBE_VFTA_SIZE; i++)
 878                 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), shadow_vfta->vfta[i]);
 879 }
 880
 881 static void
 882 ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on)
 883 {
 884         struct ixgbe_hwstrip *hwstrip =
 885                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(dev->data->dev_private);
 886
 887         if(queue >= IXGBE_MAX_RX_QUEUE_NUM)
 888                 return;
 889
 890         if (on)
 891                 IXGBE_SET_HWSTRIP(hwstrip, queue);
 892         else
 893                 IXGBE_CLEAR_HWSTRIP(hwstrip, queue);
 894 }
 895
 896 static void
 897 ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue)
 898 {
 899         struct ixgbe_hw *hw =
 900                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 901         uint32_t ctrl;
 902
 903         PMD_INIT_FUNC_TRACE();
 904
 905         if (hw->mac.type == ixgbe_mac_82598EB) {
 906                 /* No queue level support */
 907                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
 908                 return;
 909         }
 910         else {
 911                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 912                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
 913                 ctrl &= ~IXGBE_RXDCTL_VME;
 914                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
 915         }
 916         /* record those setting for HW strip per queue */
 917         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 0);
 918 }
 919
 920 static void
 921 ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue)
 922 {
 923         struct ixgbe_hw *hw =
 924                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 925         uint32_t ctrl;
 926
 927         PMD_INIT_FUNC_TRACE();
 928
 929         if (hw->mac.type == ixgbe_mac_82598EB) {
 930                 /* No queue level supported */
 931                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
 932                 return;
 933         }
 934         else {
 935                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 936                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
 937                 ctrl |= IXGBE_RXDCTL_VME;
 938                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
 939         }
 940         /* record those setting for HW strip per queue */
 941         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 1);
 942 }
 943
 944 static void
 945 ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev)
 946 {
 947         struct ixgbe_hw *hw =
 948                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 949         uint32_t ctrl;
 950         uint16_t i;
 951
 952         PMD_INIT_FUNC_TRACE();
 953
 954         if (hw->mac.type == ixgbe_mac_82598EB) {
 955                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 956                 ctrl &= ~IXGBE_VLNCTRL_VME;
 957                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
 958         }
 959         else {
 960                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 961                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
 962                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
 963                         ctrl &= ~IXGBE_RXDCTL_VME;
 964                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
 965
 966                         /* record those setting for HW strip per queue */
 967                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 0);
 968                 }
 969         }
 970 }
 971
 972 static void
 973 ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev)
 974 {
 975         struct ixgbe_hw *hw =
 976                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 977         uint32_t ctrl;
 978         uint16_t i;
 979
 980         PMD_INIT_FUNC_TRACE();
 981
 982         if (hw->mac.type == ixgbe_mac_82598EB) {
 983                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 984                 ctrl |= IXGBE_VLNCTRL_VME;
 985                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
 986         }
 987         else {
 988                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 989                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
 990                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
 991                         ctrl |= IXGBE_RXDCTL_VME;
 992                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
 993
 994                         /* record those setting for HW strip per queue */
 995                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 1);
 996                 }
 997         }
 998 }
 999
1000 static void
1001 ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1002 {
1003         struct ixgbe_hw *hw =
1004                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1005         uint32_t ctrl;
1006
1007         PMD_INIT_FUNC_TRACE();
1008
1009         /* DMATXCTRL: Geric Double VLAN Disable */
1010         ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1011         ctrl &= ~IXGBE_DMATXCTL_GDV;
1012         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1013
1014         /* CTRL_EXT: Global Double VLAN Disable */
1015         ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1016         ctrl &= ~IXGBE_EXTENDED_VLAN;
1017         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1018
1019 }
1020
1021 static void
1022 ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1023 {
1024         struct ixgbe_hw *hw =
1025                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1026         uint32_t ctrl;
1027
1028         PMD_INIT_FUNC_TRACE();
1029
1030         /* DMATXCTRL: Geric Double VLAN Enable */
1031         ctrl  = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1032         ctrl |= IXGBE_DMATXCTL_GDV;
1033         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1034
1035         /* CTRL_EXT: Global Double VLAN Enable */
1036         ctrl  = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1037         ctrl |= IXGBE_EXTENDED_VLAN;
1038         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1039
1040         /*
1041          * VET EXT field in the EXVET register = 0x8100 by default
1042          * So no need to change. Same to VT field of DMATXCTL register
1043          */
1044 }
1045
1046 static void
1047 ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1048 {
1049         if(mask & ETH_VLAN_STRIP_MASK){
1050                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1051                         ixgbe_vlan_hw_strip_enable_all(dev);
1052                 else
1053                         ixgbe_vlan_hw_strip_disable_all(dev);
1054         }
1055
1056         if(mask & ETH_VLAN_FILTER_MASK){
1057                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1058                         ixgbe_vlan_hw_filter_enable(dev);
1059                 else
1060                         ixgbe_vlan_hw_filter_disable(dev);
1061         }
1062
1063         if(mask & ETH_VLAN_EXTEND_MASK){
1064                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1065                         ixgbe_vlan_hw_extend_enable(dev);
1066                 else
1067                         ixgbe_vlan_hw_extend_disable(dev);
1068         }
1069 }
1070
1071 static int
1072 ixgbe_dev_configure(struct rte_eth_dev *dev)
1073 {
1074         struct ixgbe_interrupt *intr =
1075                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1076
1077         PMD_INIT_FUNC_TRACE();
1078
1079         /* set flag to update link status after init */
1080         intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1081
1082         return 0;
1083 }
1084
1085 /*
1086  * Configure device link speed and setup link.
1087  * It returns 0 on success.
1088  */
1089 static int
1090 ixgbe_dev_start(struct rte_eth_dev *dev)
1091 {
1092         struct ixgbe_hw *hw =
1093                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1094         int err, link_up = 0, negotiate = 0;
1095         uint32_t speed = 0;
1096
1097         PMD_INIT_FUNC_TRACE();
1098
1099         /* IXGBE devices don't support half duplex */
1100         if ((dev->data->dev_conf.link_duplex != ETH_LINK_AUTONEG_DUPLEX) &&
1101                         (dev->data->dev_conf.link_duplex != ETH_LINK_FULL_DUPLEX)) {
1102                 PMD_INIT_LOG(ERR, "Invalid link_duplex (%u) for port %u\n",
1103                                 dev->data->dev_conf.link_duplex,
1104                                 dev->data->port_id);
1105                 return -EINVAL;
1106         }
1107
1108         /* stop adapter */
1109         hw->adapter_stopped = FALSE;
1110         ixgbe_stop_adapter(hw);
1111
1112         /* reinitialize adapter
1113          * this calls reset and start */
1114         ixgbe_init_hw(hw);
1115
1116         /* initialize transmission unit */
1117         ixgbe_dev_tx_init(dev);
1118
1119         /* This can fail when allocating mbufs for descriptor rings */
1120         err = ixgbe_dev_rx_init(dev);
1121         if (err) {
1122                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware\n");
1123                 return err;
1124         }
1125
1126         ixgbe_dev_rxtx_start(dev);
1127
1128         if (ixgbe_is_sfp(hw) && hw->phy.multispeed_fiber) {
1129                 err = hw->mac.ops.setup_sfp(hw);
1130                 if (err)
1131                         goto error;
1132         }
1133
1134         /* Turn on the laser */
1135         if (hw->phy.multispeed_fiber)
1136                 ixgbe_enable_tx_laser(hw);
1137
1138         err = ixgbe_check_link(hw, &speed, &link_up, 0);
1139         if (err)
1140                 goto error;
1141         err = ixgbe_get_link_capabilities(hw, &speed, &negotiate);
1142         if (err)
1143                 goto error;
1144
1145         switch(dev->data->dev_conf.link_speed) {
1146         case ETH_LINK_SPEED_AUTONEG:
1147                 speed = (hw->mac.type != ixgbe_mac_82598EB) ?
1148                                 IXGBE_LINK_SPEED_82599_AUTONEG :
1149                                 IXGBE_LINK_SPEED_82598_AUTONEG;
1150                 break;
1151         case ETH_LINK_SPEED_100:
1152                 /*
1153                  * Invalid for 82598 but error will be detected by
1154                  * ixgbe_setup_link()
1155                  */
1156                 speed = IXGBE_LINK_SPEED_100_FULL;
1157                 break;
1158         case ETH_LINK_SPEED_1000:
1159                 speed = IXGBE_LINK_SPEED_1GB_FULL;
1160                 break;
1161         case ETH_LINK_SPEED_10000:
1162                 speed = IXGBE_LINK_SPEED_10GB_FULL;
1163                 break;
1164         default:
1165                 PMD_INIT_LOG(ERR, "Invalid link_speed (%u) for port %u\n",
1166                                 dev->data->dev_conf.link_speed, dev->data->port_id);
1167                 return -EINVAL;
1168         }
1169
1170         err = ixgbe_setup_link(hw, speed, negotiate, link_up);
1171         if (err)
1172                 goto error;
1173
1174         /* check if lsc interrupt is enabled */
1175         if (dev->data->dev_conf.intr_conf.lsc != 0) {
1176                 err = ixgbe_dev_interrupt_setup(dev);
1177                 if (err)
1178                         goto error;
1179         }
1180
1181         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
1182                 ETH_VLAN_EXTEND_MASK;
1183         ixgbe_vlan_offload_set(dev, mask);
1184
1185         /* Configure DCB hw */
1186         ixgbe_configure_dcb(dev);
1187
1188         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) {
1189                 err = ixgbe_fdir_configure(dev);
1190                 if (err)
1191                         goto error;
1192         }
1193
1194         ixgbe_restore_statistics_mapping(dev);
1195
1196         return (0);
1197
1198 error:
1199         PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err);
1200         ixgbe_dev_clear_queues(dev);
1201         return -EIO;
1202 }
1203
1204 /*
1205  * Stop device: disable rx and tx functions to allow for reconfiguring.
1206  */
1207 static void
1208 ixgbe_dev_stop(struct rte_eth_dev *dev)
1209 {
1210         struct rte_eth_link link;
1211         struct ixgbe_hw *hw =
1212                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1213
1214         PMD_INIT_FUNC_TRACE();
1215
1216         /* disable interrupts */
1217         ixgbe_disable_intr(hw);
1218
1219         /* reset the NIC */
1220         ixgbe_reset_hw(hw);
1221         hw->adapter_stopped = FALSE;
1222
1223         /* stop adapter */
1224         ixgbe_stop_adapter(hw);
1225
1226         /* Turn off the laser */
1227         if (hw->phy.multispeed_fiber)
1228                 ixgbe_disable_tx_laser(hw);
1229
1230         ixgbe_dev_clear_queues(dev);
1231
1232         /* Clear recorded link status */
1233         memset(&link, 0, sizeof(link));
1234         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1235 }
1236
1237 /*
1238  * Reest and stop device.
1239  */
1240 static void
1241 ixgbe_dev_close(struct rte_eth_dev *dev)
1242 {
1243         struct ixgbe_hw *hw =
1244                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1245
1246         PMD_INIT_FUNC_TRACE();
1247
1248         ixgbe_reset_hw(hw);
1249
1250
1251         ixgbe_dev_stop(dev);
1252         hw->adapter_stopped = 1;
1253
1254         ixgbe_disable_pcie_master(hw);
1255
1256         /* reprogram the RAR[0] in case user changed it. */
1257         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
1258 }
1259
1260 /*
1261  * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c
1262  */
1263 static void
1264 ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1265 {
1266         struct ixgbe_hw *hw =
1267                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1268         struct ixgbe_hw_stats *hw_stats =
1269                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1270         uint32_t bprc, lxon, lxoff, total;
1271         uint64_t total_missed_rx, total_qbrc, total_qprc;
1272         unsigned i;
1273
1274         total_missed_rx = 0;
1275         total_qbrc = 0;
1276         total_qprc = 0;
1277
1278         hw_stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
1279         hw_stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
1280         hw_stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
1281         hw_stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
1282
1283         for (i = 0; i < 8; i++) {
1284                 uint32_t mp;
1285                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
1286                 /* global total per queue */
1287                 hw_stats->mpc[i] += mp;
1288                 /* Running comprehensive total for stats display */
1289                 total_missed_rx += hw_stats->mpc[i];
1290                 if (hw->mac.type == ixgbe_mac_82598EB)
1291                         hw_stats->rnbc[i] +=
1292                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
1293                 hw_stats->pxontxc[i] +=
1294                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
1295                 hw_stats->pxonrxc[i] +=
1296                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
1297                 hw_stats->pxofftxc[i] +=
1298                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
1299                 hw_stats->pxoffrxc[i] +=
1300                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
1301                 hw_stats->pxon2offc[i] +=
1302                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
1303         }
1304         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1305                 hw_stats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
1306                 hw_stats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
1307                 hw_stats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
1308                 hw_stats->qbrc[i] +=
1309                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32);
1310                 hw_stats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
1311                 hw_stats->qbtc[i] +=
1312                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32);
1313                 hw_stats->qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
1314
1315                 total_qprc += hw_stats->qprc[i];
1316                 total_qbrc += hw_stats->qbrc[i];
1317         }
1318         hw_stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
1319         hw_stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
1320         hw_stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
1321
1322         /* Note that gprc counts missed packets */
1323         hw_stats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
1324
1325         if (hw->mac.type != ixgbe_mac_82598EB) {
1326                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
1327                     ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
1328                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
1329                     ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
1330                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
1331                     ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
1332                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
1333                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
1334         } else {
1335                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
1336                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
1337                 /* 82598 only has a counter in the high register */
1338                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
1339                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
1340                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
1341         }
1342
1343         /*
1344          * Workaround: mprc hardware is incorrectly counting
1345          * broadcasts, so for now we subtract those.
1346          */
1347         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
1348         hw_stats->bprc += bprc;
1349         hw_stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
1350         if (hw->mac.type == ixgbe_mac_82598EB)
1351                 hw_stats->mprc -= bprc;
1352
1353         hw_stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
1354         hw_stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
1355         hw_stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
1356         hw_stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
1357         hw_stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
1358         hw_stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
1359
1360         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
1361         hw_stats->lxontxc += lxon;
1362         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
1363         hw_stats->lxofftxc += lxoff;
1364         total = lxon + lxoff;
1365
1366         hw_stats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
1367         hw_stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
1368         hw_stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
1369         hw_stats->gptc -= total;
1370         hw_stats->mptc -= total;
1371         hw_stats->ptc64 -= total;
1372         hw_stats->gotc -= total * ETHER_MIN_LEN;
1373
1374         hw_stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
1375         hw_stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
1376         hw_stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
1377         hw_stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
1378         hw_stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
1379         hw_stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
1380         hw_stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
1381         hw_stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
1382         hw_stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
1383         hw_stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
1384         hw_stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
1385         hw_stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
1386         hw_stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
1387         hw_stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
1388         hw_stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
1389         hw_stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);
1390         hw_stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
1391         hw_stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
1392         /* Only read FCOE on 82599 */
1393         if (hw->mac.type != ixgbe_mac_82598EB) {
1394                 hw_stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
1395                 hw_stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
1396                 hw_stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
1397                 hw_stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
1398                 hw_stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
1399         }
1400
1401         if (stats == NULL)
1402                 return;
1403
1404         /* Fill out the rte_eth_stats statistics structure */
1405         stats->ipackets = total_qprc;
1406         stats->ibytes = total_qbrc;
1407         stats->opackets = hw_stats->gptc;
1408         stats->obytes = hw_stats->gotc;
1409         stats->imcasts = hw_stats->mprc;
1410
1411         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1412                 stats->q_ipackets[i] = hw_stats->qprc[i];
1413                 stats->q_opackets[i] = hw_stats->qptc[i];
1414                 stats->q_ibytes[i] = hw_stats->qbrc[i];
1415                 stats->q_obytes[i] = hw_stats->qbtc[i];
1416                 stats->q_errors[i] = hw_stats->qprdc[i];
1417         }
1418
1419         /* Rx Errors */
1420         stats->ierrors = total_missed_rx + hw_stats->crcerrs +
1421                 hw_stats->rlec;
1422
1423         stats->oerrors  = 0;
1424
1425         /* Flow Director Stats registers */
1426         hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
1427         hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
1428         stats->fdirmatch = hw_stats->fdirmatch;
1429         stats->fdirmiss = hw_stats->fdirmiss;
1430 }
1431
1432 static void
1433 ixgbe_dev_stats_reset(struct rte_eth_dev *dev)
1434 {
1435         struct ixgbe_hw_stats *stats =
1436                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1437
1438         /* HW registers are cleared on read */
1439         ixgbe_dev_stats_get(dev, NULL);
1440
1441         /* Reset software totals */
1442         memset(stats, 0, sizeof(*stats));
1443 }
1444
1445 static void
1446 ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1447 {
1448         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1449         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1450                           IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1451
1452         /* Good Rx packet, include VF loopback */
1453         UPDATE_VF_STAT(IXGBE_VFGPRC,
1454             hw_stats->last_vfgprc, hw_stats->vfgprc);
1455
1456         /* Good Rx octets, include VF loopback */
1457         UPDATE_VF_STAT_36BIT(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
1458             hw_stats->last_vfgorc, hw_stats->vfgorc);
1459
1460         /* Good Tx packet, include VF loopback */
1461         UPDATE_VF_STAT(IXGBE_VFGPTC,
1462             hw_stats->last_vfgptc, hw_stats->vfgptc);
1463
1464         /* Good Tx octets, include VF loopback */
1465         UPDATE_VF_STAT_36BIT(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
1466             hw_stats->last_vfgotc, hw_stats->vfgotc);
1467
1468         /* Rx Multicst Packet */
1469         UPDATE_VF_STAT(IXGBE_VFMPRC,
1470             hw_stats->last_vfmprc, hw_stats->vfmprc);
1471
1472         if (stats == NULL)
1473                 return;
1474
1475         memset(stats, 0, sizeof(*stats));
1476         stats->ipackets = hw_stats->vfgprc;
1477         stats->ibytes = hw_stats->vfgorc;
1478         stats->opackets = hw_stats->vfgptc;
1479         stats->obytes = hw_stats->vfgotc;
1480         stats->imcasts = hw_stats->vfmprc;
1481 }
1482
1483 static void
1484 ixgbevf_dev_stats_reset(struct rte_eth_dev *dev)
1485 {
1486         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1487                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1488
1489         /* Sync HW register to the last stats */
1490         ixgbevf_dev_stats_get(dev, NULL);
1491
1492         /* reset HW current stats*/
1493         hw_stats->vfgprc = 0;
1494         hw_stats->vfgorc = 0;
1495         hw_stats->vfgptc = 0;
1496         hw_stats->vfgotc = 0;
1497         hw_stats->vfmprc = 0;
1498
1499 }
1500
1501 static void
1502 ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1503 {
1504         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1505
1506         dev_info->max_rx_queues = hw->mac.max_rx_queues;
1507         dev_info->max_tx_queues = hw->mac.max_tx_queues;
1508         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL register */
1509         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */
1510         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
1511 }
1512
1513 /* return 0 means link status changed, -1 means not changed */
1514 static int
1515 ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1516 {
1517         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1518         struct rte_eth_link link, old;
1519         ixgbe_link_speed link_speed;
1520         int link_up;
1521         int diag;
1522
1523         link.link_status = 0;
1524         link.link_speed = 0;
1525         link.link_duplex = 0;
1526         memset(&old, 0, sizeof(old));
1527         rte_ixgbe_dev_atomic_read_link_status(dev, &old);
1528
1529         /* check if it needs to wait to complete, if lsc interrupt is enabled */
1530         if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
1531                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 0);
1532         else
1533                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 1);
1534         if (diag != 0) {
1535                 link.link_speed = ETH_LINK_SPEED_100;
1536                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1537                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1538                 if (link.link_status == old.link_status)
1539                         return -1;
1540                 return 0;
1541         }
1542
1543         if (link_up == 0) {
1544                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1545                 if (link.link_status == old.link_status)
1546                         return -1;
1547                 return 0;
1548         }
1549         link.link_status = 1;
1550         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1551
1552         switch (link_speed) {
1553         default:
1554         case IXGBE_LINK_SPEED_UNKNOWN:
1555                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1556                 link.link_speed = ETH_LINK_SPEED_100;
1557                 break;
1558
1559         case IXGBE_LINK_SPEED_100_FULL:
1560                 link.link_speed = ETH_LINK_SPEED_100;
1561                 break;
1562
1563         case IXGBE_LINK_SPEED_1GB_FULL:
1564                 link.link_speed = ETH_LINK_SPEED_1000;
1565                 break;
1566
1567         case IXGBE_LINK_SPEED_10GB_FULL:
1568                 link.link_speed = ETH_LINK_SPEED_10000;
1569                 break;
1570         }
1571         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1572
1573         if (link.link_status == old.link_status)
1574                 return -1;
1575
1576         return 0;
1577 }
1578
1579 static void
1580 ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev)
1581 {
1582         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1583         uint32_t fctrl;
1584
1585         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1586         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1587         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1588 }
1589
1590 static void
1591 ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev)
1592 {
1593         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1594         uint32_t fctrl;
1595
1596         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1597         fctrl &= (~IXGBE_FCTRL_UPE);
1598         if (dev->data->all_multicast == 1)
1599                 fctrl |= IXGBE_FCTRL_MPE;
1600         else
1601                 fctrl &= (~IXGBE_FCTRL_MPE);
1602         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1603 }
1604
1605 static void
1606 ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev)
1607 {
1608         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1609         uint32_t fctrl;
1610
1611         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1612         fctrl |= IXGBE_FCTRL_MPE;
1613         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1614 }
1615
1616 static void
1617 ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev)
1618 {
1619         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1620         uint32_t fctrl;
1621
1622         if (dev->data->promiscuous == 1)
1623                 return; /* must remain in all_multicast mode */
1624
1625         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1626         fctrl &= (~IXGBE_FCTRL_MPE);
1627         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1628 }
1629
1630 /**
1631  * It clears the interrupt causes and enables the interrupt.
1632  * It will be called once only during nic initialized.
1633  *
1634  * @param dev
1635  *  Pointer to struct rte_eth_dev.
1636  *
1637  * @return
1638  *  - On success, zero.
1639  *  - On failure, a negative value.
1640  */
1641 static int
1642 ixgbe_dev_interrupt_setup(struct rte_eth_dev *dev)
1643 {
1644         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1645
1646         ixgbe_dev_link_status_print(dev);
1647         IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_LSC);
1648         IXGBE_WRITE_FLUSH(hw);
1649         rte_intr_enable(&(dev->pci_dev->intr_handle));
1650
1651         return 0;
1652 }
1653
1654 /*
1655  * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update.
1656  *
1657  * @param dev
1658  *  Pointer to struct rte_eth_dev.
1659  *
1660  * @return
1661  *  - On success, zero.
1662  *  - On failure, a negative value.
1663  */
1664 static int
1665 ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
1666 {
1667         uint32_t eicr;
1668         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1669         struct ixgbe_interrupt *intr =
1670                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1671
1672         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_LSC);
1673         IXGBE_WRITE_FLUSH(hw);
1674
1675         /* read-on-clear nic registers here */
1676         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1677         PMD_INIT_LOG(INFO, "eicr %x", eicr);
1678         if (eicr & IXGBE_EICR_LSC) {
1679                 /* set flag for async link update */
1680                 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1681         }
1682
1683         return 0;
1684 }
1685
1686 /**
1687  * It gets and then prints the link status.
1688  *
1689  * @param dev
1690  *  Pointer to struct rte_eth_dev.
1691  *
1692  * @return
1693  *  - On success, zero.
1694  *  - On failure, a negative value.
1695  */
1696 static void
1697 ixgbe_dev_link_status_print(struct rte_eth_dev *dev)
1698 {
1699         struct rte_eth_link link;
1700
1701         memset(&link, 0, sizeof(link));
1702         rte_ixgbe_dev_atomic_read_link_status(dev, &link);
1703         if (link.link_status) {
1704                 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
1705                                         (int)(dev->data->port_id),
1706                                         (unsigned)link.link_speed,
1707                         link.link_duplex == ETH_LINK_FULL_DUPLEX ?
1708                                         "full-duplex" : "half-duplex");
1709         } else {
1710                 PMD_INIT_LOG(INFO, " Port %d: Link Down",
1711                                 (int)(dev->data->port_id));
1712         }
1713         PMD_INIT_LOG(INFO, "PCI Address: %04d:%02d:%02d:%d",
1714                                 dev->pci_dev->addr.domain,
1715                                 dev->pci_dev->addr.bus,
1716                                 dev->pci_dev->addr.devid,
1717                                 dev->pci_dev->addr.function);
1718 }
1719
1720 /*
1721  * It executes link_update after knowing an interrupt occured.
1722  *
1723  * @param dev
1724  *  Pointer to struct rte_eth_dev.
1725  *
1726  * @return
1727  *  - On success, zero.
1728  *  - On failure, a negative value.
1729  */
1730 static int
1731 ixgbe_dev_interrupt_action(struct rte_eth_dev *dev)
1732 {
1733         struct ixgbe_interrupt *intr =
1734                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1735
1736         if (!(intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE)) {
1737                 return -1;
1738         }
1739         ixgbe_dev_link_update(dev, 0);
1740
1741         return 0;
1742 }
1743
1744 /**
1745  * Interrupt handler which shall be registered for alarm callback for delayed
1746  * handling specific interrupt to wait for the stable nic state. As the
1747  * NIC interrupt state is not stable for ixgbe after link is just down,
1748  * it needs to wait 4 seconds to get the stable status.
1749  *
1750  * @param handle
1751  *  Pointer to interrupt handle.
1752  * @param param
1753  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1754  *
1755  * @return
1756  *  void
1757  */
1758 static void
1759 ixgbe_dev_interrupt_delayed_handler(void *param)
1760 {
1761         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1762         struct ixgbe_interrupt *intr =
1763                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1764         struct ixgbe_hw *hw =
1765                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1766
1767         IXGBE_READ_REG(hw, IXGBE_EICR);
1768         ixgbe_dev_interrupt_action(dev);
1769         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
1770                 intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
1771                 rte_intr_enable(&(dev->pci_dev->intr_handle));
1772                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_LSC);
1773                 IXGBE_WRITE_FLUSH(hw);
1774                 ixgbe_dev_link_status_print(dev);
1775                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
1776         }
1777 }
1778
1779 /**
1780  * Interrupt handler triggered by NIC  for handling
1781  * specific interrupt.
1782  *
1783  * @param handle
1784  *  Pointer to interrupt handle.
1785  * @param param
1786  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1787  *
1788  * @return
1789  *  void
1790  */
1791 static void
1792 ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle, void *param)
1793 {
1794         int64_t timeout;
1795         struct rte_eth_link link;
1796         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1797         struct ixgbe_interrupt *intr =
1798                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1799
1800         /* get the link status before link update, for predicting later */
1801         memset(&link, 0, sizeof(link));
1802         rte_ixgbe_dev_atomic_read_link_status(dev, &link);
1803         ixgbe_dev_interrupt_get_status(dev);
1804         ixgbe_dev_interrupt_action(dev);
1805
1806         if (!(intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE))
1807                 return;
1808
1809         /* likely to up */
1810         if (!link.link_status)
1811                 /* handle it 1 sec later, wait it being stable */
1812                 timeout = IXGBE_LINK_UP_CHECK_TIMEOUT;
1813         /* likely to down */
1814         else
1815                 /* handle it 4 sec later, wait it being stable */
1816                 timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT;
1817
1818         ixgbe_dev_link_status_print(dev);
1819         if (rte_eal_alarm_set(timeout * 1000,
1820                 ixgbe_dev_interrupt_delayed_handler, param) < 0)
1821                 PMD_INIT_LOG(ERR, "Error setting alarm");
1822 }
1823
1824 static int
1825 ixgbe_dev_led_on(struct rte_eth_dev *dev)
1826 {
1827         struct ixgbe_hw *hw;
1828
1829         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1830         return (ixgbe_led_on(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
1831 }
1832
1833 static int
1834 ixgbe_dev_led_off(struct rte_eth_dev *dev)
1835 {
1836         struct ixgbe_hw *hw;
1837
1838         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1839         return (ixgbe_led_off(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
1840 }
1841
1842 static int
1843 ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1844 {
1845         struct ixgbe_hw *hw;
1846         int err;
1847         uint32_t rx_buf_size;
1848         uint32_t max_high_water;
1849         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
1850                 ixgbe_fc_none,
1851                 ixgbe_fc_rx_pause,
1852                 ixgbe_fc_tx_pause,
1853                 ixgbe_fc_full
1854         };
1855
1856         PMD_INIT_FUNC_TRACE();
1857
1858         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1859         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0));
1860         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
1861
1862         /*
1863          * At least reserve one Ethernet frame for watermark
1864          * high_water/low_water in kilo bytes for ixgbe
1865          */
1866         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
1867         if ((fc_conf->high_water > max_high_water) ||
1868                 (fc_conf->high_water < fc_conf->low_water)) {
1869                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
1870                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
1871                 return (-EINVAL);
1872         }
1873
1874         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[fc_conf->mode];
1875         hw->fc.pause_time     = fc_conf->pause_time;
1876         hw->fc.high_water[0]  = fc_conf->high_water;
1877         hw->fc.low_water[0]   = fc_conf->low_water;
1878         hw->fc.send_xon       = fc_conf->send_xon;
1879
1880         err = ixgbe_fc_enable(hw);
1881         /* Not negotiated is not an error case */
1882         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) {
1883                 return 0;
1884         }
1885
1886         PMD_INIT_LOG(ERR, "ixgbe_fc_enable = 0x%x \n", err);
1887         return -EIO;
1888 }
1889
1890 /**
1891  *  ixgbe_pfc_enable_generic - Enable flow control
1892  *  @hw: pointer to hardware structure
1893  *  @tc_num: traffic class number
1894  *  Enable flow control according to the current settings.
1895  */
1896 static int
1897 ixgbe_dcb_pfc_enable_generic(struct ixgbe_hw *hw,uint8_t tc_num)
1898 {
1899         int ret_val = 0;
1900         uint32_t mflcn_reg, fccfg_reg;
1901         uint32_t reg;
1902         uint32_t fcrtl, fcrth;
1903         uint8_t i;
1904         uint8_t nb_rx_en;
1905
1906         /* Validate the water mark configuration */
1907         if (!hw->fc.pause_time) {
1908                 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
1909                 goto out;
1910         }
1911
1912         /* Low water mark of zero causes XOFF floods */
1913         if (hw->fc.current_mode & ixgbe_fc_tx_pause) {
1914                  /* High/Low water can not be 0 */
1915                 if( (!hw->fc.high_water[tc_num])|| (!hw->fc.low_water[tc_num])) {
1916                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
1917                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
1918                         goto out;
1919                 }
1920
1921                 if(hw->fc.low_water[tc_num] >= hw->fc.high_water[tc_num]) {
1922                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
1923                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
1924                         goto out;
1925                 }
1926         }
1927         /* Negotiate the fc mode to use */
1928         ixgbe_fc_autoneg(hw);
1929
1930         /* Disable any previous flow control settings */
1931         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
1932         mflcn_reg &= ~(IXGBE_MFLCN_RPFCE_SHIFT | IXGBE_MFLCN_RFCE|IXGBE_MFLCN_RPFCE);
1933
1934         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
1935         fccfg_reg &= ~(IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY);
1936
1937         switch (hw->fc.current_mode) {
1938         case ixgbe_fc_none:
1939                 /*
1940                  * If the count of enabled RX Priority Flow control >1,
1941                  * and the TX pause can not be disabled
1942                  */
1943                 nb_rx_en = 0;
1944                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
1945                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
1946                         if (reg & IXGBE_FCRTH_FCEN)
1947                                 nb_rx_en++;
1948                 }
1949                 if (nb_rx_en > 1)
1950                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
1951                 break;
1952         case ixgbe_fc_rx_pause:
1953                 /*
1954                  * Rx Flow control is enabled and Tx Flow control is
1955                  * disabled by software override. Since there really
1956                  * isn't a way to advertise that we are capable of RX
1957                  * Pause ONLY, we will advertise that we support both
1958                  * symmetric and asymmetric Rx PAUSE.  Later, we will
1959                  * disable the adapter's ability to send PAUSE frames.
1960                  */
1961                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
1962                 /*
1963                  * If the count of enabled RX Priority Flow control >1,
1964                  * and the TX pause can not be disabled
1965                  */
1966                 nb_rx_en = 0;
1967                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
1968                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
1969                         if (reg & IXGBE_FCRTH_FCEN)
1970                                 nb_rx_en++;
1971                 }
1972                 if (nb_rx_en > 1)
1973                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
1974                 break;
1975         case ixgbe_fc_tx_pause:
1976                 /*
1977                  * Tx Flow control is enabled, and Rx Flow control is
1978                  * disabled by software override.
1979                  */
1980                 fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
1981                 break;
1982         case ixgbe_fc_full:
1983                 /* Flow control (both Rx and Tx) is enabled by SW override. */
1984                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
1985                 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
1986                 break;
1987         default:
1988                 DEBUGOUT("Flow control param set incorrectly\n");
1989                 ret_val = IXGBE_ERR_CONFIG;
1990                 goto out;
1991                 break;
1992         }
1993
1994         /* Set 802.3x based flow control settings. */
1995         mflcn_reg |= IXGBE_MFLCN_DPF;
1996         IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg);
1997         IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg);
1998
1999         /* Set up and enable Rx high/low water mark thresholds, enable XON. */
2000         if ((hw->fc.current_mode & ixgbe_fc_tx_pause) &&
2001                 hw->fc.high_water[tc_num]) {
2002                 fcrtl = (hw->fc.low_water[tc_num] << 10) | IXGBE_FCRTL_XONE;
2003                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), fcrtl);
2004                 fcrth = (hw->fc.high_water[tc_num] << 10) | IXGBE_FCRTH_FCEN;
2005         } else {
2006                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), 0);
2007                 /*
2008                  * In order to prevent Tx hangs when the internal Tx
2009                  * switch is enabled we must set the high water mark
2010                  * to the maximum FCRTH value.  This allows the Tx
2011                  * switch to function even under heavy Rx workloads.
2012                  */
2013                 fcrth = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)) - 32;
2014         }
2015         IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(tc_num), fcrth);
2016
2017         /* Configure pause time (2 TCs per register) */
2018         reg = hw->fc.pause_time * 0x00010001;
2019         for (i = 0; i < (IXGBE_DCB_MAX_TRAFFIC_CLASS / 2); i++)
2020                 IXGBE_WRITE_REG(hw, IXGBE_FCTTV(i), reg);
2021
2022         /* Configure flow control refresh threshold value */
2023         IXGBE_WRITE_REG(hw, IXGBE_FCRTV, hw->fc.pause_time / 2);
2024
2025 out:
2026         return ret_val;
2027 }
2028
2029 static int
2030 ixgbe_dcb_pfc_enable(struct rte_eth_dev *dev,uint8_t tc_num)
2031 {
2032         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2033         int32_t ret_val = IXGBE_NOT_IMPLEMENTED;
2034
2035         if(hw->mac.type != ixgbe_mac_82598EB) {
2036                 ret_val = ixgbe_dcb_pfc_enable_generic(hw,tc_num);
2037         }
2038         return ret_val;
2039 }
2040
2041 static int
2042 ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf)
2043 {
2044         int err;
2045         uint32_t rx_buf_size;
2046         uint32_t max_high_water;
2047         uint8_t tc_num;
2048         uint8_t  map[IXGBE_DCB_MAX_USER_PRIORITY] = { 0 };
2049         struct ixgbe_hw *hw =
2050                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2051         struct ixgbe_dcb_config *dcb_config =
2052                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
2053
2054         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
2055                 ixgbe_fc_none,
2056                 ixgbe_fc_rx_pause,
2057                 ixgbe_fc_tx_pause,
2058                 ixgbe_fc_full
2059         };
2060
2061         PMD_INIT_FUNC_TRACE();
2062
2063         ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
2064         tc_num = map[pfc_conf->priority];
2065         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num));
2066         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
2067         /*
2068          * At least reserve one Ethernet frame for watermark
2069          * high_water/low_water in kilo bytes for ixgbe
2070          */
2071         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
2072         if ((pfc_conf->fc.high_water > max_high_water) ||
2073                 (pfc_conf->fc.high_water <= pfc_conf->fc.low_water)) {
2074                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
2075                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
2076                 return (-EINVAL);
2077         }
2078
2079         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[pfc_conf->fc.mode];
2080         hw->fc.pause_time = pfc_conf->fc.pause_time;
2081         hw->fc.send_xon = pfc_conf->fc.send_xon;
2082         hw->fc.low_water[tc_num] =  pfc_conf->fc.low_water;
2083         hw->fc.high_water[tc_num] = pfc_conf->fc.high_water;
2084
2085         err = ixgbe_dcb_pfc_enable(dev,tc_num);
2086
2087         /* Not negotiated is not an error case */
2088         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED))
2089                 return 0;
2090
2091         PMD_INIT_LOG(ERR, "ixgbe_dcb_pfc_enable = 0x%x \n", err);
2092         return -EIO;
2093 }
2094
2095 static void
2096 ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
2097                                 uint32_t index, uint32_t pool)
2098 {
2099         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2100         uint32_t enable_addr = 1;
2101
2102         ixgbe_set_rar(hw, index, mac_addr->addr_bytes, pool, enable_addr);
2103 }
2104
2105 static void
2106 ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
2107 {
2108         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2109
2110         ixgbe_clear_rar(hw, index);
2111 }
2112
2113 /*
2114  * Virtual Function operations
2115  */
2116 static void
2117 ixgbevf_intr_disable(struct ixgbe_hw *hw)
2118 {
2119         PMD_INIT_LOG(DEBUG, "ixgbevf_intr_disable");
2120
2121         /* Clear interrupt mask to stop from interrupts being generated */
2122         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, IXGBE_VF_IRQ_CLEAR_MASK);
2123
2124         IXGBE_WRITE_FLUSH(hw);
2125 }
2126
2127 static int
2128 ixgbevf_dev_configure(struct rte_eth_dev *dev)
2129 {
2130         struct rte_eth_conf* conf = &dev->data->dev_conf;
2131
2132
2133                 /*
2134                  * VF has no ability to enable/disable HW CRC
2135                  * Keep the persistent behavior the same as Host PF
2136                  */
2137 #ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC
2138         if (!conf->rxmode.hw_strip_crc) {
2139                 PMD_INIT_LOG(INFO, "VF can't disable HW CRC Strip\n");
2140                 conf->rxmode.hw_strip_crc = 1;
2141         }
2142 #else
2143         if (conf->rxmode.hw_strip_crc) {
2144                 PMD_INIT_LOG(INFO, "VF can't enable HW CRC Strip\n");
2145                 conf->rxmode.hw_strip_crc = 0;
2146         }
2147 #endif
2148
2149         return 0;
2150 }
2151
2152 static int
2153 ixgbevf_dev_start(struct rte_eth_dev *dev)
2154 {
2155         int err = 0;
2156         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_start");
2157
2158         ixgbevf_dev_tx_init(dev);
2159         err = ixgbevf_dev_rx_init(dev);
2160         if(err){
2161                 ixgbe_dev_clear_queues(dev);
2162                 PMD_INIT_LOG(ERR,"Unable to initialize RX hardware\n");
2163                 return err;
2164         }
2165
2166         /* Set vfta */
2167         ixgbevf_set_vfta_all(dev,1);
2168
2169         /* Set HW strip */
2170         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
2171                 ETH_VLAN_EXTEND_MASK;
2172         ixgbevf_vlan_offload_set(dev, mask);
2173
2174         ixgbevf_dev_rxtx_start(dev);
2175
2176         return 0;
2177 }
2178
2179 static void
2180 ixgbevf_dev_stop(struct rte_eth_dev *dev)
2181 {
2182         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2183
2184         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_stop");
2185
2186         ixgbe_reset_hw(hw);
2187         hw->adapter_stopped = 0;
2188         ixgbe_stop_adapter(hw);
2189         /* reprogram the RAR[0] in case user changed it. */
2190         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
2191 }
2192
2193 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on)
2194 {
2195         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2196         struct ixgbe_vfta * shadow_vfta =
2197                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2198         int i = 0, j = 0, vfta = 0, mask = 1;
2199
2200         for (i = 0; i < IXGBE_VFTA_SIZE; i++){
2201                 vfta = shadow_vfta->vfta[i];
2202                 if(vfta){
2203                         mask = 1;
2204                         for (j = 0; j < 32; j++){
2205                                 if(vfta & mask)
2206                                         ixgbe_set_vfta(hw, (i<<5)+j, 0, on);
2207                                 mask<<=1;
2208                         }
2209                 }
2210         }
2211
2212 }
2213
2214 static int
2215 ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2216 {
2217         struct ixgbe_hw *hw =
2218                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2219         struct ixgbe_vfta * shadow_vfta =
2220                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2221         uint32_t vid_idx = 0;
2222         uint32_t vid_bit = 0;
2223         int ret = 0;
2224
2225         PMD_INIT_FUNC_TRACE();
2226
2227         /* vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf */
2228         ret = ixgbe_set_vfta(hw, vlan_id, 0, !!on);
2229         if(ret){
2230                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
2231                 return ret;
2232         }
2233         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
2234         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
2235
2236         /* Save what we set and retore it after device reset */
2237         if (on)
2238                 shadow_vfta->vfta[vid_idx] |= vid_bit;
2239         else
2240                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
2241
2242         return 0;
2243 }
2244
2245 static void
2246 ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
2247 {
2248         struct ixgbe_hw *hw =
2249                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2250         uint32_t ctrl;
2251
2252         PMD_INIT_FUNC_TRACE();
2253
2254         if(queue >= hw->mac.max_rx_queues)
2255                 return;
2256
2257         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
2258         if(on)
2259                 ctrl |= IXGBE_RXDCTL_VME;
2260         else
2261                 ctrl &= ~IXGBE_RXDCTL_VME;
2262         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
2263
2264         ixgbe_vlan_hw_strip_bitmap_set( dev, queue, on);
2265 }
2266
2267 static void
2268 ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2269 {
2270         struct ixgbe_hw *hw =
2271                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2272         uint32_t i, on = 0;
2273
2274         /* VF function only support hw strip feature, others are not support */
2275         if(mask & ETH_VLAN_STRIP_MASK){
2276                 on = !!(dev->data->dev_conf.rxmode.hw_vlan_strip);
2277
2278                 for(i=0; i < hw->mac.max_rx_queues; i++)
2279                         ixgbevf_vlan_strip_queue_set(dev,i,on);
2280         }
2281 }
2282