lib/librte_pmd_ixgbe/ixgbe_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2013 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  *
  33  */
  34
  35 #include <sys/queue.h>
  36 #include <stdio.h>
  37 #include <errno.h>
  38 #include <stdint.h>
  39 #include <string.h>
  40 #include <unistd.h>
  41 #include <stdarg.h>
  42 #include <inttypes.h>
  43 #include <rte_byteorder.h>
  44 #include <rte_common.h>
  45 #include <rte_cycles.h>
  46
  47 #include <rte_interrupts.h>
  48 #include <rte_log.h>
  49 #include <rte_debug.h>
  50 #include <rte_pci.h>
  51 #include <rte_atomic.h>
  52 #include <rte_branch_prediction.h>
  53 #include <rte_memory.h>
  54 #include <rte_memzone.h>
  55 #include <rte_tailq.h>
  56 #include <rte_eal.h>
  57 #include <rte_alarm.h>
  58 #include <rte_ether.h>
  59 #include <rte_ethdev.h>
  60 #include <rte_atomic.h>
  61 #include <rte_malloc.h>
  62
  63 #include "ixgbe_logs.h"
  64 #include "ixgbe/ixgbe_api.h"
  65 #include "ixgbe/ixgbe_vf.h"
  66 #include "ixgbe/ixgbe_common.h"
  67 #include "ixgbe_ethdev.h"
  68
  69 /*
  70  * High threshold controlling when to start sending XOFF frames. Must be at
  71  * least 8 bytes less than receive packet buffer size. This value is in units
  72  * of 1024 bytes.
  73  */
  74 #define IXGBE_FC_HI    0x80
  75
  76 /*
  77  * Low threshold controlling when to start sending XON frames. This value is
  78  * in units of 1024 bytes.
  79  */
  80 #define IXGBE_FC_LO    0x40
  81
  82 /* Timer value included in XOFF frames. */
  83 #define IXGBE_FC_PAUSE 0x680
  84
  85 #define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */
  86 #define IXGBE_LINK_UP_CHECK_TIMEOUT   1000 /* ms */
  87
  88 #define IXGBE_QUEUE_STAT_COUNTERS (sizeof(hw_stats->qprc) / sizeof(hw_stats->qprc[0]))
  89
  90 static int eth_ixgbe_dev_init(struct eth_driver *eth_drv,
  91                 struct rte_eth_dev *eth_dev);
  92 static int  ixgbe_dev_configure(struct rte_eth_dev *dev);
  93 static int  ixgbe_dev_start(struct rte_eth_dev *dev);
  94 static void ixgbe_dev_stop(struct rte_eth_dev *dev);
  95 static void ixgbe_dev_close(struct rte_eth_dev *dev);
  96 static void ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev);
  97 static void ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev);
  98 static void ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev);
  99 static void ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev);
 100 static int ixgbe_dev_link_update(struct rte_eth_dev *dev,
 101                                 int wait_to_complete);
 102 static void ixgbe_dev_stats_get(struct rte_eth_dev *dev,
 103                                 struct rte_eth_stats *stats);
 104 static void ixgbe_dev_stats_reset(struct rte_eth_dev *dev);
 105 static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 106                                              uint16_t queue_id,
 107                                              uint8_t stat_idx,
 108                                              uint8_t is_rx);
 109 static void ixgbe_dev_info_get(struct rte_eth_dev *dev,
 110                                 struct rte_eth_dev_info *dev_info);
 111 static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev,
 112                 uint16_t vlan_id, int on);
 113 static void ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 114 static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev,
 115                 uint16_t queue, bool on);
 116 static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue,
 117                 int on);
 118 static void ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 119 static void ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev);
 120 static void ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev);
 121 static void ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev);
 122 static void ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev);
 123 static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue);
 124 static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue);
 125 static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 126 static void ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 127
 128 static int ixgbe_dev_led_on(struct rte_eth_dev *dev);
 129 static int ixgbe_dev_led_off(struct rte_eth_dev *dev);
 130 static int  ixgbe_flow_ctrl_set(struct rte_eth_dev *dev,
 131                 struct rte_eth_fc_conf *fc_conf);
 132 static int ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 133                 struct rte_eth_pfc_conf *pfc_conf);
 134 static int ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
 135                 struct rte_eth_rss_reta *reta_conf);
 136 static int ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
 137                 struct rte_eth_rss_reta *reta_conf);
 138 static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev);
 139 static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev);
 140 static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev);
 141 static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev);
 142 static void ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle,
 143                 void *param);
 144 static void ixgbe_dev_interrupt_delayed_handler(void *param);
 145 static void ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 146                 uint32_t index, uint32_t pool);
 147 static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index);
 148 static void ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config);
 149
 150 /* For Virtual Function support */
 151 static int eth_ixgbevf_dev_init(struct eth_driver *eth_drv,
 152                 struct rte_eth_dev *eth_dev);
 153 static int  ixgbevf_dev_configure(struct rte_eth_dev *dev);
 154 static int  ixgbevf_dev_start(struct rte_eth_dev *dev);
 155 static void ixgbevf_dev_stop(struct rte_eth_dev *dev);
 156 static void ixgbevf_dev_close(struct rte_eth_dev *dev);
 157 static void ixgbevf_intr_disable(struct ixgbe_hw *hw);
 158 static void ixgbevf_dev_stats_get(struct rte_eth_dev *dev,
 159                 struct rte_eth_stats *stats);
 160 static void ixgbevf_dev_stats_reset(struct rte_eth_dev *dev);
 161 static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev,
 162                 uint16_t vlan_id, int on);
 163 static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev,
 164                 uint16_t queue, int on);
 165 static void ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 166 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 167
 168
 169 /*
 170  * Define VF Stats MACRO for Non "cleared on read" register
 171  */
 172 #define UPDATE_VF_STAT(reg, last, cur)                          \
 173 {                                                               \
 174         u32 latest = IXGBE_READ_REG(hw, reg);                   \
 175         cur += latest - last;                                   \
 176         last = latest;                                          \
 177 }
 178
 179 #define UPDATE_VF_STAT_36BIT(lsb, msb, last, cur)                \
 180 {                                                                \
 181         u64 new_lsb = IXGBE_READ_REG(hw, lsb);                   \
 182         u64 new_msb = IXGBE_READ_REG(hw, msb);                   \
 183         u64 latest = ((new_msb << 32) | new_lsb);                \
 184         cur += (0x1000000000LL + latest - last) & 0xFFFFFFFFFLL; \
 185         last = latest;                                           \
 186 }
 187
 188 #define IXGBE_SET_HWSTRIP(h, q) do{\
 189                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 190                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 191                 (h)->bitmap[idx] |= 1 << bit;\
 192         }while(0)
 193
 194 #define IXGBE_CLEAR_HWSTRIP(h, q) do{\
 195                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 196                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 197                 (h)->bitmap[idx] &= ~(1 << bit);\
 198         }while(0)
 199
 200 #define IXGBE_GET_HWSTRIP(h, q, r) do{\
 201                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 202                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 203                 (r) = (h)->bitmap[idx] >> bit & 1;\
 204         }while(0)
 205
 206 /*
 207  * The set of PCI devices this driver supports
 208  */
 209 static struct rte_pci_id pci_id_ixgbe_map[] = {
 210
 211 #define RTE_PCI_DEV_ID_DECL_IXGBE(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 212 #include "rte_pci_dev_ids.h"
 213
 214 { .vendor_id = 0, /* sentinel */ },
 215 };
 216
 217
 218 /*
 219  * The set of PCI devices this driver supports (for 82599 VF)
 220  */
 221 static struct rte_pci_id pci_id_ixgbevf_map[] = {
 222
 223 #define RTE_PCI_DEV_ID_DECL_IXGBEVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 224 #include "rte_pci_dev_ids.h"
 225 { .vendor_id = 0, /* sentinel */ },
 226
 227 };
 228
 229 static struct eth_dev_ops ixgbe_eth_dev_ops = {
 230         .dev_configure        = ixgbe_dev_configure,
 231         .dev_start            = ixgbe_dev_start,
 232         .dev_stop             = ixgbe_dev_stop,
 233         .dev_close            = ixgbe_dev_close,
 234         .promiscuous_enable   = ixgbe_dev_promiscuous_enable,
 235         .promiscuous_disable  = ixgbe_dev_promiscuous_disable,
 236         .allmulticast_enable  = ixgbe_dev_allmulticast_enable,
 237         .allmulticast_disable = ixgbe_dev_allmulticast_disable,
 238         .link_update          = ixgbe_dev_link_update,
 239         .stats_get            = ixgbe_dev_stats_get,
 240         .stats_reset          = ixgbe_dev_stats_reset,
 241         .queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set,
 242         .dev_infos_get        = ixgbe_dev_info_get,
 243         .vlan_filter_set      = ixgbe_vlan_filter_set,
 244         .vlan_tpid_set        = ixgbe_vlan_tpid_set,
 245         .vlan_offload_set     = ixgbe_vlan_offload_set,
 246         .vlan_strip_queue_set = ixgbe_vlan_strip_queue_set,
 247         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 248         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 249         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 250         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 251         .dev_led_on           = ixgbe_dev_led_on,
 252         .dev_led_off          = ixgbe_dev_led_off,
 253         .flow_ctrl_set        = ixgbe_flow_ctrl_set,
 254         .priority_flow_ctrl_set = ixgbe_priority_flow_ctrl_set,
 255         .mac_addr_add         = ixgbe_add_rar,
 256         .mac_addr_remove      = ixgbe_remove_rar,
 257         .fdir_add_signature_filter    = ixgbe_fdir_add_signature_filter,
 258         .fdir_update_signature_filter = ixgbe_fdir_update_signature_filter,
 259         .fdir_remove_signature_filter = ixgbe_fdir_remove_signature_filter,
 260         .fdir_infos_get               = ixgbe_fdir_info_get,
 261         .fdir_add_perfect_filter      = ixgbe_fdir_add_perfect_filter,
 262         .fdir_update_perfect_filter   = ixgbe_fdir_update_perfect_filter,
 263         .fdir_remove_perfect_filter   = ixgbe_fdir_remove_perfect_filter,
 264         .fdir_set_masks               = ixgbe_fdir_set_masks,
 265         .reta_update          = ixgbe_dev_rss_reta_update,
 266         .reta_query           = ixgbe_dev_rss_reta_query,
 267 };
 268
 269 /*
 270  * dev_ops for virtual function, bare necessities for basic vf
 271  * operation have been implemented
 272  */
 273 static struct eth_dev_ops ixgbevf_eth_dev_ops = {
 274
 275         .dev_configure        = ixgbevf_dev_configure,
 276         .dev_start            = ixgbevf_dev_start,
 277         .dev_stop             = ixgbevf_dev_stop,
 278         .link_update          = ixgbe_dev_link_update,
 279         .stats_get            = ixgbevf_dev_stats_get,
 280         .stats_reset          = ixgbevf_dev_stats_reset,
 281         .dev_close            = ixgbevf_dev_close,
 282         .dev_infos_get        = ixgbe_dev_info_get,
 283         .vlan_filter_set      = ixgbevf_vlan_filter_set,
 284         .vlan_strip_queue_set = ixgbevf_vlan_strip_queue_set,
 285         .vlan_offload_set     = ixgbevf_vlan_offload_set,
 286         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 287         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 288         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 289         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 290 };
 291
 292 /**
 293  * Atomically reads the link status information from global
 294  * structure rte_eth_dev.
 295  *
 296  * @param dev
 297  *   - Pointer to the structure rte_eth_dev to read from.
 298  *   - Pointer to the buffer to be saved with the link status.
 299  *
 300  * @return
 301  *   - On success, zero.
 302  *   - On failure, negative value.
 303  */
 304 static inline int
 305 rte_ixgbe_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 306                                 struct rte_eth_link *link)
 307 {
 308         struct rte_eth_link *dst = link;
 309         struct rte_eth_link *src = &(dev->data->dev_link);
 310
 311         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 312                                         *(uint64_t *)src) == 0)
 313                 return -1;
 314
 315         return 0;
 316 }
 317
 318 /**
 319  * Atomically writes the link status information into global
 320  * structure rte_eth_dev.
 321  *
 322  * @param dev
 323  *   - Pointer to the structure rte_eth_dev to read from.
 324  *   - Pointer to the buffer to be saved with the link status.
 325  *
 326  * @return
 327  *   - On success, zero.
 328  *   - On failure, negative value.
 329  */
 330 static inline int
 331 rte_ixgbe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 332                                 struct rte_eth_link *link)
 333 {
 334         struct rte_eth_link *dst = &(dev->data->dev_link);
 335         struct rte_eth_link *src = link;
 336
 337         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 338                                         *(uint64_t *)src) == 0)
 339                 return -1;
 340
 341         return 0;
 342 }
 343
 344 /*
 345  * This function is the same as ixgbe_is_sfp() in ixgbe/ixgbe.h.
 346  */
 347 static inline int
 348 ixgbe_is_sfp(struct ixgbe_hw *hw)
 349 {
 350         switch (hw->phy.type) {
 351         case ixgbe_phy_sfp_avago:
 352         case ixgbe_phy_sfp_ftl:
 353         case ixgbe_phy_sfp_intel:
 354         case ixgbe_phy_sfp_unknown:
 355         case ixgbe_phy_sfp_passive_tyco:
 356         case ixgbe_phy_sfp_passive_unknown:
 357                 return 1;
 358         default:
 359                 return 0;
 360         }
 361 }
 362
 363 static inline void
 364 ixgbe_enable_intr(struct rte_eth_dev *dev)
 365 {
 366         struct ixgbe_interrupt *intr =
 367                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 368         struct ixgbe_hw *hw =
 369                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 370
 371         IXGBE_WRITE_REG(hw, IXGBE_EIMS, intr->mask);
 372         IXGBE_WRITE_FLUSH(hw);
 373 }
 374
 375 /*
 376  * This function is based on ixgbe_disable_intr() in ixgbe/ixgbe.h.
 377  */
 378 static void
 379 ixgbe_disable_intr(struct ixgbe_hw *hw)
 380 {
 381         PMD_INIT_FUNC_TRACE();
 382
 383         if (hw->mac.type == ixgbe_mac_82598EB) {
 384                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ~0);
 385         } else {
 386                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xFFFF0000);
 387                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), ~0);
 388                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), ~0);
 389         }
 390         IXGBE_WRITE_FLUSH(hw);
 391 }
 392
 393 /*
 394  * This function resets queue statistics mapping registers.
 395  * From Niantic datasheet, Initialization of Statistics section:
 396  * "...if software requires the queue counters, the RQSMR and TQSM registers
 397  * must be re-programmed following a device reset.
 398  */
 399 static void
 400 ixgbe_reset_qstat_mappings(struct ixgbe_hw *hw)
 401 {
 402         uint32_t i;
 403
 404         for(i = 0; i != IXGBE_NB_STAT_MAPPING_REGS; i++) {
 405                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), 0);
 406                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), 0);
 407         }
 408 }
 409
 410
 411 static int
 412 ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 413                                   uint16_t queue_id,
 414                                   uint8_t stat_idx,
 415                                   uint8_t is_rx)
 416 {
 417 #define QSM_REG_NB_BITS_PER_QMAP_FIELD 8
 418 #define NB_QMAP_FIELDS_PER_QSM_REG 4
 419 #define QMAP_FIELD_RESERVED_BITS_MASK 0x0f
 420
 421         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 422         struct ixgbe_stat_mapping_registers *stat_mappings =
 423                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(eth_dev->data->dev_private);
 424         uint32_t qsmr_mask = 0;
 425         uint32_t clearing_mask = QMAP_FIELD_RESERVED_BITS_MASK;
 426         uint32_t q_map;
 427         uint8_t n, offset;
 428
 429         if ((hw->mac.type != ixgbe_mac_82599EB) && (hw->mac.type != ixgbe_mac_X540))
 430                 return -ENOSYS;
 431
 432         PMD_INIT_LOG(INFO, "Setting port %d, %s queue_id %d to stat index %d\n",
 433                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx);
 434
 435         n = (uint8_t)(queue_id / NB_QMAP_FIELDS_PER_QSM_REG);
 436         if (n >= IXGBE_NB_STAT_MAPPING_REGS) {
 437                 PMD_INIT_LOG(ERR, "Nb of stat mapping registers exceeded\n");
 438                 return -EIO;
 439         }
 440         offset = (uint8_t)(queue_id % NB_QMAP_FIELDS_PER_QSM_REG);
 441
 442         /* Now clear any previous stat_idx set */
 443         clearing_mask <<= (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 444         if (!is_rx)
 445                 stat_mappings->tqsm[n] &= ~clearing_mask;
 446         else
 447                 stat_mappings->rqsmr[n] &= ~clearing_mask;
 448
 449         q_map = (uint32_t)stat_idx;
 450         q_map &= QMAP_FIELD_RESERVED_BITS_MASK;
 451         qsmr_mask = q_map << (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 452         if (!is_rx)
 453                 stat_mappings->tqsm[n] |= qsmr_mask;
 454         else
 455                 stat_mappings->rqsmr[n] |= qsmr_mask;
 456
 457         PMD_INIT_LOG(INFO, "Set port %d, %s queue_id %d to stat index %d\n"
 458                      "%s[%d] = 0x%08x\n",
 459                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx,
 460                      is_rx ? "RQSMR" : "TQSM",n, is_rx ? stat_mappings->rqsmr[n] : stat_mappings->tqsm[n]);
 461
 462         /* Now write the mapping in the appropriate register */
 463         if (is_rx) {
 464                 PMD_INIT_LOG(INFO, "Write 0x%x to RX IXGBE stat mapping reg:%d\n",
 465                              stat_mappings->rqsmr[n], n);
 466                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(n), stat_mappings->rqsmr[n]);
 467         }
 468         else {
 469                 PMD_INIT_LOG(INFO, "Write 0x%x to TX IXGBE stat mapping reg:%d\n",
 470                              stat_mappings->tqsm[n], n);
 471                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(n), stat_mappings->tqsm[n]);
 472         }
 473         return 0;
 474 }
 475
 476 static void
 477 ixgbe_restore_statistics_mapping(struct rte_eth_dev * dev)
 478 {
 479         struct ixgbe_stat_mapping_registers *stat_mappings =
 480                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(dev->data->dev_private);
 481         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 482         int i;
 483
 484         /* write whatever was in stat mapping table to the NIC */
 485         for (i = 0; i < IXGBE_NB_STAT_MAPPING_REGS; i++) {
 486                 /* rx */
 487                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), stat_mappings->rqsmr[i]);
 488
 489                 /* tx */
 490                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), stat_mappings->tqsm[i]);
 491         }
 492 }
 493
 494 static void
 495 ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config)
 496 {
 497         uint8_t i;
 498         struct ixgbe_dcb_tc_config *tc;
 499         uint8_t dcb_max_tc = IXGBE_DCB_MAX_TRAFFIC_CLASS;
 500
 501         dcb_config->num_tcs.pg_tcs = dcb_max_tc;
 502         dcb_config->num_tcs.pfc_tcs = dcb_max_tc;
 503         for (i = 0; i < dcb_max_tc; i++) {
 504                 tc = &dcb_config->tc_config[i];
 505                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_id = i;
 506                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
 507                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 508                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_id = i;
 509                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
 510                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 511                 tc->pfc = ixgbe_dcb_pfc_disabled;
 512         }
 513
 514         /* Initialize default user to priority mapping, UPx->TC0 */
 515         tc = &dcb_config->tc_config[0];
 516         tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
 517         tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
 518         for (i = 0; i< IXGBE_DCB_MAX_BW_GROUP; i++) {
 519                 dcb_config->bw_percentage[IXGBE_DCB_TX_CONFIG][i] = 100;
 520                 dcb_config->bw_percentage[IXGBE_DCB_RX_CONFIG][i] = 100;
 521         }
 522         dcb_config->rx_pba_cfg = ixgbe_dcb_pba_equal;
 523         dcb_config->pfc_mode_enable = false;
 524         dcb_config->vt_mode = true;
 525         dcb_config->round_robin_enable = false;
 526         /* support all DCB capabilities in 82599 */
 527         dcb_config->support.capabilities = 0xFF;
 528
 529         /*we only support 4 Tcs for X540*/
 530         if (hw->mac.type == ixgbe_mac_X540) {
 531                 dcb_config->num_tcs.pg_tcs = 4;
 532                 dcb_config->num_tcs.pfc_tcs = 4;
 533         }
 534 }
 535
 536 /*
 537  * This function is based on code in ixgbe_attach() in ixgbe/ixgbe.c.
 538  * It returns 0 on success.
 539  */
 540 static int
 541 eth_ixgbe_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 542                      struct rte_eth_dev *eth_dev)
 543 {
 544         struct rte_pci_device *pci_dev;
 545         struct ixgbe_hw *hw =
 546                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 547         struct ixgbe_vfta * shadow_vfta =
 548                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 549         struct ixgbe_hwstrip *hwstrip =
 550                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 551         struct ixgbe_dcb_config *dcb_config =
 552                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private);
 553         uint32_t ctrl_ext;
 554         uint16_t csum;
 555         int diag, i;
 556
 557         PMD_INIT_FUNC_TRACE();
 558
 559         eth_dev->dev_ops = &ixgbe_eth_dev_ops;
 560         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
 561         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
 562
 563         /* for secondary processes, we don't initialise any further as primary
 564          * has already done this work. Only check we don't need a different
 565          * RX function */
 566         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 567                 if (eth_dev->data->scattered_rx)
 568                         eth_dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
 569                 return 0;
 570         }
 571         pci_dev = eth_dev->pci_dev;
 572
 573         /* Vendor and Device ID need to be set before init of shared code */
 574         hw->device_id = pci_dev->id.device_id;
 575         hw->vendor_id = pci_dev->id.vendor_id;
 576         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 577 #ifdef RTE_LIBRTE_IXGBE_ALLOW_UNSUPPORTED_SFP
 578         hw->allow_unsupported_sfp = 1;
 579 #endif
 580
 581         /* Initialize the shared code */
 582         diag = ixgbe_init_shared_code(hw);
 583         if (diag != IXGBE_SUCCESS) {
 584                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
 585                 return -EIO;
 586         }
 587
 588         /* Initialize DCB configuration*/
 589         memset(dcb_config, 0, sizeof(struct ixgbe_dcb_config));
 590         ixgbe_dcb_init(hw,dcb_config);
 591         /* Get Hardware Flow Control setting */
 592         hw->fc.requested_mode = ixgbe_fc_full;
 593         hw->fc.current_mode = ixgbe_fc_full;
 594         hw->fc.pause_time = IXGBE_FC_PAUSE;
 595         for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
 596                 hw->fc.low_water[i] = IXGBE_FC_LO;
 597                 hw->fc.high_water[i] = IXGBE_FC_HI;
 598         }
 599         hw->fc.send_xon = 1;
 600
 601         /* Make sure we have a good EEPROM before we read from it */
 602         diag = ixgbe_validate_eeprom_checksum(hw, &csum);
 603         if (diag != IXGBE_SUCCESS) {
 604                 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag);
 605                 return -EIO;
 606         }
 607
 608         diag = ixgbe_init_hw(hw);
 609
 610         /*
 611          * Devices with copper phys will fail to initialise if ixgbe_init_hw()
 612          * is called too soon after the kernel driver unbinding/binding occurs.
 613          * The failure occurs in ixgbe_identify_phy_generic() for all devices,
 614          * but for non-copper devies, ixgbe_identify_sfp_module_generic() is
 615          * also called. See ixgbe_identify_phy_82599(). The reason for the
 616          * failure is not known, and only occuts when virtualisation features
 617          * are disabled in the bios. A delay of 100ms  was found to be enough by
 618          * trial-and-error, and is doubled to be safe.
 619          */
 620         if (diag && (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper)) {
 621                 rte_delay_ms(200);
 622                 diag = ixgbe_init_hw(hw);
 623         }
 624
 625         if (diag == IXGBE_ERR_EEPROM_VERSION) {
 626                 PMD_INIT_LOG(ERR, "This device is a pre-production adapter/"
 627                     "LOM.  Please be aware there may be issues associated "
 628                     "with your hardware.\n If you are experiencing problems "
 629                     "please contact your Intel or hardware representative "
 630                     "who provided you with this hardware.\n");
 631         } else if (diag == IXGBE_ERR_SFP_NOT_SUPPORTED)
 632                 PMD_INIT_LOG(ERR, "Unsupported SFP+ Module\n");
 633         if (diag) {
 634                 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag);
 635                 return -EIO;
 636         }
 637
 638         /* disable interrupt */
 639         ixgbe_disable_intr(hw);
 640
 641         /* pick up the PCI bus settings for reporting later */
 642         ixgbe_get_bus_info(hw);
 643
 644         /* reset mappings for queue statistics hw counters*/
 645         ixgbe_reset_qstat_mappings(hw);
 646
 647         /* Allocate memory for storing MAC addresses */
 648         eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 649                         hw->mac.num_rar_entries, 0);
 650         if (eth_dev->data->mac_addrs == NULL) {
 651                 PMD_INIT_LOG(ERR,
 652                         "Failed to allocate %d bytes needed to store MAC addresses",
 653                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 654                 return -ENOMEM;
 655         }
 656         /* Copy the permanent MAC address */
 657         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 658                         &eth_dev->data->mac_addrs[0]);
 659
 660         /* initialize the vfta */
 661         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 662
 663         /* initialize the hw strip bitmap*/
 664         memset(hwstrip, 0, sizeof(*hwstrip));
 665
 666         /* let hardware know driver is loaded */
 667         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 668         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 669         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 670
 671         if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
 672                 PMD_INIT_LOG(DEBUG,
 673                              "MAC: %d, PHY: %d, SFP+: %d<n",
 674                              (int) hw->mac.type, (int) hw->phy.type,
 675                              (int) hw->phy.sfp_type);
 676         else
 677                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d\n",
 678                              (int) hw->mac.type, (int) hw->phy.type);
 679
 680         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
 681                         eth_dev->data->port_id, pci_dev->id.vendor_id,
 682                         pci_dev->id.device_id);
 683
 684         rte_intr_callback_register(&(pci_dev->intr_handle),
 685                 ixgbe_dev_interrupt_handler, (void *)eth_dev);
 686
 687         /* enable uio intr after callback register */
 688         rte_intr_enable(&(pci_dev->intr_handle));
 689
 690         /* enable support intr */
 691         ixgbe_enable_intr(eth_dev);
 692
 693         return 0;
 694 }
 695
 696 /*
 697  * Virtual Function device init
 698  */
 699 static int
 700 eth_ixgbevf_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 701                      struct rte_eth_dev *eth_dev)
 702 {
 703         struct rte_pci_device *pci_dev;
 704         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 705         int diag;
 706         struct ixgbe_vfta * shadow_vfta =
 707                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 708         struct ixgbe_hwstrip *hwstrip =
 709                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 710
 711         PMD_INIT_LOG(DEBUG, "eth_ixgbevf_dev_init");
 712
 713         eth_dev->dev_ops = &ixgbevf_eth_dev_ops;
 714         pci_dev = eth_dev->pci_dev;
 715
 716         hw->device_id = pci_dev->id.device_id;
 717         hw->vendor_id = pci_dev->id.vendor_id;
 718         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 719
 720         /* initialize the vfta */
 721         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 722
 723         /* initialize the hw strip bitmap*/
 724         memset(hwstrip, 0, sizeof(*hwstrip));
 725
 726         /* Initialize the shared code */
 727         diag = ixgbe_init_shared_code(hw);
 728         if (diag != IXGBE_SUCCESS) {
 729                 PMD_INIT_LOG(ERR, "Shared code init failed for ixgbevf: %d", diag);
 730                 return -EIO;
 731         }
 732
 733         /* init_mailbox_params */
 734         hw->mbx.ops.init_params(hw);
 735
 736         /* Disable the interrupts for VF */
 737         ixgbevf_intr_disable(hw);
 738
 739         hw->mac.num_rar_entries = hw->mac.max_rx_queues;
 740         diag = hw->mac.ops.reset_hw(hw);
 741         if (diag != IXGBE_SUCCESS) {
 742                 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
 743                 return (diag);
 744         }
 745
 746         /* Allocate memory for storing MAC addresses */
 747         eth_dev->data->mac_addrs = rte_zmalloc("ixgbevf", ETHER_ADDR_LEN *
 748                         hw->mac.num_rar_entries, 0);
 749         if (eth_dev->data->mac_addrs == NULL) {
 750                 PMD_INIT_LOG(ERR,
 751                         "Failed to allocate %d bytes needed to store MAC addresses",
 752                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 753                 return -ENOMEM;
 754         }
 755         /* Copy the permanent MAC address */
 756         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 757                         &eth_dev->data->mac_addrs[0]);
 758
 759         /* reset the hardware with the new settings */
 760         diag = hw->mac.ops.start_hw(hw);
 761         switch (diag) {
 762                 case  0:
 763                         break;
 764
 765                 default:
 766                         PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
 767                         return (-EIO);
 768         }
 769
 770         PMD_INIT_LOG(DEBUG, "\nport %d vendorID=0x%x deviceID=0x%x mac.type=%s\n",
 771                          eth_dev->data->port_id, pci_dev->id.vendor_id, pci_dev->id.device_id,
 772                          "ixgbe_mac_82599_vf");
 773
 774         return 0;
 775 }
 776
 777 static struct eth_driver rte_ixgbe_pmd = {
 778         {
 779                 .name = "rte_ixgbe_pmd",
 780                 .id_table = pci_id_ixgbe_map,
 781 #ifdef RTE_EAL_UNBIND_PORTS
 782                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
 783 #endif
 784         },
 785         .eth_dev_init = eth_ixgbe_dev_init,
 786         .dev_private_size = sizeof(struct ixgbe_adapter),
 787 };
 788
 789 /*
 790  * virtual function driver struct
 791  */
 792 static struct eth_driver rte_ixgbevf_pmd = {
 793         {
 794                 .name = "rte_ixgbevf_pmd",
 795                 .id_table = pci_id_ixgbevf_map,
 796 #ifdef RTE_EAL_UNBIND_PORTS
 797                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
 798 #endif
 799         },
 800         .eth_dev_init = eth_ixgbevf_dev_init,
 801         .dev_private_size = sizeof(struct ixgbe_adapter),
 802 };
 803
 804 /*
 805  * Driver initialization routine.
 806  * Invoked once at EAL init time.
 807  * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
 808  */
 809 int
 810 rte_ixgbe_pmd_init(void)
 811 {
 812         PMD_INIT_FUNC_TRACE();
 813
 814         rte_eth_driver_register(&rte_ixgbe_pmd);
 815         return 0;
 816 }
 817
 818 /*
 819  * VF Driver initialization routine.
 820  * Invoked one at EAL init time.
 821  * Register itself as the [Virtual Poll Mode] Driver of PCI niantic devices.
 822  */
 823 int
 824 rte_ixgbevf_pmd_init(void)
 825 {
 826         DEBUGFUNC("rte_ixgbevf_pmd_init");
 827
 828         rte_eth_driver_register(&rte_ixgbevf_pmd);
 829         return (0);
 830 }
 831
 832 static int
 833 ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 834 {
 835         struct ixgbe_hw *hw =
 836                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 837         struct ixgbe_vfta * shadow_vfta =
 838                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
 839         uint32_t vfta;
 840         uint32_t vid_idx;
 841         uint32_t vid_bit;
 842
 843         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
 844         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
 845         vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(vid_idx));
 846         if (on)
 847                 vfta |= vid_bit;
 848         else
 849                 vfta &= ~vid_bit;
 850         IXGBE_WRITE_REG(hw, IXGBE_VFTA(vid_idx), vfta);
 851
 852         /* update local VFTA copy */
 853         shadow_vfta->vfta[vid_idx] = vfta;
 854
 855         return 0;
 856 }
 857
 858 static void
 859 ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
 860 {
 861         if (on)
 862                 ixgbe_vlan_hw_strip_enable(dev, queue);
 863         else
 864                 ixgbe_vlan_hw_strip_disable(dev, queue);
 865 }
 866
 867 static void
 868 ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
 869 {
 870         struct ixgbe_hw *hw =
 871                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 872
 873         /* Only the high 16-bits is valid */
 874         IXGBE_WRITE_REG(hw, IXGBE_EXVET, tpid << 16);
 875 }
 876
 877 static void
 878 ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev)
 879 {
 880         struct ixgbe_hw *hw =
 881                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 882         uint32_t vlnctrl;
 883
 884         PMD_INIT_FUNC_TRACE();
 885
 886         /* Filter Table Disable */
 887         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 888         vlnctrl &= ~IXGBE_VLNCTRL_VFE;
 889
 890         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
 891 }
 892
 893 static void
 894 ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev)
 895 {
 896         struct ixgbe_hw *hw =
 897                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 898         struct ixgbe_vfta * shadow_vfta =
 899                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
 900         uint32_t vlnctrl;
 901         uint16_t i;
 902
 903         PMD_INIT_FUNC_TRACE();
 904
 905         /* Filter Table Enable */
 906         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 907         vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
 908         vlnctrl |= IXGBE_VLNCTRL_VFE;
 909
 910         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
 911
 912         /* write whatever is in local vfta copy */
 913         for (i = 0; i < IXGBE_VFTA_SIZE; i++)
 914                 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), shadow_vfta->vfta[i]);
 915 }
 916
 917 static void
 918 ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on)
 919 {
 920         struct ixgbe_hwstrip *hwstrip =
 921                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(dev->data->dev_private);
 922
 923         if(queue >= IXGBE_MAX_RX_QUEUE_NUM)
 924                 return;
 925
 926         if (on)
 927                 IXGBE_SET_HWSTRIP(hwstrip, queue);
 928         else
 929                 IXGBE_CLEAR_HWSTRIP(hwstrip, queue);
 930 }
 931
 932 static void
 933 ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue)
 934 {
 935         struct ixgbe_hw *hw =
 936                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 937         uint32_t ctrl;
 938
 939         PMD_INIT_FUNC_TRACE();
 940
 941         if (hw->mac.type == ixgbe_mac_82598EB) {
 942                 /* No queue level support */
 943                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
 944                 return;
 945         }
 946         else {
 947                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 948                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
 949                 ctrl &= ~IXGBE_RXDCTL_VME;
 950                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
 951         }
 952         /* record those setting for HW strip per queue */
 953         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 0);
 954 }
 955
 956 static void
 957 ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue)
 958 {
 959         struct ixgbe_hw *hw =
 960                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 961         uint32_t ctrl;
 962
 963         PMD_INIT_FUNC_TRACE();
 964
 965         if (hw->mac.type == ixgbe_mac_82598EB) {
 966                 /* No queue level supported */
 967                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
 968                 return;
 969         }
 970         else {
 971                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 972                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
 973                 ctrl |= IXGBE_RXDCTL_VME;
 974                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
 975         }
 976         /* record those setting for HW strip per queue */
 977         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 1);
 978 }
 979
 980 static void
 981 ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev)
 982 {
 983         struct ixgbe_hw *hw =
 984                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 985         uint32_t ctrl;
 986         uint16_t i;
 987
 988         PMD_INIT_FUNC_TRACE();
 989
 990         if (hw->mac.type == ixgbe_mac_82598EB) {
 991                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 992                 ctrl &= ~IXGBE_VLNCTRL_VME;
 993                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
 994         }
 995         else {
 996                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 997                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
 998                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
 999                         ctrl &= ~IXGBE_RXDCTL_VME;
1000                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1001
1002                         /* record those setting for HW strip per queue */
1003                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 0);
1004                 }
1005         }
1006 }
1007
1008 static void
1009 ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev)
1010 {
1011         struct ixgbe_hw *hw =
1012                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1013         uint32_t ctrl;
1014         uint16_t i;
1015
1016         PMD_INIT_FUNC_TRACE();
1017
1018         if (hw->mac.type == ixgbe_mac_82598EB) {
1019                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1020                 ctrl |= IXGBE_VLNCTRL_VME;
1021                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1022         }
1023         else {
1024                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1025                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1026                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1027                         ctrl |= IXGBE_RXDCTL_VME;
1028                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1029
1030                         /* record those setting for HW strip per queue */
1031                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 1);
1032                 }
1033         }
1034 }
1035
1036 static void
1037 ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1038 {
1039         struct ixgbe_hw *hw =
1040                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1041         uint32_t ctrl;
1042
1043         PMD_INIT_FUNC_TRACE();
1044
1045         /* DMATXCTRL: Geric Double VLAN Disable */
1046         ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1047         ctrl &= ~IXGBE_DMATXCTL_GDV;
1048         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1049
1050         /* CTRL_EXT: Global Double VLAN Disable */
1051         ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1052         ctrl &= ~IXGBE_EXTENDED_VLAN;
1053         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1054
1055 }
1056
1057 static void
1058 ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1059 {
1060         struct ixgbe_hw *hw =
1061                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1062         uint32_t ctrl;
1063
1064         PMD_INIT_FUNC_TRACE();
1065
1066         /* DMATXCTRL: Geric Double VLAN Enable */
1067         ctrl  = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1068         ctrl |= IXGBE_DMATXCTL_GDV;
1069         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1070
1071         /* CTRL_EXT: Global Double VLAN Enable */
1072         ctrl  = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1073         ctrl |= IXGBE_EXTENDED_VLAN;
1074         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1075
1076         /*
1077          * VET EXT field in the EXVET register = 0x8100 by default
1078          * So no need to change. Same to VT field of DMATXCTL register
1079          */
1080 }
1081
1082 static void
1083 ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1084 {
1085         if(mask & ETH_VLAN_STRIP_MASK){
1086                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1087                         ixgbe_vlan_hw_strip_enable_all(dev);
1088                 else
1089                         ixgbe_vlan_hw_strip_disable_all(dev);
1090         }
1091
1092         if(mask & ETH_VLAN_FILTER_MASK){
1093                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1094                         ixgbe_vlan_hw_filter_enable(dev);
1095                 else
1096                         ixgbe_vlan_hw_filter_disable(dev);
1097         }
1098
1099         if(mask & ETH_VLAN_EXTEND_MASK){
1100                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1101                         ixgbe_vlan_hw_extend_enable(dev);
1102                 else
1103                         ixgbe_vlan_hw_extend_disable(dev);
1104         }
1105 }
1106
1107 static int
1108 ixgbe_dev_configure(struct rte_eth_dev *dev)
1109 {
1110         struct ixgbe_interrupt *intr =
1111                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1112
1113         PMD_INIT_FUNC_TRACE();
1114
1115         /* set flag to update link status after init */
1116         intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1117
1118         return 0;
1119 }
1120
1121 /*
1122  * Configure device link speed and setup link.
1123  * It returns 0 on success.
1124  */
1125 static int
1126 ixgbe_dev_start(struct rte_eth_dev *dev)
1127 {
1128         struct ixgbe_hw *hw =
1129                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1130         int err, link_up = 0, negotiate = 0;
1131         uint32_t speed = 0;
1132         int mask = 0;
1133
1134         PMD_INIT_FUNC_TRACE();
1135
1136         /* IXGBE devices don't support half duplex */
1137         if ((dev->data->dev_conf.link_duplex != ETH_LINK_AUTONEG_DUPLEX) &&
1138                         (dev->data->dev_conf.link_duplex != ETH_LINK_FULL_DUPLEX)) {
1139                 PMD_INIT_LOG(ERR, "Invalid link_duplex (%u) for port %u\n",
1140                                 dev->data->dev_conf.link_duplex,
1141                                 dev->data->port_id);
1142                 return -EINVAL;
1143         }
1144
1145         /* stop adapter */
1146         hw->adapter_stopped = FALSE;
1147         ixgbe_stop_adapter(hw);
1148
1149         /* reinitialize adapter
1150          * this calls reset and start */
1151         ixgbe_init_hw(hw);
1152         hw->mac.ops.start_hw(hw);
1153
1154         /* initialize transmission unit */
1155         ixgbe_dev_tx_init(dev);
1156
1157         /* This can fail when allocating mbufs for descriptor rings */
1158         err = ixgbe_dev_rx_init(dev);
1159         if (err) {
1160                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware\n");
1161                 goto error;
1162         }
1163
1164         ixgbe_dev_rxtx_start(dev);
1165
1166         if (ixgbe_is_sfp(hw) && hw->phy.multispeed_fiber) {
1167                 err = hw->mac.ops.setup_sfp(hw);
1168                 if (err)
1169                         goto error;
1170         }
1171
1172         /* Turn on the laser */
1173         if (hw->phy.multispeed_fiber)
1174                 ixgbe_enable_tx_laser(hw);
1175
1176         err = ixgbe_check_link(hw, &speed, &link_up, 0);
1177         if (err)
1178                 goto error;
1179         err = ixgbe_get_link_capabilities(hw, &speed, &negotiate);
1180         if (err)
1181                 goto error;
1182
1183         switch(dev->data->dev_conf.link_speed) {
1184         case ETH_LINK_SPEED_AUTONEG:
1185                 speed = (hw->mac.type != ixgbe_mac_82598EB) ?
1186                                 IXGBE_LINK_SPEED_82599_AUTONEG :
1187                                 IXGBE_LINK_SPEED_82598_AUTONEG;
1188                 break;
1189         case ETH_LINK_SPEED_100:
1190                 /*
1191                  * Invalid for 82598 but error will be detected by
1192                  * ixgbe_setup_link()
1193                  */
1194                 speed = IXGBE_LINK_SPEED_100_FULL;
1195                 break;
1196         case ETH_LINK_SPEED_1000:
1197                 speed = IXGBE_LINK_SPEED_1GB_FULL;
1198                 break;
1199         case ETH_LINK_SPEED_10000:
1200                 speed = IXGBE_LINK_SPEED_10GB_FULL;
1201                 break;
1202         default:
1203                 PMD_INIT_LOG(ERR, "Invalid link_speed (%u) for port %u\n",
1204                                 dev->data->dev_conf.link_speed, dev->data->port_id);
1205                 goto error;
1206         }
1207
1208         err = ixgbe_setup_link(hw, speed, negotiate, link_up);
1209         if (err)
1210                 goto error;
1211
1212         /* check if lsc interrupt is enabled */
1213         if (dev->data->dev_conf.intr_conf.lsc != 0)
1214                 ixgbe_dev_lsc_interrupt_setup(dev);
1215
1216         /* resume enabled intr since hw reset */
1217         ixgbe_enable_intr(dev);
1218
1219         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
1220                 ETH_VLAN_EXTEND_MASK;
1221         ixgbe_vlan_offload_set(dev, mask);
1222
1223         /* Configure DCB hw */
1224         ixgbe_configure_dcb(dev);
1225
1226         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) {
1227                 err = ixgbe_fdir_configure(dev);
1228                 if (err)
1229                         goto error;
1230         }
1231
1232         ixgbe_restore_statistics_mapping(dev);
1233
1234         return (0);
1235
1236 error:
1237         PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err);
1238         ixgbe_dev_clear_queues(dev);
1239         return -EIO;
1240 }
1241
1242 /*
1243  * Stop device: disable rx and tx functions to allow for reconfiguring.
1244  */
1245 static void
1246 ixgbe_dev_stop(struct rte_eth_dev *dev)
1247 {
1248         struct rte_eth_link link;
1249         struct ixgbe_hw *hw =
1250                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1251
1252         PMD_INIT_FUNC_TRACE();
1253
1254         /* disable interrupts */
1255         ixgbe_disable_intr(hw);
1256
1257         /* reset the NIC */
1258         ixgbe_reset_hw(hw);
1259         hw->adapter_stopped = FALSE;
1260
1261         /* stop adapter */
1262         ixgbe_stop_adapter(hw);
1263
1264         /* Turn off the laser */
1265         if (hw->phy.multispeed_fiber)
1266                 ixgbe_disable_tx_laser(hw);
1267
1268         ixgbe_dev_clear_queues(dev);
1269
1270         /* Clear recorded link status */
1271         memset(&link, 0, sizeof(link));
1272         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1273 }
1274
1275 /*
1276  * Reest and stop device.
1277  */
1278 static void
1279 ixgbe_dev_close(struct rte_eth_dev *dev)
1280 {
1281         struct ixgbe_hw *hw =
1282                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1283
1284         PMD_INIT_FUNC_TRACE();
1285
1286         ixgbe_reset_hw(hw);
1287
1288
1289         ixgbe_dev_stop(dev);
1290         hw->adapter_stopped = 1;
1291
1292         ixgbe_disable_pcie_master(hw);
1293
1294         /* reprogram the RAR[0] in case user changed it. */
1295         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
1296 }
1297
1298 /*
1299  * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c
1300  */
1301 static void
1302 ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1303 {
1304         struct ixgbe_hw *hw =
1305                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1306         struct ixgbe_hw_stats *hw_stats =
1307                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1308         uint32_t bprc, lxon, lxoff, total;
1309         uint64_t total_missed_rx, total_qbrc, total_qprc;
1310         unsigned i;
1311
1312         total_missed_rx = 0;
1313         total_qbrc = 0;
1314         total_qprc = 0;
1315
1316         hw_stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
1317         hw_stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
1318         hw_stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
1319         hw_stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
1320
1321         for (i = 0; i < 8; i++) {
1322                 uint32_t mp;
1323                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
1324                 /* global total per queue */
1325                 hw_stats->mpc[i] += mp;
1326                 /* Running comprehensive total for stats display */
1327                 total_missed_rx += hw_stats->mpc[i];
1328                 if (hw->mac.type == ixgbe_mac_82598EB)
1329                         hw_stats->rnbc[i] +=
1330                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
1331                 hw_stats->pxontxc[i] +=
1332                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
1333                 hw_stats->pxonrxc[i] +=
1334                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
1335                 hw_stats->pxofftxc[i] +=
1336                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
1337                 hw_stats->pxoffrxc[i] +=
1338                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
1339                 hw_stats->pxon2offc[i] +=
1340                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
1341         }
1342         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1343                 hw_stats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
1344                 hw_stats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
1345                 hw_stats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
1346                 hw_stats->qbrc[i] +=
1347                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32);
1348                 hw_stats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
1349                 hw_stats->qbtc[i] +=
1350                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32);
1351                 hw_stats->qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
1352
1353                 total_qprc += hw_stats->qprc[i];
1354                 total_qbrc += hw_stats->qbrc[i];
1355         }
1356         hw_stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
1357         hw_stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
1358         hw_stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
1359
1360         /* Note that gprc counts missed packets */
1361         hw_stats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
1362
1363         if (hw->mac.type != ixgbe_mac_82598EB) {
1364                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
1365                 hw_stats->gorc += ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
1366                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
1367                 hw_stats->gotc += ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
1368                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
1369                 hw_stats->tor += ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
1370                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
1371                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
1372         } else {
1373                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
1374                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
1375                 /* 82598 only has a counter in the high register */
1376                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
1377                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
1378                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
1379         }
1380
1381         /*
1382          * Workaround: mprc hardware is incorrectly counting
1383          * broadcasts, so for now we subtract those.
1384          */
1385         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
1386         hw_stats->bprc += bprc;
1387         hw_stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
1388         if (hw->mac.type == ixgbe_mac_82598EB)
1389                 hw_stats->mprc -= bprc;
1390
1391         hw_stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
1392         hw_stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
1393         hw_stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
1394         hw_stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
1395         hw_stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
1396         hw_stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
1397
1398         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
1399         hw_stats->lxontxc += lxon;
1400         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
1401         hw_stats->lxofftxc += lxoff;
1402         total = lxon + lxoff;
1403
1404         hw_stats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
1405         hw_stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
1406         hw_stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
1407         hw_stats->gptc -= total;
1408         hw_stats->mptc -= total;
1409         hw_stats->ptc64 -= total;
1410         hw_stats->gotc -= total * ETHER_MIN_LEN;
1411
1412         hw_stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
1413         hw_stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
1414         hw_stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
1415         hw_stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
1416         hw_stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
1417         hw_stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
1418         hw_stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
1419         hw_stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
1420         hw_stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
1421         hw_stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
1422         hw_stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
1423         hw_stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
1424         hw_stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
1425         hw_stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
1426         hw_stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
1427         hw_stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);
1428         hw_stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
1429         hw_stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
1430         /* Only read FCOE on 82599 */
1431         if (hw->mac.type != ixgbe_mac_82598EB) {
1432                 hw_stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
1433                 hw_stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
1434                 hw_stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
1435                 hw_stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
1436                 hw_stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
1437         }
1438
1439         if (stats == NULL)
1440                 return;
1441
1442         /* Fill out the rte_eth_stats statistics structure */
1443         stats->ipackets = total_qprc;
1444         stats->ibytes = total_qbrc;
1445         stats->opackets = hw_stats->gptc;
1446         stats->obytes = hw_stats->gotc;
1447         stats->imcasts = hw_stats->mprc;
1448
1449         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1450                 stats->q_ipackets[i] = hw_stats->qprc[i];
1451                 stats->q_opackets[i] = hw_stats->qptc[i];
1452                 stats->q_ibytes[i] = hw_stats->qbrc[i];
1453                 stats->q_obytes[i] = hw_stats->qbtc[i];
1454                 stats->q_errors[i] = hw_stats->qprdc[i];
1455         }
1456
1457         /* Rx Errors */
1458         stats->ierrors = total_missed_rx + hw_stats->crcerrs +
1459                 hw_stats->rlec;
1460
1461         stats->oerrors  = 0;
1462
1463         /* Flow Director Stats registers */
1464         hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
1465         hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
1466         stats->fdirmatch = hw_stats->fdirmatch;
1467         stats->fdirmiss = hw_stats->fdirmiss;
1468 }
1469
1470 static void
1471 ixgbe_dev_stats_reset(struct rte_eth_dev *dev)
1472 {
1473         struct ixgbe_hw_stats *stats =
1474                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1475
1476         /* HW registers are cleared on read */
1477         ixgbe_dev_stats_get(dev, NULL);
1478
1479         /* Reset software totals */
1480         memset(stats, 0, sizeof(*stats));
1481 }
1482
1483 static void
1484 ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1485 {
1486         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1487         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1488                           IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1489
1490         /* Good Rx packet, include VF loopback */
1491         UPDATE_VF_STAT(IXGBE_VFGPRC,
1492             hw_stats->last_vfgprc, hw_stats->vfgprc);
1493
1494         /* Good Rx octets, include VF loopback */
1495         UPDATE_VF_STAT_36BIT(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
1496             hw_stats->last_vfgorc, hw_stats->vfgorc);
1497
1498         /* Good Tx packet, include VF loopback */
1499         UPDATE_VF_STAT(IXGBE_VFGPTC,
1500             hw_stats->last_vfgptc, hw_stats->vfgptc);
1501
1502         /* Good Tx octets, include VF loopback */
1503         UPDATE_VF_STAT_36BIT(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
1504             hw_stats->last_vfgotc, hw_stats->vfgotc);
1505
1506         /* Rx Multicst Packet */
1507         UPDATE_VF_STAT(IXGBE_VFMPRC,
1508             hw_stats->last_vfmprc, hw_stats->vfmprc);
1509
1510         if (stats == NULL)
1511                 return;
1512
1513         memset(stats, 0, sizeof(*stats));
1514         stats->ipackets = hw_stats->vfgprc;
1515         stats->ibytes = hw_stats->vfgorc;
1516         stats->opackets = hw_stats->vfgptc;
1517         stats->obytes = hw_stats->vfgotc;
1518         stats->imcasts = hw_stats->vfmprc;
1519 }
1520
1521 static void
1522 ixgbevf_dev_stats_reset(struct rte_eth_dev *dev)
1523 {
1524         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1525                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1526
1527         /* Sync HW register to the last stats */
1528         ixgbevf_dev_stats_get(dev, NULL);
1529
1530         /* reset HW current stats*/
1531         hw_stats->vfgprc = 0;
1532         hw_stats->vfgorc = 0;
1533         hw_stats->vfgptc = 0;
1534         hw_stats->vfgotc = 0;
1535         hw_stats->vfmprc = 0;
1536
1537 }
1538
1539 static void
1540 ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1541 {
1542         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1543
1544         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
1545         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
1546         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL register */
1547         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */
1548         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
1549 }
1550
1551 /* return 0 means link status changed, -1 means not changed */
1552 static int
1553 ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1554 {
1555         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1556         struct rte_eth_link link, old;
1557         ixgbe_link_speed link_speed;
1558         int link_up;
1559         int diag;
1560
1561         link.link_status = 0;
1562         link.link_speed = 0;
1563         link.link_duplex = 0;
1564         memset(&old, 0, sizeof(old));
1565         rte_ixgbe_dev_atomic_read_link_status(dev, &old);
1566
1567         /* check if it needs to wait to complete, if lsc interrupt is enabled */
1568         if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
1569                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 0);
1570         else
1571                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 1);
1572         if (diag != 0) {
1573                 link.link_speed = ETH_LINK_SPEED_100;
1574                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1575                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1576                 if (link.link_status == old.link_status)
1577                         return -1;
1578                 return 0;
1579         }
1580
1581         if (link_up == 0) {
1582                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1583                 if (link.link_status == old.link_status)
1584                         return -1;
1585                 return 0;
1586         }
1587         link.link_status = 1;
1588         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1589
1590         switch (link_speed) {
1591         default:
1592         case IXGBE_LINK_SPEED_UNKNOWN:
1593                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1594                 link.link_speed = ETH_LINK_SPEED_100;
1595                 break;
1596
1597         case IXGBE_LINK_SPEED_100_FULL:
1598                 link.link_speed = ETH_LINK_SPEED_100;
1599                 break;
1600
1601         case IXGBE_LINK_SPEED_1GB_FULL:
1602                 link.link_speed = ETH_LINK_SPEED_1000;
1603                 break;
1604
1605         case IXGBE_LINK_SPEED_10GB_FULL:
1606                 link.link_speed = ETH_LINK_SPEED_10000;
1607                 break;
1608         }
1609         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1610
1611         if (link.link_status == old.link_status)
1612                 return -1;
1613
1614         return 0;
1615 }
1616
1617 static void
1618 ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev)
1619 {
1620         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1621         uint32_t fctrl;
1622
1623         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1624         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1625         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1626 }
1627
1628 static void
1629 ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev)
1630 {
1631         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1632         uint32_t fctrl;
1633
1634         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1635         fctrl &= (~IXGBE_FCTRL_UPE);
1636         if (dev->data->all_multicast == 1)
1637                 fctrl |= IXGBE_FCTRL_MPE;
1638         else
1639                 fctrl &= (~IXGBE_FCTRL_MPE);
1640         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1641 }
1642
1643 static void
1644 ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev)
1645 {
1646         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1647         uint32_t fctrl;
1648
1649         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1650         fctrl |= IXGBE_FCTRL_MPE;
1651         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1652 }
1653
1654 static void
1655 ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev)
1656 {
1657         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1658         uint32_t fctrl;
1659
1660         if (dev->data->promiscuous == 1)
1661                 return; /* must remain in all_multicast mode */
1662
1663         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1664         fctrl &= (~IXGBE_FCTRL_MPE);
1665         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1666 }
1667
1668 /**
1669  * It clears the interrupt causes and enables the interrupt.
1670  * It will be called once only during nic initialized.
1671  *
1672  * @param dev
1673  *  Pointer to struct rte_eth_dev.
1674  *
1675  * @return
1676  *  - On success, zero.
1677  *  - On failure, a negative value.
1678  */
1679 static int
1680 ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev)
1681 {
1682         struct ixgbe_interrupt *intr =
1683                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1684
1685         ixgbe_dev_link_status_print(dev);
1686         intr->mask |= IXGBE_EICR_LSC;
1687
1688         return 0;
1689 }
1690
1691 /*
1692  * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update.
1693  *
1694  * @param dev
1695  *  Pointer to struct rte_eth_dev.
1696  *
1697  * @return
1698  *  - On success, zero.
1699  *  - On failure, a negative value.
1700  */
1701 static int
1702 ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
1703 {
1704         uint32_t eicr;
1705         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1706         struct ixgbe_interrupt *intr =
1707                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1708
1709         /* clear all cause mask */
1710         ixgbe_disable_intr(hw);
1711
1712         /* read-on-clear nic registers here */
1713         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1714         PMD_DRV_LOG(INFO, "eicr %x", eicr);
1715
1716         intr->flags = 0;
1717         if (eicr & IXGBE_EICR_LSC) {
1718                 /* set flag for async link update */
1719                 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1720         }
1721
1722         return 0;
1723 }
1724
1725 /**
1726  * It gets and then prints the link status.
1727  *
1728  * @param dev
1729  *  Pointer to struct rte_eth_dev.
1730  *
1731  * @return
1732  *  - On success, zero.
1733  *  - On failure, a negative value.
1734  */
1735 static void
1736 ixgbe_dev_link_status_print(struct rte_eth_dev *dev)
1737 {
1738         struct rte_eth_link link;
1739
1740         memset(&link, 0, sizeof(link));
1741         rte_ixgbe_dev_atomic_read_link_status(dev, &link);
1742         if (link.link_status) {
1743                 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
1744                                         (int)(dev->data->port_id),
1745                                         (unsigned)link.link_speed,
1746                         link.link_duplex == ETH_LINK_FULL_DUPLEX ?
1747                                         "full-duplex" : "half-duplex");
1748         } else {
1749                 PMD_INIT_LOG(INFO, " Port %d: Link Down",
1750                                 (int)(dev->data->port_id));
1751         }
1752         PMD_INIT_LOG(INFO, "PCI Address: %04d:%02d:%02d:%d",
1753                                 dev->pci_dev->addr.domain,
1754                                 dev->pci_dev->addr.bus,
1755                                 dev->pci_dev->addr.devid,
1756                                 dev->pci_dev->addr.function);
1757 }
1758
1759 /*
1760  * It executes link_update after knowing an interrupt occured.
1761  *
1762  * @param dev
1763  *  Pointer to struct rte_eth_dev.
1764  *
1765  * @return
1766  *  - On success, zero.
1767  *  - On failure, a negative value.
1768  */
1769 static int
1770 ixgbe_dev_interrupt_action(struct rte_eth_dev *dev)
1771 {
1772         struct ixgbe_interrupt *intr =
1773                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1774         int64_t timeout;
1775         struct rte_eth_link link;
1776         int intr_enable_delay = false;
1777
1778         PMD_DRV_LOG(DEBUG, "intr action type %d\n", intr->flags);
1779
1780         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
1781                 /* get the link status before link update, for predicting later */
1782                 memset(&link, 0, sizeof(link));
1783                 rte_ixgbe_dev_atomic_read_link_status(dev, &link);
1784
1785                 ixgbe_dev_link_update(dev, 0);
1786
1787                 /* likely to up */
1788                 if (!link.link_status)
1789                         /* handle it 1 sec later, wait it being stable */
1790                         timeout = IXGBE_LINK_UP_CHECK_TIMEOUT;
1791                 /* likely to down */
1792                 else
1793                         /* handle it 4 sec later, wait it being stable */
1794                         timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT;
1795
1796                 ixgbe_dev_link_status_print(dev);
1797
1798                 intr_enable_delay = true;
1799         }
1800
1801         if (intr_enable_delay) {
1802                 if (rte_eal_alarm_set(timeout * 1000,
1803                                       ixgbe_dev_interrupt_delayed_handler, (void*)dev) < 0)
1804                         PMD_DRV_LOG(ERR, "Error setting alarm");
1805         } else {
1806                 PMD_DRV_LOG(DEBUG, "enable intr immediately");
1807                 ixgbe_enable_intr(dev);
1808                 rte_intr_enable(&(dev->pci_dev->intr_handle));
1809         }
1810
1811
1812         return 0;
1813 }
1814
1815 /**
1816  * Interrupt handler which shall be registered for alarm callback for delayed
1817  * handling specific interrupt to wait for the stable nic state. As the
1818  * NIC interrupt state is not stable for ixgbe after link is just down,
1819  * it needs to wait 4 seconds to get the stable status.
1820  *
1821  * @param handle
1822  *  Pointer to interrupt handle.
1823  * @param param
1824  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1825  *
1826  * @return
1827  *  void
1828  */
1829 static void
1830 ixgbe_dev_interrupt_delayed_handler(void *param)
1831 {
1832         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1833         struct ixgbe_interrupt *intr =
1834                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1835
1836         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
1837                 ixgbe_dev_link_update(dev, 0);
1838                 intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
1839                 ixgbe_dev_link_status_print(dev);
1840                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
1841         }
1842
1843         PMD_DRV_LOG(DEBUG, "enable intr in delayed handler\n");
1844         ixgbe_enable_intr(dev);
1845         rte_intr_enable(&(dev->pci_dev->intr_handle));
1846 }
1847
1848 /**
1849  * Interrupt handler triggered by NIC  for handling
1850  * specific interrupt.
1851  *
1852  * @param handle
1853  *  Pointer to interrupt handle.
1854  * @param param
1855  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1856  *
1857  * @return
1858  *  void
1859  */
1860 static void
1861 ixgbe_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
1862                                                         void *param)
1863 {
1864         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1865         ixgbe_dev_interrupt_get_status(dev);
1866         ixgbe_dev_interrupt_action(dev);
1867 }
1868
1869 static int
1870 ixgbe_dev_led_on(struct rte_eth_dev *dev)
1871 {
1872         struct ixgbe_hw *hw;
1873
1874         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1875         return (ixgbe_led_on(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
1876 }
1877
1878 static int
1879 ixgbe_dev_led_off(struct rte_eth_dev *dev)
1880 {
1881         struct ixgbe_hw *hw;
1882
1883         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1884         return (ixgbe_led_off(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
1885 }
1886
1887 static int
1888 ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1889 {
1890         struct ixgbe_hw *hw;
1891         int err;
1892         uint32_t rx_buf_size;
1893         uint32_t max_high_water;
1894         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
1895                 ixgbe_fc_none,
1896                 ixgbe_fc_rx_pause,
1897                 ixgbe_fc_tx_pause,
1898                 ixgbe_fc_full
1899         };
1900
1901         PMD_INIT_FUNC_TRACE();
1902
1903         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1904         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0));
1905         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
1906
1907         /*
1908          * At least reserve one Ethernet frame for watermark
1909          * high_water/low_water in kilo bytes for ixgbe
1910          */
1911         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
1912         if ((fc_conf->high_water > max_high_water) ||
1913                 (fc_conf->high_water < fc_conf->low_water)) {
1914                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
1915                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
1916                 return (-EINVAL);
1917         }
1918
1919         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[fc_conf->mode];
1920         hw->fc.pause_time     = fc_conf->pause_time;
1921         hw->fc.high_water[0]  = fc_conf->high_water;
1922         hw->fc.low_water[0]   = fc_conf->low_water;
1923         hw->fc.send_xon       = fc_conf->send_xon;
1924
1925         err = ixgbe_fc_enable(hw);
1926         /* Not negotiated is not an error case */
1927         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) {
1928                 return 0;
1929         }
1930
1931         PMD_INIT_LOG(ERR, "ixgbe_fc_enable = 0x%x \n", err);
1932         return -EIO;
1933 }
1934
1935 /**
1936  *  ixgbe_pfc_enable_generic - Enable flow control
1937  *  @hw: pointer to hardware structure
1938  *  @tc_num: traffic class number
1939  *  Enable flow control according to the current settings.
1940  */
1941 static int
1942 ixgbe_dcb_pfc_enable_generic(struct ixgbe_hw *hw,uint8_t tc_num)
1943 {
1944         int ret_val = 0;
1945         uint32_t mflcn_reg, fccfg_reg;
1946         uint32_t reg;
1947         uint32_t fcrtl, fcrth;
1948         uint8_t i;
1949         uint8_t nb_rx_en;
1950
1951         /* Validate the water mark configuration */
1952         if (!hw->fc.pause_time) {
1953                 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
1954                 goto out;
1955         }
1956
1957         /* Low water mark of zero causes XOFF floods */
1958         if (hw->fc.current_mode & ixgbe_fc_tx_pause) {
1959                  /* High/Low water can not be 0 */
1960                 if( (!hw->fc.high_water[tc_num])|| (!hw->fc.low_water[tc_num])) {
1961                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
1962                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
1963                         goto out;
1964                 }
1965
1966                 if(hw->fc.low_water[tc_num] >= hw->fc.high_water[tc_num]) {
1967                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
1968                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
1969                         goto out;
1970                 }
1971         }
1972         /* Negotiate the fc mode to use */
1973         ixgbe_fc_autoneg(hw);
1974
1975         /* Disable any previous flow control settings */
1976         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
1977         mflcn_reg &= ~(IXGBE_MFLCN_RPFCE_SHIFT | IXGBE_MFLCN_RFCE|IXGBE_MFLCN_RPFCE);
1978
1979         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
1980         fccfg_reg &= ~(IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY);
1981
1982         switch (hw->fc.current_mode) {
1983         case ixgbe_fc_none:
1984                 /*
1985                  * If the count of enabled RX Priority Flow control >1,
1986                  * and the TX pause can not be disabled
1987                  */
1988                 nb_rx_en = 0;
1989                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
1990                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
1991                         if (reg & IXGBE_FCRTH_FCEN)
1992                                 nb_rx_en++;
1993                 }
1994                 if (nb_rx_en > 1)
1995                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
1996                 break;
1997         case ixgbe_fc_rx_pause:
1998                 /*
1999                  * Rx Flow control is enabled and Tx Flow control is
2000                  * disabled by software override. Since there really
2001                  * isn't a way to advertise that we are capable of RX
2002                  * Pause ONLY, we will advertise that we support both
2003                  * symmetric and asymmetric Rx PAUSE.  Later, we will
2004                  * disable the adapter's ability to send PAUSE frames.
2005                  */
2006                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
2007                 /*
2008                  * If the count of enabled RX Priority Flow control >1,
2009                  * and the TX pause can not be disabled
2010                  */
2011                 nb_rx_en = 0;
2012                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
2013                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
2014                         if (reg & IXGBE_FCRTH_FCEN)
2015                                 nb_rx_en++;
2016                 }
2017                 if (nb_rx_en > 1)
2018                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2019                 break;
2020         case ixgbe_fc_tx_pause:
2021                 /*
2022                  * Tx Flow control is enabled, and Rx Flow control is
2023                  * disabled by software override.
2024                  */
2025                 fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2026                 break;
2027         case ixgbe_fc_full:
2028                 /* Flow control (both Rx and Tx) is enabled by SW override. */
2029                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
2030                 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
2031                 break;
2032         default:
2033                 DEBUGOUT("Flow control param set incorrectly\n");
2034                 ret_val = IXGBE_ERR_CONFIG;
2035                 goto out;
2036                 break;
2037         }
2038
2039         /* Set 802.3x based flow control settings. */
2040         mflcn_reg |= IXGBE_MFLCN_DPF;
2041         IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg);
2042         IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg);
2043
2044         /* Set up and enable Rx high/low water mark thresholds, enable XON. */
2045         if ((hw->fc.current_mode & ixgbe_fc_tx_pause) &&
2046                 hw->fc.high_water[tc_num]) {
2047                 fcrtl = (hw->fc.low_water[tc_num] << 10) | IXGBE_FCRTL_XONE;
2048                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), fcrtl);
2049                 fcrth = (hw->fc.high_water[tc_num] << 10) | IXGBE_FCRTH_FCEN;
2050         } else {
2051                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), 0);
2052                 /*
2053                  * In order to prevent Tx hangs when the internal Tx
2054                  * switch is enabled we must set the high water mark
2055                  * to the maximum FCRTH value.  This allows the Tx
2056                  * switch to function even under heavy Rx workloads.
2057                  */
2058                 fcrth = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)) - 32;
2059         }
2060         IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(tc_num), fcrth);
2061
2062         /* Configure pause time (2 TCs per register) */
2063         reg = hw->fc.pause_time * 0x00010001;
2064         for (i = 0; i < (IXGBE_DCB_MAX_TRAFFIC_CLASS / 2); i++)
2065                 IXGBE_WRITE_REG(hw, IXGBE_FCTTV(i), reg);
2066
2067         /* Configure flow control refresh threshold value */
2068         IXGBE_WRITE_REG(hw, IXGBE_FCRTV, hw->fc.pause_time / 2);
2069
2070 out:
2071         return ret_val;
2072 }
2073
2074 static int
2075 ixgbe_dcb_pfc_enable(struct rte_eth_dev *dev,uint8_t tc_num)
2076 {
2077         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2078         int32_t ret_val = IXGBE_NOT_IMPLEMENTED;
2079
2080         if(hw->mac.type != ixgbe_mac_82598EB) {
2081                 ret_val = ixgbe_dcb_pfc_enable_generic(hw,tc_num);
2082         }
2083         return ret_val;
2084 }
2085
2086 static int
2087 ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf)
2088 {
2089         int err;
2090         uint32_t rx_buf_size;
2091         uint32_t max_high_water;
2092         uint8_t tc_num;
2093         uint8_t  map[IXGBE_DCB_MAX_USER_PRIORITY] = { 0 };
2094         struct ixgbe_hw *hw =
2095                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2096         struct ixgbe_dcb_config *dcb_config =
2097                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
2098
2099         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
2100                 ixgbe_fc_none,
2101                 ixgbe_fc_rx_pause,
2102                 ixgbe_fc_tx_pause,
2103                 ixgbe_fc_full
2104         };
2105
2106         PMD_INIT_FUNC_TRACE();
2107
2108         ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
2109         tc_num = map[pfc_conf->priority];
2110         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num));
2111         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
2112         /*
2113          * At least reserve one Ethernet frame for watermark
2114          * high_water/low_water in kilo bytes for ixgbe
2115          */
2116         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
2117         if ((pfc_conf->fc.high_water > max_high_water) ||
2118                 (pfc_conf->fc.high_water <= pfc_conf->fc.low_water)) {
2119                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
2120                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
2121                 return (-EINVAL);
2122         }
2123
2124         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[pfc_conf->fc.mode];
2125         hw->fc.pause_time = pfc_conf->fc.pause_time;
2126         hw->fc.send_xon = pfc_conf->fc.send_xon;
2127         hw->fc.low_water[tc_num] =  pfc_conf->fc.low_water;
2128         hw->fc.high_water[tc_num] = pfc_conf->fc.high_water;
2129
2130         err = ixgbe_dcb_pfc_enable(dev,tc_num);
2131
2132         /* Not negotiated is not an error case */
2133         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED))
2134                 return 0;
2135
2136         PMD_INIT_LOG(ERR, "ixgbe_dcb_pfc_enable = 0x%x \n", err);
2137         return -EIO;
2138 }
2139
2140 static int
2141 ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
2142                                 struct rte_eth_rss_reta *reta_conf)
2143 {
2144         uint8_t i,j,mask;
2145         uint32_t reta;
2146         struct ixgbe_hw *hw =
2147                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2148
2149         PMD_INIT_FUNC_TRACE();
2150         /*
2151         * Update Redirection Table RETA[n],n=0...31,The redirection table has
2152         * 128-entries in 32 registers
2153          */
2154         for(i = 0; i < ETH_RSS_RETA_NUM_ENTRIES; i += 4) {
2155                 if (i < ETH_RSS_RETA_NUM_ENTRIES/2)
2156                         mask = (uint8_t)((reta_conf->mask_lo >> i) & 0xF);
2157                 else
2158                         mask = (uint8_t)((reta_conf->mask_hi >>
2159                                 (i - ETH_RSS_RETA_NUM_ENTRIES/2)) & 0xF);
2160                 if (mask != 0) {
2161                         reta = 0;
2162                         if (mask != 0xF)
2163                                 reta = IXGBE_READ_REG(hw,IXGBE_RETA(i >> 2));
2164
2165                         for (j = 0; j < 4; j++) {
2166                                 if (mask & (0x1 << j)) {
2167                                         if (mask != 0xF)
2168                                                 reta &= ~(0xFF << 8 * j);
2169                                         reta |= reta_conf->reta[i + j] << 8*j;
2170                                 }
2171                         }
2172                         IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2),reta);
2173                 }
2174         }
2175
2176         return 0;
2177 }
2178
2179 static int
2180 ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
2181                                 struct rte_eth_rss_reta *reta_conf)
2182 {
2183         uint8_t i,j,mask;
2184         uint32_t reta;
2185         struct ixgbe_hw *hw =
2186                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2187
2188         PMD_INIT_FUNC_TRACE();
2189         /*
2190          * Read Redirection Table RETA[n],n=0...31,The redirection table has
2191          * 128-entries in 32 registers
2192          */
2193         for(i = 0; i < ETH_RSS_RETA_NUM_ENTRIES; i += 4) {
2194                 if (i < ETH_RSS_RETA_NUM_ENTRIES/2)
2195                         mask = (uint8_t)((reta_conf->mask_lo >> i) & 0xF);
2196                 else
2197                         mask = (uint8_t)((reta_conf->mask_hi >>
2198                                 (i - ETH_RSS_RETA_NUM_ENTRIES/2)) & 0xF);
2199
2200                 if (mask != 0) {
2201                         reta = IXGBE_READ_REG(hw,IXGBE_RETA(i >> 2));
2202                         for (j = 0; j < 4; j++) {
2203                                 if (mask & (0x1 << j))
2204                                         reta_conf->reta[i + j] =
2205                                                 (uint8_t)((reta >> 8 * j) & 0xFF);
2206                         }
2207                 }
2208         }
2209
2210         return 0;
2211 }
2212
2213 static void
2214 ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
2215                                 uint32_t index, uint32_t pool)
2216 {
2217         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2218         uint32_t enable_addr = 1;
2219
2220         ixgbe_set_rar(hw, index, mac_addr->addr_bytes, pool, enable_addr);
2221 }
2222
2223 static void
2224 ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
2225 {
2226         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2227
2228         ixgbe_clear_rar(hw, index);
2229 }
2230
2231 /*
2232  * Virtual Function operations
2233  */
2234 static void
2235 ixgbevf_intr_disable(struct ixgbe_hw *hw)
2236 {
2237         PMD_INIT_LOG(DEBUG, "ixgbevf_intr_disable");
2238
2239         /* Clear interrupt mask to stop from interrupts being generated */
2240         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, IXGBE_VF_IRQ_CLEAR_MASK);
2241
2242         IXGBE_WRITE_FLUSH(hw);
2243 }
2244
2245 static int
2246 ixgbevf_dev_configure(struct rte_eth_dev *dev)
2247 {
2248         struct rte_eth_conf* conf = &dev->data->dev_conf;
2249
2250         PMD_INIT_LOG(DEBUG, "\nConfigured Virtual Function port id: %d\n",
2251                 dev->data->port_id);
2252
2253         /*
2254          * VF has no ability to enable/disable HW CRC
2255          * Keep the persistent behavior the same as Host PF
2256          */
2257 #ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC
2258         if (!conf->rxmode.hw_strip_crc) {
2259                 PMD_INIT_LOG(INFO, "VF can't disable HW CRC Strip\n");
2260                 conf->rxmode.hw_strip_crc = 1;
2261         }
2262 #else
2263         if (conf->rxmode.hw_strip_crc) {
2264                 PMD_INIT_LOG(INFO, "VF can't enable HW CRC Strip\n");
2265                 conf->rxmode.hw_strip_crc = 0;
2266         }
2267 #endif
2268
2269         return 0;
2270 }
2271
2272 static int
2273 ixgbevf_dev_start(struct rte_eth_dev *dev)
2274 {
2275         struct ixgbe_hw *hw =
2276                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2277         int err, mask = 0;
2278
2279         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_start");
2280
2281         hw->mac.ops.reset_hw(hw);
2282
2283         ixgbevf_dev_tx_init(dev);
2284
2285         /* This can fail when allocating mbufs for descriptor rings */
2286         err = ixgbevf_dev_rx_init(dev);
2287         if (err) {
2288                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware (%d)\n", err);
2289                 ixgbe_dev_clear_queues(dev);
2290                 return err;
2291         }
2292
2293         /* Set vfta */
2294         ixgbevf_set_vfta_all(dev,1);
2295
2296         /* Set HW strip */
2297         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
2298                 ETH_VLAN_EXTEND_MASK;
2299         ixgbevf_vlan_offload_set(dev, mask);
2300
2301         ixgbevf_dev_rxtx_start(dev);
2302
2303         return 0;
2304 }
2305
2306 static void
2307 ixgbevf_dev_stop(struct rte_eth_dev *dev)
2308 {
2309         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2310
2311         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_stop");
2312
2313         hw->adapter_stopped = TRUE;
2314         ixgbe_stop_adapter(hw);
2315
2316         /*
2317           * Clear what we set, but we still keep shadow_vfta to
2318           * restore after device starts
2319           */
2320         ixgbevf_set_vfta_all(dev,0);
2321
2322         ixgbe_dev_clear_queues(dev);
2323 }
2324
2325 static void
2326 ixgbevf_dev_close(struct rte_eth_dev *dev)
2327 {
2328         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2329
2330         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_close");
2331
2332         ixgbe_reset_hw(hw);
2333
2334         ixgbevf_dev_stop(dev);
2335
2336         /* reprogram the RAR[0] in case user changed it. */
2337         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
2338 }
2339
2340 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on)
2341 {
2342         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2343         struct ixgbe_vfta * shadow_vfta =
2344                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2345         int i = 0, j = 0, vfta = 0, mask = 1;
2346
2347         for (i = 0; i < IXGBE_VFTA_SIZE; i++){
2348                 vfta = shadow_vfta->vfta[i];
2349                 if(vfta){
2350                         mask = 1;
2351                         for (j = 0; j < 32; j++){
2352                                 if(vfta & mask)
2353                                         ixgbe_set_vfta(hw, (i<<5)+j, 0, on);
2354                                 mask<<=1;
2355                         }
2356                 }
2357         }
2358
2359 }
2360
2361 static int
2362 ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2363 {
2364         struct ixgbe_hw *hw =
2365                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2366         struct ixgbe_vfta * shadow_vfta =
2367                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2368         uint32_t vid_idx = 0;
2369         uint32_t vid_bit = 0;
2370         int ret = 0;
2371
2372         PMD_INIT_FUNC_TRACE();
2373
2374         /* vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf */
2375         ret = ixgbe_set_vfta(hw, vlan_id, 0, !!on);
2376         if(ret){
2377                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
2378                 return ret;
2379         }
2380         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
2381         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
2382
2383         /* Save what we set and retore it after device reset */
2384         if (on)
2385                 shadow_vfta->vfta[vid_idx] |= vid_bit;
2386         else
2387                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
2388
2389         return 0;
2390 }
2391
2392 static void
2393 ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
2394 {
2395         struct ixgbe_hw *hw =
2396                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2397         uint32_t ctrl;
2398
2399         PMD_INIT_FUNC_TRACE();
2400
2401         if(queue >= hw->mac.max_rx_queues)
2402                 return;
2403
2404         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
2405         if(on)
2406                 ctrl |= IXGBE_RXDCTL_VME;
2407         else
2408                 ctrl &= ~IXGBE_RXDCTL_VME;
2409         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
2410
2411         ixgbe_vlan_hw_strip_bitmap_set( dev, queue, on);
2412 }
2413
2414 static void
2415 ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2416 {
2417         struct ixgbe_hw *hw =
2418                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2419         uint16_t i;
2420         int on = 0;
2421
2422         /* VF function only support hw strip feature, others are not support */
2423         if(mask & ETH_VLAN_STRIP_MASK){
2424                 on = !!(dev->data->dev_conf.rxmode.hw_vlan_strip);
2425
2426                 for(i=0; i < hw->mac.max_rx_queues; i++)
2427                         ixgbevf_vlan_strip_queue_set(dev,i,on);
2428         }
2429 }