lib/librte_pmd_ixgbe/ixgbe_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2013 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <inttypes.h>
  42 #include <rte_byteorder.h>
  43 #include <rte_common.h>
  44 #include <rte_cycles.h>
  45
  46 #include <rte_interrupts.h>
  47 #include <rte_log.h>
  48 #include <rte_debug.h>
  49 #include <rte_pci.h>
  50 #include <rte_atomic.h>
  51 #include <rte_branch_prediction.h>
  52 #include <rte_memory.h>
  53 #include <rte_memzone.h>
  54 #include <rte_tailq.h>
  55 #include <rte_eal.h>
  56 #include <rte_alarm.h>
  57 #include <rte_ether.h>
  58 #include <rte_ethdev.h>
  59 #include <rte_atomic.h>
  60 #include <rte_malloc.h>
  61
  62 #include "ixgbe_logs.h"
  63 #include "ixgbe/ixgbe_api.h"
  64 #include "ixgbe/ixgbe_vf.h"
  65 #include "ixgbe/ixgbe_common.h"
  66 #include "ixgbe_ethdev.h"
  67
  68 /*
  69  * High threshold controlling when to start sending XOFF frames. Must be at
  70  * least 8 bytes less than receive packet buffer size. This value is in units
  71  * of 1024 bytes.
  72  */
  73 #define IXGBE_FC_HI    0x80
  74
  75 /*
  76  * Low threshold controlling when to start sending XON frames. This value is
  77  * in units of 1024 bytes.
  78  */
  79 #define IXGBE_FC_LO    0x40
  80
  81 /* Timer value included in XOFF frames. */
  82 #define IXGBE_FC_PAUSE 0x680
  83
  84 #define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */
  85 #define IXGBE_LINK_UP_CHECK_TIMEOUT   1000 /* ms */
  86
  87 #define IXGBEVF_PMD_NAME "rte_ixgbevf_pmd" /* PMD name */
  88
  89 #define IXGBE_QUEUE_STAT_COUNTERS (sizeof(hw_stats->qprc) / sizeof(hw_stats->qprc[0]))
  90
  91 static int eth_ixgbe_dev_init(struct eth_driver *eth_drv,
  92                 struct rte_eth_dev *eth_dev);
  93 static int  ixgbe_dev_configure(struct rte_eth_dev *dev);
  94 static int  ixgbe_dev_start(struct rte_eth_dev *dev);
  95 static void ixgbe_dev_stop(struct rte_eth_dev *dev);
  96 static void ixgbe_dev_close(struct rte_eth_dev *dev);
  97 static void ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev);
  98 static void ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev);
  99 static void ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev);
 100 static void ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev);
 101 static int ixgbe_dev_link_update(struct rte_eth_dev *dev,
 102                                 int wait_to_complete);
 103 static void ixgbe_dev_stats_get(struct rte_eth_dev *dev,
 104                                 struct rte_eth_stats *stats);
 105 static void ixgbe_dev_stats_reset(struct rte_eth_dev *dev);
 106 static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 107                                              uint16_t queue_id,
 108                                              uint8_t stat_idx,
 109                                              uint8_t is_rx);
 110 static void ixgbe_dev_info_get(struct rte_eth_dev *dev,
 111                                 struct rte_eth_dev_info *dev_info);
 112 static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev,
 113                 uint16_t vlan_id, int on);
 114 static void ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 115 static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev,
 116                 uint16_t queue, bool on);
 117 static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue,
 118                 int on);
 119 static void ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 120 static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue);
 121 static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue);
 122 static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 123 static void ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 124
 125 static int ixgbe_dev_led_on(struct rte_eth_dev *dev);
 126 static int ixgbe_dev_led_off(struct rte_eth_dev *dev);
 127 static int  ixgbe_flow_ctrl_set(struct rte_eth_dev *dev,
 128                 struct rte_eth_fc_conf *fc_conf);
 129 static int ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 130                 struct rte_eth_pfc_conf *pfc_conf);
 131 static int ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
 132                 struct rte_eth_rss_reta *reta_conf);
 133 static int ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
 134                 struct rte_eth_rss_reta *reta_conf);
 135 static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev);
 136 static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev);
 137 static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev);
 138 static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev);
 139 static void ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle,
 140                 void *param);
 141 static void ixgbe_dev_interrupt_delayed_handler(void *param);
 142 static void ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 143                 uint32_t index, uint32_t pool);
 144 static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index);
 145 static void ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config);
 146
 147 /* For Virtual Function support */
 148 static int eth_ixgbevf_dev_init(struct eth_driver *eth_drv,
 149                 struct rte_eth_dev *eth_dev);
 150 static int  ixgbevf_dev_configure(struct rte_eth_dev *dev);
 151 static int  ixgbevf_dev_start(struct rte_eth_dev *dev);
 152 static void ixgbevf_dev_stop(struct rte_eth_dev *dev);
 153 static void ixgbevf_dev_close(struct rte_eth_dev *dev);
 154 static void ixgbevf_intr_disable(struct ixgbe_hw *hw);
 155 static void ixgbevf_dev_stats_get(struct rte_eth_dev *dev,
 156                 struct rte_eth_stats *stats);
 157 static void ixgbevf_dev_stats_reset(struct rte_eth_dev *dev);
 158 static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev,
 159                 uint16_t vlan_id, int on);
 160 static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev,
 161                 uint16_t queue, int on);
 162 static void ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 163 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 164
 165 /*
 166  * Define VF Stats MACRO for Non "cleared on read" register
 167  */
 168 #define UPDATE_VF_STAT(reg, last, cur)                          \
 169 {                                                               \
 170         u32 latest = IXGBE_READ_REG(hw, reg);                   \
 171         cur += latest - last;                                   \
 172         last = latest;                                          \
 173 }
 174
 175 #define UPDATE_VF_STAT_36BIT(lsb, msb, last, cur)                \
 176 {                                                                \
 177         u64 new_lsb = IXGBE_READ_REG(hw, lsb);                   \
 178         u64 new_msb = IXGBE_READ_REG(hw, msb);                   \
 179         u64 latest = ((new_msb << 32) | new_lsb);                \
 180         cur += (0x1000000000LL + latest - last) & 0xFFFFFFFFFLL; \
 181         last = latest;                                           \
 182 }
 183
 184 #define IXGBE_SET_HWSTRIP(h, q) do{\
 185                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 186                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 187                 (h)->bitmap[idx] |= 1 << bit;\
 188         }while(0)
 189
 190 #define IXGBE_CLEAR_HWSTRIP(h, q) do{\
 191                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 192                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 193                 (h)->bitmap[idx] &= ~(1 << bit);\
 194         }while(0)
 195
 196 #define IXGBE_GET_HWSTRIP(h, q, r) do{\
 197                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 198                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 199                 (r) = (h)->bitmap[idx] >> bit & 1;\
 200         }while(0)
 201
 202 /*
 203  * The set of PCI devices this driver supports
 204  */
 205 static struct rte_pci_id pci_id_ixgbe_map[] = {
 206
 207 #define RTE_PCI_DEV_ID_DECL_IXGBE(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 208 #include "rte_pci_dev_ids.h"
 209
 210 { .vendor_id = 0, /* sentinel */ },
 211 };
 212
 213
 214 /*
 215  * The set of PCI devices this driver supports (for 82599 VF)
 216  */
 217 static struct rte_pci_id pci_id_ixgbevf_map[] = {
 218
 219 #define RTE_PCI_DEV_ID_DECL_IXGBEVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 220 #include "rte_pci_dev_ids.h"
 221 { .vendor_id = 0, /* sentinel */ },
 222
 223 };
 224
 225 static struct eth_dev_ops ixgbe_eth_dev_ops = {
 226         .dev_configure        = ixgbe_dev_configure,
 227         .dev_start            = ixgbe_dev_start,
 228         .dev_stop             = ixgbe_dev_stop,
 229         .dev_close            = ixgbe_dev_close,
 230         .promiscuous_enable   = ixgbe_dev_promiscuous_enable,
 231         .promiscuous_disable  = ixgbe_dev_promiscuous_disable,
 232         .allmulticast_enable  = ixgbe_dev_allmulticast_enable,
 233         .allmulticast_disable = ixgbe_dev_allmulticast_disable,
 234         .link_update          = ixgbe_dev_link_update,
 235         .stats_get            = ixgbe_dev_stats_get,
 236         .stats_reset          = ixgbe_dev_stats_reset,
 237         .queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set,
 238         .dev_infos_get        = ixgbe_dev_info_get,
 239         .vlan_filter_set      = ixgbe_vlan_filter_set,
 240         .vlan_tpid_set        = ixgbe_vlan_tpid_set,
 241         .vlan_offload_set     = ixgbe_vlan_offload_set,
 242         .vlan_strip_queue_set = ixgbe_vlan_strip_queue_set,
 243         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 244         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 245         .rx_queue_count       = ixgbe_dev_rx_queue_count,
 246         .rx_descriptor_done   = ixgbe_dev_rx_descriptor_done,
 247         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 248         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 249         .dev_led_on           = ixgbe_dev_led_on,
 250         .dev_led_off          = ixgbe_dev_led_off,
 251         .flow_ctrl_set        = ixgbe_flow_ctrl_set,
 252         .priority_flow_ctrl_set = ixgbe_priority_flow_ctrl_set,
 253         .mac_addr_add         = ixgbe_add_rar,
 254         .mac_addr_remove      = ixgbe_remove_rar,
 255         .fdir_add_signature_filter    = ixgbe_fdir_add_signature_filter,
 256         .fdir_update_signature_filter = ixgbe_fdir_update_signature_filter,
 257         .fdir_remove_signature_filter = ixgbe_fdir_remove_signature_filter,
 258         .fdir_infos_get               = ixgbe_fdir_info_get,
 259         .fdir_add_perfect_filter      = ixgbe_fdir_add_perfect_filter,
 260         .fdir_update_perfect_filter   = ixgbe_fdir_update_perfect_filter,
 261         .fdir_remove_perfect_filter   = ixgbe_fdir_remove_perfect_filter,
 262         .fdir_set_masks               = ixgbe_fdir_set_masks,
 263         .reta_update          = ixgbe_dev_rss_reta_update,
 264         .reta_query           = ixgbe_dev_rss_reta_query,
 265 };
 266
 267 /*
 268  * dev_ops for virtual function, bare necessities for basic vf
 269  * operation have been implemented
 270  */
 271 static struct eth_dev_ops ixgbevf_eth_dev_ops = {
 272
 273         .dev_configure        = ixgbevf_dev_configure,
 274         .dev_start            = ixgbevf_dev_start,
 275         .dev_stop             = ixgbevf_dev_stop,
 276         .link_update          = ixgbe_dev_link_update,
 277         .stats_get            = ixgbevf_dev_stats_get,
 278         .stats_reset          = ixgbevf_dev_stats_reset,
 279         .dev_close            = ixgbevf_dev_close,
 280         .dev_infos_get        = ixgbe_dev_info_get,
 281         .vlan_filter_set      = ixgbevf_vlan_filter_set,
 282         .vlan_strip_queue_set = ixgbevf_vlan_strip_queue_set,
 283         .vlan_offload_set     = ixgbevf_vlan_offload_set,
 284         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 285         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 286         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 287         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 288 };
 289
 290 /**
 291  * Atomically reads the link status information from global
 292  * structure rte_eth_dev.
 293  *
 294  * @param dev
 295  *   - Pointer to the structure rte_eth_dev to read from.
 296  *   - Pointer to the buffer to be saved with the link status.
 297  *
 298  * @return
 299  *   - On success, zero.
 300  *   - On failure, negative value.
 301  */
 302 static inline int
 303 rte_ixgbe_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 304                                 struct rte_eth_link *link)
 305 {
 306         struct rte_eth_link *dst = link;
 307         struct rte_eth_link *src = &(dev->data->dev_link);
 308
 309         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 310                                         *(uint64_t *)src) == 0)
 311                 return -1;
 312
 313         return 0;
 314 }
 315
 316 /**
 317  * Atomically writes the link status information into global
 318  * structure rte_eth_dev.
 319  *
 320  * @param dev
 321  *   - Pointer to the structure rte_eth_dev to read from.
 322  *   - Pointer to the buffer to be saved with the link status.
 323  *
 324  * @return
 325  *   - On success, zero.
 326  *   - On failure, negative value.
 327  */
 328 static inline int
 329 rte_ixgbe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 330                                 struct rte_eth_link *link)
 331 {
 332         struct rte_eth_link *dst = &(dev->data->dev_link);
 333         struct rte_eth_link *src = link;
 334
 335         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 336                                         *(uint64_t *)src) == 0)
 337                 return -1;
 338
 339         return 0;
 340 }
 341
 342 /*
 343  * This function is the same as ixgbe_is_sfp() in ixgbe/ixgbe.h.
 344  */
 345 static inline int
 346 ixgbe_is_sfp(struct ixgbe_hw *hw)
 347 {
 348         switch (hw->phy.type) {
 349         case ixgbe_phy_sfp_avago:
 350         case ixgbe_phy_sfp_ftl:
 351         case ixgbe_phy_sfp_intel:
 352         case ixgbe_phy_sfp_unknown:
 353         case ixgbe_phy_sfp_passive_tyco:
 354         case ixgbe_phy_sfp_passive_unknown:
 355                 return 1;
 356         default:
 357                 return 0;
 358         }
 359 }
 360
 361 static inline int32_t
 362 ixgbe_pf_reset_hw(struct ixgbe_hw *hw)
 363 {
 364         uint32_t ctrl_ext;
 365         int32_t status;
 366
 367         status = ixgbe_reset_hw(hw);
 368
 369         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 370         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 371         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
 372         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 373         IXGBE_WRITE_FLUSH(hw);
 374
 375         return status;
 376 }
 377
 378 static inline void
 379 ixgbe_enable_intr(struct rte_eth_dev *dev)
 380 {
 381         struct ixgbe_interrupt *intr =
 382                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 383         struct ixgbe_hw *hw =
 384                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 385
 386         IXGBE_WRITE_REG(hw, IXGBE_EIMS, intr->mask);
 387         IXGBE_WRITE_FLUSH(hw);
 388 }
 389
 390 /*
 391  * This function is based on ixgbe_disable_intr() in ixgbe/ixgbe.h.
 392  */
 393 static void
 394 ixgbe_disable_intr(struct ixgbe_hw *hw)
 395 {
 396         PMD_INIT_FUNC_TRACE();
 397
 398         if (hw->mac.type == ixgbe_mac_82598EB) {
 399                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ~0);
 400         } else {
 401                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xFFFF0000);
 402                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), ~0);
 403                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), ~0);
 404         }
 405         IXGBE_WRITE_FLUSH(hw);
 406 }
 407
 408 /*
 409  * This function resets queue statistics mapping registers.
 410  * From Niantic datasheet, Initialization of Statistics section:
 411  * "...if software requires the queue counters, the RQSMR and TQSM registers
 412  * must be re-programmed following a device reset.
 413  */
 414 static void
 415 ixgbe_reset_qstat_mappings(struct ixgbe_hw *hw)
 416 {
 417         uint32_t i;
 418
 419         for(i = 0; i != IXGBE_NB_STAT_MAPPING_REGS; i++) {
 420                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), 0);
 421                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), 0);
 422         }
 423 }
 424
 425
 426 static int
 427 ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 428                                   uint16_t queue_id,
 429                                   uint8_t stat_idx,
 430                                   uint8_t is_rx)
 431 {
 432 #define QSM_REG_NB_BITS_PER_QMAP_FIELD 8
 433 #define NB_QMAP_FIELDS_PER_QSM_REG 4
 434 #define QMAP_FIELD_RESERVED_BITS_MASK 0x0f
 435
 436         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 437         struct ixgbe_stat_mapping_registers *stat_mappings =
 438                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(eth_dev->data->dev_private);
 439         uint32_t qsmr_mask = 0;
 440         uint32_t clearing_mask = QMAP_FIELD_RESERVED_BITS_MASK;
 441         uint32_t q_map;
 442         uint8_t n, offset;
 443
 444         if ((hw->mac.type != ixgbe_mac_82599EB) && (hw->mac.type != ixgbe_mac_X540))
 445                 return -ENOSYS;
 446
 447         PMD_INIT_LOG(INFO, "Setting port %d, %s queue_id %d to stat index %d\n",
 448                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx);
 449
 450         n = (uint8_t)(queue_id / NB_QMAP_FIELDS_PER_QSM_REG);
 451         if (n >= IXGBE_NB_STAT_MAPPING_REGS) {
 452                 PMD_INIT_LOG(ERR, "Nb of stat mapping registers exceeded\n");
 453                 return -EIO;
 454         }
 455         offset = (uint8_t)(queue_id % NB_QMAP_FIELDS_PER_QSM_REG);
 456
 457         /* Now clear any previous stat_idx set */
 458         clearing_mask <<= (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 459         if (!is_rx)
 460                 stat_mappings->tqsm[n] &= ~clearing_mask;
 461         else
 462                 stat_mappings->rqsmr[n] &= ~clearing_mask;
 463
 464         q_map = (uint32_t)stat_idx;
 465         q_map &= QMAP_FIELD_RESERVED_BITS_MASK;
 466         qsmr_mask = q_map << (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 467         if (!is_rx)
 468                 stat_mappings->tqsm[n] |= qsmr_mask;
 469         else
 470                 stat_mappings->rqsmr[n] |= qsmr_mask;
 471
 472         PMD_INIT_LOG(INFO, "Set port %d, %s queue_id %d to stat index %d\n"
 473                      "%s[%d] = 0x%08x\n",
 474                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx,
 475                      is_rx ? "RQSMR" : "TQSM",n, is_rx ? stat_mappings->rqsmr[n] : stat_mappings->tqsm[n]);
 476
 477         /* Now write the mapping in the appropriate register */
 478         if (is_rx) {
 479                 PMD_INIT_LOG(INFO, "Write 0x%x to RX IXGBE stat mapping reg:%d\n",
 480                              stat_mappings->rqsmr[n], n);
 481                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(n), stat_mappings->rqsmr[n]);
 482         }
 483         else {
 484                 PMD_INIT_LOG(INFO, "Write 0x%x to TX IXGBE stat mapping reg:%d\n",
 485                              stat_mappings->tqsm[n], n);
 486                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(n), stat_mappings->tqsm[n]);
 487         }
 488         return 0;
 489 }
 490
 491 static void
 492 ixgbe_restore_statistics_mapping(struct rte_eth_dev * dev)
 493 {
 494         struct ixgbe_stat_mapping_registers *stat_mappings =
 495                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(dev->data->dev_private);
 496         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 497         int i;
 498
 499         /* write whatever was in stat mapping table to the NIC */
 500         for (i = 0; i < IXGBE_NB_STAT_MAPPING_REGS; i++) {
 501                 /* rx */
 502                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), stat_mappings->rqsmr[i]);
 503
 504                 /* tx */
 505                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), stat_mappings->tqsm[i]);
 506         }
 507 }
 508
 509 static void
 510 ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config)
 511 {
 512         uint8_t i;
 513         struct ixgbe_dcb_tc_config *tc;
 514         uint8_t dcb_max_tc = IXGBE_DCB_MAX_TRAFFIC_CLASS;
 515
 516         dcb_config->num_tcs.pg_tcs = dcb_max_tc;
 517         dcb_config->num_tcs.pfc_tcs = dcb_max_tc;
 518         for (i = 0; i < dcb_max_tc; i++) {
 519                 tc = &dcb_config->tc_config[i];
 520                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_id = i;
 521                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
 522                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 523                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_id = i;
 524                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
 525                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 526                 tc->pfc = ixgbe_dcb_pfc_disabled;
 527         }
 528
 529         /* Initialize default user to priority mapping, UPx->TC0 */
 530         tc = &dcb_config->tc_config[0];
 531         tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
 532         tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
 533         for (i = 0; i< IXGBE_DCB_MAX_BW_GROUP; i++) {
 534                 dcb_config->bw_percentage[IXGBE_DCB_TX_CONFIG][i] = 100;
 535                 dcb_config->bw_percentage[IXGBE_DCB_RX_CONFIG][i] = 100;
 536         }
 537         dcb_config->rx_pba_cfg = ixgbe_dcb_pba_equal;
 538         dcb_config->pfc_mode_enable = false;
 539         dcb_config->vt_mode = true;
 540         dcb_config->round_robin_enable = false;
 541         /* support all DCB capabilities in 82599 */
 542         dcb_config->support.capabilities = 0xFF;
 543
 544         /*we only support 4 Tcs for X540*/
 545         if (hw->mac.type == ixgbe_mac_X540) {
 546                 dcb_config->num_tcs.pg_tcs = 4;
 547                 dcb_config->num_tcs.pfc_tcs = 4;
 548         }
 549 }
 550
 551 /*
 552  * This function is based on code in ixgbe_attach() in ixgbe/ixgbe.c.
 553  * It returns 0 on success.
 554  */
 555 static int
 556 eth_ixgbe_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 557                      struct rte_eth_dev *eth_dev)
 558 {
 559         struct rte_pci_device *pci_dev;
 560         struct ixgbe_hw *hw =
 561                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 562         struct ixgbe_vfta * shadow_vfta =
 563                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 564         struct ixgbe_hwstrip *hwstrip =
 565                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 566         struct ixgbe_dcb_config *dcb_config =
 567                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private);
 568         uint32_t ctrl_ext;
 569         uint16_t csum;
 570         int diag, i;
 571
 572         PMD_INIT_FUNC_TRACE();
 573
 574         eth_dev->dev_ops = &ixgbe_eth_dev_ops;
 575         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
 576         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
 577
 578         /* for secondary processes, we don't initialise any further as primary
 579          * has already done this work. Only check we don't need a different
 580          * RX function */
 581         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 582                 if (eth_dev->data->scattered_rx)
 583                         eth_dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
 584                 return 0;
 585         }
 586         pci_dev = eth_dev->pci_dev;
 587
 588         /* Vendor and Device ID need to be set before init of shared code */
 589         hw->device_id = pci_dev->id.device_id;
 590         hw->vendor_id = pci_dev->id.vendor_id;
 591         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 592 #ifdef RTE_LIBRTE_IXGBE_ALLOW_UNSUPPORTED_SFP
 593         hw->allow_unsupported_sfp = 1;
 594 #endif
 595
 596         /* Initialize the shared code */
 597         diag = ixgbe_init_shared_code(hw);
 598         if (diag != IXGBE_SUCCESS) {
 599                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
 600                 return -EIO;
 601         }
 602
 603         /* Initialize DCB configuration*/
 604         memset(dcb_config, 0, sizeof(struct ixgbe_dcb_config));
 605         ixgbe_dcb_init(hw,dcb_config);
 606         /* Get Hardware Flow Control setting */
 607         hw->fc.requested_mode = ixgbe_fc_full;
 608         hw->fc.current_mode = ixgbe_fc_full;
 609         hw->fc.pause_time = IXGBE_FC_PAUSE;
 610         for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
 611                 hw->fc.low_water[i] = IXGBE_FC_LO;
 612                 hw->fc.high_water[i] = IXGBE_FC_HI;
 613         }
 614         hw->fc.send_xon = 1;
 615
 616         /* Make sure we have a good EEPROM before we read from it */
 617         diag = ixgbe_validate_eeprom_checksum(hw, &csum);
 618         if (diag != IXGBE_SUCCESS) {
 619                 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag);
 620                 return -EIO;
 621         }
 622
 623         diag = ixgbe_init_hw(hw);
 624
 625         /*
 626          * Devices with copper phys will fail to initialise if ixgbe_init_hw()
 627          * is called too soon after the kernel driver unbinding/binding occurs.
 628          * The failure occurs in ixgbe_identify_phy_generic() for all devices,
 629          * but for non-copper devies, ixgbe_identify_sfp_module_generic() is
 630          * also called. See ixgbe_identify_phy_82599(). The reason for the
 631          * failure is not known, and only occuts when virtualisation features
 632          * are disabled in the bios. A delay of 100ms  was found to be enough by
 633          * trial-and-error, and is doubled to be safe.
 634          */
 635         if (diag && (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper)) {
 636                 rte_delay_ms(200);
 637                 diag = ixgbe_init_hw(hw);
 638         }
 639
 640         if (diag == IXGBE_ERR_EEPROM_VERSION) {
 641                 PMD_INIT_LOG(ERR, "This device is a pre-production adapter/"
 642                     "LOM.  Please be aware there may be issues associated "
 643                     "with your hardware.\n If you are experiencing problems "
 644                     "please contact your Intel or hardware representative "
 645                     "who provided you with this hardware.\n");
 646         } else if (diag == IXGBE_ERR_SFP_NOT_SUPPORTED)
 647                 PMD_INIT_LOG(ERR, "Unsupported SFP+ Module\n");
 648         if (diag) {
 649                 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag);
 650                 return -EIO;
 651         }
 652
 653         /* disable interrupt */
 654         ixgbe_disable_intr(hw);
 655
 656         /* pick up the PCI bus settings for reporting later */
 657         ixgbe_get_bus_info(hw);
 658
 659         /* reset mappings for queue statistics hw counters*/
 660         ixgbe_reset_qstat_mappings(hw);
 661
 662         /* Allocate memory for storing MAC addresses */
 663         eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 664                         hw->mac.num_rar_entries, 0);
 665         if (eth_dev->data->mac_addrs == NULL) {
 666                 PMD_INIT_LOG(ERR,
 667                         "Failed to allocate %d bytes needed to store MAC addresses",
 668                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 669                 return -ENOMEM;
 670         }
 671         /* Copy the permanent MAC address */
 672         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 673                         &eth_dev->data->mac_addrs[0]);
 674
 675         /* initialize the vfta */
 676         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 677
 678         /* initialize the hw strip bitmap*/
 679         memset(hwstrip, 0, sizeof(*hwstrip));
 680
 681         /* initialize PF if max_vfs not zero */
 682         ixgbe_pf_host_init(eth_dev);
 683
 684         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 685         /* let hardware know driver is loaded */
 686         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 687         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 688         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
 689         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 690         IXGBE_WRITE_FLUSH(hw);
 691
 692         if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
 693                 PMD_INIT_LOG(DEBUG,
 694                              "MAC: %d, PHY: %d, SFP+: %d<n",
 695                              (int) hw->mac.type, (int) hw->phy.type,
 696                              (int) hw->phy.sfp_type);
 697         else
 698                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d\n",
 699                              (int) hw->mac.type, (int) hw->phy.type);
 700
 701         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
 702                         eth_dev->data->port_id, pci_dev->id.vendor_id,
 703                         pci_dev->id.device_id);
 704
 705         rte_intr_callback_register(&(pci_dev->intr_handle),
 706                 ixgbe_dev_interrupt_handler, (void *)eth_dev);
 707
 708         /* enable uio intr after callback register */
 709         rte_intr_enable(&(pci_dev->intr_handle));
 710
 711         /* enable support intr */
 712         ixgbe_enable_intr(eth_dev);
 713
 714         return 0;
 715 }
 716
 717 /*
 718  * Virtual Function device init
 719  */
 720 static int
 721 eth_ixgbevf_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 722                      struct rte_eth_dev *eth_dev)
 723 {
 724         struct rte_pci_device *pci_dev;
 725         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 726         int diag;
 727         struct ixgbe_vfta * shadow_vfta =
 728                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 729         struct ixgbe_hwstrip *hwstrip =
 730                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 731
 732         PMD_INIT_LOG(DEBUG, "eth_ixgbevf_dev_init");
 733
 734         eth_dev->dev_ops = &ixgbevf_eth_dev_ops;
 735         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
 736         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
 737
 738         /* for secondary processes, we don't initialise any further as primary
 739          * has already done this work. Only check we don't need a different
 740          * RX function */
 741         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 742                 if (eth_dev->data->scattered_rx)
 743                         eth_dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
 744                 return 0;
 745         }
 746
 747         pci_dev = eth_dev->pci_dev;
 748
 749         hw->device_id = pci_dev->id.device_id;
 750         hw->vendor_id = pci_dev->id.vendor_id;
 751         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 752
 753         /* initialize the vfta */
 754         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 755
 756         /* initialize the hw strip bitmap*/
 757         memset(hwstrip, 0, sizeof(*hwstrip));
 758
 759         /* Initialize the shared code */
 760         diag = ixgbe_init_shared_code(hw);
 761         if (diag != IXGBE_SUCCESS) {
 762                 PMD_INIT_LOG(ERR, "Shared code init failed for ixgbevf: %d", diag);
 763                 return -EIO;
 764         }
 765
 766         /* init_mailbox_params */
 767         hw->mbx.ops.init_params(hw);
 768
 769         /* Disable the interrupts for VF */
 770         ixgbevf_intr_disable(hw);
 771
 772         hw->mac.num_rar_entries = hw->mac.max_rx_queues;
 773         diag = hw->mac.ops.reset_hw(hw);
 774
 775         if (diag != IXGBE_SUCCESS) {
 776                 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
 777                         RTE_LOG(ERR, PMD, "\tThe MAC address is not valid.\n"
 778                                         "\tThe most likely cause of this error is that the VM host\n"
 779                                         "\thas not assigned a valid MAC address to this VF device.\n"
 780                                         "\tPlease consult the DPDK Release Notes (FAQ section) for\n"
 781                                         "\ta possible solution to this problem.\n");
 782                 return (diag);
 783         }
 784
 785         /* Allocate memory for storing MAC addresses */
 786         eth_dev->data->mac_addrs = rte_zmalloc("ixgbevf", ETHER_ADDR_LEN *
 787                         hw->mac.num_rar_entries, 0);
 788         if (eth_dev->data->mac_addrs == NULL) {
 789                 PMD_INIT_LOG(ERR,
 790                         "Failed to allocate %d bytes needed to store MAC addresses",
 791                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 792                 return -ENOMEM;
 793         }
 794
 795         /* Copy the permanent MAC address */
 796         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 797                         &eth_dev->data->mac_addrs[0]);
 798
 799         /* reset the hardware with the new settings */
 800         diag = hw->mac.ops.start_hw(hw);
 801         switch (diag) {
 802                 case  0:
 803                         break;
 804
 805                 default:
 806                         PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
 807                         return (-EIO);
 808         }
 809
 810         PMD_INIT_LOG(DEBUG, "\nport %d vendorID=0x%x deviceID=0x%x mac.type=%s\n",
 811                          eth_dev->data->port_id, pci_dev->id.vendor_id, pci_dev->id.device_id,
 812                          "ixgbe_mac_82599_vf");
 813
 814         return 0;
 815 }
 816
 817 static struct eth_driver rte_ixgbe_pmd = {
 818         {
 819                 .name = "rte_ixgbe_pmd",
 820                 .id_table = pci_id_ixgbe_map,
 821 #ifdef RTE_EAL_UNBIND_PORTS
 822                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
 823 #endif
 824         },
 825         .eth_dev_init = eth_ixgbe_dev_init,
 826         .dev_private_size = sizeof(struct ixgbe_adapter),
 827 };
 828
 829 /*
 830  * virtual function driver struct
 831  */
 832 static struct eth_driver rte_ixgbevf_pmd = {
 833         {
 834                 .name = "rte_ixgbevf_pmd",
 835                 .id_table = pci_id_ixgbevf_map,
 836 #ifdef RTE_EAL_UNBIND_PORTS
 837                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
 838 #endif
 839         },
 840         .eth_dev_init = eth_ixgbevf_dev_init,
 841         .dev_private_size = sizeof(struct ixgbe_adapter),
 842 };
 843
 844 /*
 845  * Driver initialization routine.
 846  * Invoked once at EAL init time.
 847  * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
 848  */
 849 int
 850 rte_ixgbe_pmd_init(void)
 851 {
 852         PMD_INIT_FUNC_TRACE();
 853
 854         rte_eth_driver_register(&rte_ixgbe_pmd);
 855         return 0;
 856 }
 857
 858 /*
 859  * VF Driver initialization routine.
 860  * Invoked one at EAL init time.
 861  * Register itself as the [Virtual Poll Mode] Driver of PCI niantic devices.
 862  */
 863 int
 864 rte_ixgbevf_pmd_init(void)
 865 {
 866         DEBUGFUNC("rte_ixgbevf_pmd_init");
 867
 868         rte_eth_driver_register(&rte_ixgbevf_pmd);
 869         return (0);
 870 }
 871
 872 static int
 873 ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 874 {
 875         struct ixgbe_hw *hw =
 876                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 877         struct ixgbe_vfta * shadow_vfta =
 878                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
 879         uint32_t vfta;
 880         uint32_t vid_idx;
 881         uint32_t vid_bit;
 882
 883         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
 884         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
 885         vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(vid_idx));
 886         if (on)
 887                 vfta |= vid_bit;
 888         else
 889                 vfta &= ~vid_bit;
 890         IXGBE_WRITE_REG(hw, IXGBE_VFTA(vid_idx), vfta);
 891
 892         /* update local VFTA copy */
 893         shadow_vfta->vfta[vid_idx] = vfta;
 894
 895         return 0;
 896 }
 897
 898 static void
 899 ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
 900 {
 901         if (on)
 902                 ixgbe_vlan_hw_strip_enable(dev, queue);
 903         else
 904                 ixgbe_vlan_hw_strip_disable(dev, queue);
 905 }
 906
 907 static void
 908 ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
 909 {
 910         struct ixgbe_hw *hw =
 911                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 912
 913         /* Only the high 16-bits is valid */
 914         IXGBE_WRITE_REG(hw, IXGBE_EXVET, tpid << 16);
 915 }
 916
 917 void
 918 ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev)
 919 {
 920         struct ixgbe_hw *hw =
 921                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 922         uint32_t vlnctrl;
 923
 924         PMD_INIT_FUNC_TRACE();
 925
 926         /* Filter Table Disable */
 927         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 928         vlnctrl &= ~IXGBE_VLNCTRL_VFE;
 929
 930         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
 931 }
 932
 933 void
 934 ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev)
 935 {
 936         struct ixgbe_hw *hw =
 937                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 938         struct ixgbe_vfta * shadow_vfta =
 939                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
 940         uint32_t vlnctrl;
 941         uint16_t i;
 942
 943         PMD_INIT_FUNC_TRACE();
 944
 945         /* Filter Table Enable */
 946         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 947         vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
 948         vlnctrl |= IXGBE_VLNCTRL_VFE;
 949
 950         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
 951
 952         /* write whatever is in local vfta copy */
 953         for (i = 0; i < IXGBE_VFTA_SIZE; i++)
 954                 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), shadow_vfta->vfta[i]);
 955 }
 956
 957 static void
 958 ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on)
 959 {
 960         struct ixgbe_hwstrip *hwstrip =
 961                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(dev->data->dev_private);
 962
 963         if(queue >= IXGBE_MAX_RX_QUEUE_NUM)
 964                 return;
 965
 966         if (on)
 967                 IXGBE_SET_HWSTRIP(hwstrip, queue);
 968         else
 969                 IXGBE_CLEAR_HWSTRIP(hwstrip, queue);
 970 }
 971
 972 static void
 973 ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue)
 974 {
 975         struct ixgbe_hw *hw =
 976                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 977         uint32_t ctrl;
 978
 979         PMD_INIT_FUNC_TRACE();
 980
 981         if (hw->mac.type == ixgbe_mac_82598EB) {
 982                 /* No queue level support */
 983                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
 984                 return;
 985         }
 986         else {
 987                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 988                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
 989                 ctrl &= ~IXGBE_RXDCTL_VME;
 990                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
 991         }
 992         /* record those setting for HW strip per queue */
 993         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 0);
 994 }
 995
 996 static void
 997 ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue)
 998 {
 999         struct ixgbe_hw *hw =
1000                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1001         uint32_t ctrl;
1002
1003         PMD_INIT_FUNC_TRACE();
1004
1005         if (hw->mac.type == ixgbe_mac_82598EB) {
1006                 /* No queue level supported */
1007                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
1008                 return;
1009         }
1010         else {
1011                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1012                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1013                 ctrl |= IXGBE_RXDCTL_VME;
1014                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
1015         }
1016         /* record those setting for HW strip per queue */
1017         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 1);
1018 }
1019
1020 void
1021 ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev)
1022 {
1023         struct ixgbe_hw *hw =
1024                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1025         uint32_t ctrl;
1026         uint16_t i;
1027
1028         PMD_INIT_FUNC_TRACE();
1029
1030         if (hw->mac.type == ixgbe_mac_82598EB) {
1031                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1032                 ctrl &= ~IXGBE_VLNCTRL_VME;
1033                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1034         }
1035         else {
1036                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1037                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1038                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1039                         ctrl &= ~IXGBE_RXDCTL_VME;
1040                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1041
1042                         /* record those setting for HW strip per queue */
1043                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 0);
1044                 }
1045         }
1046 }
1047
1048 void
1049 ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev)
1050 {
1051         struct ixgbe_hw *hw =
1052                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1053         uint32_t ctrl;
1054         uint16_t i;
1055
1056         PMD_INIT_FUNC_TRACE();
1057
1058         if (hw->mac.type == ixgbe_mac_82598EB) {
1059                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1060                 ctrl |= IXGBE_VLNCTRL_VME;
1061                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1062         }
1063         else {
1064                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1065                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1066                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1067                         ctrl |= IXGBE_RXDCTL_VME;
1068                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1069
1070                         /* record those setting for HW strip per queue */
1071                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 1);
1072                 }
1073         }
1074 }
1075
1076 static void
1077 ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1078 {
1079         struct ixgbe_hw *hw =
1080                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1081         uint32_t ctrl;
1082
1083         PMD_INIT_FUNC_TRACE();
1084
1085         /* DMATXCTRL: Geric Double VLAN Disable */
1086         ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1087         ctrl &= ~IXGBE_DMATXCTL_GDV;
1088         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1089
1090         /* CTRL_EXT: Global Double VLAN Disable */
1091         ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1092         ctrl &= ~IXGBE_EXTENDED_VLAN;
1093         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1094
1095 }
1096
1097 static void
1098 ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1099 {
1100         struct ixgbe_hw *hw =
1101                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1102         uint32_t ctrl;
1103
1104         PMD_INIT_FUNC_TRACE();
1105
1106         /* DMATXCTRL: Geric Double VLAN Enable */
1107         ctrl  = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1108         ctrl |= IXGBE_DMATXCTL_GDV;
1109         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1110
1111         /* CTRL_EXT: Global Double VLAN Enable */
1112         ctrl  = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1113         ctrl |= IXGBE_EXTENDED_VLAN;
1114         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1115
1116         /*
1117          * VET EXT field in the EXVET register = 0x8100 by default
1118          * So no need to change. Same to VT field of DMATXCTL register
1119          */
1120 }
1121
1122 static void
1123 ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1124 {
1125         if(mask & ETH_VLAN_STRIP_MASK){
1126                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1127                         ixgbe_vlan_hw_strip_enable_all(dev);
1128                 else
1129                         ixgbe_vlan_hw_strip_disable_all(dev);
1130         }
1131
1132         if(mask & ETH_VLAN_FILTER_MASK){
1133                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1134                         ixgbe_vlan_hw_filter_enable(dev);
1135                 else
1136                         ixgbe_vlan_hw_filter_disable(dev);
1137         }
1138
1139         if(mask & ETH_VLAN_EXTEND_MASK){
1140                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1141                         ixgbe_vlan_hw_extend_enable(dev);
1142                 else
1143                         ixgbe_vlan_hw_extend_disable(dev);
1144         }
1145 }
1146
1147 static int
1148 ixgbe_dev_configure(struct rte_eth_dev *dev)
1149 {
1150         struct ixgbe_interrupt *intr =
1151                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1152
1153         PMD_INIT_FUNC_TRACE();
1154
1155         /* set flag to update link status after init */
1156         intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1157
1158         return 0;
1159 }
1160
1161 /*
1162  * Configure device link speed and setup link.
1163  * It returns 0 on success.
1164  */
1165 static int
1166 ixgbe_dev_start(struct rte_eth_dev *dev)
1167 {
1168         struct ixgbe_hw *hw =
1169                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1170         int err, link_up = 0, negotiate = 0;
1171         uint32_t speed = 0;
1172         int mask = 0;
1173         int status;
1174
1175         PMD_INIT_FUNC_TRACE();
1176
1177         /* IXGBE devices don't support half duplex */
1178         if ((dev->data->dev_conf.link_duplex != ETH_LINK_AUTONEG_DUPLEX) &&
1179                         (dev->data->dev_conf.link_duplex != ETH_LINK_FULL_DUPLEX)) {
1180                 PMD_INIT_LOG(ERR, "Invalid link_duplex (%u) for port %u\n",
1181                                 dev->data->dev_conf.link_duplex,
1182                                 dev->data->port_id);
1183                 return -EINVAL;
1184         }
1185
1186         /* stop adapter */
1187         hw->adapter_stopped = FALSE;
1188         ixgbe_stop_adapter(hw);
1189
1190         /* reinitialize adapter
1191          * this calls reset and start */
1192         status = ixgbe_pf_reset_hw(hw);
1193         if (status != 0)
1194                 return -1;
1195         hw->mac.ops.start_hw(hw);
1196
1197         /* configure PF module if SRIOV enabled */
1198         ixgbe_pf_host_configure(dev);
1199
1200         /* initialize transmission unit */
1201         ixgbe_dev_tx_init(dev);
1202
1203         /* This can fail when allocating mbufs for descriptor rings */
1204         err = ixgbe_dev_rx_init(dev);
1205         if (err) {
1206                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware\n");
1207                 goto error;
1208         }
1209
1210         ixgbe_dev_rxtx_start(dev);
1211
1212         if (ixgbe_is_sfp(hw) && hw->phy.multispeed_fiber) {
1213                 err = hw->mac.ops.setup_sfp(hw);
1214                 if (err)
1215                         goto error;
1216         }
1217
1218         /* Turn on the laser */
1219         ixgbe_enable_tx_laser(hw);
1220
1221         err = ixgbe_check_link(hw, &speed, &link_up, 0);
1222         if (err)
1223                 goto error;
1224         err = ixgbe_get_link_capabilities(hw, &speed, &negotiate);
1225         if (err)
1226                 goto error;
1227
1228         switch(dev->data->dev_conf.link_speed) {
1229         case ETH_LINK_SPEED_AUTONEG:
1230                 speed = (hw->mac.type != ixgbe_mac_82598EB) ?
1231                                 IXGBE_LINK_SPEED_82599_AUTONEG :
1232                                 IXGBE_LINK_SPEED_82598_AUTONEG;
1233                 break;
1234         case ETH_LINK_SPEED_100:
1235                 /*
1236                  * Invalid for 82598 but error will be detected by
1237                  * ixgbe_setup_link()
1238                  */
1239                 speed = IXGBE_LINK_SPEED_100_FULL;
1240                 break;
1241         case ETH_LINK_SPEED_1000:
1242                 speed = IXGBE_LINK_SPEED_1GB_FULL;
1243                 break;
1244         case ETH_LINK_SPEED_10000:
1245                 speed = IXGBE_LINK_SPEED_10GB_FULL;
1246                 break;
1247         default:
1248                 PMD_INIT_LOG(ERR, "Invalid link_speed (%u) for port %u\n",
1249                                 dev->data->dev_conf.link_speed, dev->data->port_id);
1250                 goto error;
1251         }
1252
1253         err = ixgbe_setup_link(hw, speed, negotiate, link_up);
1254         if (err)
1255                 goto error;
1256
1257         /* check if lsc interrupt is enabled */
1258         if (dev->data->dev_conf.intr_conf.lsc != 0)
1259                 ixgbe_dev_lsc_interrupt_setup(dev);
1260
1261         /* resume enabled intr since hw reset */
1262         ixgbe_enable_intr(dev);
1263
1264         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
1265                 ETH_VLAN_EXTEND_MASK;
1266         ixgbe_vlan_offload_set(dev, mask);
1267
1268         /* Configure DCB hw */
1269         ixgbe_configure_dcb(dev);
1270
1271         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) {
1272                 err = ixgbe_fdir_configure(dev);
1273                 if (err)
1274                         goto error;
1275         }
1276
1277         ixgbe_restore_statistics_mapping(dev);
1278
1279         return (0);
1280
1281 error:
1282         PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err);
1283         ixgbe_dev_clear_queues(dev);
1284         return -EIO;
1285 }
1286
1287 /*
1288  * Stop device: disable rx and tx functions to allow for reconfiguring.
1289  */
1290 static void
1291 ixgbe_dev_stop(struct rte_eth_dev *dev)
1292 {
1293         struct rte_eth_link link;
1294         struct ixgbe_hw *hw =
1295                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1296
1297         PMD_INIT_FUNC_TRACE();
1298
1299         /* disable interrupts */
1300         ixgbe_disable_intr(hw);
1301
1302         /* reset the NIC */
1303         ixgbe_pf_reset_hw(hw);
1304         hw->adapter_stopped = FALSE;
1305
1306         /* stop adapter */
1307         ixgbe_stop_adapter(hw);
1308
1309         /* Turn off the laser */
1310         ixgbe_disable_tx_laser(hw);
1311
1312         ixgbe_dev_clear_queues(dev);
1313
1314         /* Clear recorded link status */
1315         memset(&link, 0, sizeof(link));
1316         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1317 }
1318
1319 /*
1320  * Reest and stop device.
1321  */
1322 static void
1323 ixgbe_dev_close(struct rte_eth_dev *dev)
1324 {
1325         struct ixgbe_hw *hw =
1326                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1327
1328         PMD_INIT_FUNC_TRACE();
1329
1330         ixgbe_pf_reset_hw(hw);
1331
1332         ixgbe_dev_stop(dev);
1333         hw->adapter_stopped = 1;
1334
1335         ixgbe_disable_pcie_master(hw);
1336
1337         /* reprogram the RAR[0] in case user changed it. */
1338         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
1339 }
1340
1341 /*
1342  * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c
1343  */
1344 static void
1345 ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1346 {
1347         struct ixgbe_hw *hw =
1348                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1349         struct ixgbe_hw_stats *hw_stats =
1350                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1351         uint32_t bprc, lxon, lxoff, total;
1352         uint64_t total_missed_rx, total_qbrc, total_qprc;
1353         unsigned i;
1354
1355         total_missed_rx = 0;
1356         total_qbrc = 0;
1357         total_qprc = 0;
1358
1359         hw_stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
1360         hw_stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
1361         hw_stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
1362         hw_stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
1363
1364         for (i = 0; i < 8; i++) {
1365                 uint32_t mp;
1366                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
1367                 /* global total per queue */
1368                 hw_stats->mpc[i] += mp;
1369                 /* Running comprehensive total for stats display */
1370                 total_missed_rx += hw_stats->mpc[i];
1371                 if (hw->mac.type == ixgbe_mac_82598EB)
1372                         hw_stats->rnbc[i] +=
1373                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
1374                 hw_stats->pxontxc[i] +=
1375                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
1376                 hw_stats->pxonrxc[i] +=
1377                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
1378                 hw_stats->pxofftxc[i] +=
1379                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
1380                 hw_stats->pxoffrxc[i] +=
1381                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
1382                 hw_stats->pxon2offc[i] +=
1383                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
1384         }
1385         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1386                 hw_stats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
1387                 hw_stats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
1388                 hw_stats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
1389                 hw_stats->qbrc[i] +=
1390                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32);
1391                 hw_stats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
1392                 hw_stats->qbtc[i] +=
1393                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32);
1394                 hw_stats->qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
1395
1396                 total_qprc += hw_stats->qprc[i];
1397                 total_qbrc += hw_stats->qbrc[i];
1398         }
1399         hw_stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
1400         hw_stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
1401         hw_stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
1402
1403         /* Note that gprc counts missed packets */
1404         hw_stats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
1405
1406         if (hw->mac.type != ixgbe_mac_82598EB) {
1407                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
1408                 hw_stats->gorc += ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
1409                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
1410                 hw_stats->gotc += ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
1411                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
1412                 hw_stats->tor += ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
1413                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
1414                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
1415         } else {
1416                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
1417                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
1418                 /* 82598 only has a counter in the high register */
1419                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
1420                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
1421                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
1422         }
1423
1424         /*
1425          * Workaround: mprc hardware is incorrectly counting
1426          * broadcasts, so for now we subtract those.
1427          */
1428         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
1429         hw_stats->bprc += bprc;
1430         hw_stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
1431         if (hw->mac.type == ixgbe_mac_82598EB)
1432                 hw_stats->mprc -= bprc;
1433
1434         hw_stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
1435         hw_stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
1436         hw_stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
1437         hw_stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
1438         hw_stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
1439         hw_stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
1440
1441         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
1442         hw_stats->lxontxc += lxon;
1443         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
1444         hw_stats->lxofftxc += lxoff;
1445         total = lxon + lxoff;
1446
1447         hw_stats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
1448         hw_stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
1449         hw_stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
1450         hw_stats->gptc -= total;
1451         hw_stats->mptc -= total;
1452         hw_stats->ptc64 -= total;
1453         hw_stats->gotc -= total * ETHER_MIN_LEN;
1454
1455         hw_stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
1456         hw_stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
1457         hw_stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
1458         hw_stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
1459         hw_stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
1460         hw_stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
1461         hw_stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
1462         hw_stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
1463         hw_stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
1464         hw_stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
1465         hw_stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
1466         hw_stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
1467         hw_stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
1468         hw_stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
1469         hw_stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
1470         hw_stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);
1471         hw_stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
1472         hw_stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
1473         /* Only read FCOE on 82599 */
1474         if (hw->mac.type != ixgbe_mac_82598EB) {
1475                 hw_stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
1476                 hw_stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
1477                 hw_stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
1478                 hw_stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
1479                 hw_stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
1480         }
1481
1482         if (stats == NULL)
1483                 return;
1484
1485         /* Fill out the rte_eth_stats statistics structure */
1486         stats->ipackets = total_qprc;
1487         stats->ibytes = total_qbrc;
1488         stats->opackets = hw_stats->gptc;
1489         stats->obytes = hw_stats->gotc;
1490         stats->imcasts = hw_stats->mprc;
1491
1492         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1493                 stats->q_ipackets[i] = hw_stats->qprc[i];
1494                 stats->q_opackets[i] = hw_stats->qptc[i];
1495                 stats->q_ibytes[i] = hw_stats->qbrc[i];
1496                 stats->q_obytes[i] = hw_stats->qbtc[i];
1497                 stats->q_errors[i] = hw_stats->qprdc[i];
1498         }
1499
1500         /* Rx Errors */
1501         stats->ierrors = total_missed_rx + hw_stats->crcerrs +
1502                 hw_stats->rlec;
1503
1504         stats->oerrors  = 0;
1505
1506         /* Flow Director Stats registers */
1507         hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
1508         hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
1509         stats->fdirmatch = hw_stats->fdirmatch;
1510         stats->fdirmiss = hw_stats->fdirmiss;
1511 }
1512
1513 static void
1514 ixgbe_dev_stats_reset(struct rte_eth_dev *dev)
1515 {
1516         struct ixgbe_hw_stats *stats =
1517                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1518
1519         /* HW registers are cleared on read */
1520         ixgbe_dev_stats_get(dev, NULL);
1521
1522         /* Reset software totals */
1523         memset(stats, 0, sizeof(*stats));
1524 }
1525
1526 static void
1527 ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1528 {
1529         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1530         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1531                           IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1532
1533         /* Good Rx packet, include VF loopback */
1534         UPDATE_VF_STAT(IXGBE_VFGPRC,
1535             hw_stats->last_vfgprc, hw_stats->vfgprc);
1536
1537         /* Good Rx octets, include VF loopback */
1538         UPDATE_VF_STAT_36BIT(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
1539             hw_stats->last_vfgorc, hw_stats->vfgorc);
1540
1541         /* Good Tx packet, include VF loopback */
1542         UPDATE_VF_STAT(IXGBE_VFGPTC,
1543             hw_stats->last_vfgptc, hw_stats->vfgptc);
1544
1545         /* Good Tx octets, include VF loopback */
1546         UPDATE_VF_STAT_36BIT(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
1547             hw_stats->last_vfgotc, hw_stats->vfgotc);
1548
1549         /* Rx Multicst Packet */
1550         UPDATE_VF_STAT(IXGBE_VFMPRC,
1551             hw_stats->last_vfmprc, hw_stats->vfmprc);
1552
1553         if (stats == NULL)
1554                 return;
1555
1556         memset(stats, 0, sizeof(*stats));
1557         stats->ipackets = hw_stats->vfgprc;
1558         stats->ibytes = hw_stats->vfgorc;
1559         stats->opackets = hw_stats->vfgptc;
1560         stats->obytes = hw_stats->vfgotc;
1561         stats->imcasts = hw_stats->vfmprc;
1562 }
1563
1564 static void
1565 ixgbevf_dev_stats_reset(struct rte_eth_dev *dev)
1566 {
1567         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1568                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1569
1570         /* Sync HW register to the last stats */
1571         ixgbevf_dev_stats_get(dev, NULL);
1572
1573         /* reset HW current stats*/
1574         hw_stats->vfgprc = 0;
1575         hw_stats->vfgorc = 0;
1576         hw_stats->vfgptc = 0;
1577         hw_stats->vfgotc = 0;
1578         hw_stats->vfmprc = 0;
1579
1580 }
1581
1582 static void
1583 ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1584 {
1585         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1586
1587         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
1588         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
1589         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL register */
1590         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */
1591         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
1592 }
1593
1594 /* return 0 means link status changed, -1 means not changed */
1595 static int
1596 ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1597 {
1598         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1599         struct rte_eth_link link, old;
1600         ixgbe_link_speed link_speed;
1601         int link_up;
1602         int diag;
1603
1604         link.link_status = 0;
1605         link.link_speed = 0;
1606         link.link_duplex = 0;
1607         memset(&old, 0, sizeof(old));
1608         rte_ixgbe_dev_atomic_read_link_status(dev, &old);
1609
1610         /* check if it needs to wait to complete, if lsc interrupt is enabled */
1611         if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
1612                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 0);
1613         else
1614                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 1);
1615         if (diag != 0) {
1616                 link.link_speed = ETH_LINK_SPEED_100;
1617                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1618                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1619                 if (link.link_status == old.link_status)
1620                         return -1;
1621                 return 0;
1622         }
1623
1624         if (link_up == 0) {
1625                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1626                 if (link.link_status == old.link_status)
1627                         return -1;
1628                 return 0;
1629         }
1630         link.link_status = 1;
1631         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1632
1633         switch (link_speed) {
1634         default:
1635         case IXGBE_LINK_SPEED_UNKNOWN:
1636                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1637                 link.link_speed = ETH_LINK_SPEED_100;
1638                 break;
1639
1640         case IXGBE_LINK_SPEED_100_FULL:
1641                 link.link_speed = ETH_LINK_SPEED_100;
1642                 break;
1643
1644         case IXGBE_LINK_SPEED_1GB_FULL:
1645                 link.link_speed = ETH_LINK_SPEED_1000;
1646                 break;
1647
1648         case IXGBE_LINK_SPEED_10GB_FULL:
1649                 link.link_speed = ETH_LINK_SPEED_10000;
1650                 break;
1651         }
1652         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1653
1654         if (link.link_status == old.link_status)
1655                 return -1;
1656
1657         return 0;
1658 }
1659
1660 static void
1661 ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev)
1662 {
1663         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1664         uint32_t fctrl;
1665
1666         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1667         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1668         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1669 }
1670
1671 static void
1672 ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev)
1673 {
1674         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1675         uint32_t fctrl;
1676
1677         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1678         fctrl &= (~IXGBE_FCTRL_UPE);
1679         if (dev->data->all_multicast == 1)
1680                 fctrl |= IXGBE_FCTRL_MPE;
1681         else
1682                 fctrl &= (~IXGBE_FCTRL_MPE);
1683         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1684 }
1685
1686 static void
1687 ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev)
1688 {
1689         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1690         uint32_t fctrl;
1691
1692         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1693         fctrl |= IXGBE_FCTRL_MPE;
1694         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1695 }
1696
1697 static void
1698 ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev)
1699 {
1700         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1701         uint32_t fctrl;
1702
1703         if (dev->data->promiscuous == 1)
1704                 return; /* must remain in all_multicast mode */
1705
1706         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1707         fctrl &= (~IXGBE_FCTRL_MPE);
1708         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1709 }
1710
1711 /**
1712  * It clears the interrupt causes and enables the interrupt.
1713  * It will be called once only during nic initialized.
1714  *
1715  * @param dev
1716  *  Pointer to struct rte_eth_dev.
1717  *
1718  * @return
1719  *  - On success, zero.
1720  *  - On failure, a negative value.
1721  */
1722 static int
1723 ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev)
1724 {
1725         struct ixgbe_interrupt *intr =
1726                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1727
1728         ixgbe_dev_link_status_print(dev);
1729         intr->mask |= IXGBE_EICR_LSC;
1730
1731         return 0;
1732 }
1733
1734 /*
1735  * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update.
1736  *
1737  * @param dev
1738  *  Pointer to struct rte_eth_dev.
1739  *
1740  * @return
1741  *  - On success, zero.
1742  *  - On failure, a negative value.
1743  */
1744 static int
1745 ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
1746 {
1747         uint32_t eicr;
1748         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1749         struct ixgbe_interrupt *intr =
1750                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1751
1752         /* clear all cause mask */
1753         ixgbe_disable_intr(hw);
1754
1755         /* read-on-clear nic registers here */
1756         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1757         PMD_DRV_LOG(INFO, "eicr %x", eicr);
1758
1759         intr->flags = 0;
1760         if (eicr & IXGBE_EICR_LSC) {
1761                 /* set flag for async link update */
1762                 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1763         }
1764
1765         if (eicr & IXGBE_EICR_MAILBOX)
1766                 intr->flags |= IXGBE_FLAG_MAILBOX;
1767
1768         return 0;
1769 }
1770
1771 /**
1772  * It gets and then prints the link status.
1773  *
1774  * @param dev
1775  *  Pointer to struct rte_eth_dev.
1776  *
1777  * @return
1778  *  - On success, zero.
1779  *  - On failure, a negative value.
1780  */
1781 static void
1782 ixgbe_dev_link_status_print(struct rte_eth_dev *dev)
1783 {
1784         struct rte_eth_link link;
1785
1786         memset(&link, 0, sizeof(link));
1787         rte_ixgbe_dev_atomic_read_link_status(dev, &link);
1788         if (link.link_status) {
1789                 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
1790                                         (int)(dev->data->port_id),
1791                                         (unsigned)link.link_speed,
1792                         link.link_duplex == ETH_LINK_FULL_DUPLEX ?
1793                                         "full-duplex" : "half-duplex");
1794         } else {
1795                 PMD_INIT_LOG(INFO, " Port %d: Link Down",
1796                                 (int)(dev->data->port_id));
1797         }
1798         PMD_INIT_LOG(INFO, "PCI Address: %04d:%02d:%02d:%d",
1799                                 dev->pci_dev->addr.domain,
1800                                 dev->pci_dev->addr.bus,
1801                                 dev->pci_dev->addr.devid,
1802                                 dev->pci_dev->addr.function);
1803 }
1804
1805 /*
1806  * It executes link_update after knowing an interrupt occured.
1807  *
1808  * @param dev
1809  *  Pointer to struct rte_eth_dev.
1810  *
1811  * @return
1812  *  - On success, zero.
1813  *  - On failure, a negative value.
1814  */
1815 static int
1816 ixgbe_dev_interrupt_action(struct rte_eth_dev *dev)
1817 {
1818         struct ixgbe_interrupt *intr =
1819                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1820         int64_t timeout;
1821         struct rte_eth_link link;
1822         int intr_enable_delay = false;
1823
1824         PMD_DRV_LOG(DEBUG, "intr action type %d\n", intr->flags);
1825
1826         if (intr->flags & IXGBE_FLAG_MAILBOX) {
1827                 ixgbe_pf_mbx_process(dev);
1828                 intr->flags &= ~IXGBE_FLAG_MAILBOX;
1829         }
1830
1831         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
1832                 /* get the link status before link update, for predicting later */
1833                 memset(&link, 0, sizeof(link));
1834                 rte_ixgbe_dev_atomic_read_link_status(dev, &link);
1835
1836                 ixgbe_dev_link_update(dev, 0);
1837
1838                 /* likely to up */
1839                 if (!link.link_status)
1840                         /* handle it 1 sec later, wait it being stable */
1841                         timeout = IXGBE_LINK_UP_CHECK_TIMEOUT;
1842                 /* likely to down */
1843                 else
1844                         /* handle it 4 sec later, wait it being stable */
1845                         timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT;
1846
1847                 ixgbe_dev_link_status_print(dev);
1848
1849                 intr_enable_delay = true;
1850         }
1851
1852         if (intr_enable_delay) {
1853                 if (rte_eal_alarm_set(timeout * 1000,
1854                                       ixgbe_dev_interrupt_delayed_handler, (void*)dev) < 0)
1855                         PMD_DRV_LOG(ERR, "Error setting alarm");
1856         } else {
1857                 PMD_DRV_LOG(DEBUG, "enable intr immediately");
1858                 ixgbe_enable_intr(dev);
1859                 rte_intr_enable(&(dev->pci_dev->intr_handle));
1860         }
1861
1862
1863         return 0;
1864 }
1865
1866 /**
1867  * Interrupt handler which shall be registered for alarm callback for delayed
1868  * handling specific interrupt to wait for the stable nic state. As the
1869  * NIC interrupt state is not stable for ixgbe after link is just down,
1870  * it needs to wait 4 seconds to get the stable status.
1871  *
1872  * @param handle
1873  *  Pointer to interrupt handle.
1874  * @param param
1875  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1876  *
1877  * @return
1878  *  void
1879  */
1880 static void
1881 ixgbe_dev_interrupt_delayed_handler(void *param)
1882 {
1883         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1884         struct ixgbe_interrupt *intr =
1885                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1886         struct ixgbe_hw *hw =
1887                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1888         uint32_t eicr;
1889
1890         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1891         if (eicr & IXGBE_EICR_MAILBOX)
1892                 ixgbe_pf_mbx_process(dev);
1893
1894         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
1895                 ixgbe_dev_link_update(dev, 0);
1896                 intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
1897                 ixgbe_dev_link_status_print(dev);
1898                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
1899         }
1900
1901         PMD_DRV_LOG(DEBUG, "enable intr in delayed handler S[%08x]\n", eicr);
1902         ixgbe_enable_intr(dev);
1903         rte_intr_enable(&(dev->pci_dev->intr_handle));
1904 }
1905
1906 /**
1907  * Interrupt handler triggered by NIC  for handling
1908  * specific interrupt.
1909  *
1910  * @param handle
1911  *  Pointer to interrupt handle.
1912  * @param param
1913  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1914  *
1915  * @return
1916  *  void
1917  */
1918 static void
1919 ixgbe_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
1920                                                         void *param)
1921 {
1922         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1923         ixgbe_dev_interrupt_get_status(dev);
1924         ixgbe_dev_interrupt_action(dev);
1925 }
1926
1927 static int
1928 ixgbe_dev_led_on(struct rte_eth_dev *dev)
1929 {
1930         struct ixgbe_hw *hw;
1931
1932         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1933         return (ixgbe_led_on(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
1934 }
1935
1936 static int
1937 ixgbe_dev_led_off(struct rte_eth_dev *dev)
1938 {
1939         struct ixgbe_hw *hw;
1940
1941         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1942         return (ixgbe_led_off(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
1943 }
1944
1945 static int
1946 ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1947 {
1948         struct ixgbe_hw *hw;
1949         int err;
1950         uint32_t rx_buf_size;
1951         uint32_t max_high_water;
1952         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
1953                 ixgbe_fc_none,
1954                 ixgbe_fc_rx_pause,
1955                 ixgbe_fc_tx_pause,
1956                 ixgbe_fc_full
1957         };
1958
1959         PMD_INIT_FUNC_TRACE();
1960
1961         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1962         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0));
1963         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
1964
1965         /*
1966          * At least reserve one Ethernet frame for watermark
1967          * high_water/low_water in kilo bytes for ixgbe
1968          */
1969         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
1970         if ((fc_conf->high_water > max_high_water) ||
1971                 (fc_conf->high_water < fc_conf->low_water)) {
1972                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
1973                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
1974                 return (-EINVAL);
1975         }
1976
1977         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[fc_conf->mode];
1978         hw->fc.pause_time     = fc_conf->pause_time;
1979         hw->fc.high_water[0]  = fc_conf->high_water;
1980         hw->fc.low_water[0]   = fc_conf->low_water;
1981         hw->fc.send_xon       = fc_conf->send_xon;
1982
1983         err = ixgbe_fc_enable(hw);
1984         /* Not negotiated is not an error case */
1985         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) {
1986                 return 0;
1987         }
1988
1989         PMD_INIT_LOG(ERR, "ixgbe_fc_enable = 0x%x \n", err);
1990         return -EIO;
1991 }
1992
1993 /**
1994  *  ixgbe_pfc_enable_generic - Enable flow control
1995  *  @hw: pointer to hardware structure
1996  *  @tc_num: traffic class number
1997  *  Enable flow control according to the current settings.
1998  */
1999 static int
2000 ixgbe_dcb_pfc_enable_generic(struct ixgbe_hw *hw,uint8_t tc_num)
2001 {
2002         int ret_val = 0;
2003         uint32_t mflcn_reg, fccfg_reg;
2004         uint32_t reg;
2005         uint32_t fcrtl, fcrth;
2006         uint8_t i;
2007         uint8_t nb_rx_en;
2008
2009         /* Validate the water mark configuration */
2010         if (!hw->fc.pause_time) {
2011                 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2012                 goto out;
2013         }
2014
2015         /* Low water mark of zero causes XOFF floods */
2016         if (hw->fc.current_mode & ixgbe_fc_tx_pause) {
2017                  /* High/Low water can not be 0 */
2018                 if( (!hw->fc.high_water[tc_num])|| (!hw->fc.low_water[tc_num])) {
2019                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
2020                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2021                         goto out;
2022                 }
2023
2024                 if(hw->fc.low_water[tc_num] >= hw->fc.high_water[tc_num]) {
2025                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
2026                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2027                         goto out;
2028                 }
2029         }
2030         /* Negotiate the fc mode to use */
2031         ixgbe_fc_autoneg(hw);
2032
2033         /* Disable any previous flow control settings */
2034         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
2035         mflcn_reg &= ~(IXGBE_MFLCN_RPFCE_SHIFT | IXGBE_MFLCN_RFCE|IXGBE_MFLCN_RPFCE);
2036
2037         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
2038         fccfg_reg &= ~(IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY);
2039
2040         switch (hw->fc.current_mode) {
2041         case ixgbe_fc_none:
2042                 /*
2043                  * If the count of enabled RX Priority Flow control >1,
2044                  * and the TX pause can not be disabled
2045                  */
2046                 nb_rx_en = 0;
2047                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
2048                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
2049                         if (reg & IXGBE_FCRTH_FCEN)
2050                                 nb_rx_en++;
2051                 }
2052                 if (nb_rx_en > 1)
2053                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2054                 break;
2055         case ixgbe_fc_rx_pause:
2056                 /*
2057                  * Rx Flow control is enabled and Tx Flow control is
2058                  * disabled by software override. Since there really
2059                  * isn't a way to advertise that we are capable of RX
2060                  * Pause ONLY, we will advertise that we support both
2061                  * symmetric and asymmetric Rx PAUSE.  Later, we will
2062                  * disable the adapter's ability to send PAUSE frames.
2063                  */
2064                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
2065                 /*
2066                  * If the count of enabled RX Priority Flow control >1,
2067                  * and the TX pause can not be disabled
2068                  */
2069                 nb_rx_en = 0;
2070                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
2071                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
2072                         if (reg & IXGBE_FCRTH_FCEN)
2073                                 nb_rx_en++;
2074                 }
2075                 if (nb_rx_en > 1)
2076                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2077                 break;
2078         case ixgbe_fc_tx_pause:
2079                 /*
2080                  * Tx Flow control is enabled, and Rx Flow control is
2081                  * disabled by software override.
2082                  */
2083                 fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2084                 break;
2085         case ixgbe_fc_full:
2086                 /* Flow control (both Rx and Tx) is enabled by SW override. */
2087                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
2088                 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
2089                 break;
2090         default:
2091                 DEBUGOUT("Flow control param set incorrectly\n");
2092                 ret_val = IXGBE_ERR_CONFIG;
2093                 goto out;
2094                 break;
2095         }
2096
2097         /* Set 802.3x based flow control settings. */
2098         mflcn_reg |= IXGBE_MFLCN_DPF;
2099         IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg);
2100         IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg);
2101
2102         /* Set up and enable Rx high/low water mark thresholds, enable XON. */
2103         if ((hw->fc.current_mode & ixgbe_fc_tx_pause) &&
2104                 hw->fc.high_water[tc_num]) {
2105                 fcrtl = (hw->fc.low_water[tc_num] << 10) | IXGBE_FCRTL_XONE;
2106                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), fcrtl);
2107                 fcrth = (hw->fc.high_water[tc_num] << 10) | IXGBE_FCRTH_FCEN;
2108         } else {
2109                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), 0);
2110                 /*
2111                  * In order to prevent Tx hangs when the internal Tx
2112                  * switch is enabled we must set the high water mark
2113                  * to the maximum FCRTH value.  This allows the Tx
2114                  * switch to function even under heavy Rx workloads.
2115                  */
2116                 fcrth = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)) - 32;
2117         }
2118         IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(tc_num), fcrth);
2119
2120         /* Configure pause time (2 TCs per register) */
2121         reg = hw->fc.pause_time * 0x00010001;
2122         for (i = 0; i < (IXGBE_DCB_MAX_TRAFFIC_CLASS / 2); i++)
2123                 IXGBE_WRITE_REG(hw, IXGBE_FCTTV(i), reg);
2124
2125         /* Configure flow control refresh threshold value */
2126         IXGBE_WRITE_REG(hw, IXGBE_FCRTV, hw->fc.pause_time / 2);
2127
2128 out:
2129         return ret_val;
2130 }
2131
2132 static int
2133 ixgbe_dcb_pfc_enable(struct rte_eth_dev *dev,uint8_t tc_num)
2134 {
2135         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2136         int32_t ret_val = IXGBE_NOT_IMPLEMENTED;
2137
2138         if(hw->mac.type != ixgbe_mac_82598EB) {
2139                 ret_val = ixgbe_dcb_pfc_enable_generic(hw,tc_num);
2140         }
2141         return ret_val;
2142 }
2143
2144 static int
2145 ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf)
2146 {
2147         int err;
2148         uint32_t rx_buf_size;
2149         uint32_t max_high_water;
2150         uint8_t tc_num;
2151         uint8_t  map[IXGBE_DCB_MAX_USER_PRIORITY] = { 0 };
2152         struct ixgbe_hw *hw =
2153                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2154         struct ixgbe_dcb_config *dcb_config =
2155                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
2156
2157         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
2158                 ixgbe_fc_none,
2159                 ixgbe_fc_rx_pause,
2160                 ixgbe_fc_tx_pause,
2161                 ixgbe_fc_full
2162         };
2163
2164         PMD_INIT_FUNC_TRACE();
2165
2166         ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
2167         tc_num = map[pfc_conf->priority];
2168         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num));
2169         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
2170         /*
2171          * At least reserve one Ethernet frame for watermark
2172          * high_water/low_water in kilo bytes for ixgbe
2173          */
2174         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
2175         if ((pfc_conf->fc.high_water > max_high_water) ||
2176                 (pfc_conf->fc.high_water <= pfc_conf->fc.low_water)) {
2177                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
2178                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
2179                 return (-EINVAL);
2180         }
2181
2182         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[pfc_conf->fc.mode];
2183         hw->fc.pause_time = pfc_conf->fc.pause_time;
2184         hw->fc.send_xon = pfc_conf->fc.send_xon;
2185         hw->fc.low_water[tc_num] =  pfc_conf->fc.low_water;
2186         hw->fc.high_water[tc_num] = pfc_conf->fc.high_water;
2187
2188         err = ixgbe_dcb_pfc_enable(dev,tc_num);
2189
2190         /* Not negotiated is not an error case */
2191         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED))
2192                 return 0;
2193
2194         PMD_INIT_LOG(ERR, "ixgbe_dcb_pfc_enable = 0x%x \n", err);
2195         return -EIO;
2196 }
2197
2198 static int
2199 ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
2200                                 struct rte_eth_rss_reta *reta_conf)
2201 {
2202         uint8_t i,j,mask;
2203         uint32_t reta;
2204         struct ixgbe_hw *hw =
2205                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2206
2207         PMD_INIT_FUNC_TRACE();
2208         /*
2209         * Update Redirection Table RETA[n],n=0...31,The redirection table has
2210         * 128-entries in 32 registers
2211          */
2212         for(i = 0; i < ETH_RSS_RETA_NUM_ENTRIES; i += 4) {
2213                 if (i < ETH_RSS_RETA_NUM_ENTRIES/2)
2214                         mask = (uint8_t)((reta_conf->mask_lo >> i) & 0xF);
2215                 else
2216                         mask = (uint8_t)((reta_conf->mask_hi >>
2217                                 (i - ETH_RSS_RETA_NUM_ENTRIES/2)) & 0xF);
2218                 if (mask != 0) {
2219                         reta = 0;
2220                         if (mask != 0xF)
2221                                 reta = IXGBE_READ_REG(hw,IXGBE_RETA(i >> 2));
2222
2223                         for (j = 0; j < 4; j++) {
2224                                 if (mask & (0x1 << j)) {
2225                                         if (mask != 0xF)
2226                                                 reta &= ~(0xFF << 8 * j);
2227                                         reta |= reta_conf->reta[i + j] << 8*j;
2228                                 }
2229                         }
2230                         IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2),reta);
2231                 }
2232         }
2233
2234         return 0;
2235 }
2236
2237 static int
2238 ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
2239                                 struct rte_eth_rss_reta *reta_conf)
2240 {
2241         uint8_t i,j,mask;
2242         uint32_t reta;
2243         struct ixgbe_hw *hw =
2244                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2245
2246         PMD_INIT_FUNC_TRACE();
2247         /*
2248          * Read Redirection Table RETA[n],n=0...31,The redirection table has
2249          * 128-entries in 32 registers
2250          */
2251         for(i = 0; i < ETH_RSS_RETA_NUM_ENTRIES; i += 4) {
2252                 if (i < ETH_RSS_RETA_NUM_ENTRIES/2)
2253                         mask = (uint8_t)((reta_conf->mask_lo >> i) & 0xF);
2254                 else
2255                         mask = (uint8_t)((reta_conf->mask_hi >>
2256                                 (i - ETH_RSS_RETA_NUM_ENTRIES/2)) & 0xF);
2257
2258                 if (mask != 0) {
2259                         reta = IXGBE_READ_REG(hw,IXGBE_RETA(i >> 2));
2260                         for (j = 0; j < 4; j++) {
2261                                 if (mask & (0x1 << j))
2262                                         reta_conf->reta[i + j] =
2263                                                 (uint8_t)((reta >> 8 * j) & 0xFF);
2264                         }
2265                 }
2266         }
2267
2268         return 0;
2269 }
2270
2271 static void
2272 ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
2273                                 uint32_t index, uint32_t pool)
2274 {
2275         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2276         uint32_t enable_addr = 1;
2277
2278         ixgbe_set_rar(hw, index, mac_addr->addr_bytes, pool, enable_addr);
2279 }
2280
2281 static void
2282 ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
2283 {
2284         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2285
2286         ixgbe_clear_rar(hw, index);
2287 }
2288
2289 /*
2290  * Virtual Function operations
2291  */
2292 static void
2293 ixgbevf_intr_disable(struct ixgbe_hw *hw)
2294 {
2295         PMD_INIT_LOG(DEBUG, "ixgbevf_intr_disable");
2296
2297         /* Clear interrupt mask to stop from interrupts being generated */
2298         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, IXGBE_VF_IRQ_CLEAR_MASK);
2299
2300         IXGBE_WRITE_FLUSH(hw);
2301 }
2302
2303 static int
2304 ixgbevf_dev_configure(struct rte_eth_dev *dev)
2305 {
2306         struct rte_eth_conf* conf = &dev->data->dev_conf;
2307
2308         PMD_INIT_LOG(DEBUG, "\nConfigured Virtual Function port id: %d\n",
2309                 dev->data->port_id);
2310
2311         /*
2312          * VF has no ability to enable/disable HW CRC
2313          * Keep the persistent behavior the same as Host PF
2314          */
2315 #ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC
2316         if (!conf->rxmode.hw_strip_crc) {
2317                 PMD_INIT_LOG(INFO, "VF can't disable HW CRC Strip\n");
2318                 conf->rxmode.hw_strip_crc = 1;
2319         }
2320 #else
2321         if (conf->rxmode.hw_strip_crc) {
2322                 PMD_INIT_LOG(INFO, "VF can't enable HW CRC Strip\n");
2323                 conf->rxmode.hw_strip_crc = 0;
2324         }
2325 #endif
2326
2327         return 0;
2328 }
2329
2330 static int
2331 ixgbevf_dev_start(struct rte_eth_dev *dev)
2332 {
2333         struct ixgbe_hw *hw =
2334                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2335         int err, mask = 0;
2336
2337         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_start");
2338
2339         hw->mac.ops.reset_hw(hw);
2340
2341         ixgbevf_dev_tx_init(dev);
2342
2343         /* This can fail when allocating mbufs for descriptor rings */
2344         err = ixgbevf_dev_rx_init(dev);
2345         if (err) {
2346                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware (%d)\n", err);
2347                 ixgbe_dev_clear_queues(dev);
2348                 return err;
2349         }
2350
2351         /* Set vfta */
2352         ixgbevf_set_vfta_all(dev,1);
2353
2354         /* Set HW strip */
2355         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
2356                 ETH_VLAN_EXTEND_MASK;
2357         ixgbevf_vlan_offload_set(dev, mask);
2358
2359         ixgbevf_dev_rxtx_start(dev);
2360
2361         return 0;
2362 }
2363
2364 static void
2365 ixgbevf_dev_stop(struct rte_eth_dev *dev)
2366 {
2367         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2368
2369         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_stop");
2370
2371         hw->adapter_stopped = TRUE;
2372         ixgbe_stop_adapter(hw);
2373
2374         /*
2375           * Clear what we set, but we still keep shadow_vfta to
2376           * restore after device starts
2377           */
2378         ixgbevf_set_vfta_all(dev,0);
2379
2380         ixgbe_dev_clear_queues(dev);
2381 }
2382
2383 static void
2384 ixgbevf_dev_close(struct rte_eth_dev *dev)
2385 {
2386         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2387
2388         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_close");
2389
2390         ixgbe_reset_hw(hw);
2391
2392         ixgbevf_dev_stop(dev);
2393
2394         /* reprogram the RAR[0] in case user changed it. */
2395         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
2396 }
2397
2398 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on)
2399 {
2400         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2401         struct ixgbe_vfta * shadow_vfta =
2402                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2403         int i = 0, j = 0, vfta = 0, mask = 1;
2404
2405         for (i = 0; i < IXGBE_VFTA_SIZE; i++){
2406                 vfta = shadow_vfta->vfta[i];
2407                 if(vfta){
2408                         mask = 1;
2409                         for (j = 0; j < 32; j++){
2410                                 if(vfta & mask)
2411                                         ixgbe_set_vfta(hw, (i<<5)+j, 0, on);
2412                                 mask<<=1;
2413                         }
2414                 }
2415         }
2416
2417 }
2418
2419 static int
2420 ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2421 {
2422         struct ixgbe_hw *hw =
2423                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2424         struct ixgbe_vfta * shadow_vfta =
2425                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2426         uint32_t vid_idx = 0;
2427         uint32_t vid_bit = 0;
2428         int ret = 0;
2429
2430         PMD_INIT_FUNC_TRACE();
2431
2432         /* vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf */
2433         ret = ixgbe_set_vfta(hw, vlan_id, 0, !!on);
2434         if(ret){
2435                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
2436                 return ret;
2437         }
2438         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
2439         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
2440
2441         /* Save what we set and retore it after device reset */
2442         if (on)
2443                 shadow_vfta->vfta[vid_idx] |= vid_bit;
2444         else
2445                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
2446
2447         return 0;
2448 }
2449
2450 static void
2451 ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
2452 {
2453         struct ixgbe_hw *hw =
2454                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2455         uint32_t ctrl;
2456
2457         PMD_INIT_FUNC_TRACE();
2458
2459         if(queue >= hw->mac.max_rx_queues)
2460                 return;
2461
2462         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
2463         if(on)
2464                 ctrl |= IXGBE_RXDCTL_VME;
2465         else
2466                 ctrl &= ~IXGBE_RXDCTL_VME;
2467         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
2468
2469         ixgbe_vlan_hw_strip_bitmap_set( dev, queue, on);
2470 }
2471
2472 static void
2473 ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2474 {
2475         struct ixgbe_hw *hw =
2476                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2477         uint16_t i;
2478         int on = 0;
2479
2480         /* VF function only support hw strip feature, others are not support */
2481         if(mask & ETH_VLAN_STRIP_MASK){
2482                 on = !!(dev->data->dev_conf.rxmode.hw_vlan_strip);
2483
2484                 for(i=0; i < hw->mac.max_rx_queues; i++)
2485                         ixgbevf_vlan_strip_queue_set(dev,i,on);
2486         }
2487 }