lib/librte_pmd_ixgbe/ixgbe_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2012 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  *
  33  */
  34
  35 #include <sys/queue.h>
  36 #include <stdio.h>
  37 #include <errno.h>
  38 #include <stdint.h>
  39 #include <string.h>
  40 #include <unistd.h>
  41 #include <stdarg.h>
  42 #include <inttypes.h>
  43 #include <rte_byteorder.h>
  44 #include <rte_common.h>
  45 #include <rte_cycles.h>
  46
  47 #include <rte_interrupts.h>
  48 #include <rte_log.h>
  49 #include <rte_debug.h>
  50 #include <rte_pci.h>
  51 #include <rte_atomic.h>
  52 #include <rte_branch_prediction.h>
  53 #include <rte_memory.h>
  54 #include <rte_memzone.h>
  55 #include <rte_tailq.h>
  56 #include <rte_eal.h>
  57 #include <rte_alarm.h>
  58 #include <rte_ether.h>
  59 #include <rte_ethdev.h>
  60 #include <rte_atomic.h>
  61 #include <rte_malloc.h>
  62
  63 #include "ixgbe_logs.h"
  64 #include "ixgbe/ixgbe_api.h"
  65 #include "ixgbe/ixgbe_vf.h"
  66 #include "ixgbe/ixgbe_common.h"
  67 #include "ixgbe_ethdev.h"
  68
  69 /*
  70  * High threshold controlling when to start sending XOFF frames. Must be at
  71  * least 8 bytes less than receive packet buffer size. This value is in units
  72  * of 1024 bytes.
  73  */
  74 #define IXGBE_FC_HI    0x80
  75
  76 /*
  77  * Low threshold controlling when to start sending XON frames. This value is
  78  * in units of 1024 bytes.
  79  */
  80 #define IXGBE_FC_LO    0x40
  81
  82 /* Timer value included in XOFF frames. */
  83 #define IXGBE_FC_PAUSE 0x680
  84
  85 #define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */
  86 #define IXGBE_LINK_UP_CHECK_TIMEOUT   1000 /* ms */
  87
  88 #define IXGBE_QUEUE_STAT_COUNTERS (sizeof(hw_stats->qprc) / sizeof(hw_stats->qprc[0]))
  89
  90 static int eth_ixgbe_dev_init(struct eth_driver *eth_drv,
  91                 struct rte_eth_dev *eth_dev);
  92 static int  ixgbe_dev_configure(struct rte_eth_dev *dev);
  93 static int  ixgbe_dev_start(struct rte_eth_dev *dev);
  94 static void ixgbe_dev_stop(struct rte_eth_dev *dev);
  95 static void ixgbe_dev_close(struct rte_eth_dev *dev);
  96 static void ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev);
  97 static void ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev);
  98 static void ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev);
  99 static void ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev);
 100 static int ixgbe_dev_link_update(struct rte_eth_dev *dev,
 101                                 int wait_to_complete);
 102 static void ixgbe_dev_stats_get(struct rte_eth_dev *dev,
 103                                 struct rte_eth_stats *stats);
 104 static void ixgbe_dev_stats_reset(struct rte_eth_dev *dev);
 105 static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 106                                              uint16_t queue_id,
 107                                              uint8_t stat_idx,
 108                                              uint8_t is_rx);
 109 static void ixgbe_dev_info_get(struct rte_eth_dev *dev,
 110                                 struct rte_eth_dev_info *dev_info);
 111 static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev,
 112                 uint16_t vlan_id, int on);
 113 static void ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 114 static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev,
 115                 uint16_t queue, bool on);
 116 static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue,
 117                                   int on);
 118 static void ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 119 static void ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev);
 120 static void ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev);
 121 static void ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev);
 122 static void ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev);
 123 static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue);
 124 static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue);
 125 static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 126 static void ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 127
 128 static int ixgbe_dev_led_on(struct rte_eth_dev *dev);
 129 static int ixgbe_dev_led_off(struct rte_eth_dev *dev);
 130 static int  ixgbe_flow_ctrl_set(struct rte_eth_dev *dev,
 131                                 struct rte_eth_fc_conf *fc_conf);
 132 static int ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 133                 struct rte_eth_pfc_conf *pfc_conf);
 134 static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev);
 135 static int ixgbe_dev_interrupt_setup(struct rte_eth_dev *dev);
 136 static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev);
 137 static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev);
 138 static void ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle,
 139                                                         void *param);
 140 static void ixgbe_dev_interrupt_delayed_handler(void *param);
 141 static void ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 142                                 uint32_t index, uint32_t pool);
 143 static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index);
 144 static void ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config);
 145
 146 /* For Virtual Function support */
 147 static int eth_ixgbevf_dev_init(struct eth_driver *eth_drv,
 148                 struct rte_eth_dev *eth_dev);
 149 static int  ixgbevf_dev_configure(struct rte_eth_dev *dev);
 150 static int  ixgbevf_dev_start(struct rte_eth_dev *dev);
 151 static void ixgbevf_dev_stop(struct rte_eth_dev *dev);
 152 static void ixgbevf_dev_close(struct rte_eth_dev *dev);
 153 static void ixgbevf_intr_disable(struct ixgbe_hw *hw);
 154 static void ixgbevf_dev_stats_get(struct rte_eth_dev *dev,
 155                 struct rte_eth_stats *stats);
 156 static void ixgbevf_dev_stats_reset(struct rte_eth_dev *dev);
 157 static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev,
 158                 uint16_t vlan_id, int on);
 159 static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev,
 160                 uint16_t queue, int on);
 161 static void ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 162 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 163
 164
 165 /*
 166  * Define VF Stats MACRO for Non "cleared on read" register
 167  */
 168 #define UPDATE_VF_STAT(reg, last, cur)                          \
 169 {                                                               \
 170         u32 latest = IXGBE_READ_REG(hw, reg);                   \
 171         cur += latest - last;                                   \
 172         last = latest;                                          \
 173 }
 174
 175 #define UPDATE_VF_STAT_36BIT(lsb, msb, last, cur)                \
 176 {                                                                \
 177         u64 new_lsb = IXGBE_READ_REG(hw, lsb);                   \
 178         u64 new_msb = IXGBE_READ_REG(hw, msb);                   \
 179         u64 latest = ((new_msb << 32) | new_lsb);                \
 180         cur += (0x1000000000LL + latest - last) & 0xFFFFFFFFFLL; \
 181         last = latest;                                           \
 182 }
 183
 184 #define IXGBE_SET_HWSTRIP(h, q) do{\
 185                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 186                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 187                 (h)->bitmap[idx] |= 1 << bit;\
 188         }while(0)
 189
 190 #define IXGBE_CLEAR_HWSTRIP(h, q) do{\
 191                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 192                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 193                 (h)->bitmap[idx] &= ~(1 << bit);\
 194         }while(0)
 195
 196 #define IXGBE_GET_HWSTRIP(h, q, r) do{\
 197                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 198                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 199                 (r) = (h)->bitmap[idx] >> bit & 1;\
 200         }while(0)
 201
 202 /*
 203  * The set of PCI devices this driver supports
 204  */
 205 static struct rte_pci_id pci_id_ixgbe_map[] = {
 206
 207 #define RTE_PCI_DEV_ID_DECL_IXGBE(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 208 #include "rte_pci_dev_ids.h"
 209
 210 { .vendor_id = 0, /* sentinel */ },
 211 };
 212
 213
 214 /*
 215  * The set of PCI devices this driver supports (for 82599 VF)
 216  */
 217 static struct rte_pci_id pci_id_ixgbevf_map[] = {
 218
 219 #define RTE_PCI_DEV_ID_DECL_IXGBEVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 220 #include "rte_pci_dev_ids.h"
 221 { .vendor_id = 0, /* sentinel */ },
 222
 223 };
 224
 225 static struct eth_dev_ops ixgbe_eth_dev_ops = {
 226         .dev_configure        = ixgbe_dev_configure,
 227         .dev_start            = ixgbe_dev_start,
 228         .dev_stop             = ixgbe_dev_stop,
 229         .dev_close            = ixgbe_dev_close,
 230         .promiscuous_enable   = ixgbe_dev_promiscuous_enable,
 231         .promiscuous_disable  = ixgbe_dev_promiscuous_disable,
 232         .allmulticast_enable  = ixgbe_dev_allmulticast_enable,
 233         .allmulticast_disable = ixgbe_dev_allmulticast_disable,
 234         .link_update          = ixgbe_dev_link_update,
 235         .stats_get            = ixgbe_dev_stats_get,
 236         .stats_reset          = ixgbe_dev_stats_reset,
 237         .queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set,
 238         .dev_infos_get        = ixgbe_dev_info_get,
 239         .vlan_filter_set      = ixgbe_vlan_filter_set,
 240         .vlan_tpid_set        = ixgbe_vlan_tpid_set,
 241         .vlan_offload_set     = ixgbe_vlan_offload_set,
 242         .vlan_strip_queue_set = ixgbe_vlan_strip_queue_set,
 243         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 244         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 245         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 246         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 247         .dev_led_on           = ixgbe_dev_led_on,
 248         .dev_led_off          = ixgbe_dev_led_off,
 249         .flow_ctrl_set        = ixgbe_flow_ctrl_set,
 250         .priority_flow_ctrl_set = ixgbe_priority_flow_ctrl_set,
 251         .mac_addr_add         = ixgbe_add_rar,
 252         .mac_addr_remove      = ixgbe_remove_rar,
 253         .fdir_add_signature_filter    = ixgbe_fdir_add_signature_filter,
 254         .fdir_update_signature_filter = ixgbe_fdir_update_signature_filter,
 255         .fdir_remove_signature_filter = ixgbe_fdir_remove_signature_filter,
 256         .fdir_infos_get               = ixgbe_fdir_info_get,
 257         .fdir_add_perfect_filter      = ixgbe_fdir_add_perfect_filter,
 258         .fdir_update_perfect_filter   = ixgbe_fdir_update_perfect_filter,
 259         .fdir_remove_perfect_filter   = ixgbe_fdir_remove_perfect_filter,
 260         .fdir_set_masks               = ixgbe_fdir_set_masks,
 261 };
 262
 263 /*
 264  * dev_ops for virtual function, bare necessities for basic vf
 265  * operation have been implemented
 266  */
 267 static struct eth_dev_ops ixgbevf_eth_dev_ops = {
 268
 269         .dev_configure        = ixgbevf_dev_configure,
 270         .dev_start            = ixgbevf_dev_start,
 271         .dev_stop             = ixgbevf_dev_stop,
 272         .link_update          = ixgbe_dev_link_update,
 273         .stats_get            = ixgbevf_dev_stats_get,
 274         .stats_reset          = ixgbevf_dev_stats_reset,
 275         .dev_close            = ixgbevf_dev_close,
 276         .dev_infos_get        = ixgbe_dev_info_get,
 277         .vlan_filter_set      = ixgbevf_vlan_filter_set,
 278         .vlan_strip_queue_set = ixgbevf_vlan_strip_queue_set,
 279         .vlan_offload_set     = ixgbevf_vlan_offload_set,
 280         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 281         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 282         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 283         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 284 };
 285
 286 /**
 287  * Atomically reads the link status information from global
 288  * structure rte_eth_dev.
 289  *
 290  * @param dev
 291  *   - Pointer to the structure rte_eth_dev to read from.
 292  *   - Pointer to the buffer to be saved with the link status.
 293  *
 294  * @return
 295  *   - On success, zero.
 296  *   - On failure, negative value.
 297  */
 298 static inline int
 299 rte_ixgbe_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 300                                 struct rte_eth_link *link)
 301 {
 302         struct rte_eth_link *dst = link;
 303         struct rte_eth_link *src = &(dev->data->dev_link);
 304
 305         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 306                                         *(uint64_t *)src) == 0)
 307                 return -1;
 308
 309         return 0;
 310 }
 311
 312 /**
 313  * Atomically writes the link status information into global
 314  * structure rte_eth_dev.
 315  *
 316  * @param dev
 317  *   - Pointer to the structure rte_eth_dev to read from.
 318  *   - Pointer to the buffer to be saved with the link status.
 319  *
 320  * @return
 321  *   - On success, zero.
 322  *   - On failure, negative value.
 323  */
 324 static inline int
 325 rte_ixgbe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 326                                 struct rte_eth_link *link)
 327 {
 328         struct rte_eth_link *dst = &(dev->data->dev_link);
 329         struct rte_eth_link *src = link;
 330
 331         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 332                                         *(uint64_t *)src) == 0)
 333                 return -1;
 334
 335         return 0;
 336 }
 337
 338 /*
 339  * This function is the same as ixgbe_is_sfp() in ixgbe/ixgbe.h.
 340  */
 341 static inline int
 342 ixgbe_is_sfp(struct ixgbe_hw *hw)
 343 {
 344         switch (hw->phy.type) {
 345         case ixgbe_phy_sfp_avago:
 346         case ixgbe_phy_sfp_ftl:
 347         case ixgbe_phy_sfp_intel:
 348         case ixgbe_phy_sfp_unknown:
 349         case ixgbe_phy_sfp_passive_tyco:
 350         case ixgbe_phy_sfp_passive_unknown:
 351                 return 1;
 352         default:
 353                 return 0;
 354         }
 355 }
 356
 357 /*
 358  * This function is based on ixgbe_disable_intr() in ixgbe/ixgbe.h.
 359  */
 360 static void
 361 ixgbe_disable_intr(struct ixgbe_hw *hw)
 362 {
 363         PMD_INIT_FUNC_TRACE();
 364
 365         if (hw->mac.type == ixgbe_mac_82598EB) {
 366                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ~0);
 367         } else {
 368                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xFFFF0000);
 369                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), ~0);
 370                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), ~0);
 371         }
 372         IXGBE_WRITE_FLUSH(hw);
 373 }
 374
 375 /*
 376  * This function resets queue statistics mapping registers.
 377  * From Niantic datasheet, Initialization of Statistics section:
 378  * "...if software requires the queue counters, the RQSMR and TQSM registers
 379  * must be re-programmed following a device reset.
 380  */
 381 static void
 382 ixgbe_reset_qstat_mappings(struct ixgbe_hw *hw)
 383 {
 384         uint32_t i;
 385
 386         for(i = 0; i != IXGBE_NB_STAT_MAPPING_REGS; i++) {
 387                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), 0);
 388                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), 0);
 389         }
 390 }
 391
 392
 393 static int
 394 ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 395                                   uint16_t queue_id,
 396                                   uint8_t stat_idx,
 397                                   uint8_t is_rx)
 398 {
 399 #define QSM_REG_NB_BITS_PER_QMAP_FIELD 8
 400 #define NB_QMAP_FIELDS_PER_QSM_REG 4
 401 #define QMAP_FIELD_RESERVED_BITS_MASK 0x0f
 402
 403         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 404         struct ixgbe_stat_mapping_registers *stat_mappings =
 405                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(eth_dev->data->dev_private);
 406         uint32_t qsmr_mask = 0;
 407         uint32_t clearing_mask = QMAP_FIELD_RESERVED_BITS_MASK;
 408         uint32_t q_map;
 409         uint8_t n, offset;
 410
 411         if ((hw->mac.type != ixgbe_mac_82599EB) && (hw->mac.type != ixgbe_mac_X540))
 412                 return -ENOSYS;
 413
 414         PMD_INIT_LOG(INFO, "Setting port %d, %s queue_id %d to stat index %d\n",
 415                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx);
 416
 417         n = queue_id / NB_QMAP_FIELDS_PER_QSM_REG;
 418         if (n >= IXGBE_NB_STAT_MAPPING_REGS) {
 419                 PMD_INIT_LOG(ERR, "Nb of stat mapping registers exceeded\n");
 420                 return -EIO;
 421         }
 422         offset = queue_id % NB_QMAP_FIELDS_PER_QSM_REG;
 423
 424         /* Now clear any previous stat_idx set */
 425         clearing_mask <<= (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 426         if (!is_rx)
 427                 stat_mappings->tqsm[n] &= ~clearing_mask;
 428         else
 429                 stat_mappings->rqsmr[n] &= ~clearing_mask;
 430
 431         q_map = (uint32_t)stat_idx;
 432         q_map &= QMAP_FIELD_RESERVED_BITS_MASK;
 433         qsmr_mask = q_map << (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 434         if (!is_rx)
 435                 stat_mappings->tqsm[n] |= qsmr_mask;
 436         else
 437                 stat_mappings->rqsmr[n] |= qsmr_mask;
 438
 439         PMD_INIT_LOG(INFO, "Set port %d, %s queue_id %d to stat index %d\n"
 440                      "%s[%d] = 0x%08x\n",
 441                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx,
 442                      is_rx ? "RQSMR" : "TQSM",n, is_rx ? stat_mappings->rqsmr[n] : stat_mappings->tqsm[n]);
 443
 444         /* Now write the mapping in the appropriate register */
 445         if (is_rx) {
 446                 PMD_INIT_LOG(INFO, "Write 0x%x to RX IXGBE stat mapping reg:%d\n",
 447                              stat_mappings->rqsmr[n], n);
 448                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(n), stat_mappings->rqsmr[n]);
 449         }
 450         else {
 451                 PMD_INIT_LOG(INFO, "Write 0x%x to TX IXGBE stat mapping reg:%d\n",
 452                              stat_mappings->tqsm[n], n);
 453                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(n), stat_mappings->tqsm[n]);
 454         }
 455         return 0;
 456 }
 457
 458 static void
 459 ixgbe_restore_statistics_mapping(struct rte_eth_dev * dev)
 460 {
 461         struct ixgbe_stat_mapping_registers *stat_mappings =
 462                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(dev->data->dev_private);
 463         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 464         int i;
 465
 466         /* write whatever was in stat mapping table to the NIC */
 467         for (i = 0; i < IXGBE_NB_STAT_MAPPING_REGS; i++) {
 468                 /* rx */
 469                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), stat_mappings->rqsmr[i]);
 470
 471                 /* tx */
 472                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), stat_mappings->tqsm[i]);
 473         }
 474 }
 475
 476 static void
 477 ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config)
 478 {
 479         uint8_t i;
 480         struct ixgbe_dcb_tc_config *tc;
 481         int dcb_max_tc = IXGBE_DCB_MAX_TRAFFIC_CLASS;
 482
 483         dcb_config->num_tcs.pg_tcs = dcb_max_tc;
 484         dcb_config->num_tcs.pfc_tcs = dcb_max_tc;
 485         for (i = 0; i < dcb_max_tc; i++) {
 486                 tc = &dcb_config->tc_config[i];
 487                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_id = i;
 488                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = 100/dcb_max_tc + (i & 1);
 489                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_id = i;
 490                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent = 100/dcb_max_tc + (i & 1);
 491                 tc->pfc = ixgbe_dcb_pfc_disabled;
 492         }
 493
 494         /* Initialize default user to priority mapping, UPx->TC0 */
 495         tc = &dcb_config->tc_config[0];
 496         tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
 497         tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
 498         for (i = 0; i< IXGBE_DCB_MAX_BW_GROUP; i++) {
 499                 dcb_config->bw_percentage[IXGBE_DCB_TX_CONFIG][i] = 100;
 500                 dcb_config->bw_percentage[IXGBE_DCB_RX_CONFIG][i] = 100;
 501         }
 502         dcb_config->rx_pba_cfg = ixgbe_dcb_pba_equal;
 503         dcb_config->pfc_mode_enable = false;
 504         dcb_config->vt_mode = true;
 505         dcb_config->round_robin_enable = false;
 506         /* support all DCB capabilities in 82599 */
 507         dcb_config->support.capabilities = 0xFF;
 508
 509         /*we only support 4 Tcs for X540*/
 510         if (hw->mac.type == ixgbe_mac_X540) {
 511                 dcb_config->num_tcs.pg_tcs = 4;
 512                 dcb_config->num_tcs.pfc_tcs = 4;
 513         }
 514 }
 515
 516 /*
 517  * This function is based on code in ixgbe_attach() in ixgbe/ixgbe.c.
 518  * It returns 0 on success.
 519  */
 520 static int
 521 eth_ixgbe_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 522                      struct rte_eth_dev *eth_dev)
 523 {
 524         struct rte_pci_device *pci_dev;
 525         struct ixgbe_hw *hw =
 526                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 527         struct ixgbe_vfta * shadow_vfta =
 528                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 529         struct ixgbe_hwstrip *hwstrip =
 530                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 531         struct ixgbe_dcb_config *dcb_config =
 532                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private);
 533         uint32_t ctrl_ext;
 534         uint16_t csum;
 535         int diag, i;
 536
 537         PMD_INIT_FUNC_TRACE();
 538
 539         eth_dev->dev_ops = &ixgbe_eth_dev_ops;
 540         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
 541         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
 542
 543         /* for secondary processes, we don't initialise any further as primary
 544          * has already done this work. Only check we don't need a different
 545          * RX function */
 546         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 547                 if (eth_dev->data->scattered_rx)
 548                         eth_dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
 549                 return 0;
 550         }
 551         pci_dev = eth_dev->pci_dev;
 552
 553         /* Vendor and Device ID need to be set before init of shared code */
 554         hw->device_id = pci_dev->id.device_id;
 555         hw->vendor_id = pci_dev->id.vendor_id;
 556         hw->hw_addr = (void *)pci_dev->mem_resource.addr;
 557
 558         /* Initialize the shared code */
 559         diag = ixgbe_init_shared_code(hw);
 560         if (diag != IXGBE_SUCCESS) {
 561                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
 562                 return -EIO;
 563         }
 564
 565         /* Initialize DCB configuration*/
 566         memset(dcb_config, 0, sizeof(struct ixgbe_dcb_config));
 567         ixgbe_dcb_init(hw,dcb_config);
 568         /* Get Hardware Flow Control setting */
 569         hw->fc.requested_mode = ixgbe_fc_full;
 570         hw->fc.current_mode = ixgbe_fc_full;
 571         hw->fc.pause_time = IXGBE_FC_PAUSE;
 572         for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
 573                 hw->fc.low_water[i] = IXGBE_FC_LO;
 574                 hw->fc.high_water[i] = IXGBE_FC_HI;
 575         }
 576         hw->fc.send_xon = 1;
 577
 578         ixgbe_disable_intr(hw);
 579
 580         /* Make sure we have a good EEPROM before we read from it */
 581         diag = ixgbe_validate_eeprom_checksum(hw, &csum);
 582         if (diag != IXGBE_SUCCESS) {
 583                 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag);
 584                 return -EIO;
 585         }
 586
 587         diag = ixgbe_init_hw(hw);
 588
 589         /*
 590          * Devices with copper phys will fail to initialise if ixgbe_init_hw()
 591          * is called too soon after the kernel driver unbinding/binding occurs.
 592          * The failure occurs in ixgbe_identify_phy_generic() for all devices,
 593          * but for non-copper devies, ixgbe_identify_sfp_module_generic() is
 594          * also called. See ixgbe_identify_phy_82599(). The reason for the
 595          * failure is not known, and only occuts when virtualisation features
 596          * are disabled in the bios. A delay of 100ms  was found to be enough by
 597          * trial-and-error, and is doubled to be safe.
 598          */
 599         if (diag && (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper)) {
 600                 rte_delay_ms(200);
 601                 diag = ixgbe_init_hw(hw);
 602         }
 603
 604         if (diag == IXGBE_ERR_EEPROM_VERSION) {
 605                 PMD_INIT_LOG(ERR, "This device is a pre-production adapter/"
 606                     "LOM.  Please be aware there may be issues associated "
 607                     "with your hardware.\n If you are experiencing problems "
 608                     "please contact your Intel or hardware representative "
 609                     "who provided you with this hardware.\n");
 610         } else if (diag == IXGBE_ERR_SFP_NOT_SUPPORTED)
 611                 PMD_INIT_LOG(ERR, "Unsupported SFP+ Module\n");
 612         if (diag) {
 613                 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag);
 614                 return -EIO;
 615         }
 616
 617         /* pick up the PCI bus settings for reporting later */
 618         ixgbe_get_bus_info(hw);
 619
 620         /* reset mappings for queue statistics hw counters*/
 621         ixgbe_reset_qstat_mappings(hw);
 622
 623         /* Allocate memory for storing MAC addresses */
 624         eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 625                         hw->mac.num_rar_entries, 0);
 626         if (eth_dev->data->mac_addrs == NULL) {
 627                 PMD_INIT_LOG(ERR,
 628                         "Failed to allocate %d bytes needed to store MAC addresses",
 629                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 630                 return -ENOMEM;
 631         }
 632         /* Copy the permanent MAC address */
 633         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 634                         &eth_dev->data->mac_addrs[0]);
 635
 636         /* initialize the vfta */
 637         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 638
 639         /* initialize the hw strip bitmap*/
 640         memset(hwstrip, 0, sizeof(*hwstrip));
 641
 642         /* let hardware know driver is loaded */
 643         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 644         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 645         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 646
 647         if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
 648                 PMD_INIT_LOG(DEBUG,
 649                              "MAC: %d, PHY: %d, SFP+: %d<n",
 650                              (int) hw->mac.type, (int) hw->phy.type,
 651                              (int) hw->phy.sfp_type);
 652         else
 653                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d\n",
 654                              (int) hw->mac.type, (int) hw->phy.type);
 655
 656         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
 657                         eth_dev->data->port_id, pci_dev->id.vendor_id,
 658                         pci_dev->id.device_id);
 659
 660         rte_intr_callback_register(&(pci_dev->intr_handle),
 661                 ixgbe_dev_interrupt_handler, (void *)eth_dev);
 662
 663         return 0;
 664 }
 665
 666 /*
 667  * Virtual Function device init
 668  */
 669 static int
 670 eth_ixgbevf_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 671                      struct rte_eth_dev *eth_dev)
 672 {
 673         struct rte_pci_device *pci_dev;
 674         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 675         int diag;
 676         struct ixgbe_vfta * shadow_vfta =
 677                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 678         struct ixgbe_hwstrip *hwstrip =
 679                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 680
 681         PMD_INIT_LOG(DEBUG, "eth_ixgbevf_dev_init");
 682
 683         eth_dev->dev_ops = &ixgbevf_eth_dev_ops;
 684         pci_dev = eth_dev->pci_dev;
 685
 686         hw->device_id = pci_dev->id.device_id;
 687         hw->vendor_id = pci_dev->id.vendor_id;
 688         hw->hw_addr = (void *)pci_dev->mem_resource.addr;
 689
 690         /* initialize the vfta */
 691         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 692
 693         /* initialize the hw strip bitmap*/
 694         memset(hwstrip, 0, sizeof(*hwstrip));
 695
 696         /* Initialize the shared code */
 697         diag = ixgbe_init_shared_code(hw);
 698         if (diag != IXGBE_SUCCESS) {
 699                 PMD_INIT_LOG(ERR, "Shared code init failed for ixgbevf: %d", diag);
 700                 return -EIO;
 701         }
 702
 703         /* init_mailbox_params */
 704         hw->mbx.ops.init_params(hw);
 705
 706         /* Disable the interrupts for VF */
 707         ixgbevf_intr_disable(hw);
 708
 709         hw->mac.num_rar_entries = hw->mac.max_rx_queues;
 710         diag = hw->mac.ops.reset_hw(hw);
 711
 712         /* Allocate memory for storing MAC addresses */
 713         eth_dev->data->mac_addrs = rte_zmalloc("ixgbevf", ETHER_ADDR_LEN *
 714                         hw->mac.num_rar_entries, 0);
 715         if (eth_dev->data->mac_addrs == NULL) {
 716                 PMD_INIT_LOG(ERR,
 717                         "Failed to allocate %d bytes needed to store MAC addresses",
 718                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 719                 return -ENOMEM;
 720         }
 721         /* Copy the permanent MAC address */
 722         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 723                         &eth_dev->data->mac_addrs[0]);
 724
 725         /* reset the hardware with the new settings */
 726         diag = hw->mac.ops.start_hw(hw);
 727         switch (diag) {
 728                 case  0:
 729                         break;
 730
 731                 default:
 732                         PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
 733                         return (-EIO);
 734         }
 735
 736         PMD_INIT_LOG(DEBUG, "\nport %d vendorID=0x%x deviceID=0x%x mac.type=%s\n",
 737                          eth_dev->data->port_id, pci_dev->id.vendor_id, pci_dev->id.device_id,
 738                          "ixgbe_mac_82599_vf");
 739
 740         return 0;
 741 }
 742
 743 static struct eth_driver rte_ixgbe_pmd = {
 744         {
 745                 .name = "rte_ixgbe_pmd",
 746                 .id_table = pci_id_ixgbe_map,
 747                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
 748         },
 749         .eth_dev_init = eth_ixgbe_dev_init,
 750         .dev_private_size = sizeof(struct ixgbe_adapter),
 751 };
 752
 753 /*
 754  * virtual function driver struct
 755  */
 756 static struct eth_driver rte_ixgbevf_pmd = {
 757         {
 758                 .name = "rte_ixgbevf_pmd",
 759                 .id_table = pci_id_ixgbevf_map,
 760                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
 761         },
 762         .eth_dev_init = eth_ixgbevf_dev_init,
 763         .dev_private_size = sizeof(struct ixgbe_adapter),
 764 };
 765
 766 /*
 767  * Driver initialization routine.
 768  * Invoked once at EAL init time.
 769  * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
 770  */
 771 int
 772 rte_ixgbe_pmd_init(void)
 773 {
 774         PMD_INIT_FUNC_TRACE();
 775
 776         rte_eth_driver_register(&rte_ixgbe_pmd);
 777         return 0;
 778 }
 779
 780 /*
 781  * VF Driver initialization routine.
 782  * Invoked one at EAL init time.
 783  * Register itself as the [Virtual Poll Mode] Driver of PCI niantic devices.
 784  */
 785 int
 786 rte_ixgbevf_pmd_init(void)
 787 {
 788         DEBUGFUNC("rte_ixgbevf_pmd_init");
 789
 790         rte_eth_driver_register(&rte_ixgbevf_pmd);
 791         return (0);
 792 }
 793
 794 static int
 795 ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 796 {
 797         struct ixgbe_hw *hw =
 798                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 799         struct ixgbe_vfta * shadow_vfta =
 800                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
 801         uint32_t vfta;
 802         uint32_t vid_idx;
 803         uint32_t vid_bit;
 804
 805         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
 806         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
 807         vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(vid_idx));
 808         if (on)
 809                 vfta |= vid_bit;
 810         else
 811                 vfta &= ~vid_bit;
 812         IXGBE_WRITE_REG(hw, IXGBE_VFTA(vid_idx), vfta);
 813
 814         /* update local VFTA copy */
 815         shadow_vfta->vfta[vid_idx] = vfta;
 816
 817         return 0;
 818 }
 819
 820 static void
 821 ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
 822 {
 823         if (on)
 824                 ixgbe_vlan_hw_strip_enable(dev, queue);
 825         else
 826                 ixgbe_vlan_hw_strip_disable(dev, queue);
 827 }
 828
 829 static void
 830 ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
 831 {
 832         struct ixgbe_hw *hw =
 833                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 834
 835         /* Only the high 16-bits is valid */
 836         IXGBE_WRITE_REG(hw, IXGBE_EXVET, tpid << 16);
 837 }
 838
 839 static void
 840 ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev)
 841 {
 842         struct ixgbe_hw *hw =
 843                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 844         uint32_t vlnctrl;
 845
 846         PMD_INIT_FUNC_TRACE();
 847
 848         /* Filter Table Disable */
 849         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 850         vlnctrl &= ~IXGBE_VLNCTRL_VFE;
 851
 852         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
 853 }
 854
 855 static void
 856 ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev)
 857 {
 858         struct ixgbe_hw *hw =
 859                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 860         struct ixgbe_vfta * shadow_vfta =
 861                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
 862         uint32_t vlnctrl;
 863         uint16_t i;
 864
 865         PMD_INIT_FUNC_TRACE();
 866
 867         /* Filter Table Enable */
 868         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 869         vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
 870         vlnctrl |= IXGBE_VLNCTRL_VFE;
 871
 872         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
 873
 874         /* write whatever is in local vfta copy */
 875         for (i = 0; i < IXGBE_VFTA_SIZE; i++)
 876                 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), shadow_vfta->vfta[i]);
 877 }
 878
 879 static void
 880 ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on)
 881 {
 882         struct ixgbe_hwstrip *hwstrip =
 883                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(dev->data->dev_private);
 884
 885         if(queue >= IXGBE_MAX_RX_QUEUE_NUM)
 886                 return;
 887
 888         if (on)
 889                 IXGBE_SET_HWSTRIP(hwstrip, queue);
 890         else
 891                 IXGBE_CLEAR_HWSTRIP(hwstrip, queue);
 892 }
 893
 894 static void
 895 ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue)
 896 {
 897         struct ixgbe_hw *hw =
 898                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 899         uint32_t ctrl;
 900
 901         PMD_INIT_FUNC_TRACE();
 902
 903         if (hw->mac.type == ixgbe_mac_82598EB) {
 904                 /* No queue level support */
 905                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
 906                 return;
 907         }
 908         else {
 909                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 910                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
 911                 ctrl &= ~IXGBE_RXDCTL_VME;
 912                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
 913         }
 914         /* record those setting for HW strip per queue */
 915         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 0);
 916 }
 917
 918 static void
 919 ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue)
 920 {
 921         struct ixgbe_hw *hw =
 922                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 923         uint32_t ctrl;
 924
 925         PMD_INIT_FUNC_TRACE();
 926
 927         if (hw->mac.type == ixgbe_mac_82598EB) {
 928                 /* No queue level supported */
 929                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
 930                 return;
 931         }
 932         else {
 933                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 934                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
 935                 ctrl |= IXGBE_RXDCTL_VME;
 936                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
 937         }
 938         /* record those setting for HW strip per queue */
 939         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 1);
 940 }
 941
 942 static void
 943 ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev)
 944 {
 945         struct ixgbe_hw *hw =
 946                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 947         uint32_t ctrl;
 948         uint16_t i;
 949
 950         PMD_INIT_FUNC_TRACE();
 951
 952         if (hw->mac.type == ixgbe_mac_82598EB) {
 953                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 954                 ctrl &= ~IXGBE_VLNCTRL_VME;
 955                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
 956         }
 957         else {
 958                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 959                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
 960                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
 961                         ctrl &= ~IXGBE_RXDCTL_VME;
 962                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
 963
 964                         /* record those setting for HW strip per queue */
 965                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 0);
 966                 }
 967         }
 968 }
 969
 970 static void
 971 ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev)
 972 {
 973         struct ixgbe_hw *hw =
 974                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 975         uint32_t ctrl;
 976         uint16_t i;
 977
 978         PMD_INIT_FUNC_TRACE();
 979
 980         if (hw->mac.type == ixgbe_mac_82598EB) {
 981                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 982                 ctrl |= IXGBE_VLNCTRL_VME;
 983                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
 984         }
 985         else {
 986                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 987                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
 988                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
 989                         ctrl |= IXGBE_RXDCTL_VME;
 990                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
 991
 992                         /* record those setting for HW strip per queue */
 993                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 1);
 994                 }
 995         }
 996 }
 997
 998 static void
 999 ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1000 {
1001         struct ixgbe_hw *hw =
1002                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1003         uint32_t ctrl;
1004
1005         PMD_INIT_FUNC_TRACE();
1006
1007         /* DMATXCTRL: Geric Double VLAN Disable */
1008         ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1009         ctrl &= ~IXGBE_DMATXCTL_GDV;
1010         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1011
1012         /* CTRL_EXT: Global Double VLAN Disable */
1013         ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1014         ctrl &= ~IXGBE_EXTENDED_VLAN;
1015         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1016
1017 }
1018
1019 static void
1020 ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1021 {
1022         struct ixgbe_hw *hw =
1023                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1024         uint32_t ctrl;
1025
1026         PMD_INIT_FUNC_TRACE();
1027
1028         /* DMATXCTRL: Geric Double VLAN Enable */
1029         ctrl  = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1030         ctrl |= IXGBE_DMATXCTL_GDV;
1031         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1032
1033         /* CTRL_EXT: Global Double VLAN Enable */
1034         ctrl  = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1035         ctrl |= IXGBE_EXTENDED_VLAN;
1036         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1037
1038         /*
1039          * VET EXT field in the EXVET register = 0x8100 by default
1040          * So no need to change. Same to VT field of DMATXCTL register
1041          */
1042 }
1043
1044 static void
1045 ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1046 {
1047         if(mask & ETH_VLAN_STRIP_MASK){
1048                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1049                         ixgbe_vlan_hw_strip_enable_all(dev);
1050                 else
1051                         ixgbe_vlan_hw_strip_disable_all(dev);
1052         }
1053
1054         if(mask & ETH_VLAN_FILTER_MASK){
1055                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1056                         ixgbe_vlan_hw_filter_enable(dev);
1057                 else
1058                         ixgbe_vlan_hw_filter_disable(dev);
1059         }
1060
1061         if(mask & ETH_VLAN_EXTEND_MASK){
1062                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1063                         ixgbe_vlan_hw_extend_enable(dev);
1064                 else
1065                         ixgbe_vlan_hw_extend_disable(dev);
1066         }
1067 }
1068
1069 static int
1070 ixgbe_dev_configure(struct rte_eth_dev *dev)
1071 {
1072         struct ixgbe_interrupt *intr =
1073                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1074
1075         PMD_INIT_FUNC_TRACE();
1076
1077         /* set flag to update link status after init */
1078         intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1079
1080         return 0;
1081 }
1082
1083 /*
1084  * Configure device link speed and setup link.
1085  * It returns 0 on success.
1086  */
1087 static int
1088 ixgbe_dev_start(struct rte_eth_dev *dev)
1089 {
1090         struct ixgbe_hw *hw =
1091                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1092         int err, link_up = 0, negotiate = 0;
1093         uint32_t speed = 0;
1094         int mask = 0;
1095
1096         PMD_INIT_FUNC_TRACE();
1097
1098         /* IXGBE devices don't support half duplex */
1099         if ((dev->data->dev_conf.link_duplex != ETH_LINK_AUTONEG_DUPLEX) &&
1100                         (dev->data->dev_conf.link_duplex != ETH_LINK_FULL_DUPLEX)) {
1101                 PMD_INIT_LOG(ERR, "Invalid link_duplex (%u) for port %u\n",
1102                                 dev->data->dev_conf.link_duplex,
1103                                 dev->data->port_id);
1104                 return -EINVAL;
1105         }
1106
1107         /* stop adapter */
1108         hw->adapter_stopped = FALSE;
1109         ixgbe_stop_adapter(hw);
1110
1111         /* reinitialize adapter
1112          * this calls reset and start */
1113         ixgbe_init_hw(hw);
1114
1115         /* initialize transmission unit */
1116         ixgbe_dev_tx_init(dev);
1117
1118         /* This can fail when allocating mbufs for descriptor rings */
1119         err = ixgbe_dev_rx_init(dev);
1120         if (err) {
1121                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware\n");
1122                 goto error;
1123         }
1124
1125         ixgbe_dev_rxtx_start(dev);
1126
1127         if (ixgbe_is_sfp(hw) && hw->phy.multispeed_fiber) {
1128                 err = hw->mac.ops.setup_sfp(hw);
1129                 if (err)
1130                         goto error;
1131         }
1132
1133         /* Turn on the laser */
1134         if (hw->phy.multispeed_fiber)
1135                 ixgbe_enable_tx_laser(hw);
1136
1137         err = ixgbe_check_link(hw, &speed, &link_up, 0);
1138         if (err)
1139                 goto error;
1140         err = ixgbe_get_link_capabilities(hw, &speed, &negotiate);
1141         if (err)
1142                 goto error;
1143
1144         switch(dev->data->dev_conf.link_speed) {
1145         case ETH_LINK_SPEED_AUTONEG:
1146                 speed = (hw->mac.type != ixgbe_mac_82598EB) ?
1147                                 IXGBE_LINK_SPEED_82599_AUTONEG :
1148                                 IXGBE_LINK_SPEED_82598_AUTONEG;
1149                 break;
1150         case ETH_LINK_SPEED_100:
1151                 /*
1152                  * Invalid for 82598 but error will be detected by
1153                  * ixgbe_setup_link()
1154                  */
1155                 speed = IXGBE_LINK_SPEED_100_FULL;
1156                 break;
1157         case ETH_LINK_SPEED_1000:
1158                 speed = IXGBE_LINK_SPEED_1GB_FULL;
1159                 break;
1160         case ETH_LINK_SPEED_10000:
1161                 speed = IXGBE_LINK_SPEED_10GB_FULL;
1162                 break;
1163         default:
1164                 PMD_INIT_LOG(ERR, "Invalid link_speed (%u) for port %u\n",
1165                                 dev->data->dev_conf.link_speed, dev->data->port_id);
1166                 goto error;
1167         }
1168
1169         err = ixgbe_setup_link(hw, speed, negotiate, link_up);
1170         if (err)
1171                 goto error;
1172
1173         /* check if lsc interrupt is enabled */
1174         if (dev->data->dev_conf.intr_conf.lsc != 0) {
1175                 err = ixgbe_dev_interrupt_setup(dev);
1176                 if (err)
1177                         goto error;
1178         }
1179
1180         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
1181                 ETH_VLAN_EXTEND_MASK;
1182         ixgbe_vlan_offload_set(dev, mask);
1183
1184         /* Configure DCB hw */
1185         ixgbe_configure_dcb(dev);
1186
1187         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) {
1188                 err = ixgbe_fdir_configure(dev);
1189                 if (err)
1190                         goto error;
1191         }
1192
1193         ixgbe_restore_statistics_mapping(dev);
1194
1195         return (0);
1196
1197 error:
1198         PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err);
1199         ixgbe_dev_clear_queues(dev);
1200         return -EIO;
1201 }
1202
1203 /*
1204  * Stop device: disable rx and tx functions to allow for reconfiguring.
1205  */
1206 static void
1207 ixgbe_dev_stop(struct rte_eth_dev *dev)
1208 {
1209         struct rte_eth_link link;
1210         struct ixgbe_hw *hw =
1211                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1212
1213         PMD_INIT_FUNC_TRACE();
1214
1215         /* disable interrupts */
1216         ixgbe_disable_intr(hw);
1217
1218         /* reset the NIC */
1219         ixgbe_reset_hw(hw);
1220         hw->adapter_stopped = FALSE;
1221
1222         /* stop adapter */
1223         ixgbe_stop_adapter(hw);
1224
1225         /* Turn off the laser */
1226         if (hw->phy.multispeed_fiber)
1227                 ixgbe_disable_tx_laser(hw);
1228
1229         ixgbe_dev_clear_queues(dev);
1230
1231         /* Clear recorded link status */
1232         memset(&link, 0, sizeof(link));
1233         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1234 }
1235
1236 /*
1237  * Reest and stop device.
1238  */
1239 static void
1240 ixgbe_dev_close(struct rte_eth_dev *dev)
1241 {
1242         struct ixgbe_hw *hw =
1243                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1244
1245         PMD_INIT_FUNC_TRACE();
1246
1247         ixgbe_reset_hw(hw);
1248
1249
1250         ixgbe_dev_stop(dev);
1251         hw->adapter_stopped = 1;
1252
1253         ixgbe_disable_pcie_master(hw);
1254
1255         /* reprogram the RAR[0] in case user changed it. */
1256         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
1257 }
1258
1259 /*
1260  * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c
1261  */
1262 static void
1263 ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1264 {
1265         struct ixgbe_hw *hw =
1266                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1267         struct ixgbe_hw_stats *hw_stats =
1268                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1269         uint32_t bprc, lxon, lxoff, total;
1270         uint64_t total_missed_rx, total_qbrc, total_qprc;
1271         unsigned i;
1272
1273         total_missed_rx = 0;
1274         total_qbrc = 0;
1275         total_qprc = 0;
1276
1277         hw_stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
1278         hw_stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
1279         hw_stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
1280         hw_stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
1281
1282         for (i = 0; i < 8; i++) {
1283                 uint32_t mp;
1284                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
1285                 /* global total per queue */
1286                 hw_stats->mpc[i] += mp;
1287                 /* Running comprehensive total for stats display */
1288                 total_missed_rx += hw_stats->mpc[i];
1289                 if (hw->mac.type == ixgbe_mac_82598EB)
1290                         hw_stats->rnbc[i] +=
1291                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
1292                 hw_stats->pxontxc[i] +=
1293                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
1294                 hw_stats->pxonrxc[i] +=
1295                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
1296                 hw_stats->pxofftxc[i] +=
1297                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
1298                 hw_stats->pxoffrxc[i] +=
1299                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
1300                 hw_stats->pxon2offc[i] +=
1301                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
1302         }
1303         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1304                 hw_stats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
1305                 hw_stats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
1306                 hw_stats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
1307                 hw_stats->qbrc[i] +=
1308                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32);
1309                 hw_stats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
1310                 hw_stats->qbtc[i] +=
1311                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32);
1312                 hw_stats->qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
1313
1314                 total_qprc += hw_stats->qprc[i];
1315                 total_qbrc += hw_stats->qbrc[i];
1316         }
1317         hw_stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
1318         hw_stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
1319         hw_stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
1320
1321         /* Note that gprc counts missed packets */
1322         hw_stats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
1323
1324         if (hw->mac.type != ixgbe_mac_82598EB) {
1325                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
1326                     ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
1327                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
1328                     ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
1329                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
1330                     ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
1331                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
1332                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
1333         } else {
1334                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
1335                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
1336                 /* 82598 only has a counter in the high register */
1337                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
1338                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
1339                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
1340         }
1341
1342         /*
1343          * Workaround: mprc hardware is incorrectly counting
1344          * broadcasts, so for now we subtract those.
1345          */
1346         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
1347         hw_stats->bprc += bprc;
1348         hw_stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
1349         if (hw->mac.type == ixgbe_mac_82598EB)
1350                 hw_stats->mprc -= bprc;
1351
1352         hw_stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
1353         hw_stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
1354         hw_stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
1355         hw_stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
1356         hw_stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
1357         hw_stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
1358
1359         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
1360         hw_stats->lxontxc += lxon;
1361         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
1362         hw_stats->lxofftxc += lxoff;
1363         total = lxon + lxoff;
1364
1365         hw_stats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
1366         hw_stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
1367         hw_stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
1368         hw_stats->gptc -= total;
1369         hw_stats->mptc -= total;
1370         hw_stats->ptc64 -= total;
1371         hw_stats->gotc -= total * ETHER_MIN_LEN;
1372
1373         hw_stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
1374         hw_stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
1375         hw_stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
1376         hw_stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
1377         hw_stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
1378         hw_stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
1379         hw_stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
1380         hw_stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
1381         hw_stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
1382         hw_stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
1383         hw_stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
1384         hw_stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
1385         hw_stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
1386         hw_stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
1387         hw_stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
1388         hw_stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);
1389         hw_stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
1390         hw_stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
1391         /* Only read FCOE on 82599 */
1392         if (hw->mac.type != ixgbe_mac_82598EB) {
1393                 hw_stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
1394                 hw_stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
1395                 hw_stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
1396                 hw_stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
1397                 hw_stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
1398         }
1399
1400         if (stats == NULL)
1401                 return;
1402
1403         /* Fill out the rte_eth_stats statistics structure */
1404         stats->ipackets = total_qprc;
1405         stats->ibytes = total_qbrc;
1406         stats->opackets = hw_stats->gptc;
1407         stats->obytes = hw_stats->gotc;
1408         stats->imcasts = hw_stats->mprc;
1409
1410         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1411                 stats->q_ipackets[i] = hw_stats->qprc[i];
1412                 stats->q_opackets[i] = hw_stats->qptc[i];
1413                 stats->q_ibytes[i] = hw_stats->qbrc[i];
1414                 stats->q_obytes[i] = hw_stats->qbtc[i];
1415                 stats->q_errors[i] = hw_stats->qprdc[i];
1416         }
1417
1418         /* Rx Errors */
1419         stats->ierrors = total_missed_rx + hw_stats->crcerrs +
1420                 hw_stats->rlec;
1421
1422         stats->oerrors  = 0;
1423
1424         /* Flow Director Stats registers */
1425         hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
1426         hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
1427         stats->fdirmatch = hw_stats->fdirmatch;
1428         stats->fdirmiss = hw_stats->fdirmiss;
1429 }
1430
1431 static void
1432 ixgbe_dev_stats_reset(struct rte_eth_dev *dev)
1433 {
1434         struct ixgbe_hw_stats *stats =
1435                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1436
1437         /* HW registers are cleared on read */
1438         ixgbe_dev_stats_get(dev, NULL);
1439
1440         /* Reset software totals */
1441         memset(stats, 0, sizeof(*stats));
1442 }
1443
1444 static void
1445 ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1446 {
1447         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1448         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1449                           IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1450
1451         /* Good Rx packet, include VF loopback */
1452         UPDATE_VF_STAT(IXGBE_VFGPRC,
1453             hw_stats->last_vfgprc, hw_stats->vfgprc);
1454
1455         /* Good Rx octets, include VF loopback */
1456         UPDATE_VF_STAT_36BIT(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
1457             hw_stats->last_vfgorc, hw_stats->vfgorc);
1458
1459         /* Good Tx packet, include VF loopback */
1460         UPDATE_VF_STAT(IXGBE_VFGPTC,
1461             hw_stats->last_vfgptc, hw_stats->vfgptc);
1462
1463         /* Good Tx octets, include VF loopback */
1464         UPDATE_VF_STAT_36BIT(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
1465             hw_stats->last_vfgotc, hw_stats->vfgotc);
1466
1467         /* Rx Multicst Packet */
1468         UPDATE_VF_STAT(IXGBE_VFMPRC,
1469             hw_stats->last_vfmprc, hw_stats->vfmprc);
1470
1471         if (stats == NULL)
1472                 return;
1473
1474         memset(stats, 0, sizeof(*stats));
1475         stats->ipackets = hw_stats->vfgprc;
1476         stats->ibytes = hw_stats->vfgorc;
1477         stats->opackets = hw_stats->vfgptc;
1478         stats->obytes = hw_stats->vfgotc;
1479         stats->imcasts = hw_stats->vfmprc;
1480 }
1481
1482 static void
1483 ixgbevf_dev_stats_reset(struct rte_eth_dev *dev)
1484 {
1485         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1486                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1487
1488         /* Sync HW register to the last stats */
1489         ixgbevf_dev_stats_get(dev, NULL);
1490
1491         /* reset HW current stats*/
1492         hw_stats->vfgprc = 0;
1493         hw_stats->vfgorc = 0;
1494         hw_stats->vfgptc = 0;
1495         hw_stats->vfgotc = 0;
1496         hw_stats->vfmprc = 0;
1497
1498 }
1499
1500 static void
1501 ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1502 {
1503         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1504
1505         dev_info->max_rx_queues = hw->mac.max_rx_queues;
1506         dev_info->max_tx_queues = hw->mac.max_tx_queues;
1507         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL register */
1508         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */
1509         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
1510 }
1511
1512 /* return 0 means link status changed, -1 means not changed */
1513 static int
1514 ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1515 {
1516         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1517         struct rte_eth_link link, old;
1518         ixgbe_link_speed link_speed;
1519         int link_up;
1520         int diag;
1521
1522         link.link_status = 0;
1523         link.link_speed = 0;
1524         link.link_duplex = 0;
1525         memset(&old, 0, sizeof(old));
1526         rte_ixgbe_dev_atomic_read_link_status(dev, &old);
1527
1528         /* check if it needs to wait to complete, if lsc interrupt is enabled */
1529         if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
1530                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 0);
1531         else
1532                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 1);
1533         if (diag != 0) {
1534                 link.link_speed = ETH_LINK_SPEED_100;
1535                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1536                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1537                 if (link.link_status == old.link_status)
1538                         return -1;
1539                 return 0;
1540         }
1541
1542         if (link_up == 0) {
1543                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1544                 if (link.link_status == old.link_status)
1545                         return -1;
1546                 return 0;
1547         }
1548         link.link_status = 1;
1549         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1550
1551         switch (link_speed) {
1552         default:
1553         case IXGBE_LINK_SPEED_UNKNOWN:
1554                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1555                 link.link_speed = ETH_LINK_SPEED_100;
1556                 break;
1557
1558         case IXGBE_LINK_SPEED_100_FULL:
1559                 link.link_speed = ETH_LINK_SPEED_100;
1560                 break;
1561
1562         case IXGBE_LINK_SPEED_1GB_FULL:
1563                 link.link_speed = ETH_LINK_SPEED_1000;
1564                 break;
1565
1566         case IXGBE_LINK_SPEED_10GB_FULL:
1567                 link.link_speed = ETH_LINK_SPEED_10000;
1568                 break;
1569         }
1570         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1571
1572         if (link.link_status == old.link_status)
1573                 return -1;
1574
1575         return 0;
1576 }
1577
1578 static void
1579 ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev)
1580 {
1581         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1582         uint32_t fctrl;
1583
1584         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1585         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1586         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1587 }
1588
1589 static void
1590 ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev)
1591 {
1592         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1593         uint32_t fctrl;
1594
1595         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1596         fctrl &= (~IXGBE_FCTRL_UPE);
1597         if (dev->data->all_multicast == 1)
1598                 fctrl |= IXGBE_FCTRL_MPE;
1599         else
1600                 fctrl &= (~IXGBE_FCTRL_MPE);
1601         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1602 }
1603
1604 static void
1605 ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev)
1606 {
1607         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1608         uint32_t fctrl;
1609
1610         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1611         fctrl |= IXGBE_FCTRL_MPE;
1612         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1613 }
1614
1615 static void
1616 ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev)
1617 {
1618         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1619         uint32_t fctrl;
1620
1621         if (dev->data->promiscuous == 1)
1622                 return; /* must remain in all_multicast mode */
1623
1624         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1625         fctrl &= (~IXGBE_FCTRL_MPE);
1626         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1627 }
1628
1629 /**
1630  * It clears the interrupt causes and enables the interrupt.
1631  * It will be called once only during nic initialized.
1632  *
1633  * @param dev
1634  *  Pointer to struct rte_eth_dev.
1635  *
1636  * @return
1637  *  - On success, zero.
1638  *  - On failure, a negative value.
1639  */
1640 static int
1641 ixgbe_dev_interrupt_setup(struct rte_eth_dev *dev)
1642 {
1643         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1644
1645         ixgbe_dev_link_status_print(dev);
1646         IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_LSC);
1647         IXGBE_WRITE_FLUSH(hw);
1648         rte_intr_enable(&(dev->pci_dev->intr_handle));
1649
1650         return 0;
1651 }
1652
1653 /*
1654  * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update.
1655  *
1656  * @param dev
1657  *  Pointer to struct rte_eth_dev.
1658  *
1659  * @return
1660  *  - On success, zero.
1661  *  - On failure, a negative value.
1662  */
1663 static int
1664 ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
1665 {
1666         uint32_t eicr;
1667         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1668         struct ixgbe_interrupt *intr =
1669                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1670
1671         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_LSC);
1672         IXGBE_WRITE_FLUSH(hw);
1673
1674         /* read-on-clear nic registers here */
1675         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1676         PMD_INIT_LOG(INFO, "eicr %x", eicr);
1677         if (eicr & IXGBE_EICR_LSC) {
1678                 /* set flag for async link update */
1679                 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1680         }
1681
1682         return 0;
1683 }
1684
1685 /**
1686  * It gets and then prints the link status.
1687  *
1688  * @param dev
1689  *  Pointer to struct rte_eth_dev.
1690  *
1691  * @return
1692  *  - On success, zero.
1693  *  - On failure, a negative value.
1694  */
1695 static void
1696 ixgbe_dev_link_status_print(struct rte_eth_dev *dev)
1697 {
1698         struct rte_eth_link link;
1699
1700         memset(&link, 0, sizeof(link));
1701         rte_ixgbe_dev_atomic_read_link_status(dev, &link);
1702         if (link.link_status) {
1703                 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
1704                                         (int)(dev->data->port_id),
1705                                         (unsigned)link.link_speed,
1706                         link.link_duplex == ETH_LINK_FULL_DUPLEX ?
1707                                         "full-duplex" : "half-duplex");
1708         } else {
1709                 PMD_INIT_LOG(INFO, " Port %d: Link Down",
1710                                 (int)(dev->data->port_id));
1711         }
1712         PMD_INIT_LOG(INFO, "PCI Address: %04d:%02d:%02d:%d",
1713                                 dev->pci_dev->addr.domain,
1714                                 dev->pci_dev->addr.bus,
1715                                 dev->pci_dev->addr.devid,
1716                                 dev->pci_dev->addr.function);
1717 }
1718
1719 /*
1720  * It executes link_update after knowing an interrupt occured.
1721  *
1722  * @param dev
1723  *  Pointer to struct rte_eth_dev.
1724  *
1725  * @return
1726  *  - On success, zero.
1727  *  - On failure, a negative value.
1728  */
1729 static int
1730 ixgbe_dev_interrupt_action(struct rte_eth_dev *dev)
1731 {
1732         struct ixgbe_interrupt *intr =
1733                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1734
1735         if (!(intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE)) {
1736                 return -1;
1737         }
1738         ixgbe_dev_link_update(dev, 0);
1739
1740         return 0;
1741 }
1742
1743 /**
1744  * Interrupt handler which shall be registered for alarm callback for delayed
1745  * handling specific interrupt to wait for the stable nic state. As the
1746  * NIC interrupt state is not stable for ixgbe after link is just down,
1747  * it needs to wait 4 seconds to get the stable status.
1748  *
1749  * @param handle
1750  *  Pointer to interrupt handle.
1751  * @param param
1752  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1753  *
1754  * @return
1755  *  void
1756  */
1757 static void
1758 ixgbe_dev_interrupt_delayed_handler(void *param)
1759 {
1760         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1761         struct ixgbe_interrupt *intr =
1762                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1763         struct ixgbe_hw *hw =
1764                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1765
1766         IXGBE_READ_REG(hw, IXGBE_EICR);
1767         ixgbe_dev_interrupt_action(dev);
1768         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
1769                 intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
1770                 rte_intr_enable(&(dev->pci_dev->intr_handle));
1771                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_LSC);
1772                 IXGBE_WRITE_FLUSH(hw);
1773                 ixgbe_dev_link_status_print(dev);
1774                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
1775         }
1776 }
1777
1778 /**
1779  * Interrupt handler triggered by NIC  for handling
1780  * specific interrupt.
1781  *
1782  * @param handle
1783  *  Pointer to interrupt handle.
1784  * @param param
1785  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1786  *
1787  * @return
1788  *  void
1789  */
1790 static void
1791 ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle, void *param)
1792 {
1793         int64_t timeout;
1794         struct rte_eth_link link;
1795         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1796         struct ixgbe_interrupt *intr =
1797                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1798
1799         /* get the link status before link update, for predicting later */
1800         memset(&link, 0, sizeof(link));
1801         rte_ixgbe_dev_atomic_read_link_status(dev, &link);
1802         ixgbe_dev_interrupt_get_status(dev);
1803         ixgbe_dev_interrupt_action(dev);
1804
1805         if (!(intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE))
1806                 return;
1807
1808         /* likely to up */
1809         if (!link.link_status)
1810                 /* handle it 1 sec later, wait it being stable */
1811                 timeout = IXGBE_LINK_UP_CHECK_TIMEOUT;
1812         /* likely to down */
1813         else
1814                 /* handle it 4 sec later, wait it being stable */
1815                 timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT;
1816
1817         ixgbe_dev_link_status_print(dev);
1818         if (rte_eal_alarm_set(timeout * 1000,
1819                 ixgbe_dev_interrupt_delayed_handler, param) < 0)
1820                 PMD_INIT_LOG(ERR, "Error setting alarm");
1821 }
1822
1823 static int
1824 ixgbe_dev_led_on(struct rte_eth_dev *dev)
1825 {
1826         struct ixgbe_hw *hw;
1827
1828         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1829         return (ixgbe_led_on(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
1830 }
1831
1832 static int
1833 ixgbe_dev_led_off(struct rte_eth_dev *dev)
1834 {
1835         struct ixgbe_hw *hw;
1836
1837         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1838         return (ixgbe_led_off(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
1839 }
1840
1841 static int
1842 ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1843 {
1844         struct ixgbe_hw *hw;
1845         int err;
1846         uint32_t rx_buf_size;
1847         uint32_t max_high_water;
1848         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
1849                 ixgbe_fc_none,
1850                 ixgbe_fc_rx_pause,
1851                 ixgbe_fc_tx_pause,
1852                 ixgbe_fc_full
1853         };
1854
1855         PMD_INIT_FUNC_TRACE();
1856
1857         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1858         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0));
1859         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
1860
1861         /*
1862          * At least reserve one Ethernet frame for watermark
1863          * high_water/low_water in kilo bytes for ixgbe
1864          */
1865         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
1866         if ((fc_conf->high_water > max_high_water) ||
1867                 (fc_conf->high_water < fc_conf->low_water)) {
1868                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
1869                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
1870                 return (-EINVAL);
1871         }
1872
1873         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[fc_conf->mode];
1874         hw->fc.pause_time     = fc_conf->pause_time;
1875         hw->fc.high_water[0]  = fc_conf->high_water;
1876         hw->fc.low_water[0]   = fc_conf->low_water;
1877         hw->fc.send_xon       = fc_conf->send_xon;
1878
1879         err = ixgbe_fc_enable(hw);
1880         /* Not negotiated is not an error case */
1881         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) {
1882                 return 0;
1883         }
1884
1885         PMD_INIT_LOG(ERR, "ixgbe_fc_enable = 0x%x \n", err);
1886         return -EIO;
1887 }
1888
1889 /**
1890  *  ixgbe_pfc_enable_generic - Enable flow control
1891  *  @hw: pointer to hardware structure
1892  *  @tc_num: traffic class number
1893  *  Enable flow control according to the current settings.
1894  */
1895 static int
1896 ixgbe_dcb_pfc_enable_generic(struct ixgbe_hw *hw,uint8_t tc_num)
1897 {
1898         int ret_val = 0;
1899         uint32_t mflcn_reg, fccfg_reg;
1900         uint32_t reg;
1901         uint32_t fcrtl, fcrth;
1902         uint8_t i;
1903         uint8_t nb_rx_en;
1904
1905         /* Validate the water mark configuration */
1906         if (!hw->fc.pause_time) {
1907                 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
1908                 goto out;
1909         }
1910
1911         /* Low water mark of zero causes XOFF floods */
1912         if (hw->fc.current_mode & ixgbe_fc_tx_pause) {
1913                  /* High/Low water can not be 0 */
1914                 if( (!hw->fc.high_water[tc_num])|| (!hw->fc.low_water[tc_num])) {
1915                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
1916                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
1917                         goto out;
1918                 }
1919
1920                 if(hw->fc.low_water[tc_num] >= hw->fc.high_water[tc_num]) {
1921                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
1922                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
1923                         goto out;
1924                 }
1925         }
1926         /* Negotiate the fc mode to use */
1927         ixgbe_fc_autoneg(hw);
1928
1929         /* Disable any previous flow control settings */
1930         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
1931         mflcn_reg &= ~(IXGBE_MFLCN_RPFCE_SHIFT | IXGBE_MFLCN_RFCE|IXGBE_MFLCN_RPFCE);
1932
1933         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
1934         fccfg_reg &= ~(IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY);
1935
1936         switch (hw->fc.current_mode) {
1937         case ixgbe_fc_none:
1938                 /*
1939                  * If the count of enabled RX Priority Flow control >1,
1940                  * and the TX pause can not be disabled
1941                  */
1942                 nb_rx_en = 0;
1943                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
1944                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
1945                         if (reg & IXGBE_FCRTH_FCEN)
1946                                 nb_rx_en++;
1947                 }
1948                 if (nb_rx_en > 1)
1949                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
1950                 break;
1951         case ixgbe_fc_rx_pause:
1952                 /*
1953                  * Rx Flow control is enabled and Tx Flow control is
1954                  * disabled by software override. Since there really
1955                  * isn't a way to advertise that we are capable of RX
1956                  * Pause ONLY, we will advertise that we support both
1957                  * symmetric and asymmetric Rx PAUSE.  Later, we will
1958                  * disable the adapter's ability to send PAUSE frames.
1959                  */
1960                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
1961                 /*
1962                  * If the count of enabled RX Priority Flow control >1,
1963                  * and the TX pause can not be disabled
1964                  */
1965                 nb_rx_en = 0;
1966                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
1967                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
1968                         if (reg & IXGBE_FCRTH_FCEN)
1969                                 nb_rx_en++;
1970                 }
1971                 if (nb_rx_en > 1)
1972                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
1973                 break;
1974         case ixgbe_fc_tx_pause:
1975                 /*
1976                  * Tx Flow control is enabled, and Rx Flow control is
1977                  * disabled by software override.
1978                  */
1979                 fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
1980                 break;
1981         case ixgbe_fc_full:
1982                 /* Flow control (both Rx and Tx) is enabled by SW override. */
1983                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
1984                 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
1985                 break;
1986         default:
1987                 DEBUGOUT("Flow control param set incorrectly\n");
1988                 ret_val = IXGBE_ERR_CONFIG;
1989                 goto out;
1990                 break;
1991         }
1992
1993         /* Set 802.3x based flow control settings. */
1994         mflcn_reg |= IXGBE_MFLCN_DPF;
1995         IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg);
1996         IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg);
1997
1998         /* Set up and enable Rx high/low water mark thresholds, enable XON. */
1999         if ((hw->fc.current_mode & ixgbe_fc_tx_pause) &&
2000                 hw->fc.high_water[tc_num]) {
2001                 fcrtl = (hw->fc.low_water[tc_num] << 10) | IXGBE_FCRTL_XONE;
2002                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), fcrtl);
2003                 fcrth = (hw->fc.high_water[tc_num] << 10) | IXGBE_FCRTH_FCEN;
2004         } else {
2005                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), 0);
2006                 /*
2007                  * In order to prevent Tx hangs when the internal Tx
2008                  * switch is enabled we must set the high water mark
2009                  * to the maximum FCRTH value.  This allows the Tx
2010                  * switch to function even under heavy Rx workloads.
2011                  */
2012                 fcrth = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)) - 32;
2013         }
2014         IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(tc_num), fcrth);
2015
2016         /* Configure pause time (2 TCs per register) */
2017         reg = hw->fc.pause_time * 0x00010001;
2018         for (i = 0; i < (IXGBE_DCB_MAX_TRAFFIC_CLASS / 2); i++)
2019                 IXGBE_WRITE_REG(hw, IXGBE_FCTTV(i), reg);
2020
2021         /* Configure flow control refresh threshold value */
2022         IXGBE_WRITE_REG(hw, IXGBE_FCRTV, hw->fc.pause_time / 2);
2023
2024 out:
2025         return ret_val;
2026 }
2027
2028 static int
2029 ixgbe_dcb_pfc_enable(struct rte_eth_dev *dev,uint8_t tc_num)
2030 {
2031         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2032         int32_t ret_val = IXGBE_NOT_IMPLEMENTED;
2033
2034         if(hw->mac.type != ixgbe_mac_82598EB) {
2035                 ret_val = ixgbe_dcb_pfc_enable_generic(hw,tc_num);
2036         }
2037         return ret_val;
2038 }
2039
2040 static int
2041 ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf)
2042 {
2043         int err;
2044         uint32_t rx_buf_size;
2045         uint32_t max_high_water;
2046         uint8_t tc_num;
2047         uint8_t  map[IXGBE_DCB_MAX_USER_PRIORITY] = { 0 };
2048         struct ixgbe_hw *hw =
2049                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2050         struct ixgbe_dcb_config *dcb_config =
2051                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
2052
2053         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
2054                 ixgbe_fc_none,
2055                 ixgbe_fc_rx_pause,
2056                 ixgbe_fc_tx_pause,
2057                 ixgbe_fc_full
2058         };
2059
2060         PMD_INIT_FUNC_TRACE();
2061
2062         ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
2063         tc_num = map[pfc_conf->priority];
2064         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num));
2065         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
2066         /*
2067          * At least reserve one Ethernet frame for watermark
2068          * high_water/low_water in kilo bytes for ixgbe
2069          */
2070         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
2071         if ((pfc_conf->fc.high_water > max_high_water) ||
2072                 (pfc_conf->fc.high_water <= pfc_conf->fc.low_water)) {
2073                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
2074                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
2075                 return (-EINVAL);
2076         }
2077
2078         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[pfc_conf->fc.mode];
2079         hw->fc.pause_time = pfc_conf->fc.pause_time;
2080         hw->fc.send_xon = pfc_conf->fc.send_xon;
2081         hw->fc.low_water[tc_num] =  pfc_conf->fc.low_water;
2082         hw->fc.high_water[tc_num] = pfc_conf->fc.high_water;
2083
2084         err = ixgbe_dcb_pfc_enable(dev,tc_num);
2085
2086         /* Not negotiated is not an error case */
2087         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED))
2088                 return 0;
2089
2090         PMD_INIT_LOG(ERR, "ixgbe_dcb_pfc_enable = 0x%x \n", err);
2091         return -EIO;
2092 }
2093
2094 static void
2095 ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
2096                                 uint32_t index, uint32_t pool)
2097 {
2098         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2099         uint32_t enable_addr = 1;
2100
2101         ixgbe_set_rar(hw, index, mac_addr->addr_bytes, pool, enable_addr);
2102 }
2103
2104 static void
2105 ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
2106 {
2107         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2108
2109         ixgbe_clear_rar(hw, index);
2110 }
2111
2112 /*
2113  * Virtual Function operations
2114  */
2115 static void
2116 ixgbevf_intr_disable(struct ixgbe_hw *hw)
2117 {
2118         PMD_INIT_LOG(DEBUG, "ixgbevf_intr_disable");
2119
2120         /* Clear interrupt mask to stop from interrupts being generated */
2121         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, IXGBE_VF_IRQ_CLEAR_MASK);
2122
2123         IXGBE_WRITE_FLUSH(hw);
2124 }
2125
2126 static int
2127 ixgbevf_dev_configure(struct rte_eth_dev *dev)
2128 {
2129         struct rte_eth_conf* conf = &dev->data->dev_conf;
2130
2131         PMD_INIT_LOG(DEBUG, "\nConfigured Virtual Function port id: %d\n",
2132                 dev->data->port_id);
2133
2134                 /*
2135                  * VF has no ability to enable/disable HW CRC
2136                  * Keep the persistent behavior the same as Host PF
2137                  */
2138 #ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC
2139         if (!conf->rxmode.hw_strip_crc) {
2140                 PMD_INIT_LOG(INFO, "VF can't disable HW CRC Strip\n");
2141                 conf->rxmode.hw_strip_crc = 1;
2142         }
2143 #else
2144         if (conf->rxmode.hw_strip_crc) {
2145                 PMD_INIT_LOG(INFO, "VF can't enable HW CRC Strip\n");
2146                 conf->rxmode.hw_strip_crc = 0;
2147         }
2148 #endif
2149
2150         return 0;
2151 }
2152
2153 static int
2154 ixgbevf_dev_start(struct rte_eth_dev *dev)
2155 {
2156         int err, mask = 0;
2157
2158         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_start");
2159
2160         ixgbevf_dev_tx_init(dev);
2161
2162         /* This can fail when allocating mbufs for descriptor rings */
2163         err = ixgbevf_dev_rx_init(dev);
2164         if (err) {
2165                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware (%d)\n", err);
2166                 ixgbe_dev_clear_queues(dev);
2167                 return err;
2168         }
2169
2170         /* Set vfta */
2171         ixgbevf_set_vfta_all(dev,1);
2172
2173         /* Set HW strip */
2174         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
2175                 ETH_VLAN_EXTEND_MASK;
2176         ixgbevf_vlan_offload_set(dev, mask);
2177
2178         ixgbevf_dev_rxtx_start(dev);
2179
2180         return 0;
2181 }
2182
2183 static void
2184 ixgbevf_dev_stop(struct rte_eth_dev *dev)
2185 {
2186         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2187
2188         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_stop");
2189
2190         hw->adapter_stopped = TRUE;
2191         ixgbe_stop_adapter(hw);
2192
2193         /*
2194           * Clear what we set, but we still keep shadow_vfta to
2195           * restore after device starts
2196           */
2197         ixgbevf_set_vfta_all(dev,0);
2198
2199         ixgbe_dev_clear_queues(dev);
2200 }
2201
2202 static void
2203 ixgbevf_dev_close(struct rte_eth_dev *dev)
2204 {
2205         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2206
2207         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_close");
2208
2209         ixgbe_reset_hw(hw);
2210
2211         ixgbevf_dev_stop(dev);
2212
2213         /* reprogram the RAR[0] in case user changed it. */
2214         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
2215 }
2216
2217 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on)
2218 {
2219         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2220         struct ixgbe_vfta * shadow_vfta =
2221                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2222         int i = 0, j = 0, vfta = 0, mask = 1;
2223
2224         for (i = 0; i < IXGBE_VFTA_SIZE; i++){
2225                 vfta = shadow_vfta->vfta[i];
2226                 if(vfta){
2227                         mask = 1;
2228                         for (j = 0; j < 32; j++){
2229                                 if(vfta & mask)
2230                                         ixgbe_set_vfta(hw, (i<<5)+j, 0, on);
2231                                 mask<<=1;
2232                         }
2233                 }
2234         }
2235
2236 }
2237
2238 static int
2239 ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2240 {
2241         struct ixgbe_hw *hw =
2242                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2243         struct ixgbe_vfta * shadow_vfta =
2244                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2245         uint32_t vid_idx = 0;
2246         uint32_t vid_bit = 0;
2247         int ret = 0;
2248
2249         PMD_INIT_FUNC_TRACE();
2250
2251         /* vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf */
2252         ret = ixgbe_set_vfta(hw, vlan_id, 0, !!on);
2253         if(ret){
2254                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
2255                 return ret;
2256         }
2257         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
2258         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
2259
2260         /* Save what we set and retore it after device reset */
2261         if (on)
2262                 shadow_vfta->vfta[vid_idx] |= vid_bit;
2263         else
2264                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
2265
2266         return 0;
2267 }
2268
2269 static void
2270 ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
2271 {
2272         struct ixgbe_hw *hw =
2273                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2274         uint32_t ctrl;
2275
2276         PMD_INIT_FUNC_TRACE();
2277
2278         if(queue >= hw->mac.max_rx_queues)
2279                 return;
2280
2281         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
2282         if(on)
2283                 ctrl |= IXGBE_RXDCTL_VME;
2284         else
2285                 ctrl &= ~IXGBE_RXDCTL_VME;
2286         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
2287
2288         ixgbe_vlan_hw_strip_bitmap_set( dev, queue, on);
2289 }
2290
2291 static void
2292 ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2293 {
2294         struct ixgbe_hw *hw =
2295                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2296         uint32_t i, on = 0;
2297
2298         /* VF function only support hw strip feature, others are not support */
2299         if(mask & ETH_VLAN_STRIP_MASK){
2300                 on = !!(dev->data->dev_conf.rxmode.hw_vlan_strip);
2301
2302                 for(i=0; i < hw->mac.max_rx_queues; i++)
2303                         ixgbevf_vlan_strip_queue_set(dev,i,on);
2304         }
2305 }
2306