lib/librte_pmd_ixgbe/ixgbe_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2013 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  *
  33  */
  34
  35 #include <sys/queue.h>
  36 #include <stdio.h>
  37 #include <errno.h>
  38 #include <stdint.h>
  39 #include <string.h>
  40 #include <unistd.h>
  41 #include <stdarg.h>
  42 #include <inttypes.h>
  43 #include <rte_byteorder.h>
  44 #include <rte_common.h>
  45 #include <rte_cycles.h>
  46
  47 #include <rte_interrupts.h>
  48 #include <rte_log.h>
  49 #include <rte_debug.h>
  50 #include <rte_pci.h>
  51 #include <rte_atomic.h>
  52 #include <rte_branch_prediction.h>
  53 #include <rte_memory.h>
  54 #include <rte_memzone.h>
  55 #include <rte_tailq.h>
  56 #include <rte_eal.h>
  57 #include <rte_alarm.h>
  58 #include <rte_ether.h>
  59 #include <rte_ethdev.h>
  60 #include <rte_atomic.h>
  61 #include <rte_malloc.h>
  62
  63 #include "ixgbe_logs.h"
  64 #include "ixgbe/ixgbe_api.h"
  65 #include "ixgbe/ixgbe_vf.h"
  66 #include "ixgbe/ixgbe_common.h"
  67 #include "ixgbe_ethdev.h"
  68
  69 /*
  70  * High threshold controlling when to start sending XOFF frames. Must be at
  71  * least 8 bytes less than receive packet buffer size. This value is in units
  72  * of 1024 bytes.
  73  */
  74 #define IXGBE_FC_HI    0x80
  75
  76 /*
  77  * Low threshold controlling when to start sending XON frames. This value is
  78  * in units of 1024 bytes.
  79  */
  80 #define IXGBE_FC_LO    0x40
  81
  82 /* Timer value included in XOFF frames. */
  83 #define IXGBE_FC_PAUSE 0x680
  84
  85 #define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */
  86 #define IXGBE_LINK_UP_CHECK_TIMEOUT   1000 /* ms */
  87
  88 #define IXGBE_QUEUE_STAT_COUNTERS (sizeof(hw_stats->qprc) / sizeof(hw_stats->qprc[0]))
  89
  90 static int eth_ixgbe_dev_init(struct eth_driver *eth_drv,
  91                 struct rte_eth_dev *eth_dev);
  92 static int  ixgbe_dev_configure(struct rte_eth_dev *dev);
  93 static int  ixgbe_dev_start(struct rte_eth_dev *dev);
  94 static void ixgbe_dev_stop(struct rte_eth_dev *dev);
  95 static void ixgbe_dev_close(struct rte_eth_dev *dev);
  96 static void ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev);
  97 static void ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev);
  98 static void ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev);
  99 static void ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev);
 100 static int ixgbe_dev_link_update(struct rte_eth_dev *dev,
 101                                 int wait_to_complete);
 102 static void ixgbe_dev_stats_get(struct rte_eth_dev *dev,
 103                                 struct rte_eth_stats *stats);
 104 static void ixgbe_dev_stats_reset(struct rte_eth_dev *dev);
 105 static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 106                                              uint16_t queue_id,
 107                                              uint8_t stat_idx,
 108                                              uint8_t is_rx);
 109 static void ixgbe_dev_info_get(struct rte_eth_dev *dev,
 110                                 struct rte_eth_dev_info *dev_info);
 111 static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev,
 112                 uint16_t vlan_id, int on);
 113 static void ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 114 static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev,
 115                 uint16_t queue, bool on);
 116 static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue,
 117                 int on);
 118 static void ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 119 static void ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev);
 120 static void ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev);
 121 static void ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev);
 122 static void ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev);
 123 static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue);
 124 static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue);
 125 static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 126 static void ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 127
 128 static int ixgbe_dev_led_on(struct rte_eth_dev *dev);
 129 static int ixgbe_dev_led_off(struct rte_eth_dev *dev);
 130 static int  ixgbe_flow_ctrl_set(struct rte_eth_dev *dev,
 131                 struct rte_eth_fc_conf *fc_conf);
 132 static int ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 133                 struct rte_eth_pfc_conf *pfc_conf);
 134 static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev);
 135 static int ixgbe_dev_interrupt_setup(struct rte_eth_dev *dev);
 136 static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev);
 137 static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev);
 138 static void ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle,
 139                 void *param);
 140 static void ixgbe_dev_interrupt_delayed_handler(void *param);
 141 static void ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 142                 uint32_t index, uint32_t pool);
 143 static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index);
 144 static void ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config);
 145
 146 /* For Virtual Function support */
 147 static int eth_ixgbevf_dev_init(struct eth_driver *eth_drv,
 148                 struct rte_eth_dev *eth_dev);
 149 static int  ixgbevf_dev_configure(struct rte_eth_dev *dev);
 150 static int  ixgbevf_dev_start(struct rte_eth_dev *dev);
 151 static void ixgbevf_dev_stop(struct rte_eth_dev *dev);
 152 static void ixgbevf_dev_close(struct rte_eth_dev *dev);
 153 static void ixgbevf_intr_disable(struct ixgbe_hw *hw);
 154 static void ixgbevf_dev_stats_get(struct rte_eth_dev *dev,
 155                 struct rte_eth_stats *stats);
 156 static void ixgbevf_dev_stats_reset(struct rte_eth_dev *dev);
 157 static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev,
 158                 uint16_t vlan_id, int on);
 159 static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev,
 160                 uint16_t queue, int on);
 161 static void ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 162 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 163
 164
 165 /*
 166  * Define VF Stats MACRO for Non "cleared on read" register
 167  */
 168 #define UPDATE_VF_STAT(reg, last, cur)                          \
 169 {                                                               \
 170         u32 latest = IXGBE_READ_REG(hw, reg);                   \
 171         cur += latest - last;                                   \
 172         last = latest;                                          \
 173 }
 174
 175 #define UPDATE_VF_STAT_36BIT(lsb, msb, last, cur)                \
 176 {                                                                \
 177         u64 new_lsb = IXGBE_READ_REG(hw, lsb);                   \
 178         u64 new_msb = IXGBE_READ_REG(hw, msb);                   \
 179         u64 latest = ((new_msb << 32) | new_lsb);                \
 180         cur += (0x1000000000LL + latest - last) & 0xFFFFFFFFFLL; \
 181         last = latest;                                           \
 182 }
 183
 184 #define IXGBE_SET_HWSTRIP(h, q) do{\
 185                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 186                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 187                 (h)->bitmap[idx] |= 1 << bit;\
 188         }while(0)
 189
 190 #define IXGBE_CLEAR_HWSTRIP(h, q) do{\
 191                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 192                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 193                 (h)->bitmap[idx] &= ~(1 << bit);\
 194         }while(0)
 195
 196 #define IXGBE_GET_HWSTRIP(h, q, r) do{\
 197                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 198                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 199                 (r) = (h)->bitmap[idx] >> bit & 1;\
 200         }while(0)
 201
 202 /*
 203  * The set of PCI devices this driver supports
 204  */
 205 static struct rte_pci_id pci_id_ixgbe_map[] = {
 206
 207 #define RTE_PCI_DEV_ID_DECL_IXGBE(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 208 #include "rte_pci_dev_ids.h"
 209
 210 { .vendor_id = 0, /* sentinel */ },
 211 };
 212
 213
 214 /*
 215  * The set of PCI devices this driver supports (for 82599 VF)
 216  */
 217 static struct rte_pci_id pci_id_ixgbevf_map[] = {
 218
 219 #define RTE_PCI_DEV_ID_DECL_IXGBEVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 220 #include "rte_pci_dev_ids.h"
 221 { .vendor_id = 0, /* sentinel */ },
 222
 223 };
 224
 225 static struct eth_dev_ops ixgbe_eth_dev_ops = {
 226         .dev_configure        = ixgbe_dev_configure,
 227         .dev_start            = ixgbe_dev_start,
 228         .dev_stop             = ixgbe_dev_stop,
 229         .dev_close            = ixgbe_dev_close,
 230         .promiscuous_enable   = ixgbe_dev_promiscuous_enable,
 231         .promiscuous_disable  = ixgbe_dev_promiscuous_disable,
 232         .allmulticast_enable  = ixgbe_dev_allmulticast_enable,
 233         .allmulticast_disable = ixgbe_dev_allmulticast_disable,
 234         .link_update          = ixgbe_dev_link_update,
 235         .stats_get            = ixgbe_dev_stats_get,
 236         .stats_reset          = ixgbe_dev_stats_reset,
 237         .queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set,
 238         .dev_infos_get        = ixgbe_dev_info_get,
 239         .vlan_filter_set      = ixgbe_vlan_filter_set,
 240         .vlan_tpid_set        = ixgbe_vlan_tpid_set,
 241         .vlan_offload_set     = ixgbe_vlan_offload_set,
 242         .vlan_strip_queue_set = ixgbe_vlan_strip_queue_set,
 243         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 244         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 245         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 246         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 247         .dev_led_on           = ixgbe_dev_led_on,
 248         .dev_led_off          = ixgbe_dev_led_off,
 249         .flow_ctrl_set        = ixgbe_flow_ctrl_set,
 250         .priority_flow_ctrl_set = ixgbe_priority_flow_ctrl_set,
 251         .mac_addr_add         = ixgbe_add_rar,
 252         .mac_addr_remove      = ixgbe_remove_rar,
 253         .fdir_add_signature_filter    = ixgbe_fdir_add_signature_filter,
 254         .fdir_update_signature_filter = ixgbe_fdir_update_signature_filter,
 255         .fdir_remove_signature_filter = ixgbe_fdir_remove_signature_filter,
 256         .fdir_infos_get               = ixgbe_fdir_info_get,
 257         .fdir_add_perfect_filter      = ixgbe_fdir_add_perfect_filter,
 258         .fdir_update_perfect_filter   = ixgbe_fdir_update_perfect_filter,
 259         .fdir_remove_perfect_filter   = ixgbe_fdir_remove_perfect_filter,
 260         .fdir_set_masks               = ixgbe_fdir_set_masks,
 261 };
 262
 263 /*
 264  * dev_ops for virtual function, bare necessities for basic vf
 265  * operation have been implemented
 266  */
 267 static struct eth_dev_ops ixgbevf_eth_dev_ops = {
 268
 269         .dev_configure        = ixgbevf_dev_configure,
 270         .dev_start            = ixgbevf_dev_start,
 271         .dev_stop             = ixgbevf_dev_stop,
 272         .link_update          = ixgbe_dev_link_update,
 273         .stats_get            = ixgbevf_dev_stats_get,
 274         .stats_reset          = ixgbevf_dev_stats_reset,
 275         .dev_close            = ixgbevf_dev_close,
 276         .dev_infos_get        = ixgbe_dev_info_get,
 277         .vlan_filter_set      = ixgbevf_vlan_filter_set,
 278         .vlan_strip_queue_set = ixgbevf_vlan_strip_queue_set,
 279         .vlan_offload_set     = ixgbevf_vlan_offload_set,
 280         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 281         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 282         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 283         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 284 };
 285
 286 /**
 287  * Atomically reads the link status information from global
 288  * structure rte_eth_dev.
 289  *
 290  * @param dev
 291  *   - Pointer to the structure rte_eth_dev to read from.
 292  *   - Pointer to the buffer to be saved with the link status.
 293  *
 294  * @return
 295  *   - On success, zero.
 296  *   - On failure, negative value.
 297  */
 298 static inline int
 299 rte_ixgbe_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 300                                 struct rte_eth_link *link)
 301 {
 302         struct rte_eth_link *dst = link;
 303         struct rte_eth_link *src = &(dev->data->dev_link);
 304
 305         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 306                                         *(uint64_t *)src) == 0)
 307                 return -1;
 308
 309         return 0;
 310 }
 311
 312 /**
 313  * Atomically writes the link status information into global
 314  * structure rte_eth_dev.
 315  *
 316  * @param dev
 317  *   - Pointer to the structure rte_eth_dev to read from.
 318  *   - Pointer to the buffer to be saved with the link status.
 319  *
 320  * @return
 321  *   - On success, zero.
 322  *   - On failure, negative value.
 323  */
 324 static inline int
 325 rte_ixgbe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 326                                 struct rte_eth_link *link)
 327 {
 328         struct rte_eth_link *dst = &(dev->data->dev_link);
 329         struct rte_eth_link *src = link;
 330
 331         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 332                                         *(uint64_t *)src) == 0)
 333                 return -1;
 334
 335         return 0;
 336 }
 337
 338 /*
 339  * This function is the same as ixgbe_is_sfp() in ixgbe/ixgbe.h.
 340  */
 341 static inline int
 342 ixgbe_is_sfp(struct ixgbe_hw *hw)
 343 {
 344         switch (hw->phy.type) {
 345         case ixgbe_phy_sfp_avago:
 346         case ixgbe_phy_sfp_ftl:
 347         case ixgbe_phy_sfp_intel:
 348         case ixgbe_phy_sfp_unknown:
 349         case ixgbe_phy_sfp_passive_tyco:
 350         case ixgbe_phy_sfp_passive_unknown:
 351                 return 1;
 352         default:
 353                 return 0;
 354         }
 355 }
 356
 357 /*
 358  * This function is based on ixgbe_disable_intr() in ixgbe/ixgbe.h.
 359  */
 360 static void
 361 ixgbe_disable_intr(struct ixgbe_hw *hw)
 362 {
 363         PMD_INIT_FUNC_TRACE();
 364
 365         if (hw->mac.type == ixgbe_mac_82598EB) {
 366                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ~0);
 367         } else {
 368                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xFFFF0000);
 369                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), ~0);
 370                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), ~0);
 371         }
 372         IXGBE_WRITE_FLUSH(hw);
 373 }
 374
 375 /*
 376  * This function resets queue statistics mapping registers.
 377  * From Niantic datasheet, Initialization of Statistics section:
 378  * "...if software requires the queue counters, the RQSMR and TQSM registers
 379  * must be re-programmed following a device reset.
 380  */
 381 static void
 382 ixgbe_reset_qstat_mappings(struct ixgbe_hw *hw)
 383 {
 384         uint32_t i;
 385
 386         for(i = 0; i != IXGBE_NB_STAT_MAPPING_REGS; i++) {
 387                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), 0);
 388                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), 0);
 389         }
 390 }
 391
 392
 393 static int
 394 ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 395                                   uint16_t queue_id,
 396                                   uint8_t stat_idx,
 397                                   uint8_t is_rx)
 398 {
 399 #define QSM_REG_NB_BITS_PER_QMAP_FIELD 8
 400 #define NB_QMAP_FIELDS_PER_QSM_REG 4
 401 #define QMAP_FIELD_RESERVED_BITS_MASK 0x0f
 402
 403         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 404         struct ixgbe_stat_mapping_registers *stat_mappings =
 405                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(eth_dev->data->dev_private);
 406         uint32_t qsmr_mask = 0;
 407         uint32_t clearing_mask = QMAP_FIELD_RESERVED_BITS_MASK;
 408         uint32_t q_map;
 409         uint8_t n, offset;
 410
 411         if ((hw->mac.type != ixgbe_mac_82599EB) && (hw->mac.type != ixgbe_mac_X540))
 412                 return -ENOSYS;
 413
 414         PMD_INIT_LOG(INFO, "Setting port %d, %s queue_id %d to stat index %d\n",
 415                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx);
 416
 417         n = queue_id / NB_QMAP_FIELDS_PER_QSM_REG;
 418         if (n >= IXGBE_NB_STAT_MAPPING_REGS) {
 419                 PMD_INIT_LOG(ERR, "Nb of stat mapping registers exceeded\n");
 420                 return -EIO;
 421         }
 422         offset = queue_id % NB_QMAP_FIELDS_PER_QSM_REG;
 423
 424         /* Now clear any previous stat_idx set */
 425         clearing_mask <<= (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 426         if (!is_rx)
 427                 stat_mappings->tqsm[n] &= ~clearing_mask;
 428         else
 429                 stat_mappings->rqsmr[n] &= ~clearing_mask;
 430
 431         q_map = (uint32_t)stat_idx;
 432         q_map &= QMAP_FIELD_RESERVED_BITS_MASK;
 433         qsmr_mask = q_map << (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 434         if (!is_rx)
 435                 stat_mappings->tqsm[n] |= qsmr_mask;
 436         else
 437                 stat_mappings->rqsmr[n] |= qsmr_mask;
 438
 439         PMD_INIT_LOG(INFO, "Set port %d, %s queue_id %d to stat index %d\n"
 440                      "%s[%d] = 0x%08x\n",
 441                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx,
 442                      is_rx ? "RQSMR" : "TQSM",n, is_rx ? stat_mappings->rqsmr[n] : stat_mappings->tqsm[n]);
 443
 444         /* Now write the mapping in the appropriate register */
 445         if (is_rx) {
 446                 PMD_INIT_LOG(INFO, "Write 0x%x to RX IXGBE stat mapping reg:%d\n",
 447                              stat_mappings->rqsmr[n], n);
 448                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(n), stat_mappings->rqsmr[n]);
 449         }
 450         else {
 451                 PMD_INIT_LOG(INFO, "Write 0x%x to TX IXGBE stat mapping reg:%d\n",
 452                              stat_mappings->tqsm[n], n);
 453                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(n), stat_mappings->tqsm[n]);
 454         }
 455         return 0;
 456 }
 457
 458 static void
 459 ixgbe_restore_statistics_mapping(struct rte_eth_dev * dev)
 460 {
 461         struct ixgbe_stat_mapping_registers *stat_mappings =
 462                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(dev->data->dev_private);
 463         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 464         int i;
 465
 466         /* write whatever was in stat mapping table to the NIC */
 467         for (i = 0; i < IXGBE_NB_STAT_MAPPING_REGS; i++) {
 468                 /* rx */
 469                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), stat_mappings->rqsmr[i]);
 470
 471                 /* tx */
 472                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), stat_mappings->tqsm[i]);
 473         }
 474 }
 475
 476 static void
 477 ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config)
 478 {
 479         uint8_t i;
 480         struct ixgbe_dcb_tc_config *tc;
 481         int dcb_max_tc = IXGBE_DCB_MAX_TRAFFIC_CLASS;
 482
 483         dcb_config->num_tcs.pg_tcs = dcb_max_tc;
 484         dcb_config->num_tcs.pfc_tcs = dcb_max_tc;
 485         for (i = 0; i < dcb_max_tc; i++) {
 486                 tc = &dcb_config->tc_config[i];
 487                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_id = i;
 488                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = 100/dcb_max_tc + (i & 1);
 489                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_id = i;
 490                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent = 100/dcb_max_tc + (i & 1);
 491                 tc->pfc = ixgbe_dcb_pfc_disabled;
 492         }
 493
 494         /* Initialize default user to priority mapping, UPx->TC0 */
 495         tc = &dcb_config->tc_config[0];
 496         tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
 497         tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
 498         for (i = 0; i< IXGBE_DCB_MAX_BW_GROUP; i++) {
 499                 dcb_config->bw_percentage[IXGBE_DCB_TX_CONFIG][i] = 100;
 500                 dcb_config->bw_percentage[IXGBE_DCB_RX_CONFIG][i] = 100;
 501         }
 502         dcb_config->rx_pba_cfg = ixgbe_dcb_pba_equal;
 503         dcb_config->pfc_mode_enable = false;
 504         dcb_config->vt_mode = true;
 505         dcb_config->round_robin_enable = false;
 506         /* support all DCB capabilities in 82599 */
 507         dcb_config->support.capabilities = 0xFF;
 508
 509         /*we only support 4 Tcs for X540*/
 510         if (hw->mac.type == ixgbe_mac_X540) {
 511                 dcb_config->num_tcs.pg_tcs = 4;
 512                 dcb_config->num_tcs.pfc_tcs = 4;
 513         }
 514 }
 515
 516 /*
 517  * This function is based on code in ixgbe_attach() in ixgbe/ixgbe.c.
 518  * It returns 0 on success.
 519  */
 520 static int
 521 eth_ixgbe_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 522                      struct rte_eth_dev *eth_dev)
 523 {
 524         struct rte_pci_device *pci_dev;
 525         struct ixgbe_hw *hw =
 526                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 527         struct ixgbe_vfta * shadow_vfta =
 528                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 529         struct ixgbe_hwstrip *hwstrip =
 530                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 531         struct ixgbe_dcb_config *dcb_config =
 532                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private);
 533         uint32_t ctrl_ext;
 534         uint16_t csum;
 535         int diag, i;
 536
 537         PMD_INIT_FUNC_TRACE();
 538
 539         eth_dev->dev_ops = &ixgbe_eth_dev_ops;
 540         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
 541         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
 542
 543         /* for secondary processes, we don't initialise any further as primary
 544          * has already done this work. Only check we don't need a different
 545          * RX function */
 546         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 547                 if (eth_dev->data->scattered_rx)
 548                         eth_dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
 549                 return 0;
 550         }
 551         pci_dev = eth_dev->pci_dev;
 552
 553         /* Vendor and Device ID need to be set before init of shared code */
 554         hw->device_id = pci_dev->id.device_id;
 555         hw->vendor_id = pci_dev->id.vendor_id;
 556         hw->hw_addr = (void *)pci_dev->mem_resource.addr;
 557
 558         /* Initialize the shared code */
 559         diag = ixgbe_init_shared_code(hw);
 560         if (diag != IXGBE_SUCCESS) {
 561                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
 562                 return -EIO;
 563         }
 564
 565         /* Initialize DCB configuration*/
 566         memset(dcb_config, 0, sizeof(struct ixgbe_dcb_config));
 567         ixgbe_dcb_init(hw,dcb_config);
 568         /* Get Hardware Flow Control setting */
 569         hw->fc.requested_mode = ixgbe_fc_full;
 570         hw->fc.current_mode = ixgbe_fc_full;
 571         hw->fc.pause_time = IXGBE_FC_PAUSE;
 572         for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
 573                 hw->fc.low_water[i] = IXGBE_FC_LO;
 574                 hw->fc.high_water[i] = IXGBE_FC_HI;
 575         }
 576         hw->fc.send_xon = 1;
 577
 578         ixgbe_disable_intr(hw);
 579
 580         /* Make sure we have a good EEPROM before we read from it */
 581         diag = ixgbe_validate_eeprom_checksum(hw, &csum);
 582         if (diag != IXGBE_SUCCESS) {
 583                 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag);
 584                 return -EIO;
 585         }
 586
 587         diag = ixgbe_init_hw(hw);
 588
 589         /*
 590          * Devices with copper phys will fail to initialise if ixgbe_init_hw()
 591          * is called too soon after the kernel driver unbinding/binding occurs.
 592          * The failure occurs in ixgbe_identify_phy_generic() for all devices,
 593          * but for non-copper devies, ixgbe_identify_sfp_module_generic() is
 594          * also called. See ixgbe_identify_phy_82599(). The reason for the
 595          * failure is not known, and only occuts when virtualisation features
 596          * are disabled in the bios. A delay of 100ms  was found to be enough by
 597          * trial-and-error, and is doubled to be safe.
 598          */
 599         if (diag && (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper)) {
 600                 rte_delay_ms(200);
 601                 diag = ixgbe_init_hw(hw);
 602         }
 603
 604         if (diag == IXGBE_ERR_EEPROM_VERSION) {
 605                 PMD_INIT_LOG(ERR, "This device is a pre-production adapter/"
 606                     "LOM.  Please be aware there may be issues associated "
 607                     "with your hardware.\n If you are experiencing problems "
 608                     "please contact your Intel or hardware representative "
 609                     "who provided you with this hardware.\n");
 610         } else if (diag == IXGBE_ERR_SFP_NOT_SUPPORTED)
 611                 PMD_INIT_LOG(ERR, "Unsupported SFP+ Module\n");
 612         if (diag) {
 613                 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag);
 614                 return -EIO;
 615         }
 616
 617         /* pick up the PCI bus settings for reporting later */
 618         ixgbe_get_bus_info(hw);
 619
 620         /* reset mappings for queue statistics hw counters*/
 621         ixgbe_reset_qstat_mappings(hw);
 622
 623         /* Allocate memory for storing MAC addresses */
 624         eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 625                         hw->mac.num_rar_entries, 0);
 626         if (eth_dev->data->mac_addrs == NULL) {
 627                 PMD_INIT_LOG(ERR,
 628                         "Failed to allocate %d bytes needed to store MAC addresses",
 629                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 630                 return -ENOMEM;
 631         }
 632         /* Copy the permanent MAC address */
 633         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 634                         &eth_dev->data->mac_addrs[0]);
 635
 636         /* initialize the vfta */
 637         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 638
 639         /* initialize the hw strip bitmap*/
 640         memset(hwstrip, 0, sizeof(*hwstrip));
 641
 642         /* let hardware know driver is loaded */
 643         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 644         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 645         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 646
 647         if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
 648                 PMD_INIT_LOG(DEBUG,
 649                              "MAC: %d, PHY: %d, SFP+: %d<n",
 650                              (int) hw->mac.type, (int) hw->phy.type,
 651                              (int) hw->phy.sfp_type);
 652         else
 653                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d\n",
 654                              (int) hw->mac.type, (int) hw->phy.type);
 655
 656         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
 657                         eth_dev->data->port_id, pci_dev->id.vendor_id,
 658                         pci_dev->id.device_id);
 659
 660         rte_intr_callback_register(&(pci_dev->intr_handle),
 661                 ixgbe_dev_interrupt_handler, (void *)eth_dev);
 662
 663         return 0;
 664 }
 665
 666 /*
 667  * Virtual Function device init
 668  */
 669 static int
 670 eth_ixgbevf_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 671                      struct rte_eth_dev *eth_dev)
 672 {
 673         struct rte_pci_device *pci_dev;
 674         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 675         int diag;
 676         struct ixgbe_vfta * shadow_vfta =
 677                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 678         struct ixgbe_hwstrip *hwstrip =
 679                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 680
 681         PMD_INIT_LOG(DEBUG, "eth_ixgbevf_dev_init");
 682
 683         eth_dev->dev_ops = &ixgbevf_eth_dev_ops;
 684         pci_dev = eth_dev->pci_dev;
 685
 686         hw->device_id = pci_dev->id.device_id;
 687         hw->vendor_id = pci_dev->id.vendor_id;
 688         hw->hw_addr = (void *)pci_dev->mem_resource.addr;
 689
 690         /* initialize the vfta */
 691         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 692
 693         /* initialize the hw strip bitmap*/
 694         memset(hwstrip, 0, sizeof(*hwstrip));
 695
 696         /* Initialize the shared code */
 697         diag = ixgbe_init_shared_code(hw);
 698         if (diag != IXGBE_SUCCESS) {
 699                 PMD_INIT_LOG(ERR, "Shared code init failed for ixgbevf: %d", diag);
 700                 return -EIO;
 701         }
 702
 703         /* init_mailbox_params */
 704         hw->mbx.ops.init_params(hw);
 705
 706         /* Disable the interrupts for VF */
 707         ixgbevf_intr_disable(hw);
 708
 709         hw->mac.num_rar_entries = hw->mac.max_rx_queues;
 710         diag = hw->mac.ops.reset_hw(hw);
 711         if (diag != IXGBE_SUCCESS) {
 712                 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
 713                 return (diag);
 714         }
 715
 716         /* Allocate memory for storing MAC addresses */
 717         eth_dev->data->mac_addrs = rte_zmalloc("ixgbevf", ETHER_ADDR_LEN *
 718                         hw->mac.num_rar_entries, 0);
 719         if (eth_dev->data->mac_addrs == NULL) {
 720                 PMD_INIT_LOG(ERR,
 721                         "Failed to allocate %d bytes needed to store MAC addresses",
 722                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 723                 return -ENOMEM;
 724         }
 725         /* Copy the permanent MAC address */
 726         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 727                         &eth_dev->data->mac_addrs[0]);
 728
 729         /* reset the hardware with the new settings */
 730         diag = hw->mac.ops.start_hw(hw);
 731         switch (diag) {
 732                 case  0:
 733                         break;
 734
 735                 default:
 736                         PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
 737                         return (-EIO);
 738         }
 739
 740         PMD_INIT_LOG(DEBUG, "\nport %d vendorID=0x%x deviceID=0x%x mac.type=%s\n",
 741                          eth_dev->data->port_id, pci_dev->id.vendor_id, pci_dev->id.device_id,
 742                          "ixgbe_mac_82599_vf");
 743
 744         return 0;
 745 }
 746
 747 static struct eth_driver rte_ixgbe_pmd = {
 748         {
 749                 .name = "rte_ixgbe_pmd",
 750                 .id_table = pci_id_ixgbe_map,
 751                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
 752         },
 753         .eth_dev_init = eth_ixgbe_dev_init,
 754         .dev_private_size = sizeof(struct ixgbe_adapter),
 755 };
 756
 757 /*
 758  * virtual function driver struct
 759  */
 760 static struct eth_driver rte_ixgbevf_pmd = {
 761         {
 762                 .name = "rte_ixgbevf_pmd",
 763                 .id_table = pci_id_ixgbevf_map,
 764                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
 765         },
 766         .eth_dev_init = eth_ixgbevf_dev_init,
 767         .dev_private_size = sizeof(struct ixgbe_adapter),
 768 };
 769
 770 /*
 771  * Driver initialization routine.
 772  * Invoked once at EAL init time.
 773  * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
 774  */
 775 int
 776 rte_ixgbe_pmd_init(void)
 777 {
 778         PMD_INIT_FUNC_TRACE();
 779
 780         rte_eth_driver_register(&rte_ixgbe_pmd);
 781         return 0;
 782 }
 783
 784 /*
 785  * VF Driver initialization routine.
 786  * Invoked one at EAL init time.
 787  * Register itself as the [Virtual Poll Mode] Driver of PCI niantic devices.
 788  */
 789 int
 790 rte_ixgbevf_pmd_init(void)
 791 {
 792         DEBUGFUNC("rte_ixgbevf_pmd_init");
 793
 794         rte_eth_driver_register(&rte_ixgbevf_pmd);
 795         return (0);
 796 }
 797
 798 static int
 799 ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 800 {
 801         struct ixgbe_hw *hw =
 802                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 803         struct ixgbe_vfta * shadow_vfta =
 804                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
 805         uint32_t vfta;
 806         uint32_t vid_idx;
 807         uint32_t vid_bit;
 808
 809         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
 810         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
 811         vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(vid_idx));
 812         if (on)
 813                 vfta |= vid_bit;
 814         else
 815                 vfta &= ~vid_bit;
 816         IXGBE_WRITE_REG(hw, IXGBE_VFTA(vid_idx), vfta);
 817
 818         /* update local VFTA copy */
 819         shadow_vfta->vfta[vid_idx] = vfta;
 820
 821         return 0;
 822 }
 823
 824 static void
 825 ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
 826 {
 827         if (on)
 828                 ixgbe_vlan_hw_strip_enable(dev, queue);
 829         else
 830                 ixgbe_vlan_hw_strip_disable(dev, queue);
 831 }
 832
 833 static void
 834 ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
 835 {
 836         struct ixgbe_hw *hw =
 837                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 838
 839         /* Only the high 16-bits is valid */
 840         IXGBE_WRITE_REG(hw, IXGBE_EXVET, tpid << 16);
 841 }
 842
 843 static void
 844 ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev)
 845 {
 846         struct ixgbe_hw *hw =
 847                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 848         uint32_t vlnctrl;
 849
 850         PMD_INIT_FUNC_TRACE();
 851
 852         /* Filter Table Disable */
 853         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 854         vlnctrl &= ~IXGBE_VLNCTRL_VFE;
 855
 856         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
 857 }
 858
 859 static void
 860 ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev)
 861 {
 862         struct ixgbe_hw *hw =
 863                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 864         struct ixgbe_vfta * shadow_vfta =
 865                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
 866         uint32_t vlnctrl;
 867         uint16_t i;
 868
 869         PMD_INIT_FUNC_TRACE();
 870
 871         /* Filter Table Enable */
 872         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 873         vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
 874         vlnctrl |= IXGBE_VLNCTRL_VFE;
 875
 876         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
 877
 878         /* write whatever is in local vfta copy */
 879         for (i = 0; i < IXGBE_VFTA_SIZE; i++)
 880                 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), shadow_vfta->vfta[i]);
 881 }
 882
 883 static void
 884 ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on)
 885 {
 886         struct ixgbe_hwstrip *hwstrip =
 887                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(dev->data->dev_private);
 888
 889         if(queue >= IXGBE_MAX_RX_QUEUE_NUM)
 890                 return;
 891
 892         if (on)
 893                 IXGBE_SET_HWSTRIP(hwstrip, queue);
 894         else
 895                 IXGBE_CLEAR_HWSTRIP(hwstrip, queue);
 896 }
 897
 898 static void
 899 ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue)
 900 {
 901         struct ixgbe_hw *hw =
 902                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 903         uint32_t ctrl;
 904
 905         PMD_INIT_FUNC_TRACE();
 906
 907         if (hw->mac.type == ixgbe_mac_82598EB) {
 908                 /* No queue level support */
 909                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
 910                 return;
 911         }
 912         else {
 913                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 914                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
 915                 ctrl &= ~IXGBE_RXDCTL_VME;
 916                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
 917         }
 918         /* record those setting for HW strip per queue */
 919         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 0);
 920 }
 921
 922 static void
 923 ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue)
 924 {
 925         struct ixgbe_hw *hw =
 926                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 927         uint32_t ctrl;
 928
 929         PMD_INIT_FUNC_TRACE();
 930
 931         if (hw->mac.type == ixgbe_mac_82598EB) {
 932                 /* No queue level supported */
 933                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
 934                 return;
 935         }
 936         else {
 937                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 938                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
 939                 ctrl |= IXGBE_RXDCTL_VME;
 940                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
 941         }
 942         /* record those setting for HW strip per queue */
 943         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 1);
 944 }
 945
 946 static void
 947 ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev)
 948 {
 949         struct ixgbe_hw *hw =
 950                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 951         uint32_t ctrl;
 952         uint16_t i;
 953
 954         PMD_INIT_FUNC_TRACE();
 955
 956         if (hw->mac.type == ixgbe_mac_82598EB) {
 957                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 958                 ctrl &= ~IXGBE_VLNCTRL_VME;
 959                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
 960         }
 961         else {
 962                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 963                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
 964                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
 965                         ctrl &= ~IXGBE_RXDCTL_VME;
 966                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
 967
 968                         /* record those setting for HW strip per queue */
 969                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 0);
 970                 }
 971         }
 972 }
 973
 974 static void
 975 ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev)
 976 {
 977         struct ixgbe_hw *hw =
 978                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 979         uint32_t ctrl;
 980         uint16_t i;
 981
 982         PMD_INIT_FUNC_TRACE();
 983
 984         if (hw->mac.type == ixgbe_mac_82598EB) {
 985                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 986                 ctrl |= IXGBE_VLNCTRL_VME;
 987                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
 988         }
 989         else {
 990                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 991                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
 992                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
 993                         ctrl |= IXGBE_RXDCTL_VME;
 994                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
 995
 996                         /* record those setting for HW strip per queue */
 997                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 1);
 998                 }
 999         }
1000 }
1001
1002 static void
1003 ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1004 {
1005         struct ixgbe_hw *hw =
1006                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1007         uint32_t ctrl;
1008
1009         PMD_INIT_FUNC_TRACE();
1010
1011         /* DMATXCTRL: Geric Double VLAN Disable */
1012         ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1013         ctrl &= ~IXGBE_DMATXCTL_GDV;
1014         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1015
1016         /* CTRL_EXT: Global Double VLAN Disable */
1017         ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1018         ctrl &= ~IXGBE_EXTENDED_VLAN;
1019         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1020
1021 }
1022
1023 static void
1024 ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1025 {
1026         struct ixgbe_hw *hw =
1027                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1028         uint32_t ctrl;
1029
1030         PMD_INIT_FUNC_TRACE();
1031
1032         /* DMATXCTRL: Geric Double VLAN Enable */
1033         ctrl  = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1034         ctrl |= IXGBE_DMATXCTL_GDV;
1035         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1036
1037         /* CTRL_EXT: Global Double VLAN Enable */
1038         ctrl  = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1039         ctrl |= IXGBE_EXTENDED_VLAN;
1040         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1041
1042         /*
1043          * VET EXT field in the EXVET register = 0x8100 by default
1044          * So no need to change. Same to VT field of DMATXCTL register
1045          */
1046 }
1047
1048 static void
1049 ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1050 {
1051         if(mask & ETH_VLAN_STRIP_MASK){
1052                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1053                         ixgbe_vlan_hw_strip_enable_all(dev);
1054                 else
1055                         ixgbe_vlan_hw_strip_disable_all(dev);
1056         }
1057
1058         if(mask & ETH_VLAN_FILTER_MASK){
1059                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1060                         ixgbe_vlan_hw_filter_enable(dev);
1061                 else
1062                         ixgbe_vlan_hw_filter_disable(dev);
1063         }
1064
1065         if(mask & ETH_VLAN_EXTEND_MASK){
1066                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1067                         ixgbe_vlan_hw_extend_enable(dev);
1068                 else
1069                         ixgbe_vlan_hw_extend_disable(dev);
1070         }
1071 }
1072
1073 static int
1074 ixgbe_dev_configure(struct rte_eth_dev *dev)
1075 {
1076         struct ixgbe_interrupt *intr =
1077                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1078
1079         PMD_INIT_FUNC_TRACE();
1080
1081         /* set flag to update link status after init */
1082         intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1083
1084         return 0;
1085 }
1086
1087 /*
1088  * Configure device link speed and setup link.
1089  * It returns 0 on success.
1090  */
1091 static int
1092 ixgbe_dev_start(struct rte_eth_dev *dev)
1093 {
1094         struct ixgbe_hw *hw =
1095                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1096         int err, link_up = 0, negotiate = 0;
1097         uint32_t speed = 0;
1098         int mask = 0;
1099
1100         PMD_INIT_FUNC_TRACE();
1101
1102         /* IXGBE devices don't support half duplex */
1103         if ((dev->data->dev_conf.link_duplex != ETH_LINK_AUTONEG_DUPLEX) &&
1104                         (dev->data->dev_conf.link_duplex != ETH_LINK_FULL_DUPLEX)) {
1105                 PMD_INIT_LOG(ERR, "Invalid link_duplex (%u) for port %u\n",
1106                                 dev->data->dev_conf.link_duplex,
1107                                 dev->data->port_id);
1108                 return -EINVAL;
1109         }
1110
1111         /* stop adapter */
1112         hw->adapter_stopped = FALSE;
1113         ixgbe_stop_adapter(hw);
1114
1115         /* reinitialize adapter
1116          * this calls reset and start */
1117         ixgbe_init_hw(hw);
1118
1119         /* initialize transmission unit */
1120         ixgbe_dev_tx_init(dev);
1121
1122         /* This can fail when allocating mbufs for descriptor rings */
1123         err = ixgbe_dev_rx_init(dev);
1124         if (err) {
1125                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware\n");
1126                 goto error;
1127         }
1128
1129         ixgbe_dev_rxtx_start(dev);
1130
1131         if (ixgbe_is_sfp(hw) && hw->phy.multispeed_fiber) {
1132                 err = hw->mac.ops.setup_sfp(hw);
1133                 if (err)
1134                         goto error;
1135         }
1136
1137         /* Turn on the laser */
1138         if (hw->phy.multispeed_fiber)
1139                 ixgbe_enable_tx_laser(hw);
1140
1141         err = ixgbe_check_link(hw, &speed, &link_up, 0);
1142         if (err)
1143                 goto error;
1144         err = ixgbe_get_link_capabilities(hw, &speed, &negotiate);
1145         if (err)
1146                 goto error;
1147
1148         switch(dev->data->dev_conf.link_speed) {
1149         case ETH_LINK_SPEED_AUTONEG:
1150                 speed = (hw->mac.type != ixgbe_mac_82598EB) ?
1151                                 IXGBE_LINK_SPEED_82599_AUTONEG :
1152                                 IXGBE_LINK_SPEED_82598_AUTONEG;
1153                 break;
1154         case ETH_LINK_SPEED_100:
1155                 /*
1156                  * Invalid for 82598 but error will be detected by
1157                  * ixgbe_setup_link()
1158                  */
1159                 speed = IXGBE_LINK_SPEED_100_FULL;
1160                 break;
1161         case ETH_LINK_SPEED_1000:
1162                 speed = IXGBE_LINK_SPEED_1GB_FULL;
1163                 break;
1164         case ETH_LINK_SPEED_10000:
1165                 speed = IXGBE_LINK_SPEED_10GB_FULL;
1166                 break;
1167         default:
1168                 PMD_INIT_LOG(ERR, "Invalid link_speed (%u) for port %u\n",
1169                                 dev->data->dev_conf.link_speed, dev->data->port_id);
1170                 goto error;
1171         }
1172
1173         err = ixgbe_setup_link(hw, speed, negotiate, link_up);
1174         if (err)
1175                 goto error;
1176
1177         /* check if lsc interrupt is enabled */
1178         if (dev->data->dev_conf.intr_conf.lsc != 0) {
1179                 err = ixgbe_dev_interrupt_setup(dev);
1180                 if (err)
1181                         goto error;
1182         }
1183
1184         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
1185                 ETH_VLAN_EXTEND_MASK;
1186         ixgbe_vlan_offload_set(dev, mask);
1187
1188         /* Configure DCB hw */
1189         ixgbe_configure_dcb(dev);
1190
1191         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) {
1192                 err = ixgbe_fdir_configure(dev);
1193                 if (err)
1194                         goto error;
1195         }
1196
1197         ixgbe_restore_statistics_mapping(dev);
1198
1199         return (0);
1200
1201 error:
1202         PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err);
1203         ixgbe_dev_clear_queues(dev);
1204         return -EIO;
1205 }
1206
1207 /*
1208  * Stop device: disable rx and tx functions to allow for reconfiguring.
1209  */
1210 static void
1211 ixgbe_dev_stop(struct rte_eth_dev *dev)
1212 {
1213         struct rte_eth_link link;
1214         struct ixgbe_hw *hw =
1215                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1216
1217         PMD_INIT_FUNC_TRACE();
1218
1219         /* disable interrupts */
1220         ixgbe_disable_intr(hw);
1221
1222         /* reset the NIC */
1223         ixgbe_reset_hw(hw);
1224         hw->adapter_stopped = FALSE;
1225
1226         /* stop adapter */
1227         ixgbe_stop_adapter(hw);
1228
1229         /* Turn off the laser */
1230         if (hw->phy.multispeed_fiber)
1231                 ixgbe_disable_tx_laser(hw);
1232
1233         ixgbe_dev_clear_queues(dev);
1234
1235         /* Clear recorded link status */
1236         memset(&link, 0, sizeof(link));
1237         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1238 }
1239
1240 /*
1241  * Reest and stop device.
1242  */
1243 static void
1244 ixgbe_dev_close(struct rte_eth_dev *dev)
1245 {
1246         struct ixgbe_hw *hw =
1247                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1248
1249         PMD_INIT_FUNC_TRACE();
1250
1251         ixgbe_reset_hw(hw);
1252
1253
1254         ixgbe_dev_stop(dev);
1255         hw->adapter_stopped = 1;
1256
1257         ixgbe_disable_pcie_master(hw);
1258
1259         /* reprogram the RAR[0] in case user changed it. */
1260         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
1261 }
1262
1263 /*
1264  * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c
1265  */
1266 static void
1267 ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1268 {
1269         struct ixgbe_hw *hw =
1270                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1271         struct ixgbe_hw_stats *hw_stats =
1272                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1273         uint32_t bprc, lxon, lxoff, total;
1274         uint64_t total_missed_rx, total_qbrc, total_qprc;
1275         unsigned i;
1276
1277         total_missed_rx = 0;
1278         total_qbrc = 0;
1279         total_qprc = 0;
1280
1281         hw_stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
1282         hw_stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
1283         hw_stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
1284         hw_stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
1285
1286         for (i = 0; i < 8; i++) {
1287                 uint32_t mp;
1288                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
1289                 /* global total per queue */
1290                 hw_stats->mpc[i] += mp;
1291                 /* Running comprehensive total for stats display */
1292                 total_missed_rx += hw_stats->mpc[i];
1293                 if (hw->mac.type == ixgbe_mac_82598EB)
1294                         hw_stats->rnbc[i] +=
1295                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
1296                 hw_stats->pxontxc[i] +=
1297                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
1298                 hw_stats->pxonrxc[i] +=
1299                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
1300                 hw_stats->pxofftxc[i] +=
1301                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
1302                 hw_stats->pxoffrxc[i] +=
1303                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
1304                 hw_stats->pxon2offc[i] +=
1305                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
1306         }
1307         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1308                 hw_stats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
1309                 hw_stats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
1310                 hw_stats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
1311                 hw_stats->qbrc[i] +=
1312                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32);
1313                 hw_stats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
1314                 hw_stats->qbtc[i] +=
1315                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32);
1316                 hw_stats->qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
1317
1318                 total_qprc += hw_stats->qprc[i];
1319                 total_qbrc += hw_stats->qbrc[i];
1320         }
1321         hw_stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
1322         hw_stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
1323         hw_stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
1324
1325         /* Note that gprc counts missed packets */
1326         hw_stats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
1327
1328         if (hw->mac.type != ixgbe_mac_82598EB) {
1329                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
1330                     ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
1331                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
1332                     ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
1333                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
1334                     ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
1335                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
1336                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
1337         } else {
1338                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
1339                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
1340                 /* 82598 only has a counter in the high register */
1341                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
1342                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
1343                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
1344         }
1345
1346         /*
1347          * Workaround: mprc hardware is incorrectly counting
1348          * broadcasts, so for now we subtract those.
1349          */
1350         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
1351         hw_stats->bprc += bprc;
1352         hw_stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
1353         if (hw->mac.type == ixgbe_mac_82598EB)
1354                 hw_stats->mprc -= bprc;
1355
1356         hw_stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
1357         hw_stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
1358         hw_stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
1359         hw_stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
1360         hw_stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
1361         hw_stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
1362
1363         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
1364         hw_stats->lxontxc += lxon;
1365         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
1366         hw_stats->lxofftxc += lxoff;
1367         total = lxon + lxoff;
1368
1369         hw_stats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
1370         hw_stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
1371         hw_stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
1372         hw_stats->gptc -= total;
1373         hw_stats->mptc -= total;
1374         hw_stats->ptc64 -= total;
1375         hw_stats->gotc -= total * ETHER_MIN_LEN;
1376
1377         hw_stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
1378         hw_stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
1379         hw_stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
1380         hw_stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
1381         hw_stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
1382         hw_stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
1383         hw_stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
1384         hw_stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
1385         hw_stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
1386         hw_stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
1387         hw_stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
1388         hw_stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
1389         hw_stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
1390         hw_stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
1391         hw_stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
1392         hw_stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);
1393         hw_stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
1394         hw_stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
1395         /* Only read FCOE on 82599 */
1396         if (hw->mac.type != ixgbe_mac_82598EB) {
1397                 hw_stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
1398                 hw_stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
1399                 hw_stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
1400                 hw_stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
1401                 hw_stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
1402         }
1403
1404         if (stats == NULL)
1405                 return;
1406
1407         /* Fill out the rte_eth_stats statistics structure */
1408         stats->ipackets = total_qprc;
1409         stats->ibytes = total_qbrc;
1410         stats->opackets = hw_stats->gptc;
1411         stats->obytes = hw_stats->gotc;
1412         stats->imcasts = hw_stats->mprc;
1413
1414         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1415                 stats->q_ipackets[i] = hw_stats->qprc[i];
1416                 stats->q_opackets[i] = hw_stats->qptc[i];
1417                 stats->q_ibytes[i] = hw_stats->qbrc[i];
1418                 stats->q_obytes[i] = hw_stats->qbtc[i];
1419                 stats->q_errors[i] = hw_stats->qprdc[i];
1420         }
1421
1422         /* Rx Errors */
1423         stats->ierrors = total_missed_rx + hw_stats->crcerrs +
1424                 hw_stats->rlec;
1425
1426         stats->oerrors  = 0;
1427
1428         /* Flow Director Stats registers */
1429         hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
1430         hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
1431         stats->fdirmatch = hw_stats->fdirmatch;
1432         stats->fdirmiss = hw_stats->fdirmiss;
1433 }
1434
1435 static void
1436 ixgbe_dev_stats_reset(struct rte_eth_dev *dev)
1437 {
1438         struct ixgbe_hw_stats *stats =
1439                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1440
1441         /* HW registers are cleared on read */
1442         ixgbe_dev_stats_get(dev, NULL);
1443
1444         /* Reset software totals */
1445         memset(stats, 0, sizeof(*stats));
1446 }
1447
1448 static void
1449 ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1450 {
1451         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1452         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1453                           IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1454
1455         /* Good Rx packet, include VF loopback */
1456         UPDATE_VF_STAT(IXGBE_VFGPRC,
1457             hw_stats->last_vfgprc, hw_stats->vfgprc);
1458
1459         /* Good Rx octets, include VF loopback */
1460         UPDATE_VF_STAT_36BIT(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
1461             hw_stats->last_vfgorc, hw_stats->vfgorc);
1462
1463         /* Good Tx packet, include VF loopback */
1464         UPDATE_VF_STAT(IXGBE_VFGPTC,
1465             hw_stats->last_vfgptc, hw_stats->vfgptc);
1466
1467         /* Good Tx octets, include VF loopback */
1468         UPDATE_VF_STAT_36BIT(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
1469             hw_stats->last_vfgotc, hw_stats->vfgotc);
1470
1471         /* Rx Multicst Packet */
1472         UPDATE_VF_STAT(IXGBE_VFMPRC,
1473             hw_stats->last_vfmprc, hw_stats->vfmprc);
1474
1475         if (stats == NULL)
1476                 return;
1477
1478         memset(stats, 0, sizeof(*stats));
1479         stats->ipackets = hw_stats->vfgprc;
1480         stats->ibytes = hw_stats->vfgorc;
1481         stats->opackets = hw_stats->vfgptc;
1482         stats->obytes = hw_stats->vfgotc;
1483         stats->imcasts = hw_stats->vfmprc;
1484 }
1485
1486 static void
1487 ixgbevf_dev_stats_reset(struct rte_eth_dev *dev)
1488 {
1489         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1490                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1491
1492         /* Sync HW register to the last stats */
1493         ixgbevf_dev_stats_get(dev, NULL);
1494
1495         /* reset HW current stats*/
1496         hw_stats->vfgprc = 0;
1497         hw_stats->vfgorc = 0;
1498         hw_stats->vfgptc = 0;
1499         hw_stats->vfgotc = 0;
1500         hw_stats->vfmprc = 0;
1501
1502 }
1503
1504 static void
1505 ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1506 {
1507         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1508
1509         dev_info->max_rx_queues = hw->mac.max_rx_queues;
1510         dev_info->max_tx_queues = hw->mac.max_tx_queues;
1511         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL register */
1512         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */
1513         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
1514 }
1515
1516 /* return 0 means link status changed, -1 means not changed */
1517 static int
1518 ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1519 {
1520         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1521         struct rte_eth_link link, old;
1522         ixgbe_link_speed link_speed;
1523         int link_up;
1524         int diag;
1525
1526         link.link_status = 0;
1527         link.link_speed = 0;
1528         link.link_duplex = 0;
1529         memset(&old, 0, sizeof(old));
1530         rte_ixgbe_dev_atomic_read_link_status(dev, &old);
1531
1532         /* check if it needs to wait to complete, if lsc interrupt is enabled */
1533         if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
1534                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 0);
1535         else
1536                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 1);
1537         if (diag != 0) {
1538                 link.link_speed = ETH_LINK_SPEED_100;
1539                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1540                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1541                 if (link.link_status == old.link_status)
1542                         return -1;
1543                 return 0;
1544         }
1545
1546         if (link_up == 0) {
1547                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1548                 if (link.link_status == old.link_status)
1549                         return -1;
1550                 return 0;
1551         }
1552         link.link_status = 1;
1553         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1554
1555         switch (link_speed) {
1556         default:
1557         case IXGBE_LINK_SPEED_UNKNOWN:
1558                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1559                 link.link_speed = ETH_LINK_SPEED_100;
1560                 break;
1561
1562         case IXGBE_LINK_SPEED_100_FULL:
1563                 link.link_speed = ETH_LINK_SPEED_100;
1564                 break;
1565
1566         case IXGBE_LINK_SPEED_1GB_FULL:
1567                 link.link_speed = ETH_LINK_SPEED_1000;
1568                 break;
1569
1570         case IXGBE_LINK_SPEED_10GB_FULL:
1571                 link.link_speed = ETH_LINK_SPEED_10000;
1572                 break;
1573         }
1574         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1575
1576         if (link.link_status == old.link_status)
1577                 return -1;
1578
1579         return 0;
1580 }
1581
1582 static void
1583 ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev)
1584 {
1585         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1586         uint32_t fctrl;
1587
1588         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1589         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1590         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1591 }
1592
1593 static void
1594 ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev)
1595 {
1596         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1597         uint32_t fctrl;
1598
1599         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1600         fctrl &= (~IXGBE_FCTRL_UPE);
1601         if (dev->data->all_multicast == 1)
1602                 fctrl |= IXGBE_FCTRL_MPE;
1603         else
1604                 fctrl &= (~IXGBE_FCTRL_MPE);
1605         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1606 }
1607
1608 static void
1609 ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev)
1610 {
1611         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1612         uint32_t fctrl;
1613
1614         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1615         fctrl |= IXGBE_FCTRL_MPE;
1616         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1617 }
1618
1619 static void
1620 ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev)
1621 {
1622         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1623         uint32_t fctrl;
1624
1625         if (dev->data->promiscuous == 1)
1626                 return; /* must remain in all_multicast mode */
1627
1628         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1629         fctrl &= (~IXGBE_FCTRL_MPE);
1630         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1631 }
1632
1633 /**
1634  * It clears the interrupt causes and enables the interrupt.
1635  * It will be called once only during nic initialized.
1636  *
1637  * @param dev
1638  *  Pointer to struct rte_eth_dev.
1639  *
1640  * @return
1641  *  - On success, zero.
1642  *  - On failure, a negative value.
1643  */
1644 static int
1645 ixgbe_dev_interrupt_setup(struct rte_eth_dev *dev)
1646 {
1647         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1648
1649         ixgbe_dev_link_status_print(dev);
1650         IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_LSC);
1651         IXGBE_WRITE_FLUSH(hw);
1652         rte_intr_enable(&(dev->pci_dev->intr_handle));
1653
1654         return 0;
1655 }
1656
1657 /*
1658  * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update.
1659  *
1660  * @param dev
1661  *  Pointer to struct rte_eth_dev.
1662  *
1663  * @return
1664  *  - On success, zero.
1665  *  - On failure, a negative value.
1666  */
1667 static int
1668 ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
1669 {
1670         uint32_t eicr;
1671         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1672         struct ixgbe_interrupt *intr =
1673                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1674
1675         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_LSC);
1676         IXGBE_WRITE_FLUSH(hw);
1677
1678         /* read-on-clear nic registers here */
1679         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1680         PMD_INIT_LOG(INFO, "eicr %x", eicr);
1681         if (eicr & IXGBE_EICR_LSC) {
1682                 /* set flag for async link update */
1683                 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1684         }
1685
1686         return 0;
1687 }
1688
1689 /**
1690  * It gets and then prints the link status.
1691  *
1692  * @param dev
1693  *  Pointer to struct rte_eth_dev.
1694  *
1695  * @return
1696  *  - On success, zero.
1697  *  - On failure, a negative value.
1698  */
1699 static void
1700 ixgbe_dev_link_status_print(struct rte_eth_dev *dev)
1701 {
1702         struct rte_eth_link link;
1703
1704         memset(&link, 0, sizeof(link));
1705         rte_ixgbe_dev_atomic_read_link_status(dev, &link);
1706         if (link.link_status) {
1707                 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
1708                                         (int)(dev->data->port_id),
1709                                         (unsigned)link.link_speed,
1710                         link.link_duplex == ETH_LINK_FULL_DUPLEX ?
1711                                         "full-duplex" : "half-duplex");
1712         } else {
1713                 PMD_INIT_LOG(INFO, " Port %d: Link Down",
1714                                 (int)(dev->data->port_id));
1715         }
1716         PMD_INIT_LOG(INFO, "PCI Address: %04d:%02d:%02d:%d",
1717                                 dev->pci_dev->addr.domain,
1718                                 dev->pci_dev->addr.bus,
1719                                 dev->pci_dev->addr.devid,
1720                                 dev->pci_dev->addr.function);
1721 }
1722
1723 /*
1724  * It executes link_update after knowing an interrupt occured.
1725  *
1726  * @param dev
1727  *  Pointer to struct rte_eth_dev.
1728  *
1729  * @return
1730  *  - On success, zero.
1731  *  - On failure, a negative value.
1732  */
1733 static int
1734 ixgbe_dev_interrupt_action(struct rte_eth_dev *dev)
1735 {
1736         struct ixgbe_interrupt *intr =
1737                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1738
1739         if (!(intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE)) {
1740                 return -1;
1741         }
1742         ixgbe_dev_link_update(dev, 0);
1743
1744         return 0;
1745 }
1746
1747 /**
1748  * Interrupt handler which shall be registered for alarm callback for delayed
1749  * handling specific interrupt to wait for the stable nic state. As the
1750  * NIC interrupt state is not stable for ixgbe after link is just down,
1751  * it needs to wait 4 seconds to get the stable status.
1752  *
1753  * @param handle
1754  *  Pointer to interrupt handle.
1755  * @param param
1756  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1757  *
1758  * @return
1759  *  void
1760  */
1761 static void
1762 ixgbe_dev_interrupt_delayed_handler(void *param)
1763 {
1764         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1765         struct ixgbe_interrupt *intr =
1766                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1767         struct ixgbe_hw *hw =
1768                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1769
1770         IXGBE_READ_REG(hw, IXGBE_EICR);
1771         ixgbe_dev_interrupt_action(dev);
1772         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
1773                 intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
1774                 rte_intr_enable(&(dev->pci_dev->intr_handle));
1775                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_LSC);
1776                 IXGBE_WRITE_FLUSH(hw);
1777                 ixgbe_dev_link_status_print(dev);
1778                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
1779         }
1780 }
1781
1782 /**
1783  * Interrupt handler triggered by NIC  for handling
1784  * specific interrupt.
1785  *
1786  * @param handle
1787  *  Pointer to interrupt handle.
1788  * @param param
1789  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1790  *
1791  * @return
1792  *  void
1793  */
1794 static void
1795 ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle, void *param)
1796 {
1797         int64_t timeout;
1798         struct rte_eth_link link;
1799         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1800         struct ixgbe_interrupt *intr =
1801                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1802
1803         /* get the link status before link update, for predicting later */
1804         memset(&link, 0, sizeof(link));
1805         rte_ixgbe_dev_atomic_read_link_status(dev, &link);
1806         ixgbe_dev_interrupt_get_status(dev);
1807         ixgbe_dev_interrupt_action(dev);
1808
1809         if (!(intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE))
1810                 return;
1811
1812         /* likely to up */
1813         if (!link.link_status)
1814                 /* handle it 1 sec later, wait it being stable */
1815                 timeout = IXGBE_LINK_UP_CHECK_TIMEOUT;
1816         /* likely to down */
1817         else
1818                 /* handle it 4 sec later, wait it being stable */
1819                 timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT;
1820
1821         ixgbe_dev_link_status_print(dev);
1822         if (rte_eal_alarm_set(timeout * 1000,
1823                 ixgbe_dev_interrupt_delayed_handler, param) < 0)
1824                 PMD_INIT_LOG(ERR, "Error setting alarm");
1825 }
1826
1827 static int
1828 ixgbe_dev_led_on(struct rte_eth_dev *dev)
1829 {
1830         struct ixgbe_hw *hw;
1831
1832         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1833         return (ixgbe_led_on(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
1834 }
1835
1836 static int
1837 ixgbe_dev_led_off(struct rte_eth_dev *dev)
1838 {
1839         struct ixgbe_hw *hw;
1840
1841         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1842         return (ixgbe_led_off(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
1843 }
1844
1845 static int
1846 ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1847 {
1848         struct ixgbe_hw *hw;
1849         int err;
1850         uint32_t rx_buf_size;
1851         uint32_t max_high_water;
1852         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
1853                 ixgbe_fc_none,
1854                 ixgbe_fc_rx_pause,
1855                 ixgbe_fc_tx_pause,
1856                 ixgbe_fc_full
1857         };
1858
1859         PMD_INIT_FUNC_TRACE();
1860
1861         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1862         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0));
1863         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
1864
1865         /*
1866          * At least reserve one Ethernet frame for watermark
1867          * high_water/low_water in kilo bytes for ixgbe
1868          */
1869         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
1870         if ((fc_conf->high_water > max_high_water) ||
1871                 (fc_conf->high_water < fc_conf->low_water)) {
1872                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
1873                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
1874                 return (-EINVAL);
1875         }
1876
1877         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[fc_conf->mode];
1878         hw->fc.pause_time     = fc_conf->pause_time;
1879         hw->fc.high_water[0]  = fc_conf->high_water;
1880         hw->fc.low_water[0]   = fc_conf->low_water;
1881         hw->fc.send_xon       = fc_conf->send_xon;
1882
1883         err = ixgbe_fc_enable(hw);
1884         /* Not negotiated is not an error case */
1885         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) {
1886                 return 0;
1887         }
1888
1889         PMD_INIT_LOG(ERR, "ixgbe_fc_enable = 0x%x \n", err);
1890         return -EIO;
1891 }
1892
1893 /**
1894  *  ixgbe_pfc_enable_generic - Enable flow control
1895  *  @hw: pointer to hardware structure
1896  *  @tc_num: traffic class number
1897  *  Enable flow control according to the current settings.
1898  */
1899 static int
1900 ixgbe_dcb_pfc_enable_generic(struct ixgbe_hw *hw,uint8_t tc_num)
1901 {
1902         int ret_val = 0;
1903         uint32_t mflcn_reg, fccfg_reg;
1904         uint32_t reg;
1905         uint32_t fcrtl, fcrth;
1906         uint8_t i;
1907         uint8_t nb_rx_en;
1908
1909         /* Validate the water mark configuration */
1910         if (!hw->fc.pause_time) {
1911                 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
1912                 goto out;
1913         }
1914
1915         /* Low water mark of zero causes XOFF floods */
1916         if (hw->fc.current_mode & ixgbe_fc_tx_pause) {
1917                  /* High/Low water can not be 0 */
1918                 if( (!hw->fc.high_water[tc_num])|| (!hw->fc.low_water[tc_num])) {
1919                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
1920                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
1921                         goto out;
1922                 }
1923
1924                 if(hw->fc.low_water[tc_num] >= hw->fc.high_water[tc_num]) {
1925                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
1926                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
1927                         goto out;
1928                 }
1929         }
1930         /* Negotiate the fc mode to use */
1931         ixgbe_fc_autoneg(hw);
1932
1933         /* Disable any previous flow control settings */
1934         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
1935         mflcn_reg &= ~(IXGBE_MFLCN_RPFCE_SHIFT | IXGBE_MFLCN_RFCE|IXGBE_MFLCN_RPFCE);
1936
1937         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
1938         fccfg_reg &= ~(IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY);
1939
1940         switch (hw->fc.current_mode) {
1941         case ixgbe_fc_none:
1942                 /*
1943                  * If the count of enabled RX Priority Flow control >1,
1944                  * and the TX pause can not be disabled
1945                  */
1946                 nb_rx_en = 0;
1947                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
1948                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
1949                         if (reg & IXGBE_FCRTH_FCEN)
1950                                 nb_rx_en++;
1951                 }
1952                 if (nb_rx_en > 1)
1953                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
1954                 break;
1955         case ixgbe_fc_rx_pause:
1956                 /*
1957                  * Rx Flow control is enabled and Tx Flow control is
1958                  * disabled by software override. Since there really
1959                  * isn't a way to advertise that we are capable of RX
1960                  * Pause ONLY, we will advertise that we support both
1961                  * symmetric and asymmetric Rx PAUSE.  Later, we will
1962                  * disable the adapter's ability to send PAUSE frames.
1963                  */
1964                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
1965                 /*
1966                  * If the count of enabled RX Priority Flow control >1,
1967                  * and the TX pause can not be disabled
1968                  */
1969                 nb_rx_en = 0;
1970                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
1971                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
1972                         if (reg & IXGBE_FCRTH_FCEN)
1973                                 nb_rx_en++;
1974                 }
1975                 if (nb_rx_en > 1)
1976                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
1977                 break;
1978         case ixgbe_fc_tx_pause:
1979                 /*
1980                  * Tx Flow control is enabled, and Rx Flow control is
1981                  * disabled by software override.
1982                  */
1983                 fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
1984                 break;
1985         case ixgbe_fc_full:
1986                 /* Flow control (both Rx and Tx) is enabled by SW override. */
1987                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
1988                 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
1989                 break;
1990         default:
1991                 DEBUGOUT("Flow control param set incorrectly\n");
1992                 ret_val = IXGBE_ERR_CONFIG;
1993                 goto out;
1994                 break;
1995         }
1996
1997         /* Set 802.3x based flow control settings. */
1998         mflcn_reg |= IXGBE_MFLCN_DPF;
1999         IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg);
2000         IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg);
2001
2002         /* Set up and enable Rx high/low water mark thresholds, enable XON. */
2003         if ((hw->fc.current_mode & ixgbe_fc_tx_pause) &&
2004                 hw->fc.high_water[tc_num]) {
2005                 fcrtl = (hw->fc.low_water[tc_num] << 10) | IXGBE_FCRTL_XONE;
2006                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), fcrtl);
2007                 fcrth = (hw->fc.high_water[tc_num] << 10) | IXGBE_FCRTH_FCEN;
2008         } else {
2009                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), 0);
2010                 /*
2011                  * In order to prevent Tx hangs when the internal Tx
2012                  * switch is enabled we must set the high water mark
2013                  * to the maximum FCRTH value.  This allows the Tx
2014                  * switch to function even under heavy Rx workloads.
2015                  */
2016                 fcrth = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)) - 32;
2017         }
2018         IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(tc_num), fcrth);
2019
2020         /* Configure pause time (2 TCs per register) */
2021         reg = hw->fc.pause_time * 0x00010001;
2022         for (i = 0; i < (IXGBE_DCB_MAX_TRAFFIC_CLASS / 2); i++)
2023                 IXGBE_WRITE_REG(hw, IXGBE_FCTTV(i), reg);
2024
2025         /* Configure flow control refresh threshold value */
2026         IXGBE_WRITE_REG(hw, IXGBE_FCRTV, hw->fc.pause_time / 2);
2027
2028 out:
2029         return ret_val;
2030 }
2031
2032 static int
2033 ixgbe_dcb_pfc_enable(struct rte_eth_dev *dev,uint8_t tc_num)
2034 {
2035         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2036         int32_t ret_val = IXGBE_NOT_IMPLEMENTED;
2037
2038         if(hw->mac.type != ixgbe_mac_82598EB) {
2039                 ret_val = ixgbe_dcb_pfc_enable_generic(hw,tc_num);
2040         }
2041         return ret_val;
2042 }
2043
2044 static int
2045 ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf)
2046 {
2047         int err;
2048         uint32_t rx_buf_size;
2049         uint32_t max_high_water;
2050         uint8_t tc_num;
2051         uint8_t  map[IXGBE_DCB_MAX_USER_PRIORITY] = { 0 };
2052         struct ixgbe_hw *hw =
2053                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2054         struct ixgbe_dcb_config *dcb_config =
2055                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
2056
2057         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
2058                 ixgbe_fc_none,
2059                 ixgbe_fc_rx_pause,
2060                 ixgbe_fc_tx_pause,
2061                 ixgbe_fc_full
2062         };
2063
2064         PMD_INIT_FUNC_TRACE();
2065
2066         ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
2067         tc_num = map[pfc_conf->priority];
2068         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num));
2069         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
2070         /*
2071          * At least reserve one Ethernet frame for watermark
2072          * high_water/low_water in kilo bytes for ixgbe
2073          */
2074         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
2075         if ((pfc_conf->fc.high_water > max_high_water) ||
2076                 (pfc_conf->fc.high_water <= pfc_conf->fc.low_water)) {
2077                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
2078                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
2079                 return (-EINVAL);
2080         }
2081
2082         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[pfc_conf->fc.mode];
2083         hw->fc.pause_time = pfc_conf->fc.pause_time;
2084         hw->fc.send_xon = pfc_conf->fc.send_xon;
2085         hw->fc.low_water[tc_num] =  pfc_conf->fc.low_water;
2086         hw->fc.high_water[tc_num] = pfc_conf->fc.high_water;
2087
2088         err = ixgbe_dcb_pfc_enable(dev,tc_num);
2089
2090         /* Not negotiated is not an error case */
2091         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED))
2092                 return 0;
2093
2094         PMD_INIT_LOG(ERR, "ixgbe_dcb_pfc_enable = 0x%x \n", err);
2095         return -EIO;
2096 }
2097
2098 static void
2099 ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
2100                                 uint32_t index, uint32_t pool)
2101 {
2102         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2103         uint32_t enable_addr = 1;
2104
2105         ixgbe_set_rar(hw, index, mac_addr->addr_bytes, pool, enable_addr);
2106 }
2107
2108 static void
2109 ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
2110 {
2111         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2112
2113         ixgbe_clear_rar(hw, index);
2114 }
2115
2116 /*
2117  * Virtual Function operations
2118  */
2119 static void
2120 ixgbevf_intr_disable(struct ixgbe_hw *hw)
2121 {
2122         PMD_INIT_LOG(DEBUG, "ixgbevf_intr_disable");
2123
2124         /* Clear interrupt mask to stop from interrupts being generated */
2125         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, IXGBE_VF_IRQ_CLEAR_MASK);
2126
2127         IXGBE_WRITE_FLUSH(hw);
2128 }
2129
2130 static int
2131 ixgbevf_dev_configure(struct rte_eth_dev *dev)
2132 {
2133         struct rte_eth_conf* conf = &dev->data->dev_conf;
2134
2135         PMD_INIT_LOG(DEBUG, "\nConfigured Virtual Function port id: %d\n",
2136                 dev->data->port_id);
2137
2138         /*
2139          * VF has no ability to enable/disable HW CRC
2140          * Keep the persistent behavior the same as Host PF
2141          */
2142 #ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC
2143         if (!conf->rxmode.hw_strip_crc) {
2144                 PMD_INIT_LOG(INFO, "VF can't disable HW CRC Strip\n");
2145                 conf->rxmode.hw_strip_crc = 1;
2146         }
2147 #else
2148         if (conf->rxmode.hw_strip_crc) {
2149                 PMD_INIT_LOG(INFO, "VF can't enable HW CRC Strip\n");
2150                 conf->rxmode.hw_strip_crc = 0;
2151         }
2152 #endif
2153
2154         return 0;
2155 }
2156
2157 static int
2158 ixgbevf_dev_start(struct rte_eth_dev *dev)
2159 {
2160         int err, mask = 0;
2161
2162         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_start");
2163
2164         ixgbevf_dev_tx_init(dev);
2165
2166         /* This can fail when allocating mbufs for descriptor rings */
2167         err = ixgbevf_dev_rx_init(dev);
2168         if (err) {
2169                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware (%d)\n", err);
2170                 ixgbe_dev_clear_queues(dev);
2171                 return err;
2172         }
2173
2174         /* Set vfta */
2175         ixgbevf_set_vfta_all(dev,1);
2176
2177         /* Set HW strip */
2178         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
2179                 ETH_VLAN_EXTEND_MASK;
2180         ixgbevf_vlan_offload_set(dev, mask);
2181
2182         ixgbevf_dev_rxtx_start(dev);
2183
2184         return 0;
2185 }
2186
2187 static void
2188 ixgbevf_dev_stop(struct rte_eth_dev *dev)
2189 {
2190         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2191
2192         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_stop");
2193
2194         hw->adapter_stopped = TRUE;
2195         ixgbe_stop_adapter(hw);
2196
2197         /*
2198           * Clear what we set, but we still keep shadow_vfta to
2199           * restore after device starts
2200           */
2201         ixgbevf_set_vfta_all(dev,0);
2202
2203         ixgbe_dev_clear_queues(dev);
2204 }
2205
2206 static void
2207 ixgbevf_dev_close(struct rte_eth_dev *dev)
2208 {
2209         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2210
2211         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_close");
2212
2213         ixgbe_reset_hw(hw);
2214
2215         ixgbevf_dev_stop(dev);
2216
2217         /* reprogram the RAR[0] in case user changed it. */
2218         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
2219 }
2220
2221 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on)
2222 {
2223         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2224         struct ixgbe_vfta * shadow_vfta =
2225                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2226         int i = 0, j = 0, vfta = 0, mask = 1;
2227
2228         for (i = 0; i < IXGBE_VFTA_SIZE; i++){
2229                 vfta = shadow_vfta->vfta[i];
2230                 if(vfta){
2231                         mask = 1;
2232                         for (j = 0; j < 32; j++){
2233                                 if(vfta & mask)
2234                                         ixgbe_set_vfta(hw, (i<<5)+j, 0, on);
2235                                 mask<<=1;
2236                         }
2237                 }
2238         }
2239
2240 }
2241
2242 static int
2243 ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2244 {
2245         struct ixgbe_hw *hw =
2246                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2247         struct ixgbe_vfta * shadow_vfta =
2248                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2249         uint32_t vid_idx = 0;
2250         uint32_t vid_bit = 0;
2251         int ret = 0;
2252
2253         PMD_INIT_FUNC_TRACE();
2254
2255         /* vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf */
2256         ret = ixgbe_set_vfta(hw, vlan_id, 0, !!on);
2257         if(ret){
2258                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
2259                 return ret;
2260         }
2261         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
2262         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
2263
2264         /* Save what we set and retore it after device reset */
2265         if (on)
2266                 shadow_vfta->vfta[vid_idx] |= vid_bit;
2267         else
2268                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
2269
2270         return 0;
2271 }
2272
2273 static void
2274 ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
2275 {
2276         struct ixgbe_hw *hw =
2277                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2278         uint32_t ctrl;
2279
2280         PMD_INIT_FUNC_TRACE();
2281
2282         if(queue >= hw->mac.max_rx_queues)
2283                 return;
2284
2285         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
2286         if(on)
2287                 ctrl |= IXGBE_RXDCTL_VME;
2288         else
2289                 ctrl &= ~IXGBE_RXDCTL_VME;
2290         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
2291
2292         ixgbe_vlan_hw_strip_bitmap_set( dev, queue, on);
2293 }
2294
2295 static void
2296 ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2297 {
2298         struct ixgbe_hw *hw =
2299                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2300         uint32_t i, on = 0;
2301
2302         /* VF function only support hw strip feature, others are not support */
2303         if(mask & ETH_VLAN_STRIP_MASK){
2304                 on = !!(dev->data->dev_conf.rxmode.hw_vlan_strip);
2305
2306                 for(i=0; i < hw->mac.max_rx_queues; i++)
2307                         ixgbevf_vlan_strip_queue_set(dev,i,on);
2308         }
2309 }
2310