lib/librte_pmd_ixgbe/ixgbe_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2012 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  *
  33  */
  34
  35 #include <sys/queue.h>
  36 #include <stdio.h>
  37 #include <errno.h>
  38 #include <stdint.h>
  39 #include <string.h>
  40 #include <unistd.h>
  41 #include <stdarg.h>
  42 #include <inttypes.h>
  43 #include <rte_byteorder.h>
  44 #include <rte_common.h>
  45 #include <rte_cycles.h>
  46
  47 #include <rte_interrupts.h>
  48 #include <rte_log.h>
  49 #include <rte_debug.h>
  50 #include <rte_pci.h>
  51 #include <rte_atomic.h>
  52 #include <rte_branch_prediction.h>
  53 #include <rte_memory.h>
  54 #include <rte_memzone.h>
  55 #include <rte_tailq.h>
  56 #include <rte_eal.h>
  57 #include <rte_alarm.h>
  58 #include <rte_ether.h>
  59 #include <rte_ethdev.h>
  60 #include <rte_atomic.h>
  61 #include <rte_malloc.h>
  62
  63 #include "ixgbe_logs.h"
  64 #include "ixgbe/ixgbe_api.h"
  65 #include "ixgbe/ixgbe_vf.h"
  66 #include "ixgbe/ixgbe_common.h"
  67 #include "ixgbe_ethdev.h"
  68
  69 /*
  70  * High threshold controlling when to start sending XOFF frames. Must be at
  71  * least 8 bytes less than receive packet buffer size. This value is in units
  72  * of 1024 bytes.
  73  */
  74 #define IXGBE_FC_HI    0x80
  75
  76 /*
  77  * Low threshold controlling when to start sending XON frames. This value is
  78  * in units of 1024 bytes.
  79  */
  80 #define IXGBE_FC_LO    0x40
  81
  82 /* Timer value included in XOFF frames. */
  83 #define IXGBE_FC_PAUSE 0x680
  84
  85 #define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */
  86 #define IXGBE_LINK_UP_CHECK_TIMEOUT   1000 /* ms */
  87
  88 #define IXGBE_QUEUE_STAT_COUNTERS (sizeof(hw_stats->qprc) / sizeof(hw_stats->qprc[0]))
  89
  90 static int eth_ixgbe_dev_init(struct eth_driver *eth_drv,
  91                 struct rte_eth_dev *eth_dev);
  92 static int  ixgbe_dev_configure(struct rte_eth_dev *dev);
  93 static int  ixgbe_dev_start(struct rte_eth_dev *dev);
  94 static void ixgbe_dev_stop(struct rte_eth_dev *dev);
  95 static void ixgbe_dev_close(struct rte_eth_dev *dev);
  96 static void ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev);
  97 static void ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev);
  98 static void ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev);
  99 static void ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev);
 100 static int ixgbe_dev_link_update(struct rte_eth_dev *dev,
 101                                 int wait_to_complete);
 102 static void ixgbe_dev_stats_get(struct rte_eth_dev *dev,
 103                                 struct rte_eth_stats *stats);
 104 static void ixgbe_dev_stats_reset(struct rte_eth_dev *dev);
 105 static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 106                                              uint16_t queue_id,
 107                                              uint8_t stat_idx,
 108                                              uint8_t is_rx);
 109 static void ixgbe_dev_info_get(struct rte_eth_dev *dev,
 110                                 struct rte_eth_dev_info *dev_info);
 111 static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev,
 112                 uint16_t vlan_id, int on);
 113 static void ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 114 static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev,
 115                 uint16_t queue, bool on);
 116 static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue,
 117                                   int on);
 118 static void ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 119 static void ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev);
 120 static void ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev);
 121 static void ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev);
 122 static void ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev);
 123 static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue);
 124 static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue);
 125 static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 126 static void ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 127
 128 static int ixgbe_dev_led_on(struct rte_eth_dev *dev);
 129 static int ixgbe_dev_led_off(struct rte_eth_dev *dev);
 130 static int  ixgbe_flow_ctrl_set(struct rte_eth_dev *dev,
 131                                 struct rte_eth_fc_conf *fc_conf);
 132 static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev);
 133 static int ixgbe_dev_interrupt_setup(struct rte_eth_dev *dev);
 134 static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev);
 135 static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev);
 136 static void ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle,
 137                                                         void *param);
 138 static void ixgbe_dev_interrupt_delayed_handler(void *param);
 139 static void ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 140                                 uint32_t index, uint32_t pool);
 141 static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index);
 142
 143 /* For Virtual Function support */
 144 static int eth_ixgbevf_dev_init(struct eth_driver *eth_drv,
 145                 struct rte_eth_dev *eth_dev);
 146 static int  ixgbevf_dev_configure(struct rte_eth_dev *dev);
 147 static int  ixgbevf_dev_start(struct rte_eth_dev *dev);
 148 static void ixgbevf_dev_stop(struct rte_eth_dev *dev);
 149 static void ixgbevf_intr_disable(struct ixgbe_hw *hw);
 150 static void ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats);
 151 static void ixgbevf_dev_stats_reset(struct rte_eth_dev *dev);
 152 static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev,
 153                 uint16_t vlan_id, int on);
 154 static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev,
 155                 uint16_t queue, int on);
 156 static void ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 157 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 158
 159
 160 /*
 161  *  * Define VF Stats MACRO for Non "cleared on read" register
 162  *   */
 163 #define UPDATE_VF_STAT(reg, last, cur)                          \
 164 {                                                               \
 165         u32 latest = IXGBE_READ_REG(hw, reg);                   \
 166         cur += latest - last;                                   \
 167         last = latest;                                          \
 168 }
 169
 170 #define UPDATE_VF_STAT_36BIT(lsb, msb, last, cur)                \
 171 {                                                                \
 172         u64 new_lsb = IXGBE_READ_REG(hw, lsb);                   \
 173         u64 new_msb = IXGBE_READ_REG(hw, msb);                   \
 174         u64 latest = ((new_msb << 32) | new_lsb);                \
 175         cur += (0x1000000000LL + latest - last) & 0xFFFFFFFFFLL; \
 176         last = latest;                                           \
 177 }
 178
 179 #define IXGBE_SET_HWSTRIP(h, q) do{\
 180                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 181                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 182                 (h)->bitmap[idx] |= 1 << bit;\
 183         }while(0)
 184
 185 #define IXGBE_CLEAR_HWSTRIP(h, q) do{\
 186                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 187                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 188                 (h)->bitmap[idx] &= ~(1 << bit);\
 189         }while(0)
 190
 191 #define IXGBE_GET_HWSTRIP(h, q, r) do{\
 192                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 193                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 194                 (r) = (h)->bitmap[idx] >> bit & 1;\
 195         }while(0)
 196
 197 /*
 198  * The set of PCI devices this driver supports
 199  */
 200 static struct rte_pci_id pci_id_ixgbe_map[] = {
 201
 202 #undef RTE_LIBRTE_IGB_PMD
 203 #define RTE_PCI_DEV_ID_DECL(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 204 #include "rte_pci_dev_ids.h"
 205
 206 { .vendor_id = 0, /* sentinel */ },
 207 };
 208
 209
 210 /*
 211  * The set of PCI devices this driver supports (for 82599 VF)
 212  */
 213 static struct rte_pci_id pci_id_ixgbevf_map[] = {
 214 {
 215         .vendor_id = PCI_VENDOR_ID_INTEL,
 216         .device_id = IXGBE_DEV_ID_82599_VF,
 217         .subsystem_vendor_id = PCI_ANY_ID,
 218         .subsystem_device_id = PCI_ANY_ID,
 219 },
 220 { .vendor_id = 0, /* sentinel */ },
 221 };
 222
 223 static struct eth_dev_ops ixgbe_eth_dev_ops = {
 224         .dev_configure        = ixgbe_dev_configure,
 225         .dev_start            = ixgbe_dev_start,
 226         .dev_stop             = ixgbe_dev_stop,
 227         .dev_close            = ixgbe_dev_close,
 228         .promiscuous_enable   = ixgbe_dev_promiscuous_enable,
 229         .promiscuous_disable  = ixgbe_dev_promiscuous_disable,
 230         .allmulticast_enable  = ixgbe_dev_allmulticast_enable,
 231         .allmulticast_disable = ixgbe_dev_allmulticast_disable,
 232         .link_update          = ixgbe_dev_link_update,
 233         .stats_get            = ixgbe_dev_stats_get,
 234         .stats_reset          = ixgbe_dev_stats_reset,
 235         .queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set,
 236         .dev_infos_get        = ixgbe_dev_info_get,
 237         .vlan_filter_set      = ixgbe_vlan_filter_set,
 238         .vlan_tpid_set        = ixgbe_vlan_tpid_set,
 239         .vlan_offload_set     = ixgbe_vlan_offload_set,
 240         .vlan_strip_queue_set = ixgbe_vlan_strip_queue_set,
 241         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 242         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 243         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 244         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 245         .dev_led_on           = ixgbe_dev_led_on,
 246         .dev_led_off          = ixgbe_dev_led_off,
 247         .flow_ctrl_set        = ixgbe_flow_ctrl_set,
 248         .mac_addr_add         = ixgbe_add_rar,
 249         .mac_addr_remove      = ixgbe_remove_rar,
 250         .fdir_add_signature_filter    = ixgbe_fdir_add_signature_filter,
 251         .fdir_update_signature_filter = ixgbe_fdir_update_signature_filter,
 252         .fdir_remove_signature_filter = ixgbe_fdir_remove_signature_filter,
 253         .fdir_infos_get               = ixgbe_fdir_info_get,
 254         .fdir_add_perfect_filter      = ixgbe_fdir_add_perfect_filter,
 255         .fdir_update_perfect_filter   = ixgbe_fdir_update_perfect_filter,
 256         .fdir_remove_perfect_filter   = ixgbe_fdir_remove_perfect_filter,
 257         .fdir_set_masks               = ixgbe_fdir_set_masks,
 258 };
 259
 260 /*
 261  * dev_ops for virtual function, bare necessities for basic vf
 262  * operation have been implemented
 263  */
 264 static struct eth_dev_ops ixgbevf_eth_dev_ops = {
 265
 266         .dev_configure        = ixgbevf_dev_configure,
 267         .dev_start            = ixgbevf_dev_start,
 268         .dev_stop             = ixgbevf_dev_stop,
 269         .link_update          = ixgbe_dev_link_update,
 270         .stats_get            = ixgbevf_dev_stats_get,
 271         .stats_reset          = ixgbevf_dev_stats_reset,
 272         .dev_close            = ixgbevf_dev_stop,
 273
 274         .dev_infos_get        = ixgbe_dev_info_get,
 275         .vlan_filter_set      = ixgbevf_vlan_filter_set,
 276         .vlan_strip_queue_set = ixgbevf_vlan_strip_queue_set,
 277         .vlan_offload_set     = ixgbevf_vlan_offload_set,
 278         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 279         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 280         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 281         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 282 };
 283
 284 /**
 285  * Atomically reads the link status information from global
 286  * structure rte_eth_dev.
 287  *
 288  * @param dev
 289  *   - Pointer to the structure rte_eth_dev to read from.
 290  *   - Pointer to the buffer to be saved with the link status.
 291  *
 292  * @return
 293  *   - On success, zero.
 294  *   - On failure, negative value.
 295  */
 296 static inline int
 297 rte_ixgbe_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 298                                 struct rte_eth_link *link)
 299 {
 300         struct rte_eth_link *dst = link;
 301         struct rte_eth_link *src = &(dev->data->dev_link);
 302
 303         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 304                                         *(uint64_t *)src) == 0)
 305                 return -1;
 306
 307         return 0;
 308 }
 309
 310 /**
 311  * Atomically writes the link status information into global
 312  * structure rte_eth_dev.
 313  *
 314  * @param dev
 315  *   - Pointer to the structure rte_eth_dev to read from.
 316  *   - Pointer to the buffer to be saved with the link status.
 317  *
 318  * @return
 319  *   - On success, zero.
 320  *   - On failure, negative value.
 321  */
 322 static inline int
 323 rte_ixgbe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 324                                 struct rte_eth_link *link)
 325 {
 326         struct rte_eth_link *dst = &(dev->data->dev_link);
 327         struct rte_eth_link *src = link;
 328
 329         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 330                                         *(uint64_t *)src) == 0)
 331                 return -1;
 332
 333         return 0;
 334 }
 335
 336 /*
 337  * This function is the same as ixgbe_is_sfp() in ixgbe/ixgbe.h.
 338  */
 339 static inline int
 340 ixgbe_is_sfp(struct ixgbe_hw *hw)
 341 {
 342         switch (hw->phy.type) {
 343         case ixgbe_phy_sfp_avago:
 344         case ixgbe_phy_sfp_ftl:
 345         case ixgbe_phy_sfp_intel:
 346         case ixgbe_phy_sfp_unknown:
 347         case ixgbe_phy_sfp_passive_tyco:
 348         case ixgbe_phy_sfp_passive_unknown:
 349                 return 1;
 350         default:
 351                 return 0;
 352         }
 353 }
 354
 355 /*
 356  * This function is based on ixgbe_disable_intr() in ixgbe/ixgbe.h.
 357  */
 358 static void
 359 ixgbe_disable_intr(struct ixgbe_hw *hw)
 360 {
 361         PMD_INIT_FUNC_TRACE();
 362
 363         if (hw->mac.type == ixgbe_mac_82598EB) {
 364                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ~0);
 365         } else {
 366                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xFFFF0000);
 367                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), ~0);
 368                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), ~0);
 369         }
 370         IXGBE_WRITE_FLUSH(hw);
 371 }
 372
 373 /*
 374  * This function resets queue statistics mapping registers.
 375  * From Niantic datasheet, Initialization of Statistics section:
 376  * "...if software requires the queue counters, the RQSMR and TQSM registers
 377  * must be re-programmed following a device reset.
 378  */
 379 static void
 380 ixgbe_reset_qstat_mappings(struct ixgbe_hw *hw)
 381 {
 382         uint32_t i;
 383
 384         for(i = 0; i != IXGBE_NB_STAT_MAPPING_REGS; i++) {
 385                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), 0);
 386                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), 0);
 387         }
 388 }
 389
 390
 391 static int
 392 ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 393                                   uint16_t queue_id,
 394                                   uint8_t stat_idx,
 395                                   uint8_t is_rx)
 396 {
 397 #define QSM_REG_NB_BITS_PER_QMAP_FIELD 8
 398 #define NB_QMAP_FIELDS_PER_QSM_REG 4
 399 #define QMAP_FIELD_RESERVED_BITS_MASK 0x0f
 400
 401         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 402         struct ixgbe_stat_mapping_registers *stat_mappings =
 403                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(eth_dev->data->dev_private);
 404         uint32_t qsmr_mask = 0;
 405         uint32_t clearing_mask = QMAP_FIELD_RESERVED_BITS_MASK;
 406         uint32_t q_map;
 407         uint8_t n, offset;
 408
 409         if ((hw->mac.type != ixgbe_mac_82599EB) && (hw->mac.type != ixgbe_mac_X540))
 410                 return -ENOSYS;
 411
 412         PMD_INIT_LOG(INFO, "Setting port %d, %s queue_id %d to stat index %d\n",
 413                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx);
 414
 415         n = queue_id / NB_QMAP_FIELDS_PER_QSM_REG;
 416         if (n >= IXGBE_NB_STAT_MAPPING_REGS) {
 417                 PMD_INIT_LOG(ERR, "Nb of stat mapping registers exceeded\n");
 418                 return -EIO;
 419         }
 420         offset = queue_id % NB_QMAP_FIELDS_PER_QSM_REG;
 421
 422         /* Now clear any previous stat_idx set */
 423         clearing_mask <<= (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 424         if (!is_rx)
 425                 stat_mappings->tqsm[n] &= ~clearing_mask;
 426         else
 427                 stat_mappings->rqsmr[n] &= ~clearing_mask;
 428
 429         q_map = (uint32_t)stat_idx;
 430         q_map &= QMAP_FIELD_RESERVED_BITS_MASK;
 431         qsmr_mask = q_map << (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 432         if (!is_rx)
 433                 stat_mappings->tqsm[n] |= qsmr_mask;
 434         else
 435                 stat_mappings->rqsmr[n] |= qsmr_mask;
 436
 437         PMD_INIT_LOG(INFO, "Set port %d, %s queue_id %d to stat index %d\n"
 438                      "%s[%d] = 0x%08x\n",
 439                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx,
 440                      is_rx ? "RQSMR" : "TQSM",n, is_rx ? stat_mappings->rqsmr[n] : stat_mappings->tqsm[n]);
 441
 442         /* Now write the mapping in the appropriate register */
 443         if (is_rx) {
 444                 PMD_INIT_LOG(INFO, "Write 0x%x to RX IXGBE stat mapping reg:%d\n",
 445                              stat_mappings->rqsmr[n], n);
 446                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(n), stat_mappings->rqsmr[n]);
 447         }
 448         else {
 449                 PMD_INIT_LOG(INFO, "Write 0x%x to TX IXGBE stat mapping reg:%d\n",
 450                              stat_mappings->tqsm[n], n);
 451                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(n), stat_mappings->tqsm[n]);
 452         }
 453         return 0;
 454 }
 455
 456 static void
 457 ixgbe_restore_statistics_mapping(struct rte_eth_dev * dev)
 458 {
 459         struct ixgbe_stat_mapping_registers *stat_mappings =
 460                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(dev->data->dev_private);
 461         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 462         int i;
 463
 464         /* write whatever was in stat mapping table to the NIC */
 465         for (i = 0; i < IXGBE_NB_STAT_MAPPING_REGS; i++) {
 466                 /* rx */
 467                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), stat_mappings->rqsmr[i]);
 468
 469                 /* tx */
 470                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), stat_mappings->tqsm[i]);
 471         }
 472 }
 473
 474 /*
 475  * This function is based on code in ixgbe_attach() in ixgbe/ixgbe.c.
 476  * It returns 0 on success.
 477  */
 478 static int
 479 eth_ixgbe_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 480                      struct rte_eth_dev *eth_dev)
 481 {
 482         struct rte_pci_device *pci_dev;
 483         struct ixgbe_hw *hw =
 484                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 485         struct ixgbe_vfta * shadow_vfta =
 486                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 487         struct ixgbe_hwstrip *hwstrip =
 488                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 489         struct ixgbe_dcb_config *dcb_config =
 490                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private);
 491         uint32_t ctrl_ext;
 492         uint16_t csum;
 493         int diag, i;
 494
 495         PMD_INIT_FUNC_TRACE();
 496
 497         eth_dev->dev_ops = &ixgbe_eth_dev_ops;
 498         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
 499         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
 500
 501         /* for secondary processes, we don't initialise any further as primary
 502          * has already done this work. Only check we don't need a different
 503          * RX function */
 504         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 505                 if (eth_dev->data->scattered_rx)
 506                         eth_dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
 507                 return 0;
 508         }
 509         pci_dev = eth_dev->pci_dev;
 510
 511         /* Vendor and Device ID need to be set before init of shared code */
 512         hw->device_id = pci_dev->id.device_id;
 513         hw->vendor_id = pci_dev->id.vendor_id;
 514         hw->hw_addr = (void *)pci_dev->mem_resource.addr;
 515
 516         /* Initialize the shared code */
 517         diag = ixgbe_init_shared_code(hw);
 518         if (diag != IXGBE_SUCCESS) {
 519                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
 520                 return -EIO;
 521         }
 522
 523         /* Get Hardware Flow Control setting */
 524         hw->fc.requested_mode = ixgbe_fc_full;
 525         hw->fc.current_mode = ixgbe_fc_full;
 526         hw->fc.pause_time = IXGBE_FC_PAUSE;
 527         hw->fc.low_water = IXGBE_FC_LO;
 528         for (i = 0; i < MAX_TRAFFIC_CLASS; i++)
 529                 hw->fc.high_water[i] = IXGBE_FC_HI;
 530         hw->fc.send_xon = 1;
 531
 532         ixgbe_disable_intr(hw);
 533
 534         /* Make sure we have a good EEPROM before we read from it */
 535         diag = ixgbe_validate_eeprom_checksum(hw, &csum);
 536         if (diag != IXGBE_SUCCESS) {
 537                 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag);
 538                 return -EIO;
 539         }
 540
 541         diag = ixgbe_init_hw(hw);
 542
 543         /*
 544          * Devices with copper phys will fail to initialise if ixgbe_init_hw()
 545          * is called too soon after the kernel driver unbinding/binding occurs.
 546          * The failure occurs in ixgbe_identify_phy_generic() for all devices,
 547          * but for non-copper devies, ixgbe_identify_sfp_module_generic() is
 548          * also called. See ixgbe_identify_phy_82599(). The reason for the
 549          * failure is not known, and only occuts when virtualisation features
 550          * are disabled in the bios. A delay of 100ms  was found to be enough by
 551          * trial-and-error, and is doubled to be safe.
 552          */
 553         if (diag && (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper)) {
 554                 rte_delay_ms(200);
 555                 diag = ixgbe_init_hw(hw);
 556         }
 557
 558         if (diag == IXGBE_ERR_EEPROM_VERSION) {
 559                 PMD_INIT_LOG(ERR, "This device is a pre-production adapter/"
 560                     "LOM.  Please be aware there may be issues associated "
 561                     "with your hardware.\n If you are experiencing problems "
 562                     "please contact your Intel or hardware representative "
 563                     "who provided you with this hardware.\n");
 564         } else if (diag == IXGBE_ERR_SFP_NOT_SUPPORTED)
 565                 PMD_INIT_LOG(ERR, "Unsupported SFP+ Module\n");
 566         if (diag) {
 567                 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag);
 568                 return -EIO;
 569         }
 570
 571         /* pick up the PCI bus settings for reporting later */
 572         ixgbe_get_bus_info(hw);
 573
 574         /* reset mappings for queue statistics hw counters*/
 575         ixgbe_reset_qstat_mappings(hw);
 576
 577         /* Allocate memory for storing MAC addresses */
 578         eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 579                         hw->mac.num_rar_entries, 0);
 580         if (eth_dev->data->mac_addrs == NULL) {
 581                 PMD_INIT_LOG(ERR,
 582                         "Failed to allocate %d bytes needed to store MAC addresses",
 583                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 584                 return -ENOMEM;
 585         }
 586         /* Copy the permanent MAC address */
 587         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 588                         &eth_dev->data->mac_addrs[0]);
 589
 590         /* initialize the vfta */
 591         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 592
 593         /* initialize the hw strip bitmap*/
 594         memset(hwstrip, 0, sizeof(*hwstrip));
 595
 596         /* let hardware know driver is loaded */
 597         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 598         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 599         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 600
 601         if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
 602                 PMD_INIT_LOG(DEBUG,
 603                              "MAC: %d, PHY: %d, SFP+: %d<n",
 604                              (int) hw->mac.type, (int) hw->phy.type,
 605                              (int) hw->phy.sfp_type);
 606         else
 607                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d\n",
 608                              (int) hw->mac.type, (int) hw->phy.type);
 609
 610         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
 611                         eth_dev->data->port_id, pci_dev->id.vendor_id,
 612                         pci_dev->id.device_id);
 613
 614         rte_intr_callback_register(&(pci_dev->intr_handle),
 615                 ixgbe_dev_interrupt_handler, (void *)eth_dev);
 616
 617         return 0;
 618 }
 619
 620 /*
 621  * Virtual Function device init
 622  */
 623 static int
 624 eth_ixgbevf_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 625                      struct rte_eth_dev *eth_dev)
 626 {
 627         struct rte_pci_device *pci_dev;
 628         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 629         int diag;
 630         struct ixgbe_vfta * shadow_vfta =
 631                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 632         struct ixgbe_hwstrip *hwstrip =
 633                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 634
 635         PMD_INIT_LOG(DEBUG, "eth_ixgbevf_dev_init");
 636
 637         eth_dev->dev_ops = &ixgbevf_eth_dev_ops;
 638         pci_dev = eth_dev->pci_dev;
 639
 640         hw->device_id = pci_dev->id.device_id;
 641         hw->vendor_id = pci_dev->id.vendor_id;
 642         hw->hw_addr = (void *)pci_dev->mem_resource.addr;
 643
 644         /* initialize the vfta */
 645         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 646
 647         /* initialize the hw strip bitmap*/
 648         memset(hwstrip, 0, sizeof(*hwstrip));
 649
 650         /* Initialize the shared code */
 651         diag = ixgbe_init_shared_code(hw);
 652         if (diag != IXGBE_SUCCESS) {
 653                 PMD_INIT_LOG(ERR, "Shared code init failed for ixgbevf: %d", diag);
 654                 return -EIO;
 655         }
 656
 657         /* init_mailbox_params */
 658         hw->mbx.ops.init_params(hw);
 659
 660         /* Disable the interrupts for VF */
 661         ixgbevf_intr_disable(hw);
 662
 663         hw->mac.num_rar_entries = hw->mac.max_rx_queues;
 664         diag = hw->mac.ops.reset_hw(hw);
 665
 666         /* Allocate memory for storing MAC addresses */
 667         eth_dev->data->mac_addrs = rte_zmalloc("ixgbevf", ETHER_ADDR_LEN *
 668                         hw->mac.num_rar_entries, 0);
 669         if (eth_dev->data->mac_addrs == NULL) {
 670                 PMD_INIT_LOG(ERR,
 671                         "Failed to allocate %d bytes needed to store MAC addresses",
 672                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 673                 return -ENOMEM;
 674         }
 675         /* Copy the permanent MAC address */
 676         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 677                         &eth_dev->data->mac_addrs[0]);
 678
 679         /* reset the hardware with the new settings */
 680         diag = hw->mac.ops.start_hw(hw);
 681         switch (diag) {
 682                 case  0:
 683                         break;
 684
 685                 default:
 686                         PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
 687                         return (diag);
 688         }
 689
 690         PMD_INIT_LOG(DEBUG, "\nport %d vendorID=0x%x deviceID=0x%x mac.type=%s\n",
 691                          eth_dev->data->port_id, pci_dev->id.vendor_id, pci_dev->id.device_id,
 692                          "ixgbe_mac_82599_vf");
 693
 694         return 0;
 695 }
 696
 697 static struct eth_driver rte_ixgbe_pmd = {
 698         {
 699                 .name = "rte_ixgbe_pmd",
 700                 .id_table = pci_id_ixgbe_map,
 701                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
 702         },
 703         .eth_dev_init = eth_ixgbe_dev_init,
 704         .dev_private_size = sizeof(struct ixgbe_adapter),
 705 };
 706
 707 /*
 708  * virtual function driver struct
 709  */
 710 static struct eth_driver rte_ixgbevf_pmd = {
 711         {
 712                 .name = "rte_ixgbevf_pmd",
 713                 .id_table = pci_id_ixgbevf_map,
 714                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
 715         },
 716         .eth_dev_init = eth_ixgbevf_dev_init,
 717         .dev_private_size = sizeof(struct ixgbe_adapter),
 718 };
 719
 720 /*
 721  * Driver initialization routine.
 722  * Invoked once at EAL init time.
 723  * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
 724  */
 725 int
 726 rte_ixgbe_pmd_init(void)
 727 {
 728         PMD_INIT_FUNC_TRACE();
 729
 730         rte_eth_driver_register(&rte_ixgbe_pmd);
 731         return 0;
 732 }
 733
 734 /*
 735  * VF Driver initialization routine.
 736  * Invoked one at EAL init time.
 737  * Register itself as the [Virtual Poll Mode] Driver of PCI niantic devices.
 738  */
 739 int
 740 rte_ixgbevf_pmd_init(void)
 741 {
 742         DEBUGFUNC("rte_ixgbevf_pmd_init");
 743
 744         rte_eth_driver_register(&rte_ixgbevf_pmd);
 745         return (0);
 746 }
 747
 748 static int
 749 ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 750 {
 751         struct ixgbe_hw *hw =
 752                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 753         struct ixgbe_vfta * shadow_vfta =
 754                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
 755         uint32_t vfta;
 756         uint32_t vid_idx;
 757         uint32_t vid_bit;
 758
 759         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
 760         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
 761         vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(vid_idx));
 762         if (on)
 763                 vfta |= vid_bit;
 764         else
 765                 vfta &= ~vid_bit;
 766         IXGBE_WRITE_REG(hw, IXGBE_VFTA(vid_idx), vfta);
 767
 768         /* update local VFTA copy */
 769         shadow_vfta->vfta[vid_idx] = vfta;
 770
 771         return 0;
 772 }
 773
 774 static void
 775 ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
 776 {
 777         if (on)
 778                 ixgbe_vlan_hw_strip_enable(dev, queue);
 779         else
 780                 ixgbe_vlan_hw_strip_disable(dev, queue);
 781 }
 782
 783 static void
 784 ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
 785 {
 786         struct ixgbe_hw *hw =
 787                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 788
 789         /* Only the high 16-bits is valid */
 790         IXGBE_WRITE_REG(hw, IXGBE_EXVET, tpid << 16);
 791 }
 792
 793 static void
 794 ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev)
 795 {
 796         struct ixgbe_hw *hw =
 797                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 798         uint32_t vlnctrl;
 799
 800         PMD_INIT_FUNC_TRACE();
 801
 802         /* Filter Table Disable */
 803         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 804         vlnctrl &= ~IXGBE_VLNCTRL_VFE;
 805
 806         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
 807 }
 808
 809 static void
 810 ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev)
 811 {
 812         struct ixgbe_hw *hw =
 813                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 814         struct ixgbe_vfta * shadow_vfta =
 815                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
 816         uint32_t vlnctrl;
 817         uint16_t i;
 818
 819         PMD_INIT_FUNC_TRACE();
 820
 821         /* Filter Table Enable */
 822         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 823         vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
 824         vlnctrl |= IXGBE_VLNCTRL_VFE;
 825
 826         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
 827
 828         /* write whatever is in local vfta copy */
 829         for (i = 0; i < IXGBE_VFTA_SIZE; i++)
 830                 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), shadow_vfta->vfta[i]);
 831 }
 832
 833 static void
 834 ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on)
 835 {
 836         struct ixgbe_hwstrip *hwstrip =
 837                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(dev->data->dev_private);
 838
 839         if(queue >= IXGBE_MAX_RX_QUEUE_NUM)
 840                 return;
 841
 842         if (on)
 843                 IXGBE_SET_HWSTRIP(hwstrip, queue);
 844         else
 845                 IXGBE_CLEAR_HWSTRIP(hwstrip, queue);
 846 }
 847
 848 static void
 849 ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue)
 850 {
 851         struct ixgbe_hw *hw =
 852                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 853         uint32_t ctrl;
 854
 855         PMD_INIT_FUNC_TRACE();
 856
 857         if (hw->mac.type == ixgbe_mac_82598EB) {
 858                 /* No queue level support */
 859                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
 860                 return;
 861         }
 862         else {
 863                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 864                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
 865                 ctrl &= ~IXGBE_RXDCTL_VME;
 866                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
 867         }
 868         /* record those setting for HW strip per queue */
 869         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 0);
 870 }
 871
 872 static void
 873 ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue)
 874 {
 875         struct ixgbe_hw *hw =
 876                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 877         uint32_t ctrl;
 878
 879         PMD_INIT_FUNC_TRACE();
 880
 881         if (hw->mac.type == ixgbe_mac_82598EB) {
 882                 /* No queue level supported */
 883                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
 884                 return;
 885         }
 886         else {
 887                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 888                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
 889                 ctrl |= IXGBE_RXDCTL_VME;
 890                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
 891         }
 892         /* record those setting for HW strip per queue */
 893         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 1);
 894 }
 895
 896 static void
 897 ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev)
 898 {
 899         struct ixgbe_hw *hw =
 900                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 901         uint32_t ctrl;
 902         uint16_t i;
 903
 904         PMD_INIT_FUNC_TRACE();
 905
 906         if (hw->mac.type == ixgbe_mac_82598EB) {
 907                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 908                 ctrl &= ~IXGBE_VLNCTRL_VME;
 909                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
 910         }
 911         else {
 912                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 913                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
 914                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
 915                         ctrl &= ~IXGBE_RXDCTL_VME;
 916                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
 917
 918                         /* record those setting for HW strip per queue */
 919                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 0);
 920                 }
 921         }
 922 }
 923
 924 static void
 925 ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev)
 926 {
 927         struct ixgbe_hw *hw =
 928                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 929         uint32_t ctrl;
 930         uint16_t i;
 931
 932         PMD_INIT_FUNC_TRACE();
 933
 934         if (hw->mac.type == ixgbe_mac_82598EB) {
 935                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 936                 ctrl |= IXGBE_VLNCTRL_VME;
 937                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
 938         }
 939         else {
 940                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
 941                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
 942                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
 943                         ctrl |= IXGBE_RXDCTL_VME;
 944                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
 945
 946                         /* record those setting for HW strip per queue */
 947                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 1);
 948                 }
 949         }
 950 }
 951
 952 static void
 953 ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev)
 954 {
 955         struct ixgbe_hw *hw =
 956                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 957         uint32_t ctrl;
 958
 959         PMD_INIT_FUNC_TRACE();
 960
 961         /* DMATXCTRL: Geric Double VLAN Disable */
 962         ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
 963         ctrl &= ~IXGBE_DMATXCTL_GDV;
 964         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
 965
 966         /* CTRL_EXT: Global Double VLAN Disable */
 967         ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 968         ctrl &= ~IXGBE_EXTENDED_VLAN;
 969         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
 970
 971 }
 972
 973 static void
 974 ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev)
 975 {
 976         struct ixgbe_hw *hw =
 977                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 978         uint32_t ctrl;
 979
 980         PMD_INIT_FUNC_TRACE();
 981
 982         /* DMATXCTRL: Geric Double VLAN Enable */
 983         ctrl  = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
 984         ctrl |= IXGBE_DMATXCTL_GDV;
 985         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
 986
 987         /* CTRL_EXT: Global Double VLAN Enable */
 988         ctrl  = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 989         ctrl |= IXGBE_EXTENDED_VLAN;
 990         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
 991
 992         /*
 993          * VET EXT field in the EXVET register = 0x8100 by default
 994          * So no need to change. Same to VT field of DMATXCTL register
 995          */
 996 }
 997
 998 static void
 999 ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1000 {
1001         if(mask & ETH_VLAN_STRIP_MASK){
1002                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1003                         ixgbe_vlan_hw_strip_enable_all(dev);
1004                 else
1005                         ixgbe_vlan_hw_strip_disable_all(dev);
1006         }
1007
1008         if(mask & ETH_VLAN_FILTER_MASK){
1009                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1010                         ixgbe_vlan_hw_filter_enable(dev);
1011                 else
1012                         ixgbe_vlan_hw_filter_disable(dev);
1013         }
1014
1015         if(mask & ETH_VLAN_EXTEND_MASK){
1016                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1017                         ixgbe_vlan_hw_extend_enable(dev);
1018                 else
1019                         ixgbe_vlan_hw_extend_disable(dev);
1020         }
1021 }
1022
1023 static int
1024 ixgbe_dev_configure(struct rte_eth_dev *dev)
1025 {
1026         struct ixgbe_interrupt *intr =
1027                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1028
1029         PMD_INIT_FUNC_TRACE();
1030
1031         /* set flag to update link status after init */
1032         intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1033
1034         return 0;
1035 }
1036
1037 /*
1038  * Configure device link speed and setup link.
1039  * It returns 0 on success.
1040  */
1041 static int
1042 ixgbe_dev_start(struct rte_eth_dev *dev)
1043 {
1044         struct ixgbe_hw *hw =
1045                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1046         int err, link_up = 0, negotiate = 0;
1047         uint32_t speed = 0;
1048
1049         PMD_INIT_FUNC_TRACE();
1050
1051         /* IXGBE devices don't support half duplex */
1052         if ((dev->data->dev_conf.link_duplex != ETH_LINK_AUTONEG_DUPLEX) &&
1053                         (dev->data->dev_conf.link_duplex != ETH_LINK_FULL_DUPLEX)) {
1054                 PMD_INIT_LOG(ERR, "Invalid link_duplex (%u) for port %u\n",
1055                                 dev->data->dev_conf.link_duplex,
1056                                 dev->data->port_id);
1057                 return -EINVAL;
1058         }
1059
1060         /* stop adapter */
1061         hw->adapter_stopped = FALSE;
1062         ixgbe_stop_adapter(hw);
1063
1064         /* reinitialize adapter
1065          * this calls reset and start */
1066         ixgbe_init_hw(hw);
1067
1068         /* initialize transmission unit */
1069         ixgbe_dev_tx_init(dev);
1070
1071         /* This can fail when allocating mbufs for descriptor rings */
1072         err = ixgbe_dev_rx_init(dev);
1073         if (err) {
1074                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware\n");
1075                 return err;
1076         }
1077
1078         ixgbe_dev_rxtx_start(dev);
1079
1080         if (ixgbe_is_sfp(hw) && hw->phy.multispeed_fiber) {
1081                 err = hw->mac.ops.setup_sfp(hw);
1082                 if (err)
1083                         goto error;
1084         }
1085
1086         /* Turn on the laser */
1087         if (hw->phy.multispeed_fiber)
1088                 ixgbe_enable_tx_laser(hw);
1089
1090         err = ixgbe_check_link(hw, &speed, &link_up, 0);
1091         if (err)
1092                 goto error;
1093         err = ixgbe_get_link_capabilities(hw, &speed, &negotiate);
1094         if (err)
1095                 goto error;
1096
1097         switch(dev->data->dev_conf.link_speed) {
1098         case ETH_LINK_SPEED_AUTONEG:
1099                 speed = (hw->mac.type != ixgbe_mac_82598EB) ?
1100                                 IXGBE_LINK_SPEED_82599_AUTONEG :
1101                                 IXGBE_LINK_SPEED_82598_AUTONEG;
1102                 break;
1103         case ETH_LINK_SPEED_100:
1104                 /*
1105                  * Invalid for 82598 but error will be detected by
1106                  * ixgbe_setup_link()
1107                  */
1108                 speed = IXGBE_LINK_SPEED_100_FULL;
1109                 break;
1110         case ETH_LINK_SPEED_1000:
1111                 speed = IXGBE_LINK_SPEED_1GB_FULL;
1112                 break;
1113         case ETH_LINK_SPEED_10000:
1114                 speed = IXGBE_LINK_SPEED_10GB_FULL;
1115                 break;
1116         default:
1117                 PMD_INIT_LOG(ERR, "Invalid link_speed (%u) for port %u\n",
1118                                 dev->data->dev_conf.link_speed, dev->data->port_id);
1119                 return -EINVAL;
1120         }
1121
1122         err = ixgbe_setup_link(hw, speed, negotiate, link_up);
1123         if (err)
1124                 goto error;
1125
1126         /* check if lsc interrupt is enabled */
1127         if (dev->data->dev_conf.intr_conf.lsc != 0) {
1128                 err = ixgbe_dev_interrupt_setup(dev);
1129                 if (err)
1130                         goto error;
1131         }
1132
1133         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
1134                 ETH_VLAN_EXTEND_MASK;
1135         ixgbe_vlan_offload_set(dev, mask);
1136
1137         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) {
1138                 err = ixgbe_fdir_configure(dev);
1139                 if (err)
1140                         goto error;
1141         }
1142
1143         ixgbe_restore_statistics_mapping(dev);
1144
1145         return (0);
1146
1147 error:
1148         PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err);
1149         ixgbe_dev_clear_queues(dev);
1150         return -EIO;
1151 }
1152
1153 /*
1154  * Stop device: disable rx and tx functions to allow for reconfiguring.
1155  */
1156 static void
1157 ixgbe_dev_stop(struct rte_eth_dev *dev)
1158 {
1159         struct rte_eth_link link;
1160         struct ixgbe_hw *hw =
1161                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1162
1163         PMD_INIT_FUNC_TRACE();
1164
1165         /* disable interrupts */
1166         ixgbe_disable_intr(hw);
1167
1168         /* reset the NIC */
1169         ixgbe_reset_hw(hw);
1170         hw->adapter_stopped = FALSE;
1171
1172         /* stop adapter */
1173         ixgbe_stop_adapter(hw);
1174
1175         /* Turn off the laser */
1176         if (hw->phy.multispeed_fiber)
1177                 ixgbe_disable_tx_laser(hw);
1178
1179         ixgbe_dev_clear_queues(dev);
1180
1181         /* Clear recorded link status */
1182         memset(&link, 0, sizeof(link));
1183         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1184 }
1185
1186 /*
1187  * Reest and stop device.
1188  */
1189 static void
1190 ixgbe_dev_close(struct rte_eth_dev *dev)
1191 {
1192         struct ixgbe_hw *hw =
1193                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1194
1195         PMD_INIT_FUNC_TRACE();
1196
1197         ixgbe_reset_hw(hw);
1198
1199
1200         ixgbe_dev_stop(dev);
1201         hw->adapter_stopped = 1;
1202
1203         ixgbe_disable_pcie_master(hw);
1204
1205         /* reprogram the RAR[0] in case user changed it. */
1206         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
1207 }
1208
1209 /*
1210  * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c
1211  */
1212 static void
1213 ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1214 {
1215         struct ixgbe_hw *hw =
1216                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1217         struct ixgbe_hw_stats *hw_stats =
1218                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1219         uint32_t bprc, lxon, lxoff, total;
1220         uint64_t total_missed_rx, total_qbrc, total_qprc;
1221         unsigned i;
1222
1223         total_missed_rx = 0;
1224         total_qbrc = 0;
1225         total_qprc = 0;
1226
1227         hw_stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
1228         hw_stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
1229         hw_stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
1230         hw_stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
1231
1232         for (i = 0; i < 8; i++) {
1233                 uint32_t mp;
1234                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
1235                 /* global total per queue */
1236                 hw_stats->mpc[i] += mp;
1237                 /* Running comprehensive total for stats display */
1238                 total_missed_rx += hw_stats->mpc[i];
1239                 if (hw->mac.type == ixgbe_mac_82598EB)
1240                         hw_stats->rnbc[i] +=
1241                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
1242                 hw_stats->pxontxc[i] +=
1243                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
1244                 hw_stats->pxonrxc[i] +=
1245                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
1246                 hw_stats->pxofftxc[i] +=
1247                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
1248                 hw_stats->pxoffrxc[i] +=
1249                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
1250                 hw_stats->pxon2offc[i] +=
1251                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
1252         }
1253         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1254                 hw_stats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
1255                 hw_stats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
1256                 hw_stats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
1257                 hw_stats->qbrc[i] +=
1258                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32);
1259                 hw_stats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
1260                 hw_stats->qbtc[i] +=
1261                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32);
1262                 hw_stats->qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
1263
1264                 total_qprc += hw_stats->qprc[i];
1265                 total_qbrc += hw_stats->qbrc[i];
1266         }
1267         hw_stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
1268         hw_stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
1269         hw_stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
1270
1271         /* Note that gprc counts missed packets */
1272         hw_stats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
1273
1274         if (hw->mac.type != ixgbe_mac_82598EB) {
1275                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
1276                     ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
1277                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
1278                     ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
1279                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
1280                     ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
1281                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
1282                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
1283         } else {
1284                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
1285                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
1286                 /* 82598 only has a counter in the high register */
1287                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
1288                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
1289                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
1290         }
1291
1292         /*
1293          * Workaround: mprc hardware is incorrectly counting
1294          * broadcasts, so for now we subtract those.
1295          */
1296         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
1297         hw_stats->bprc += bprc;
1298         hw_stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
1299         if (hw->mac.type == ixgbe_mac_82598EB)
1300                 hw_stats->mprc -= bprc;
1301
1302         hw_stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
1303         hw_stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
1304         hw_stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
1305         hw_stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
1306         hw_stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
1307         hw_stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
1308
1309         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
1310         hw_stats->lxontxc += lxon;
1311         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
1312         hw_stats->lxofftxc += lxoff;
1313         total = lxon + lxoff;
1314
1315         hw_stats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
1316         hw_stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
1317         hw_stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
1318         hw_stats->gptc -= total;
1319         hw_stats->mptc -= total;
1320         hw_stats->ptc64 -= total;
1321         hw_stats->gotc -= total * ETHER_MIN_LEN;
1322
1323         hw_stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
1324         hw_stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
1325         hw_stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
1326         hw_stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
1327         hw_stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
1328         hw_stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
1329         hw_stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
1330         hw_stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
1331         hw_stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
1332         hw_stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
1333         hw_stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
1334         hw_stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
1335         hw_stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
1336         hw_stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
1337         hw_stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
1338         hw_stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);
1339         hw_stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
1340         hw_stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
1341         /* Only read FCOE on 82599 */
1342         if (hw->mac.type != ixgbe_mac_82598EB) {
1343                 hw_stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
1344                 hw_stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
1345                 hw_stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
1346                 hw_stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
1347                 hw_stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
1348         }
1349
1350         if (stats == NULL)
1351                 return;
1352
1353         /* Fill out the rte_eth_stats statistics structure */
1354         stats->ipackets = total_qprc;
1355         stats->ibytes = total_qbrc;
1356         stats->opackets = hw_stats->gptc;
1357         stats->obytes = hw_stats->gotc;
1358         stats->imcasts = hw_stats->mprc;
1359
1360         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1361                 stats->q_ipackets[i] = hw_stats->qprc[i];
1362                 stats->q_opackets[i] = hw_stats->qptc[i];
1363                 stats->q_ibytes[i] = hw_stats->qbrc[i];
1364                 stats->q_obytes[i] = hw_stats->qbtc[i];
1365                 stats->q_errors[i] = hw_stats->qprdc[i];
1366         }
1367
1368         /* Rx Errors */
1369         stats->ierrors = total_missed_rx + hw_stats->crcerrs +
1370                 hw_stats->rlec;
1371
1372         stats->oerrors  = 0;
1373
1374         /* Flow Director Stats registers */
1375         hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
1376         hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
1377         stats->fdirmatch = hw_stats->fdirmatch;
1378         stats->fdirmiss = hw_stats->fdirmiss;
1379 }
1380
1381 static void
1382 ixgbe_dev_stats_reset(struct rte_eth_dev *dev)
1383 {
1384         struct ixgbe_hw_stats *stats =
1385                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1386
1387         /* HW registers are cleared on read */
1388         ixgbe_dev_stats_get(dev, NULL);
1389
1390         /* Reset software totals */
1391         memset(stats, 0, sizeof(*stats));
1392 }
1393
1394 static void
1395 ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1396 {
1397         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1398         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1399                           IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1400
1401         /* Good Rx packet, include VF loopback */
1402         UPDATE_VF_STAT(IXGBE_VFGPRC,
1403             hw_stats->last_vfgprc, hw_stats->vfgprc);
1404
1405         /* Good Rx octets, include VF loopback */
1406         UPDATE_VF_STAT_36BIT(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
1407             hw_stats->last_vfgorc, hw_stats->vfgorc);
1408
1409         /* Good Tx packet, include VF loopback */
1410         UPDATE_VF_STAT(IXGBE_VFGPTC,
1411             hw_stats->last_vfgptc, hw_stats->vfgptc);
1412
1413         /* Good Tx octets, include VF loopback */
1414         UPDATE_VF_STAT_36BIT(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
1415             hw_stats->last_vfgotc, hw_stats->vfgotc);
1416
1417         /* Rx Multicst Packet */
1418         UPDATE_VF_STAT(IXGBE_VFMPRC,
1419             hw_stats->last_vfmprc, hw_stats->vfmprc);
1420
1421         if (stats == NULL)
1422                 return;
1423
1424         memset(stats, 0, sizeof(*stats));
1425         stats->ipackets = hw_stats->vfgprc;
1426         stats->ibytes = hw_stats->vfgorc;
1427         stats->opackets = hw_stats->vfgptc;
1428         stats->obytes = hw_stats->vfgotc;
1429         stats->imcasts = hw_stats->vfmprc;
1430 }
1431
1432 static void
1433 ixgbevf_dev_stats_reset(struct rte_eth_dev *dev)
1434 {
1435         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1436                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1437
1438         /* Sync HW register to the last stats */
1439         ixgbevf_dev_stats_get(dev, NULL);
1440
1441         /* reset HW current stats*/
1442         hw_stats->vfgprc = 0;
1443         hw_stats->vfgorc = 0;
1444         hw_stats->vfgptc = 0;
1445         hw_stats->vfgotc = 0;
1446         hw_stats->vfmprc = 0;
1447
1448 }
1449
1450 static void
1451 ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1452 {
1453         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1454
1455         dev_info->max_rx_queues = hw->mac.max_rx_queues;
1456         dev_info->max_tx_queues = hw->mac.max_tx_queues;
1457         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL register */
1458         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */
1459         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
1460 }
1461
1462 /* return 0 means link status changed, -1 means not changed */
1463 static int
1464 ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1465 {
1466         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1467         struct rte_eth_link link, old;
1468         ixgbe_link_speed link_speed;
1469         int link_up;
1470         int diag;
1471
1472         link.link_status = 0;
1473         link.link_speed = 0;
1474         link.link_duplex = 0;
1475         memset(&old, 0, sizeof(old));
1476         rte_ixgbe_dev_atomic_read_link_status(dev, &old);
1477
1478         /* check if it needs to wait to complete, if lsc interrupt is enabled */
1479         if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
1480                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 0);
1481         else
1482                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 1);
1483         if (diag != 0) {
1484                 link.link_speed = ETH_LINK_SPEED_100;
1485                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1486                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1487                 if (link.link_status == old.link_status)
1488                         return -1;
1489                 return 0;
1490         }
1491
1492         if (link_up == 0) {
1493                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1494                 if (link.link_status == old.link_status)
1495                         return -1;
1496                 return 0;
1497         }
1498         link.link_status = 1;
1499         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1500
1501         switch (link_speed) {
1502         default:
1503         case IXGBE_LINK_SPEED_UNKNOWN:
1504                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1505                 link.link_speed = ETH_LINK_SPEED_100;
1506                 break;
1507
1508         case IXGBE_LINK_SPEED_100_FULL:
1509                 link.link_speed = ETH_LINK_SPEED_100;
1510                 break;
1511
1512         case IXGBE_LINK_SPEED_1GB_FULL:
1513                 link.link_speed = ETH_LINK_SPEED_1000;
1514                 break;
1515
1516         case IXGBE_LINK_SPEED_10GB_FULL:
1517                 link.link_speed = ETH_LINK_SPEED_10000;
1518                 break;
1519         }
1520         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1521
1522         if (link.link_status == old.link_status)
1523                 return -1;
1524
1525         return 0;
1526 }
1527
1528 static void
1529 ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev)
1530 {
1531         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1532         uint32_t fctrl;
1533
1534         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1535         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1536         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1537 }
1538
1539 static void
1540 ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev)
1541 {
1542         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1543         uint32_t fctrl;
1544
1545         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1546         fctrl &= (~IXGBE_FCTRL_UPE);
1547         if (dev->data->all_multicast == 1)
1548                 fctrl |= IXGBE_FCTRL_MPE;
1549         else
1550                 fctrl &= (~IXGBE_FCTRL_MPE);
1551         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1552 }
1553
1554 static void
1555 ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev)
1556 {
1557         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1558         uint32_t fctrl;
1559
1560         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1561         fctrl |= IXGBE_FCTRL_MPE;
1562         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1563 }
1564
1565 static void
1566 ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev)
1567 {
1568         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1569         uint32_t fctrl;
1570
1571         if (dev->data->promiscuous == 1)
1572                 return; /* must remain in all_multicast mode */
1573
1574         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
1575         fctrl &= (~IXGBE_FCTRL_MPE);
1576         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
1577 }
1578
1579 /**
1580  * It clears the interrupt causes and enables the interrupt.
1581  * It will be called once only during nic initialized.
1582  *
1583  * @param dev
1584  *  Pointer to struct rte_eth_dev.
1585  *
1586  * @return
1587  *  - On success, zero.
1588  *  - On failure, a negative value.
1589  */
1590 static int
1591 ixgbe_dev_interrupt_setup(struct rte_eth_dev *dev)
1592 {
1593         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1594
1595         ixgbe_dev_link_status_print(dev);
1596         IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_LSC);
1597         IXGBE_WRITE_FLUSH(hw);
1598         rte_intr_enable(&(dev->pci_dev->intr_handle));
1599
1600         return 0;
1601 }
1602
1603 /*
1604  * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update.
1605  *
1606  * @param dev
1607  *  Pointer to struct rte_eth_dev.
1608  *
1609  * @return
1610  *  - On success, zero.
1611  *  - On failure, a negative value.
1612  */
1613 static int
1614 ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
1615 {
1616         uint32_t eicr;
1617         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1618         struct ixgbe_interrupt *intr =
1619                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1620
1621         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_LSC);
1622         IXGBE_WRITE_FLUSH(hw);
1623
1624         /* read-on-clear nic registers here */
1625         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1626         PMD_INIT_LOG(INFO, "eicr %x", eicr);
1627         if (eicr & IXGBE_EICR_LSC) {
1628                 /* set flag for async link update */
1629                 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1630         }
1631
1632         return 0;
1633 }
1634
1635 /**
1636  * It gets and then prints the link status.
1637  *
1638  * @param dev
1639  *  Pointer to struct rte_eth_dev.
1640  *
1641  * @return
1642  *  - On success, zero.
1643  *  - On failure, a negative value.
1644  */
1645 static void
1646 ixgbe_dev_link_status_print(struct rte_eth_dev *dev)
1647 {
1648         struct rte_eth_link link;
1649
1650         memset(&link, 0, sizeof(link));
1651         rte_ixgbe_dev_atomic_read_link_status(dev, &link);
1652         if (link.link_status) {
1653                 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
1654                                         (int)(dev->data->port_id),
1655                                         (unsigned)link.link_speed,
1656                         link.link_duplex == ETH_LINK_FULL_DUPLEX ?
1657                                         "full-duplex" : "half-duplex");
1658         } else {
1659                 PMD_INIT_LOG(INFO, " Port %d: Link Down",
1660                                 (int)(dev->data->port_id));
1661         }
1662         PMD_INIT_LOG(INFO, "PCI Address: %04d:%02d:%02d:%d",
1663                                 dev->pci_dev->addr.domain,
1664                                 dev->pci_dev->addr.bus,
1665                                 dev->pci_dev->addr.devid,
1666                                 dev->pci_dev->addr.function);
1667 }
1668
1669 /*
1670  * It executes link_update after knowing an interrupt occured.
1671  *
1672  * @param dev
1673  *  Pointer to struct rte_eth_dev.
1674  *
1675  * @return
1676  *  - On success, zero.
1677  *  - On failure, a negative value.
1678  */
1679 static int
1680 ixgbe_dev_interrupt_action(struct rte_eth_dev *dev)
1681 {
1682         struct ixgbe_interrupt *intr =
1683                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1684
1685         if (!(intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE)) {
1686                 return -1;
1687         }
1688         ixgbe_dev_link_update(dev, 0);
1689
1690         return 0;
1691 }
1692
1693 /**
1694  * Interrupt handler which shall be registered for alarm callback for delayed
1695  * handling specific interrupt to wait for the stable nic state. As the
1696  * NIC interrupt state is not stable for ixgbe after link is just down,
1697  * it needs to wait 4 seconds to get the stable status.
1698  *
1699  * @param handle
1700  *  Pointer to interrupt handle.
1701  * @param param
1702  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1703  *
1704  * @return
1705  *  void
1706  */
1707 static void
1708 ixgbe_dev_interrupt_delayed_handler(void *param)
1709 {
1710         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1711         struct ixgbe_interrupt *intr =
1712                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1713         struct ixgbe_hw *hw =
1714                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1715
1716         IXGBE_READ_REG(hw, IXGBE_EICR);
1717         ixgbe_dev_interrupt_action(dev);
1718         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
1719                 intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
1720                 rte_intr_enable(&(dev->pci_dev->intr_handle));
1721                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_LSC);
1722                 IXGBE_WRITE_FLUSH(hw);
1723                 ixgbe_dev_link_status_print(dev);
1724                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
1725         }
1726 }
1727
1728 /**
1729  * Interrupt handler triggered by NIC  for handling
1730  * specific interrupt.
1731  *
1732  * @param handle
1733  *  Pointer to interrupt handle.
1734  * @param param
1735  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1736  *
1737  * @return
1738  *  void
1739  */
1740 static void
1741 ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle, void *param)
1742 {
1743         int64_t timeout;
1744         struct rte_eth_link link;
1745         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1746         struct ixgbe_interrupt *intr =
1747                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1748
1749         /* get the link status before link update, for predicting later */
1750         memset(&link, 0, sizeof(link));
1751         rte_ixgbe_dev_atomic_read_link_status(dev, &link);
1752         ixgbe_dev_interrupt_get_status(dev);
1753         ixgbe_dev_interrupt_action(dev);
1754
1755         if (!(intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE))
1756                 return;
1757
1758         /* likely to up */
1759         if (!link.link_status)
1760                 /* handle it 1 sec later, wait it being stable */
1761                 timeout = IXGBE_LINK_UP_CHECK_TIMEOUT;
1762         /* likely to down */
1763         else
1764                 /* handle it 4 sec later, wait it being stable */
1765                 timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT;
1766
1767         ixgbe_dev_link_status_print(dev);
1768         if (rte_eal_alarm_set(timeout * 1000,
1769                 ixgbe_dev_interrupt_delayed_handler, param) < 0)
1770                 PMD_INIT_LOG(ERR, "Error setting alarm");
1771 }
1772
1773 static int
1774 ixgbe_dev_led_on(struct rte_eth_dev *dev)
1775 {
1776         struct ixgbe_hw *hw;
1777
1778         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1779         return (ixgbe_led_on(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
1780 }
1781
1782 static int
1783 ixgbe_dev_led_off(struct rte_eth_dev *dev)
1784 {
1785         struct ixgbe_hw *hw;
1786
1787         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1788         return (ixgbe_led_off(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
1789 }
1790
1791 static int
1792 ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1793 {
1794         struct ixgbe_hw *hw;
1795         int err;
1796         uint32_t rx_buf_size;
1797         uint32_t max_high_water;
1798         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
1799                 ixgbe_fc_none,
1800                 ixgbe_fc_rx_pause,
1801                 ixgbe_fc_tx_pause,
1802                 ixgbe_fc_full
1803         };
1804
1805         PMD_INIT_FUNC_TRACE();
1806
1807         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1808         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0));
1809         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
1810
1811         /*
1812          * At least reserve one Ethernet frame for watermark
1813          * high_water/low_water in kilo bytes for ixgbe
1814          */
1815         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
1816         if ((fc_conf->high_water > max_high_water) ||
1817                 (fc_conf->high_water < fc_conf->low_water)) {
1818                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
1819                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
1820                 return (-EINVAL);
1821         }
1822
1823         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[fc_conf->mode];
1824         hw->fc.pause_time     = fc_conf->pause_time;
1825         hw->fc.high_water[0]  = fc_conf->high_water;
1826         hw->fc.low_water      = fc_conf->low_water;
1827         hw->fc.send_xon       = fc_conf->send_xon;
1828
1829         err = ixgbe_fc_enable(hw, 0);
1830         /* Not negotiated is not an error case */
1831         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) {
1832                 return 0;
1833         }
1834
1835         PMD_INIT_LOG(ERR, "ixgbe_fc_enable = 0x%x \n", err);
1836         return -EIO;
1837 }
1838
1839 static void
1840 ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
1841                                 uint32_t index, uint32_t pool)
1842 {
1843         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1844         uint32_t enable_addr = 1;
1845
1846         ixgbe_set_rar(hw, index, mac_addr->addr_bytes, pool, enable_addr);
1847 }
1848
1849 static void
1850 ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
1851 {
1852         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1853
1854         ixgbe_clear_rar(hw, index);
1855 }
1856
1857 /*
1858  * Virtual Function operations
1859  */
1860 static void
1861 ixgbevf_intr_disable(struct ixgbe_hw *hw)
1862 {
1863         PMD_INIT_LOG(DEBUG, "ixgbevf_intr_disable");
1864
1865         /* Clear interrupt mask to stop from interrupts being generated */
1866         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, IXGBE_VF_IRQ_CLEAR_MASK);
1867
1868         IXGBE_WRITE_FLUSH(hw);
1869 }
1870
1871 static int
1872 ixgbevf_dev_configure(struct rte_eth_dev *dev)
1873 {
1874         struct rte_eth_conf* conf = &dev->data->dev_conf;
1875
1876
1877         if (!conf->rxmode.hw_strip_crc) {
1878                 /*
1879                  * VF has no ability to enable/disable HW CRC
1880                  * Keep the persistent behavior the same as Host PF
1881                  */
1882                 PMD_INIT_LOG(INFO, "VF can't disable HW CRC Strip\n");
1883                 conf->rxmode.hw_strip_crc = 1;
1884         }
1885
1886         return 0;
1887 }
1888
1889 static int
1890 ixgbevf_dev_start(struct rte_eth_dev *dev)
1891 {
1892         int err = 0;
1893         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_start");
1894
1895         ixgbevf_dev_tx_init(dev);
1896         err = ixgbevf_dev_rx_init(dev);
1897         if(err){
1898                 ixgbe_dev_clear_queues(dev);
1899                 PMD_INIT_LOG(ERR,"Unable to initialize RX hardware\n");
1900                 return err;
1901         }
1902
1903         /* Set vfta */
1904         ixgbevf_set_vfta_all(dev,1);
1905
1906         /* Set HW strip */
1907         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
1908                 ETH_VLAN_EXTEND_MASK;
1909         ixgbevf_vlan_offload_set(dev, mask);
1910
1911         ixgbevf_dev_rxtx_start(dev);
1912
1913         return 0;
1914 }
1915
1916 static void
1917 ixgbevf_dev_stop(struct rte_eth_dev *dev)
1918 {
1919         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1920
1921         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_stop");
1922
1923         ixgbe_reset_hw(hw);
1924         hw->adapter_stopped = 0;
1925         ixgbe_stop_adapter(hw);
1926         /* reprogram the RAR[0] in case user changed it. */
1927         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
1928 }
1929
1930 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on)
1931 {
1932         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1933         struct ixgbe_vfta * shadow_vfta =
1934                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1935         int i = 0, j = 0, vfta = 0, mask = 1;
1936
1937         for (i = 0; i < IXGBE_VFTA_SIZE; i++){
1938                 vfta = shadow_vfta->vfta[i];
1939                 if(vfta){
1940                         mask = 1;
1941                         for (j = 0; j < 32; j++){
1942                                 if(vfta & mask)
1943                                         ixgbe_set_vfta(hw, (i<<5)+j, 0, on);
1944                                 mask<<=1;
1945                         }
1946                 }
1947         }
1948
1949 }
1950
1951 static int
1952 ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1953 {
1954         struct ixgbe_hw *hw =
1955                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1956         struct ixgbe_vfta * shadow_vfta =
1957                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1958         uint32_t vid_idx = 0;
1959         uint32_t vid_bit = 0;
1960         int ret = 0;
1961
1962         PMD_INIT_FUNC_TRACE();
1963
1964         /* vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf */
1965         ret = ixgbe_set_vfta(hw, vlan_id, 0, !!on);
1966         if(ret){
1967                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
1968                 return ret;
1969         }
1970         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
1971         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
1972
1973         /* Save what we set and retore it after device reset */
1974         if (on)
1975                 shadow_vfta->vfta[vid_idx] |= vid_bit;
1976         else
1977                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
1978
1979         return 0;
1980 }
1981
1982 static void
1983 ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
1984 {
1985         struct ixgbe_hw *hw =
1986                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1987         uint32_t ctrl;
1988
1989         PMD_INIT_FUNC_TRACE();
1990
1991         if(queue >= hw->mac.max_rx_queues)
1992                 return;
1993
1994         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1995         if(on)
1996                 ctrl |= IXGBE_RXDCTL_VME;
1997         else
1998                 ctrl &= ~IXGBE_RXDCTL_VME;
1999         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
2000
2001         ixgbe_vlan_hw_strip_bitmap_set( dev, queue, on);
2002 }
2003
2004 static void
2005 ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2006 {
2007         struct ixgbe_hw *hw =
2008                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2009         uint32_t i, on = 0;
2010
2011         /* VF function only support hw strip feature, others are not support */
2012         if(mask & ETH_VLAN_STRIP_MASK){
2013                 on = !!(dev->data->dev_conf.rxmode.hw_vlan_strip);
2014
2015                 for(i=0; i < hw->mac.max_rx_queues; i++)
2016                         ixgbevf_vlan_strip_queue_set(dev,i,on);
2017         }
2018 }
2019