drivers/net/ixgbe/ixgbe_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <inttypes.h>
  42 #include <netinet/in.h>
  43 #include <rte_byteorder.h>
  44 #include <rte_common.h>
  45 #include <rte_cycles.h>
  46
  47 #include <rte_interrupts.h>
  48 #include <rte_log.h>
  49 #include <rte_debug.h>
  50 #include <rte_pci.h>
  51 #include <rte_atomic.h>
  52 #include <rte_branch_prediction.h>
  53 #include <rte_memory.h>
  54 #include <rte_memzone.h>
  55 #include <rte_eal.h>
  56 #include <rte_alarm.h>
  57 #include <rte_ether.h>
  58 #include <rte_ethdev.h>
  59 #include <rte_atomic.h>
  60 #include <rte_malloc.h>
  61 #include <rte_random.h>
  62 #include <rte_dev.h>
  63
  64 #include "ixgbe_logs.h"
  65 #include "base/ixgbe_api.h"
  66 #include "base/ixgbe_vf.h"
  67 #include "base/ixgbe_common.h"
  68 #include "ixgbe_ethdev.h"
  69 #include "ixgbe_bypass.h"
  70 #include "ixgbe_rxtx.h"
  71 #include "base/ixgbe_type.h"
  72 #include "base/ixgbe_phy.h"
  73 #include "ixgbe_regs.h"
  74
  75 /*
  76  * High threshold controlling when to start sending XOFF frames. Must be at
  77  * least 8 bytes less than receive packet buffer size. This value is in units
  78  * of 1024 bytes.
  79  */
  80 #define IXGBE_FC_HI    0x80
  81
  82 /*
  83  * Low threshold controlling when to start sending XON frames. This value is
  84  * in units of 1024 bytes.
  85  */
  86 #define IXGBE_FC_LO    0x40
  87
  88 /* Default minimum inter-interrupt interval for EITR configuration */
  89 #define IXGBE_MIN_INTER_INTERRUPT_INTERVAL_DEFAULT    0x79E
  90
  91 /* Timer value included in XOFF frames. */
  92 #define IXGBE_FC_PAUSE 0x680
  93
  94 #define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */
  95 #define IXGBE_LINK_UP_CHECK_TIMEOUT   1000 /* ms */
  96 #define IXGBE_VMDQ_NUM_UC_MAC         4096 /* Maximum nb. of UC MAC addr. */
  97
  98 #define IXGBE_MMW_SIZE_DEFAULT        0x4
  99 #define IXGBE_MMW_SIZE_JUMBO_FRAME    0x14
 100 #define IXGBE_MAX_RING_DESC           4096 /* replicate define from rxtx */
 101
 102 /*
 103  *  Default values for RX/TX configuration
 104  */
 105 #define IXGBE_DEFAULT_RX_FREE_THRESH  32
 106 #define IXGBE_DEFAULT_RX_PTHRESH      8
 107 #define IXGBE_DEFAULT_RX_HTHRESH      8
 108 #define IXGBE_DEFAULT_RX_WTHRESH      0
 109
 110 #define IXGBE_DEFAULT_TX_FREE_THRESH  32
 111 #define IXGBE_DEFAULT_TX_PTHRESH      32
 112 #define IXGBE_DEFAULT_TX_HTHRESH      0
 113 #define IXGBE_DEFAULT_TX_WTHRESH      0
 114 #define IXGBE_DEFAULT_TX_RSBIT_THRESH 32
 115
 116 /* Bit shift and mask */
 117 #define IXGBE_4_BIT_WIDTH  (CHAR_BIT / 2)
 118 #define IXGBE_4_BIT_MASK   RTE_LEN2MASK(IXGBE_4_BIT_WIDTH, uint8_t)
 119 #define IXGBE_8_BIT_WIDTH  CHAR_BIT
 120 #define IXGBE_8_BIT_MASK   UINT8_MAX
 121
 122 #define IXGBEVF_PMD_NAME "rte_ixgbevf_pmd" /* PMD name */
 123
 124 #define IXGBE_QUEUE_STAT_COUNTERS (sizeof(hw_stats->qprc) / sizeof(hw_stats->qprc[0]))
 125
 126 #define IXGBE_HKEY_MAX_INDEX 10
 127
 128 /* Additional timesync values. */
 129 #define IXGBE_TIMINCA_16NS_SHIFT 24
 130 #define IXGBE_TIMINCA_INCVALUE   16000000
 131 #define IXGBE_TIMINCA_INIT       ((0x02 << IXGBE_TIMINCA_16NS_SHIFT) \
 132                                   | IXGBE_TIMINCA_INCVALUE)
 133
 134 static int eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev);
 135 static int eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev);
 136 static int  ixgbe_dev_configure(struct rte_eth_dev *dev);
 137 static int  ixgbe_dev_start(struct rte_eth_dev *dev);
 138 static void ixgbe_dev_stop(struct rte_eth_dev *dev);
 139 static int  ixgbe_dev_set_link_up(struct rte_eth_dev *dev);
 140 static int  ixgbe_dev_set_link_down(struct rte_eth_dev *dev);
 141 static void ixgbe_dev_close(struct rte_eth_dev *dev);
 142 static void ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev);
 143 static void ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev);
 144 static void ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev);
 145 static void ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev);
 146 static int ixgbe_dev_link_update(struct rte_eth_dev *dev,
 147                                 int wait_to_complete);
 148 static void ixgbe_dev_stats_get(struct rte_eth_dev *dev,
 149                                 struct rte_eth_stats *stats);
 150 static int ixgbe_dev_xstats_get(struct rte_eth_dev *dev,
 151                                 struct rte_eth_xstats *xstats, unsigned n);
 152 static void ixgbe_dev_stats_reset(struct rte_eth_dev *dev);
 153 static void ixgbe_dev_xstats_reset(struct rte_eth_dev *dev);
 154 static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 155                                              uint16_t queue_id,
 156                                              uint8_t stat_idx,
 157                                              uint8_t is_rx);
 158 static void ixgbe_dev_info_get(struct rte_eth_dev *dev,
 159                                struct rte_eth_dev_info *dev_info);
 160 static void ixgbevf_dev_info_get(struct rte_eth_dev *dev,
 161                                  struct rte_eth_dev_info *dev_info);
 162 static int ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 163
 164 static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev,
 165                 uint16_t vlan_id, int on);
 166 static void ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 167 static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev,
 168                 uint16_t queue, bool on);
 169 static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue,
 170                 int on);
 171 static void ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 172 static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue);
 173 static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue);
 174 static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 175 static void ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 176
 177 static int ixgbe_dev_led_on(struct rte_eth_dev *dev);
 178 static int ixgbe_dev_led_off(struct rte_eth_dev *dev);
 179 static int ixgbe_flow_ctrl_get(struct rte_eth_dev *dev,
 180                                struct rte_eth_fc_conf *fc_conf);
 181 static int ixgbe_flow_ctrl_set(struct rte_eth_dev *dev,
 182                                struct rte_eth_fc_conf *fc_conf);
 183 static int ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 184                 struct rte_eth_pfc_conf *pfc_conf);
 185 static int ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
 186                         struct rte_eth_rss_reta_entry64 *reta_conf,
 187                         uint16_t reta_size);
 188 static int ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
 189                         struct rte_eth_rss_reta_entry64 *reta_conf,
 190                         uint16_t reta_size);
 191 static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev);
 192 static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev);
 193 static int ixgbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 194 static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev);
 195 static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev);
 196 static void ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle,
 197                 void *param);
 198 static void ixgbe_dev_interrupt_delayed_handler(void *param);
 199 static void ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 200                 uint32_t index, uint32_t pool);
 201 static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index);
 202 static void ixgbe_set_default_mac_addr(struct rte_eth_dev *dev,
 203                                            struct ether_addr *mac_addr);
 204 static void ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config);
 205
 206 /* For Virtual Function support */
 207 static int eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev);
 208 static int eth_ixgbevf_dev_uninit(struct rte_eth_dev *eth_dev);
 209 static int ixgbevf_dev_interrupt_get_status(struct rte_eth_dev *dev);
 210 static int ixgbevf_dev_interrupt_action(struct rte_eth_dev *dev);
 211 static int  ixgbevf_dev_configure(struct rte_eth_dev *dev);
 212 static int  ixgbevf_dev_start(struct rte_eth_dev *dev);
 213 static void ixgbevf_dev_stop(struct rte_eth_dev *dev);
 214 static void ixgbevf_dev_close(struct rte_eth_dev *dev);
 215 static void ixgbevf_intr_disable(struct ixgbe_hw *hw);
 216 static void ixgbevf_intr_enable(struct ixgbe_hw *hw);
 217 static void ixgbevf_dev_stats_get(struct rte_eth_dev *dev,
 218                 struct rte_eth_stats *stats);
 219 static void ixgbevf_dev_stats_reset(struct rte_eth_dev *dev);
 220 static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev,
 221                 uint16_t vlan_id, int on);
 222 static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev,
 223                 uint16_t queue, int on);
 224 static void ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 225 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 226 static void ixgbevf_dev_interrupt_handler(struct rte_intr_handle *handle,
 227                                           void *param);
 228 static int ixgbevf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev,
 229                                             uint16_t queue_id);
 230 static int ixgbevf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev,
 231                                              uint16_t queue_id);
 232 static void ixgbevf_set_ivar_map(struct ixgbe_hw *hw, int8_t direction,
 233                                  uint8_t queue, uint8_t msix_vector);
 234 static void ixgbevf_configure_msix(struct rte_eth_dev *dev);
 235
 236 /* For Eth VMDQ APIs support */
 237 static int ixgbe_uc_hash_table_set(struct rte_eth_dev *dev, struct
 238                 ether_addr* mac_addr,uint8_t on);
 239 static int ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev,uint8_t on);
 240 static int  ixgbe_set_pool_rx_mode(struct rte_eth_dev *dev,  uint16_t pool,
 241                 uint16_t rx_mask, uint8_t on);
 242 static int ixgbe_set_pool_rx(struct rte_eth_dev *dev,uint16_t pool,uint8_t on);
 243 static int ixgbe_set_pool_tx(struct rte_eth_dev *dev,uint16_t pool,uint8_t on);
 244 static int ixgbe_set_pool_vlan_filter(struct rte_eth_dev *dev, uint16_t vlan,
 245                 uint64_t pool_mask,uint8_t vlan_on);
 246 static int ixgbe_mirror_rule_set(struct rte_eth_dev *dev,
 247                 struct rte_eth_mirror_conf *mirror_conf,
 248                 uint8_t rule_id, uint8_t on);
 249 static int ixgbe_mirror_rule_reset(struct rte_eth_dev *dev,
 250                 uint8_t rule_id);
 251 static int ixgbe_dev_rx_queue_intr_enable(struct rte_eth_dev *dev,
 252                                           uint16_t queue_id);
 253 static int ixgbe_dev_rx_queue_intr_disable(struct rte_eth_dev *dev,
 254                                            uint16_t queue_id);
 255 static void ixgbe_set_ivar_map(struct ixgbe_hw *hw, int8_t direction,
 256                                uint8_t queue, uint8_t msix_vector);
 257 static void ixgbe_configure_msix(struct rte_eth_dev *dev);
 258
 259 static int ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev,
 260                 uint16_t queue_idx, uint16_t tx_rate);
 261 static int ixgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf,
 262                 uint16_t tx_rate, uint64_t q_msk);
 263
 264 static void ixgbevf_add_mac_addr(struct rte_eth_dev *dev,
 265                                  struct ether_addr *mac_addr,
 266                                  uint32_t index, uint32_t pool);
 267 static void ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index);
 268 static void ixgbevf_set_default_mac_addr(struct rte_eth_dev *dev,
 269                                              struct ether_addr *mac_addr);
 270 static int ixgbe_syn_filter_set(struct rte_eth_dev *dev,
 271                         struct rte_eth_syn_filter *filter,
 272                         bool add);
 273 static int ixgbe_syn_filter_get(struct rte_eth_dev *dev,
 274                         struct rte_eth_syn_filter *filter);
 275 static int ixgbe_syn_filter_handle(struct rte_eth_dev *dev,
 276                         enum rte_filter_op filter_op,
 277                         void *arg);
 278 static int ixgbe_add_5tuple_filter(struct rte_eth_dev *dev,
 279                         struct ixgbe_5tuple_filter *filter);
 280 static void ixgbe_remove_5tuple_filter(struct rte_eth_dev *dev,
 281                         struct ixgbe_5tuple_filter *filter);
 282 static int ixgbe_add_del_ntuple_filter(struct rte_eth_dev *dev,
 283                         struct rte_eth_ntuple_filter *filter,
 284                         bool add);
 285 static int ixgbe_ntuple_filter_handle(struct rte_eth_dev *dev,
 286                                 enum rte_filter_op filter_op,
 287                                 void *arg);
 288 static int ixgbe_get_ntuple_filter(struct rte_eth_dev *dev,
 289                         struct rte_eth_ntuple_filter *filter);
 290 static int ixgbe_add_del_ethertype_filter(struct rte_eth_dev *dev,
 291                         struct rte_eth_ethertype_filter *filter,
 292                         bool add);
 293 static int ixgbe_ethertype_filter_handle(struct rte_eth_dev *dev,
 294                                 enum rte_filter_op filter_op,
 295                                 void *arg);
 296 static int ixgbe_get_ethertype_filter(struct rte_eth_dev *dev,
 297                         struct rte_eth_ethertype_filter *filter);
 298 static int ixgbe_dev_filter_ctrl(struct rte_eth_dev *dev,
 299                      enum rte_filter_type filter_type,
 300                      enum rte_filter_op filter_op,
 301                      void *arg);
 302 static int ixgbevf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu);
 303
 304 static int ixgbe_dev_set_mc_addr_list(struct rte_eth_dev *dev,
 305                                       struct ether_addr *mc_addr_set,
 306                                       uint32_t nb_mc_addr);
 307 static int ixgbe_dev_get_dcb_info(struct rte_eth_dev *dev,
 308                                    struct rte_eth_dcb_info *dcb_info);
 309
 310 static int ixgbe_get_reg_length(struct rte_eth_dev *dev);
 311 static int ixgbe_get_regs(struct rte_eth_dev *dev,
 312                             struct rte_dev_reg_info *regs);
 313 static int ixgbe_get_eeprom_length(struct rte_eth_dev *dev);
 314 static int ixgbe_get_eeprom(struct rte_eth_dev *dev,
 315                                 struct rte_dev_eeprom_info *eeprom);
 316 static int ixgbe_set_eeprom(struct rte_eth_dev *dev,
 317                                 struct rte_dev_eeprom_info *eeprom);
 318
 319 static int ixgbevf_get_reg_length(struct rte_eth_dev *dev);
 320 static int ixgbevf_get_regs(struct rte_eth_dev *dev,
 321                                 struct rte_dev_reg_info *regs);
 322
 323 static int ixgbe_timesync_enable(struct rte_eth_dev *dev);
 324 static int ixgbe_timesync_disable(struct rte_eth_dev *dev);
 325 static int ixgbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
 326                                             struct timespec *timestamp,
 327                                             uint32_t flags);
 328 static int ixgbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
 329                                             struct timespec *timestamp);
 330
 331 /*
 332  * Define VF Stats MACRO for Non "cleared on read" register
 333  */
 334 #define UPDATE_VF_STAT(reg, last, cur)                          \
 335 {                                                               \
 336         uint32_t latest = IXGBE_READ_REG(hw, reg);              \
 337         cur += (latest - last) & UINT_MAX;                      \
 338         last = latest;                                          \
 339 }
 340
 341 #define UPDATE_VF_STAT_36BIT(lsb, msb, last, cur)                \
 342 {                                                                \
 343         u64 new_lsb = IXGBE_READ_REG(hw, lsb);                   \
 344         u64 new_msb = IXGBE_READ_REG(hw, msb);                   \
 345         u64 latest = ((new_msb << 32) | new_lsb);                \
 346         cur += (0x1000000000LL + latest - last) & 0xFFFFFFFFFLL; \
 347         last = latest;                                           \
 348 }
 349
 350 #define IXGBE_SET_HWSTRIP(h, q) do{\
 351                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 352                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 353                 (h)->bitmap[idx] |= 1 << bit;\
 354         }while(0)
 355
 356 #define IXGBE_CLEAR_HWSTRIP(h, q) do{\
 357                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 358                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 359                 (h)->bitmap[idx] &= ~(1 << bit);\
 360         }while(0)
 361
 362 #define IXGBE_GET_HWSTRIP(h, q, r) do{\
 363                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 364                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 365                 (r) = (h)->bitmap[idx] >> bit & 1;\
 366         }while(0)
 367
 368 /*
 369  * The set of PCI devices this driver supports
 370  */
 371 static const struct rte_pci_id pci_id_ixgbe_map[] = {
 372
 373 #define RTE_PCI_DEV_ID_DECL_IXGBE(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 374 #include "rte_pci_dev_ids.h"
 375
 376 { .vendor_id = 0, /* sentinel */ },
 377 };
 378
 379
 380 /*
 381  * The set of PCI devices this driver supports (for 82599 VF)
 382  */
 383 static const struct rte_pci_id pci_id_ixgbevf_map[] = {
 384
 385 #define RTE_PCI_DEV_ID_DECL_IXGBEVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 386 #include "rte_pci_dev_ids.h"
 387 { .vendor_id = 0, /* sentinel */ },
 388
 389 };
 390
 391 static const struct eth_dev_ops ixgbe_eth_dev_ops = {
 392         .dev_configure        = ixgbe_dev_configure,
 393         .dev_start            = ixgbe_dev_start,
 394         .dev_stop             = ixgbe_dev_stop,
 395         .dev_set_link_up    = ixgbe_dev_set_link_up,
 396         .dev_set_link_down  = ixgbe_dev_set_link_down,
 397         .dev_close            = ixgbe_dev_close,
 398         .promiscuous_enable   = ixgbe_dev_promiscuous_enable,
 399         .promiscuous_disable  = ixgbe_dev_promiscuous_disable,
 400         .allmulticast_enable  = ixgbe_dev_allmulticast_enable,
 401         .allmulticast_disable = ixgbe_dev_allmulticast_disable,
 402         .link_update          = ixgbe_dev_link_update,
 403         .stats_get            = ixgbe_dev_stats_get,
 404         .xstats_get           = ixgbe_dev_xstats_get,
 405         .stats_reset          = ixgbe_dev_stats_reset,
 406         .xstats_reset         = ixgbe_dev_xstats_reset,
 407         .queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set,
 408         .dev_infos_get        = ixgbe_dev_info_get,
 409         .mtu_set              = ixgbe_dev_mtu_set,
 410         .vlan_filter_set      = ixgbe_vlan_filter_set,
 411         .vlan_tpid_set        = ixgbe_vlan_tpid_set,
 412         .vlan_offload_set     = ixgbe_vlan_offload_set,
 413         .vlan_strip_queue_set = ixgbe_vlan_strip_queue_set,
 414         .rx_queue_start       = ixgbe_dev_rx_queue_start,
 415         .rx_queue_stop        = ixgbe_dev_rx_queue_stop,
 416         .tx_queue_start       = ixgbe_dev_tx_queue_start,
 417         .tx_queue_stop        = ixgbe_dev_tx_queue_stop,
 418         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 419         .rx_queue_intr_enable = ixgbe_dev_rx_queue_intr_enable,
 420         .rx_queue_intr_disable = ixgbe_dev_rx_queue_intr_disable,
 421         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 422         .rx_queue_count       = ixgbe_dev_rx_queue_count,
 423         .rx_descriptor_done   = ixgbe_dev_rx_descriptor_done,
 424         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 425         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 426         .dev_led_on           = ixgbe_dev_led_on,
 427         .dev_led_off          = ixgbe_dev_led_off,
 428         .flow_ctrl_get        = ixgbe_flow_ctrl_get,
 429         .flow_ctrl_set        = ixgbe_flow_ctrl_set,
 430         .priority_flow_ctrl_set = ixgbe_priority_flow_ctrl_set,
 431         .mac_addr_add         = ixgbe_add_rar,
 432         .mac_addr_remove      = ixgbe_remove_rar,
 433         .mac_addr_set         = ixgbe_set_default_mac_addr,
 434         .uc_hash_table_set    = ixgbe_uc_hash_table_set,
 435         .uc_all_hash_table_set  = ixgbe_uc_all_hash_table_set,
 436         .mirror_rule_set      = ixgbe_mirror_rule_set,
 437         .mirror_rule_reset    = ixgbe_mirror_rule_reset,
 438         .set_vf_rx_mode       = ixgbe_set_pool_rx_mode,
 439         .set_vf_rx            = ixgbe_set_pool_rx,
 440         .set_vf_tx            = ixgbe_set_pool_tx,
 441         .set_vf_vlan_filter   = ixgbe_set_pool_vlan_filter,
 442         .set_queue_rate_limit = ixgbe_set_queue_rate_limit,
 443         .set_vf_rate_limit    = ixgbe_set_vf_rate_limit,
 444         .reta_update          = ixgbe_dev_rss_reta_update,
 445         .reta_query           = ixgbe_dev_rss_reta_query,
 446 #ifdef RTE_NIC_BYPASS
 447         .bypass_init          = ixgbe_bypass_init,
 448         .bypass_state_set     = ixgbe_bypass_state_store,
 449         .bypass_state_show    = ixgbe_bypass_state_show,
 450         .bypass_event_set     = ixgbe_bypass_event_store,
 451         .bypass_event_show    = ixgbe_bypass_event_show,
 452         .bypass_wd_timeout_set  = ixgbe_bypass_wd_timeout_store,
 453         .bypass_wd_timeout_show = ixgbe_bypass_wd_timeout_show,
 454         .bypass_ver_show      = ixgbe_bypass_ver_show,
 455         .bypass_wd_reset      = ixgbe_bypass_wd_reset,
 456 #endif /* RTE_NIC_BYPASS */
 457         .rss_hash_update      = ixgbe_dev_rss_hash_update,
 458         .rss_hash_conf_get    = ixgbe_dev_rss_hash_conf_get,
 459         .filter_ctrl          = ixgbe_dev_filter_ctrl,
 460         .set_mc_addr_list     = ixgbe_dev_set_mc_addr_list,
 461         .timesync_enable      = ixgbe_timesync_enable,
 462         .timesync_disable     = ixgbe_timesync_disable,
 463         .timesync_read_rx_timestamp = ixgbe_timesync_read_rx_timestamp,
 464         .timesync_read_tx_timestamp = ixgbe_timesync_read_tx_timestamp,
 465         .get_reg_length       = ixgbe_get_reg_length,
 466         .get_reg              = ixgbe_get_regs,
 467         .get_eeprom_length    = ixgbe_get_eeprom_length,
 468         .get_eeprom           = ixgbe_get_eeprom,
 469         .set_eeprom           = ixgbe_set_eeprom,
 470         .get_dcb_info         = ixgbe_dev_get_dcb_info,
 471 };
 472
 473 /*
 474  * dev_ops for virtual function, bare necessities for basic vf
 475  * operation have been implemented
 476  */
 477 static const struct eth_dev_ops ixgbevf_eth_dev_ops = {
 478         .dev_configure        = ixgbevf_dev_configure,
 479         .dev_start            = ixgbevf_dev_start,
 480         .dev_stop             = ixgbevf_dev_stop,
 481         .link_update          = ixgbe_dev_link_update,
 482         .stats_get            = ixgbevf_dev_stats_get,
 483         .stats_reset          = ixgbevf_dev_stats_reset,
 484         .dev_close            = ixgbevf_dev_close,
 485         .dev_infos_get        = ixgbevf_dev_info_get,
 486         .mtu_set              = ixgbevf_dev_set_mtu,
 487         .vlan_filter_set      = ixgbevf_vlan_filter_set,
 488         .vlan_strip_queue_set = ixgbevf_vlan_strip_queue_set,
 489         .vlan_offload_set     = ixgbevf_vlan_offload_set,
 490         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 491         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 492         .rx_descriptor_done   = ixgbe_dev_rx_descriptor_done,
 493         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 494         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 495         .rx_queue_intr_enable = ixgbevf_dev_rx_queue_intr_enable,
 496         .rx_queue_intr_disable = ixgbevf_dev_rx_queue_intr_disable,
 497         .mac_addr_add         = ixgbevf_add_mac_addr,
 498         .mac_addr_remove      = ixgbevf_remove_mac_addr,
 499         .set_mc_addr_list     = ixgbe_dev_set_mc_addr_list,
 500         .mac_addr_set         = ixgbevf_set_default_mac_addr,
 501         .get_reg_length       = ixgbevf_get_reg_length,
 502         .get_reg              = ixgbevf_get_regs,
 503         .reta_update          = ixgbe_dev_rss_reta_update,
 504         .reta_query           = ixgbe_dev_rss_reta_query,
 505         .rss_hash_update      = ixgbe_dev_rss_hash_update,
 506         .rss_hash_conf_get    = ixgbe_dev_rss_hash_conf_get,
 507 };
 508
 509 /* store statistics names and its offset in stats structure */
 510 struct rte_ixgbe_xstats_name_off {
 511         char name[RTE_ETH_XSTATS_NAME_SIZE];
 512         unsigned offset;
 513 };
 514
 515 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_stats_strings[] = {
 516         {"rx_illegal_byte_err", offsetof(struct ixgbe_hw_stats, errbc)},
 517         {"rx_len_err", offsetof(struct ixgbe_hw_stats, rlec)},
 518         {"rx_undersize_count", offsetof(struct ixgbe_hw_stats, ruc)},
 519         {"rx_oversize_count", offsetof(struct ixgbe_hw_stats, roc)},
 520         {"rx_fragment_count", offsetof(struct ixgbe_hw_stats, rfc)},
 521         {"rx_jabber_count", offsetof(struct ixgbe_hw_stats, rjc)},
 522         {"l3_l4_xsum_error", offsetof(struct ixgbe_hw_stats, xec)},
 523         {"mac_local_fault", offsetof(struct ixgbe_hw_stats, mlfc)},
 524         {"mac_remote_fault", offsetof(struct ixgbe_hw_stats, mrfc)},
 525         {"mac_short_pkt_discard", offsetof(struct ixgbe_hw_stats, mspdc)},
 526         {"fccrc_error", offsetof(struct ixgbe_hw_stats, fccrc)},
 527         {"fcoe_drop", offsetof(struct ixgbe_hw_stats, fcoerpdc)},
 528         {"fc_last_error", offsetof(struct ixgbe_hw_stats, fclast)},
 529         {"rx_broadcast_packets", offsetof(struct ixgbe_hw_stats, bprc)},
 530         {"rx_phy_multicast_packets", offsetof(struct ixgbe_hw_stats, mprc)},
 531         {"mgmt_pkts_dropped", offsetof(struct ixgbe_hw_stats, mngpdc)},
 532         {"rx_crc_errors", offsetof(struct ixgbe_hw_stats, crcerrs)},
 533         {"fdir_match", offsetof(struct ixgbe_hw_stats, fdirmatch)},
 534         {"fdir_miss", offsetof(struct ixgbe_hw_stats, fdirmiss)},
 535         {"tx_flow_control_xon", offsetof(struct ixgbe_hw_stats, lxontxc)},
 536         {"rx_flow_control_xon", offsetof(struct ixgbe_hw_stats, lxonrxc)},
 537         {"tx_flow_control_xoff", offsetof(struct ixgbe_hw_stats, lxofftxc)},
 538         {"rx_flow_control_xoff", offsetof(struct ixgbe_hw_stats, lxoffrxc)},
 539 };
 540
 541 #define IXGBE_NB_XSTATS (sizeof(rte_ixgbe_stats_strings) /      \
 542                 sizeof(rte_ixgbe_stats_strings[0]))
 543
 544 /**
 545  * Atomically reads the link status information from global
 546  * structure rte_eth_dev.
 547  *
 548  * @param dev
 549  *   - Pointer to the structure rte_eth_dev to read from.
 550  *   - Pointer to the buffer to be saved with the link status.
 551  *
 552  * @return
 553  *   - On success, zero.
 554  *   - On failure, negative value.
 555  */
 556 static inline int
 557 rte_ixgbe_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 558                                 struct rte_eth_link *link)
 559 {
 560         struct rte_eth_link *dst = link;
 561         struct rte_eth_link *src = &(dev->data->dev_link);
 562
 563         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 564                                         *(uint64_t *)src) == 0)
 565                 return -1;
 566
 567         return 0;
 568 }
 569
 570 /**
 571  * Atomically writes the link status information into global
 572  * structure rte_eth_dev.
 573  *
 574  * @param dev
 575  *   - Pointer to the structure rte_eth_dev to read from.
 576  *   - Pointer to the buffer to be saved with the link status.
 577  *
 578  * @return
 579  *   - On success, zero.
 580  *   - On failure, negative value.
 581  */
 582 static inline int
 583 rte_ixgbe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 584                                 struct rte_eth_link *link)
 585 {
 586         struct rte_eth_link *dst = &(dev->data->dev_link);
 587         struct rte_eth_link *src = link;
 588
 589         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 590                                         *(uint64_t *)src) == 0)
 591                 return -1;
 592
 593         return 0;
 594 }
 595
 596 /*
 597  * This function is the same as ixgbe_is_sfp() in base/ixgbe.h.
 598  */
 599 static inline int
 600 ixgbe_is_sfp(struct ixgbe_hw *hw)
 601 {
 602         switch (hw->phy.type) {
 603         case ixgbe_phy_sfp_avago:
 604         case ixgbe_phy_sfp_ftl:
 605         case ixgbe_phy_sfp_intel:
 606         case ixgbe_phy_sfp_unknown:
 607         case ixgbe_phy_sfp_passive_tyco:
 608         case ixgbe_phy_sfp_passive_unknown:
 609                 return 1;
 610         default:
 611                 return 0;
 612         }
 613 }
 614
 615 static inline int32_t
 616 ixgbe_pf_reset_hw(struct ixgbe_hw *hw)
 617 {
 618         uint32_t ctrl_ext;
 619         int32_t status;
 620
 621         status = ixgbe_reset_hw(hw);
 622
 623         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 624         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 625         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
 626         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 627         IXGBE_WRITE_FLUSH(hw);
 628
 629         return status;
 630 }
 631
 632 static inline void
 633 ixgbe_enable_intr(struct rte_eth_dev *dev)
 634 {
 635         struct ixgbe_interrupt *intr =
 636                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 637         struct ixgbe_hw *hw =
 638                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 639
 640         IXGBE_WRITE_REG(hw, IXGBE_EIMS, intr->mask);
 641         IXGBE_WRITE_FLUSH(hw);
 642 }
 643
 644 /*
 645  * This function is based on ixgbe_disable_intr() in base/ixgbe.h.
 646  */
 647 static void
 648 ixgbe_disable_intr(struct ixgbe_hw *hw)
 649 {
 650         PMD_INIT_FUNC_TRACE();
 651
 652         if (hw->mac.type == ixgbe_mac_82598EB) {
 653                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ~0);
 654         } else {
 655                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xFFFF0000);
 656                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), ~0);
 657                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), ~0);
 658         }
 659         IXGBE_WRITE_FLUSH(hw);
 660 }
 661
 662 /*
 663  * This function resets queue statistics mapping registers.
 664  * From Niantic datasheet, Initialization of Statistics section:
 665  * "...if software requires the queue counters, the RQSMR and TQSM registers
 666  * must be re-programmed following a device reset.
 667  */
 668 static void
 669 ixgbe_reset_qstat_mappings(struct ixgbe_hw *hw)
 670 {
 671         uint32_t i;
 672
 673         for(i = 0; i != IXGBE_NB_STAT_MAPPING_REGS; i++) {
 674                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), 0);
 675                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), 0);
 676         }
 677 }
 678
 679
 680 static int
 681 ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 682                                   uint16_t queue_id,
 683                                   uint8_t stat_idx,
 684                                   uint8_t is_rx)
 685 {
 686 #define QSM_REG_NB_BITS_PER_QMAP_FIELD 8
 687 #define NB_QMAP_FIELDS_PER_QSM_REG 4
 688 #define QMAP_FIELD_RESERVED_BITS_MASK 0x0f
 689
 690         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 691         struct ixgbe_stat_mapping_registers *stat_mappings =
 692                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(eth_dev->data->dev_private);
 693         uint32_t qsmr_mask = 0;
 694         uint32_t clearing_mask = QMAP_FIELD_RESERVED_BITS_MASK;
 695         uint32_t q_map;
 696         uint8_t n, offset;
 697
 698         if ((hw->mac.type != ixgbe_mac_82599EB) &&
 699                 (hw->mac.type != ixgbe_mac_X540) &&
 700                 (hw->mac.type != ixgbe_mac_X550) &&
 701                 (hw->mac.type != ixgbe_mac_X550EM_x))
 702                 return -ENOSYS;
 703
 704         PMD_INIT_LOG(DEBUG, "Setting port %d, %s queue_id %d to stat index %d",
 705                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX",
 706                      queue_id, stat_idx);
 707
 708         n = (uint8_t)(queue_id / NB_QMAP_FIELDS_PER_QSM_REG);
 709         if (n >= IXGBE_NB_STAT_MAPPING_REGS) {
 710                 PMD_INIT_LOG(ERR, "Nb of stat mapping registers exceeded");
 711                 return -EIO;
 712         }
 713         offset = (uint8_t)(queue_id % NB_QMAP_FIELDS_PER_QSM_REG);
 714
 715         /* Now clear any previous stat_idx set */
 716         clearing_mask <<= (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 717         if (!is_rx)
 718                 stat_mappings->tqsm[n] &= ~clearing_mask;
 719         else
 720                 stat_mappings->rqsmr[n] &= ~clearing_mask;
 721
 722         q_map = (uint32_t)stat_idx;
 723         q_map &= QMAP_FIELD_RESERVED_BITS_MASK;
 724         qsmr_mask = q_map << (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 725         if (!is_rx)
 726                 stat_mappings->tqsm[n] |= qsmr_mask;
 727         else
 728                 stat_mappings->rqsmr[n] |= qsmr_mask;
 729
 730         PMD_INIT_LOG(DEBUG, "Set port %d, %s queue_id %d to stat index %d",
 731                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX",
 732                      queue_id, stat_idx);
 733         PMD_INIT_LOG(DEBUG, "%s[%d] = 0x%08x", is_rx ? "RQSMR" : "TQSM", n,
 734                      is_rx ? stat_mappings->rqsmr[n] : stat_mappings->tqsm[n]);
 735
 736         /* Now write the mapping in the appropriate register */
 737         if (is_rx) {
 738                 PMD_INIT_LOG(DEBUG, "Write 0x%x to RX IXGBE stat mapping reg:%d",
 739                              stat_mappings->rqsmr[n], n);
 740                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(n), stat_mappings->rqsmr[n]);
 741         }
 742         else {
 743                 PMD_INIT_LOG(DEBUG, "Write 0x%x to TX IXGBE stat mapping reg:%d",
 744                              stat_mappings->tqsm[n], n);
 745                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(n), stat_mappings->tqsm[n]);
 746         }
 747         return 0;
 748 }
 749
 750 static void
 751 ixgbe_restore_statistics_mapping(struct rte_eth_dev * dev)
 752 {
 753         struct ixgbe_stat_mapping_registers *stat_mappings =
 754                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(dev->data->dev_private);
 755         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 756         int i;
 757
 758         /* write whatever was in stat mapping table to the NIC */
 759         for (i = 0; i < IXGBE_NB_STAT_MAPPING_REGS; i++) {
 760                 /* rx */
 761                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), stat_mappings->rqsmr[i]);
 762
 763                 /* tx */
 764                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), stat_mappings->tqsm[i]);
 765         }
 766 }
 767
 768 static void
 769 ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config)
 770 {
 771         uint8_t i;
 772         struct ixgbe_dcb_tc_config *tc;
 773         uint8_t dcb_max_tc = IXGBE_DCB_MAX_TRAFFIC_CLASS;
 774
 775         dcb_config->num_tcs.pg_tcs = dcb_max_tc;
 776         dcb_config->num_tcs.pfc_tcs = dcb_max_tc;
 777         for (i = 0; i < dcb_max_tc; i++) {
 778                 tc = &dcb_config->tc_config[i];
 779                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_id = i;
 780                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
 781                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 782                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_id = i;
 783                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
 784                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 785                 tc->pfc = ixgbe_dcb_pfc_disabled;
 786         }
 787
 788         /* Initialize default user to priority mapping, UPx->TC0 */
 789         tc = &dcb_config->tc_config[0];
 790         tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
 791         tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
 792         for (i = 0; i< IXGBE_DCB_MAX_BW_GROUP; i++) {
 793                 dcb_config->bw_percentage[IXGBE_DCB_TX_CONFIG][i] = 100;
 794                 dcb_config->bw_percentage[IXGBE_DCB_RX_CONFIG][i] = 100;
 795         }
 796         dcb_config->rx_pba_cfg = ixgbe_dcb_pba_equal;
 797         dcb_config->pfc_mode_enable = false;
 798         dcb_config->vt_mode = true;
 799         dcb_config->round_robin_enable = false;
 800         /* support all DCB capabilities in 82599 */
 801         dcb_config->support.capabilities = 0xFF;
 802
 803         /*we only support 4 Tcs for X540, X550 */
 804         if (hw->mac.type == ixgbe_mac_X540 ||
 805                 hw->mac.type == ixgbe_mac_X550 ||
 806                 hw->mac.type == ixgbe_mac_X550EM_x) {
 807                 dcb_config->num_tcs.pg_tcs = 4;
 808                 dcb_config->num_tcs.pfc_tcs = 4;
 809         }
 810 }
 811
 812 /*
 813  * Ensure that all locks are released before first NVM or PHY access
 814  */
 815 static void
 816 ixgbe_swfw_lock_reset(struct ixgbe_hw *hw)
 817 {
 818         uint16_t mask;
 819
 820         /*
 821          * Phy lock should not fail in this early stage. If this is the case,
 822          * it is due to an improper exit of the application.
 823          * So force the release of the faulty lock. Release of common lock
 824          * is done automatically by swfw_sync function.
 825          */
 826         mask = IXGBE_GSSR_PHY0_SM << hw->bus.func;
 827         if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) {
 828                 PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released", hw->bus.func);
 829         }
 830         ixgbe_release_swfw_semaphore(hw, mask);
 831
 832         /*
 833          * These ones are more tricky since they are common to all ports; but
 834          * swfw_sync retries last long enough (1s) to be almost sure that if
 835          * lock can not be taken it is due to an improper lock of the
 836          * semaphore.
 837          */
 838         mask = IXGBE_GSSR_EEP_SM | IXGBE_GSSR_MAC_CSR_SM | IXGBE_GSSR_SW_MNG_SM;
 839         if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) {
 840                 PMD_DRV_LOG(DEBUG, "SWFW common locks released");
 841         }
 842         ixgbe_release_swfw_semaphore(hw, mask);
 843 }
 844
 845 /*
 846  * This function is based on code in ixgbe_attach() in base/ixgbe.c.
 847  * It returns 0 on success.
 848  */
 849 static int
 850 eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev)
 851 {
 852         struct rte_pci_device *pci_dev;
 853         struct ixgbe_hw *hw =
 854                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 855         struct ixgbe_vfta * shadow_vfta =
 856                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 857         struct ixgbe_hwstrip *hwstrip =
 858                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 859         struct ixgbe_dcb_config *dcb_config =
 860                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private);
 861         struct ixgbe_filter_info *filter_info =
 862                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private);
 863         uint32_t ctrl_ext;
 864         uint16_t csum;
 865         int diag, i;
 866
 867         PMD_INIT_FUNC_TRACE();
 868
 869         eth_dev->dev_ops = &ixgbe_eth_dev_ops;
 870         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
 871         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
 872
 873         /*
 874          * For secondary processes, we don't initialise any further as primary
 875          * has already done this work. Only check we don't need a different
 876          * RX and TX function.
 877          */
 878         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 879                 struct ixgbe_tx_queue *txq;
 880                 /* TX queue function in primary, set by last queue initialized
 881                  * Tx queue may not initialized by primary process */
 882                 if (eth_dev->data->tx_queues) {
 883                         txq = eth_dev->data->tx_queues[eth_dev->data->nb_tx_queues-1];
 884                         ixgbe_set_tx_function(eth_dev, txq);
 885                 } else {
 886                         /* Use default TX function if we get here */
 887                         PMD_INIT_LOG(NOTICE, "No TX queues configured yet. "
 888                                              "Using default TX function.");
 889                 }
 890
 891                 ixgbe_set_rx_function(eth_dev);
 892
 893                 return 0;
 894         }
 895         pci_dev = eth_dev->pci_dev;
 896
 897         /* Vendor and Device ID need to be set before init of shared code */
 898         hw->device_id = pci_dev->id.device_id;
 899         hw->vendor_id = pci_dev->id.vendor_id;
 900         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 901         hw->allow_unsupported_sfp = 1;
 902
 903         /* Initialize the shared code (base driver) */
 904 #ifdef RTE_NIC_BYPASS
 905         diag = ixgbe_bypass_init_shared_code(hw);
 906 #else
 907         diag = ixgbe_init_shared_code(hw);
 908 #endif /* RTE_NIC_BYPASS */
 909
 910         if (diag != IXGBE_SUCCESS) {
 911                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
 912                 return -EIO;
 913         }
 914
 915         /* pick up the PCI bus settings for reporting later */
 916         ixgbe_get_bus_info(hw);
 917
 918         /* Unlock any pending hardware semaphore */
 919         ixgbe_swfw_lock_reset(hw);
 920
 921         /* Initialize DCB configuration*/
 922         memset(dcb_config, 0, sizeof(struct ixgbe_dcb_config));
 923         ixgbe_dcb_init(hw,dcb_config);
 924         /* Get Hardware Flow Control setting */
 925         hw->fc.requested_mode = ixgbe_fc_full;
 926         hw->fc.current_mode = ixgbe_fc_full;
 927         hw->fc.pause_time = IXGBE_FC_PAUSE;
 928         for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
 929                 hw->fc.low_water[i] = IXGBE_FC_LO;
 930                 hw->fc.high_water[i] = IXGBE_FC_HI;
 931         }
 932         hw->fc.send_xon = 1;
 933
 934         /* Make sure we have a good EEPROM before we read from it */
 935         diag = ixgbe_validate_eeprom_checksum(hw, &csum);
 936         if (diag != IXGBE_SUCCESS) {
 937                 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag);
 938                 return -EIO;
 939         }
 940
 941 #ifdef RTE_NIC_BYPASS
 942         diag = ixgbe_bypass_init_hw(hw);
 943 #else
 944         diag = ixgbe_init_hw(hw);
 945 #endif /* RTE_NIC_BYPASS */
 946
 947         /*
 948          * Devices with copper phys will fail to initialise if ixgbe_init_hw()
 949          * is called too soon after the kernel driver unbinding/binding occurs.
 950          * The failure occurs in ixgbe_identify_phy_generic() for all devices,
 951          * but for non-copper devies, ixgbe_identify_sfp_module_generic() is
 952          * also called. See ixgbe_identify_phy_82599(). The reason for the
 953          * failure is not known, and only occuts when virtualisation features
 954          * are disabled in the bios. A delay of 100ms  was found to be enough by
 955          * trial-and-error, and is doubled to be safe.
 956          */
 957         if (diag && (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper)) {
 958                 rte_delay_ms(200);
 959                 diag = ixgbe_init_hw(hw);
 960         }
 961
 962         if (diag == IXGBE_ERR_EEPROM_VERSION) {
 963                 PMD_INIT_LOG(ERR, "This device is a pre-production adapter/"
 964                     "LOM.  Please be aware there may be issues associated "
 965                     "with your hardware.");
 966                 PMD_INIT_LOG(ERR, "If you are experiencing problems "
 967                     "please contact your Intel or hardware representative "
 968                     "who provided you with this hardware.");
 969         } else if (diag == IXGBE_ERR_SFP_NOT_SUPPORTED)
 970                 PMD_INIT_LOG(ERR, "Unsupported SFP+ Module");
 971         if (diag) {
 972                 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag);
 973                 return -EIO;
 974         }
 975
 976         /* Reset the hw statistics */
 977         ixgbe_dev_stats_reset(eth_dev);
 978
 979         /* disable interrupt */
 980         ixgbe_disable_intr(hw);
 981
 982         /* reset mappings for queue statistics hw counters*/
 983         ixgbe_reset_qstat_mappings(hw);
 984
 985         /* Allocate memory for storing MAC addresses */
 986         eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 987                         hw->mac.num_rar_entries, 0);
 988         if (eth_dev->data->mac_addrs == NULL) {
 989                 PMD_INIT_LOG(ERR,
 990                         "Failed to allocate %u bytes needed to store "
 991                         "MAC addresses",
 992                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 993                 return -ENOMEM;
 994         }
 995         /* Copy the permanent MAC address */
 996         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 997                         &eth_dev->data->mac_addrs[0]);
 998
 999         /* Allocate memory for storing hash filter MAC addresses */
1000         eth_dev->data->hash_mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
1001                         IXGBE_VMDQ_NUM_UC_MAC, 0);
1002         if (eth_dev->data->hash_mac_addrs == NULL) {
1003                 PMD_INIT_LOG(ERR,
1004                         "Failed to allocate %d bytes needed to store MAC addresses",
1005                         ETHER_ADDR_LEN * IXGBE_VMDQ_NUM_UC_MAC);
1006                 return -ENOMEM;
1007         }
1008
1009         /* initialize the vfta */
1010         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
1011
1012         /* initialize the hw strip bitmap*/
1013         memset(hwstrip, 0, sizeof(*hwstrip));
1014
1015         /* initialize PF if max_vfs not zero */
1016         ixgbe_pf_host_init(eth_dev);
1017
1018         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1019         /* let hardware know driver is loaded */
1020         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
1021         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
1022         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
1023         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
1024         IXGBE_WRITE_FLUSH(hw);
1025
1026         if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
1027                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d, SFP+: %d",
1028                              (int) hw->mac.type, (int) hw->phy.type,
1029                              (int) hw->phy.sfp_type);
1030         else
1031                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d",
1032                              (int) hw->mac.type, (int) hw->phy.type);
1033
1034         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
1035                         eth_dev->data->port_id, pci_dev->id.vendor_id,
1036                         pci_dev->id.device_id);
1037
1038         /* enable support intr */
1039         ixgbe_enable_intr(eth_dev);
1040
1041         /* initialize 5tuple filter list */
1042         TAILQ_INIT(&filter_info->fivetuple_list);
1043         memset(filter_info->fivetuple_mask, 0,
1044                 sizeof(uint32_t) * IXGBE_5TUPLE_ARRAY_SIZE);
1045
1046         return 0;
1047 }
1048
1049 static int
1050 eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev)
1051 {
1052         struct rte_pci_device *pci_dev;
1053         struct ixgbe_hw *hw;
1054
1055         PMD_INIT_FUNC_TRACE();
1056
1057         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1058                 return -EPERM;
1059
1060         hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1061         pci_dev = eth_dev->pci_dev;
1062
1063         if (hw->adapter_stopped == 0)
1064                 ixgbe_dev_close(eth_dev);
1065
1066         eth_dev->dev_ops = NULL;
1067         eth_dev->rx_pkt_burst = NULL;
1068         eth_dev->tx_pkt_burst = NULL;
1069
1070         /* Unlock any pending hardware semaphore */
1071         ixgbe_swfw_lock_reset(hw);
1072
1073         /* disable uio intr before callback unregister */
1074         rte_intr_disable(&(pci_dev->intr_handle));
1075         rte_intr_callback_unregister(&(pci_dev->intr_handle),
1076                 ixgbe_dev_interrupt_handler, (void *)eth_dev);
1077
1078         /* uninitialize PF if max_vfs not zero */
1079         ixgbe_pf_host_uninit(eth_dev);
1080
1081         rte_free(eth_dev->data->mac_addrs);
1082         eth_dev->data->mac_addrs = NULL;
1083
1084         rte_free(eth_dev->data->hash_mac_addrs);
1085         eth_dev->data->hash_mac_addrs = NULL;
1086
1087         return 0;
1088 }
1089
1090 /*
1091  * Negotiate mailbox API version with the PF.
1092  * After reset API version is always set to the basic one (ixgbe_mbox_api_10).
1093  * Then we try to negotiate starting with the most recent one.
1094  * If all negotiation attempts fail, then we will proceed with
1095  * the default one (ixgbe_mbox_api_10).
1096  */
1097 static void
1098 ixgbevf_negotiate_api(struct ixgbe_hw *hw)
1099 {
1100         int32_t i;
1101
1102         /* start with highest supported, proceed down */
1103         static const enum ixgbe_pfvf_api_rev sup_ver[] = {
1104                 ixgbe_mbox_api_11,
1105                 ixgbe_mbox_api_10,
1106         };
1107
1108         for (i = 0;
1109                         i != RTE_DIM(sup_ver) &&
1110                         ixgbevf_negotiate_api_version(hw, sup_ver[i]) != 0;
1111                         i++)
1112                 ;
1113 }
1114
1115 static void
1116 generate_random_mac_addr(struct ether_addr *mac_addr)
1117 {
1118         uint64_t random;
1119
1120         /* Set Organizationally Unique Identifier (OUI) prefix. */
1121         mac_addr->addr_bytes[0] = 0x00;
1122         mac_addr->addr_bytes[1] = 0x09;
1123         mac_addr->addr_bytes[2] = 0xC0;
1124         /* Force indication of locally assigned MAC address. */
1125         mac_addr->addr_bytes[0] |= ETHER_LOCAL_ADMIN_ADDR;
1126         /* Generate the last 3 bytes of the MAC address with a random number. */
1127         random = rte_rand();
1128         memcpy(&mac_addr->addr_bytes[3], &random, 3);
1129 }
1130
1131 /*
1132  * Virtual Function device init
1133  */
1134 static int
1135 eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev)
1136 {
1137         int diag;
1138         uint32_t tc, tcs;
1139         struct rte_pci_device *pci_dev;
1140         struct ixgbe_hw *hw =
1141                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1142         struct ixgbe_vfta * shadow_vfta =
1143                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
1144         struct ixgbe_hwstrip *hwstrip =
1145                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
1146         struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr;
1147
1148         PMD_INIT_FUNC_TRACE();
1149
1150         eth_dev->dev_ops = &ixgbevf_eth_dev_ops;
1151         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
1152         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
1153
1154         /* for secondary processes, we don't initialise any further as primary
1155          * has already done this work. Only check we don't need a different
1156          * RX function */
1157         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
1158                 if (eth_dev->data->scattered_rx)
1159                         eth_dev->rx_pkt_burst = ixgbe_recv_pkts_lro_single_alloc;
1160                 return 0;
1161         }
1162
1163         pci_dev = eth_dev->pci_dev;
1164
1165         hw->device_id = pci_dev->id.device_id;
1166         hw->vendor_id = pci_dev->id.vendor_id;
1167         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
1168
1169         /* initialize the vfta */
1170         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
1171
1172         /* initialize the hw strip bitmap*/
1173         memset(hwstrip, 0, sizeof(*hwstrip));
1174
1175         /* Initialize the shared code (base driver) */
1176         diag = ixgbe_init_shared_code(hw);
1177         if (diag != IXGBE_SUCCESS) {
1178                 PMD_INIT_LOG(ERR, "Shared code init failed for ixgbevf: %d", diag);
1179                 return -EIO;
1180         }
1181
1182         /* init_mailbox_params */
1183         hw->mbx.ops.init_params(hw);
1184
1185         /* Reset the hw statistics */
1186         ixgbevf_dev_stats_reset(eth_dev);
1187
1188         /* Disable the interrupts for VF */
1189         ixgbevf_intr_disable(hw);
1190
1191         hw->mac.num_rar_entries = 128; /* The MAX of the underlying PF */
1192         diag = hw->mac.ops.reset_hw(hw);
1193
1194         /*
1195          * The VF reset operation returns the IXGBE_ERR_INVALID_MAC_ADDR when
1196          * the underlying PF driver has not assigned a MAC address to the VF.
1197          * In this case, assign a random MAC address.
1198          */
1199         if ((diag != IXGBE_SUCCESS) && (diag != IXGBE_ERR_INVALID_MAC_ADDR)) {
1200                 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
1201                 return (diag);
1202         }
1203
1204         /* negotiate mailbox API version to use with the PF. */
1205         ixgbevf_negotiate_api(hw);
1206
1207         /* Get Rx/Tx queue count via mailbox, which is ready after reset_hw */
1208         ixgbevf_get_queues(hw, &tcs, &tc);
1209
1210         /* Allocate memory for storing MAC addresses */
1211         eth_dev->data->mac_addrs = rte_zmalloc("ixgbevf", ETHER_ADDR_LEN *
1212                         hw->mac.num_rar_entries, 0);
1213         if (eth_dev->data->mac_addrs == NULL) {
1214                 PMD_INIT_LOG(ERR,
1215                         "Failed to allocate %u bytes needed to store "
1216                         "MAC addresses",
1217                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
1218                 return -ENOMEM;
1219         }
1220
1221         /* Generate a random MAC address, if none was assigned by PF. */
1222         if (is_zero_ether_addr(perm_addr)) {
1223                 generate_random_mac_addr(perm_addr);
1224                 diag = ixgbe_set_rar_vf(hw, 1, perm_addr->addr_bytes, 0, 1);
1225                 if (diag) {
1226                         rte_free(eth_dev->data->mac_addrs);
1227                         eth_dev->data->mac_addrs = NULL;
1228                         return diag;
1229                 }
1230                 PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF");
1231                 PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address "
1232                              "%02x:%02x:%02x:%02x:%02x:%02x",
1233                              perm_addr->addr_bytes[0],
1234                              perm_addr->addr_bytes[1],
1235                              perm_addr->addr_bytes[2],
1236                              perm_addr->addr_bytes[3],
1237                              perm_addr->addr_bytes[4],
1238                              perm_addr->addr_bytes[5]);
1239         }
1240
1241         /* Copy the permanent MAC address */
1242         ether_addr_copy(perm_addr, &eth_dev->data->mac_addrs[0]);
1243
1244         /* reset the hardware with the new settings */
1245         diag = hw->mac.ops.start_hw(hw);
1246         switch (diag) {
1247                 case  0:
1248                         break;
1249
1250                 default:
1251                         PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
1252                         return (-EIO);
1253         }
1254
1255         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x mac.type=%s",
1256                      eth_dev->data->port_id, pci_dev->id.vendor_id,
1257                      pci_dev->id.device_id, "ixgbe_mac_82599_vf");
1258
1259         return 0;
1260 }
1261
1262 /* Virtual Function device uninit */
1263
1264 static int
1265 eth_ixgbevf_dev_uninit(struct rte_eth_dev *eth_dev)
1266 {
1267         struct ixgbe_hw *hw;
1268         unsigned i;
1269
1270         PMD_INIT_FUNC_TRACE();
1271
1272         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1273                 return -EPERM;
1274
1275         hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1276
1277         if (hw->adapter_stopped == 0)
1278                 ixgbevf_dev_close(eth_dev);
1279
1280         eth_dev->dev_ops = NULL;
1281         eth_dev->rx_pkt_burst = NULL;
1282         eth_dev->tx_pkt_burst = NULL;
1283
1284         /* Disable the interrupts for VF */
1285         ixgbevf_intr_disable(hw);
1286
1287         for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
1288                 ixgbe_dev_rx_queue_release(eth_dev->data->rx_queues[i]);
1289                 eth_dev->data->rx_queues[i] = NULL;
1290         }
1291         eth_dev->data->nb_rx_queues = 0;
1292
1293         for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
1294                 ixgbe_dev_tx_queue_release(eth_dev->data->tx_queues[i]);
1295                 eth_dev->data->tx_queues[i] = NULL;
1296         }
1297         eth_dev->data->nb_tx_queues = 0;
1298
1299         rte_free(eth_dev->data->mac_addrs);
1300         eth_dev->data->mac_addrs = NULL;
1301
1302         return 0;
1303 }
1304
1305 static struct eth_driver rte_ixgbe_pmd = {
1306         .pci_drv = {
1307                 .name = "rte_ixgbe_pmd",
1308                 .id_table = pci_id_ixgbe_map,
1309                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
1310                         RTE_PCI_DRV_DETACHABLE,
1311         },
1312         .eth_dev_init = eth_ixgbe_dev_init,
1313         .eth_dev_uninit = eth_ixgbe_dev_uninit,
1314         .dev_private_size = sizeof(struct ixgbe_adapter),
1315 };
1316
1317 /*
1318  * virtual function driver struct
1319  */
1320 static struct eth_driver rte_ixgbevf_pmd = {
1321         .pci_drv = {
1322                 .name = "rte_ixgbevf_pmd",
1323                 .id_table = pci_id_ixgbevf_map,
1324                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
1325         },
1326         .eth_dev_init = eth_ixgbevf_dev_init,
1327         .eth_dev_uninit = eth_ixgbevf_dev_uninit,
1328         .dev_private_size = sizeof(struct ixgbe_adapter),
1329 };
1330
1331 /*
1332  * Driver initialization routine.
1333  * Invoked once at EAL init time.
1334  * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
1335  */
1336 static int
1337 rte_ixgbe_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
1338 {
1339         PMD_INIT_FUNC_TRACE();
1340
1341         rte_eth_driver_register(&rte_ixgbe_pmd);
1342         return 0;
1343 }
1344
1345 /*
1346  * VF Driver initialization routine.
1347  * Invoked one at EAL init time.
1348  * Register itself as the [Virtual Poll Mode] Driver of PCI niantic devices.
1349  */
1350 static int
1351 rte_ixgbevf_pmd_init(const char *name __rte_unused, const char *param __rte_unused)
1352 {
1353         PMD_INIT_FUNC_TRACE();
1354
1355         rte_eth_driver_register(&rte_ixgbevf_pmd);
1356         return (0);
1357 }
1358
1359 static int
1360 ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1361 {
1362         struct ixgbe_hw *hw =
1363                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1364         struct ixgbe_vfta * shadow_vfta =
1365                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1366         uint32_t vfta;
1367         uint32_t vid_idx;
1368         uint32_t vid_bit;
1369
1370         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
1371         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
1372         vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(vid_idx));
1373         if (on)
1374                 vfta |= vid_bit;
1375         else
1376                 vfta &= ~vid_bit;
1377         IXGBE_WRITE_REG(hw, IXGBE_VFTA(vid_idx), vfta);
1378
1379         /* update local VFTA copy */
1380         shadow_vfta->vfta[vid_idx] = vfta;
1381
1382         return 0;
1383 }
1384
1385 static void
1386 ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
1387 {
1388         if (on)
1389                 ixgbe_vlan_hw_strip_enable(dev, queue);
1390         else
1391                 ixgbe_vlan_hw_strip_disable(dev, queue);
1392 }
1393
1394 static void
1395 ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
1396 {
1397         struct ixgbe_hw *hw =
1398                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1399
1400         /* Only the high 16-bits is valid */
1401         IXGBE_WRITE_REG(hw, IXGBE_EXVET, tpid << 16);
1402 }
1403
1404 void
1405 ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev)
1406 {
1407         struct ixgbe_hw *hw =
1408                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1409         uint32_t vlnctrl;
1410
1411         PMD_INIT_FUNC_TRACE();
1412
1413         /* Filter Table Disable */
1414         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1415         vlnctrl &= ~IXGBE_VLNCTRL_VFE;
1416
1417         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
1418 }
1419
1420 void
1421 ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1422 {
1423         struct ixgbe_hw *hw =
1424                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1425         struct ixgbe_vfta * shadow_vfta =
1426                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1427         uint32_t vlnctrl;
1428         uint16_t i;
1429
1430         PMD_INIT_FUNC_TRACE();
1431
1432         /* Filter Table Enable */
1433         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1434         vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
1435         vlnctrl |= IXGBE_VLNCTRL_VFE;
1436
1437         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
1438
1439         /* write whatever is in local vfta copy */
1440         for (i = 0; i < IXGBE_VFTA_SIZE; i++)
1441                 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), shadow_vfta->vfta[i]);
1442 }
1443
1444 static void
1445 ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on)
1446 {
1447         struct ixgbe_hwstrip *hwstrip =
1448                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(dev->data->dev_private);
1449
1450         if(queue >= IXGBE_MAX_RX_QUEUE_NUM)
1451                 return;
1452
1453         if (on)
1454                 IXGBE_SET_HWSTRIP(hwstrip, queue);
1455         else
1456                 IXGBE_CLEAR_HWSTRIP(hwstrip, queue);
1457 }
1458
1459 static void
1460 ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue)
1461 {
1462         struct ixgbe_hw *hw =
1463                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1464         uint32_t ctrl;
1465
1466         PMD_INIT_FUNC_TRACE();
1467
1468         if (hw->mac.type == ixgbe_mac_82598EB) {
1469                 /* No queue level support */
1470                 PMD_INIT_LOG(NOTICE, "82598EB not support queue level hw strip");
1471                 return;
1472         }
1473         else {
1474                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1475                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1476                 ctrl &= ~IXGBE_RXDCTL_VME;
1477                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
1478         }
1479         /* record those setting for HW strip per queue */
1480         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 0);
1481 }
1482
1483 static void
1484 ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue)
1485 {
1486         struct ixgbe_hw *hw =
1487                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1488         uint32_t ctrl;
1489
1490         PMD_INIT_FUNC_TRACE();
1491
1492         if (hw->mac.type == ixgbe_mac_82598EB) {
1493                 /* No queue level supported */
1494                 PMD_INIT_LOG(NOTICE, "82598EB not support queue level hw strip");
1495                 return;
1496         }
1497         else {
1498                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1499                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1500                 ctrl |= IXGBE_RXDCTL_VME;
1501                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
1502         }
1503         /* record those setting for HW strip per queue */
1504         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 1);
1505 }
1506
1507 void
1508 ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev)
1509 {
1510         struct ixgbe_hw *hw =
1511                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1512         uint32_t ctrl;
1513         uint16_t i;
1514
1515         PMD_INIT_FUNC_TRACE();
1516
1517         if (hw->mac.type == ixgbe_mac_82598EB) {
1518                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1519                 ctrl &= ~IXGBE_VLNCTRL_VME;
1520                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1521         }
1522         else {
1523                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1524                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1525                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1526                         ctrl &= ~IXGBE_RXDCTL_VME;
1527                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1528
1529                         /* record those setting for HW strip per queue */
1530                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 0);
1531                 }
1532         }
1533 }
1534
1535 void
1536 ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev)
1537 {
1538         struct ixgbe_hw *hw =
1539                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1540         uint32_t ctrl;
1541         uint16_t i;
1542
1543         PMD_INIT_FUNC_TRACE();
1544
1545         if (hw->mac.type == ixgbe_mac_82598EB) {
1546                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1547                 ctrl |= IXGBE_VLNCTRL_VME;
1548                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1549         }
1550         else {
1551                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1552                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1553                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1554                         ctrl |= IXGBE_RXDCTL_VME;
1555                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1556
1557                         /* record those setting for HW strip per queue */
1558                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 1);
1559                 }
1560         }
1561 }
1562
1563 static void
1564 ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1565 {
1566         struct ixgbe_hw *hw =
1567                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1568         uint32_t ctrl;
1569
1570         PMD_INIT_FUNC_TRACE();
1571
1572         /* DMATXCTRL: Geric Double VLAN Disable */
1573         ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1574         ctrl &= ~IXGBE_DMATXCTL_GDV;
1575         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1576
1577         /* CTRL_EXT: Global Double VLAN Disable */
1578         ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1579         ctrl &= ~IXGBE_EXTENDED_VLAN;
1580         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1581
1582 }
1583
1584 static void
1585 ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1586 {
1587         struct ixgbe_hw *hw =
1588                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1589         uint32_t ctrl;
1590
1591         PMD_INIT_FUNC_TRACE();
1592
1593         /* DMATXCTRL: Geric Double VLAN Enable */
1594         ctrl  = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1595         ctrl |= IXGBE_DMATXCTL_GDV;
1596         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1597
1598         /* CTRL_EXT: Global Double VLAN Enable */
1599         ctrl  = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1600         ctrl |= IXGBE_EXTENDED_VLAN;
1601         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1602
1603         /*
1604          * VET EXT field in the EXVET register = 0x8100 by default
1605          * So no need to change. Same to VT field of DMATXCTL register
1606          */
1607 }
1608
1609 static void
1610 ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1611 {
1612         if(mask & ETH_VLAN_STRIP_MASK){
1613                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1614                         ixgbe_vlan_hw_strip_enable_all(dev);
1615                 else
1616                         ixgbe_vlan_hw_strip_disable_all(dev);
1617         }
1618
1619         if(mask & ETH_VLAN_FILTER_MASK){
1620                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1621                         ixgbe_vlan_hw_filter_enable(dev);
1622                 else
1623                         ixgbe_vlan_hw_filter_disable(dev);
1624         }
1625
1626         if(mask & ETH_VLAN_EXTEND_MASK){
1627                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1628                         ixgbe_vlan_hw_extend_enable(dev);
1629                 else
1630                         ixgbe_vlan_hw_extend_disable(dev);
1631         }
1632 }
1633
1634 static void
1635 ixgbe_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1636 {
1637         struct ixgbe_hw *hw =
1638                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1639         /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
1640         uint32_t vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1641         vlanctrl |= IXGBE_VLNCTRL_VFE ; /* enable vlan filters */
1642         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
1643 }
1644
1645 static int
1646 ixgbe_check_vf_rss_rxq_num(struct rte_eth_dev *dev, uint16_t nb_rx_q)
1647 {
1648         switch (nb_rx_q) {
1649         case 1:
1650         case 2:
1651                 RTE_ETH_DEV_SRIOV(dev).active = ETH_64_POOLS;
1652                 break;
1653         case 4:
1654                 RTE_ETH_DEV_SRIOV(dev).active = ETH_32_POOLS;
1655                 break;
1656         default:
1657                 return -EINVAL;
1658         }
1659
1660         RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = nb_rx_q;
1661         RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx = dev->pci_dev->max_vfs * nb_rx_q;
1662
1663         return 0;
1664 }
1665
1666 static int
1667 ixgbe_check_mq_mode(struct rte_eth_dev *dev)
1668 {
1669         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
1670         uint16_t nb_rx_q = dev->data->nb_rx_queues;
1671         uint16_t nb_tx_q = dev->data->nb_rx_queues;
1672
1673         if (RTE_ETH_DEV_SRIOV(dev).active != 0) {
1674                 /* check multi-queue mode */
1675                 switch (dev_conf->rxmode.mq_mode) {
1676                 case ETH_MQ_RX_VMDQ_DCB:
1677                 case ETH_MQ_RX_VMDQ_DCB_RSS:
1678                         /* DCB/RSS VMDQ in SRIOV mode, not implement yet */
1679                         PMD_INIT_LOG(ERR, "SRIOV active,"
1680                                         " unsupported mq_mode rx %d.",
1681                                         dev_conf->rxmode.mq_mode);
1682                         return -EINVAL;
1683                 case ETH_MQ_RX_RSS:
1684                 case ETH_MQ_RX_VMDQ_RSS:
1685                         dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_RSS;
1686                         if (nb_rx_q <= RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool)
1687                                 if (ixgbe_check_vf_rss_rxq_num(dev, nb_rx_q)) {
1688                                         PMD_INIT_LOG(ERR, "SRIOV is active,"
1689                                                 " invalid queue number"
1690                                                 " for VMDQ RSS, allowed"
1691                                                 " value are 1, 2 or 4.");
1692                                         return -EINVAL;
1693                                 }
1694                         break;
1695                 case ETH_MQ_RX_VMDQ_ONLY:
1696                 case ETH_MQ_RX_NONE:
1697                         /* if nothing mq mode configure, use default scheme */
1698                         dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY;
1699                         if (RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool > 1)
1700                                 RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 1;
1701                         break;
1702                 default: /* ETH_MQ_RX_DCB, ETH_MQ_RX_DCB_RSS or ETH_MQ_TX_DCB*/
1703                         /* SRIOV only works in VMDq enable mode */
1704                         PMD_INIT_LOG(ERR, "SRIOV is active,"
1705                                         " wrong mq_mode rx %d.",
1706                                         dev_conf->rxmode.mq_mode);
1707                         return -EINVAL;
1708                 }
1709
1710                 switch (dev_conf->txmode.mq_mode) {
1711                 case ETH_MQ_TX_VMDQ_DCB:
1712                         /* DCB VMDQ in SRIOV mode, not implement yet */
1713                         PMD_INIT_LOG(ERR, "SRIOV is active,"
1714                                         " unsupported VMDQ mq_mode tx %d.",
1715                                         dev_conf->txmode.mq_mode);
1716                         return -EINVAL;
1717                 default: /* ETH_MQ_TX_VMDQ_ONLY or ETH_MQ_TX_NONE */
1718                         dev->data->dev_conf.txmode.mq_mode = ETH_MQ_TX_VMDQ_ONLY;
1719                         break;
1720                 }
1721
1722                 /* check valid queue number */
1723                 if ((nb_rx_q > RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool) ||
1724                     (nb_tx_q > RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool)) {
1725                         PMD_INIT_LOG(ERR, "SRIOV is active,"
1726                                         " queue number must less equal to %d.",
1727                                         RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool);
1728                         return -EINVAL;
1729                 }
1730         } else {
1731                 /* check configuration for vmdb+dcb mode */
1732                 if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_VMDQ_DCB) {
1733                         const struct rte_eth_vmdq_dcb_conf *conf;
1734
1735                         if (nb_rx_q != IXGBE_VMDQ_DCB_NB_QUEUES) {
1736                                 PMD_INIT_LOG(ERR, "VMDQ+DCB, nb_rx_q != %d.",
1737                                                 IXGBE_VMDQ_DCB_NB_QUEUES);
1738                                 return -EINVAL;
1739                         }
1740                         conf = &dev_conf->rx_adv_conf.vmdq_dcb_conf;
1741                         if (!(conf->nb_queue_pools == ETH_16_POOLS ||
1742                                conf->nb_queue_pools == ETH_32_POOLS)) {
1743                                 PMD_INIT_LOG(ERR, "VMDQ+DCB selected,"
1744                                                 " nb_queue_pools must be %d or %d.",
1745                                                 ETH_16_POOLS, ETH_32_POOLS);
1746                                 return -EINVAL;
1747                         }
1748                 }
1749                 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_VMDQ_DCB) {
1750                         const struct rte_eth_vmdq_dcb_tx_conf *conf;
1751
1752                         if (nb_tx_q != IXGBE_VMDQ_DCB_NB_QUEUES) {
1753                                 PMD_INIT_LOG(ERR, "VMDQ+DCB, nb_tx_q != %d",
1754                                                  IXGBE_VMDQ_DCB_NB_QUEUES);
1755                                 return -EINVAL;
1756                         }
1757                         conf = &dev_conf->tx_adv_conf.vmdq_dcb_tx_conf;
1758                         if (!(conf->nb_queue_pools == ETH_16_POOLS ||
1759                                conf->nb_queue_pools == ETH_32_POOLS)) {
1760                                 PMD_INIT_LOG(ERR, "VMDQ+DCB selected,"
1761                                                 " nb_queue_pools != %d and"
1762                                                 " nb_queue_pools != %d.",
1763                                                 ETH_16_POOLS, ETH_32_POOLS);
1764                                 return -EINVAL;
1765                         }
1766                 }
1767
1768                 /* For DCB mode check our configuration before we go further */
1769                 if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_DCB) {
1770                         const struct rte_eth_dcb_rx_conf *conf;
1771
1772                         if (nb_rx_q != IXGBE_DCB_NB_QUEUES) {
1773                                 PMD_INIT_LOG(ERR, "DCB selected, nb_rx_q != %d.",
1774                                                  IXGBE_DCB_NB_QUEUES);
1775                                 return -EINVAL;
1776                         }
1777                         conf = &dev_conf->rx_adv_conf.dcb_rx_conf;
1778                         if (!(conf->nb_tcs == ETH_4_TCS ||
1779                                conf->nb_tcs == ETH_8_TCS)) {
1780                                 PMD_INIT_LOG(ERR, "DCB selected, nb_tcs != %d"
1781                                                 " and nb_tcs != %d.",
1782                                                 ETH_4_TCS, ETH_8_TCS);
1783                                 return -EINVAL;
1784                         }
1785                 }
1786
1787                 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_DCB) {
1788                         const struct rte_eth_dcb_tx_conf *conf;
1789
1790                         if (nb_tx_q != IXGBE_DCB_NB_QUEUES) {
1791                                 PMD_INIT_LOG(ERR, "DCB, nb_tx_q != %d.",
1792                                                  IXGBE_DCB_NB_QUEUES);
1793                                 return -EINVAL;
1794                         }
1795                         conf = &dev_conf->tx_adv_conf.dcb_tx_conf;
1796                         if (!(conf->nb_tcs == ETH_4_TCS ||
1797                                conf->nb_tcs == ETH_8_TCS)) {
1798                                 PMD_INIT_LOG(ERR, "DCB selected, nb_tcs != %d"
1799                                                 " and nb_tcs != %d.",
1800                                                 ETH_4_TCS, ETH_8_TCS);
1801                                 return -EINVAL;
1802                         }
1803                 }
1804         }
1805         return 0;
1806 }
1807
1808 static int
1809 ixgbe_dev_configure(struct rte_eth_dev *dev)
1810 {
1811         struct ixgbe_interrupt *intr =
1812                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1813         struct ixgbe_adapter *adapter =
1814                 (struct ixgbe_adapter *)dev->data->dev_private;
1815         int ret;
1816
1817         PMD_INIT_FUNC_TRACE();
1818         /* multipe queue mode checking */
1819         ret  = ixgbe_check_mq_mode(dev);
1820         if (ret != 0) {
1821                 PMD_DRV_LOG(ERR, "ixgbe_check_mq_mode fails with %d.",
1822                             ret);
1823                 return ret;
1824         }
1825
1826         /* set flag to update link status after init */
1827         intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1828
1829         /*
1830          * Initialize to TRUE. If any of Rx queues doesn't meet the bulk
1831          * allocation or vector Rx preconditions we will reset it.
1832          */
1833         adapter->rx_bulk_alloc_allowed = true;
1834         adapter->rx_vec_allowed = true;
1835
1836         return 0;
1837 }
1838
1839 /*
1840  * Configure device link speed and setup link.
1841  * It returns 0 on success.
1842  */
1843 static int
1844 ixgbe_dev_start(struct rte_eth_dev *dev)
1845 {
1846         struct ixgbe_hw *hw =
1847                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1848         struct ixgbe_vf_info *vfinfo =
1849                 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private);
1850         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1851         uint32_t intr_vector = 0;
1852         int err, link_up = 0, negotiate = 0;
1853         uint32_t speed = 0;
1854         int mask = 0;
1855         int status;
1856         uint16_t vf, idx;
1857
1858         PMD_INIT_FUNC_TRACE();
1859
1860         /* IXGBE devices don't support half duplex */
1861         if ((dev->data->dev_conf.link_duplex != ETH_LINK_AUTONEG_DUPLEX) &&
1862                         (dev->data->dev_conf.link_duplex != ETH_LINK_FULL_DUPLEX)) {
1863                 PMD_INIT_LOG(ERR, "Invalid link_duplex (%hu) for port %hhu",
1864                              dev->data->dev_conf.link_duplex,
1865                              dev->data->port_id);
1866                 return -EINVAL;
1867         }
1868
1869         /* stop adapter */
1870         hw->adapter_stopped = 0;
1871         ixgbe_stop_adapter(hw);
1872
1873         /* reinitialize adapter
1874          * this calls reset and start */
1875         status = ixgbe_pf_reset_hw(hw);
1876         if (status != 0)
1877                 return -1;
1878         hw->mac.ops.start_hw(hw);
1879         hw->mac.get_link_status = true;
1880
1881         /* configure PF module if SRIOV enabled */
1882         ixgbe_pf_host_configure(dev);
1883
1884         /* check and configure queue intr-vector mapping */
1885         if (dev->data->dev_conf.intr_conf.rxq != 0)
1886                 intr_vector = dev->data->nb_rx_queues;
1887
1888         if (rte_intr_efd_enable(intr_handle, intr_vector))
1889                 return -1;
1890
1891         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
1892                 intr_handle->intr_vec =
1893                         rte_zmalloc("intr_vec",
1894                                     dev->data->nb_rx_queues * sizeof(int),
1895                                     0);
1896                 if (intr_handle->intr_vec == NULL) {
1897                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
1898                                      " intr_vec\n", dev->data->nb_rx_queues);
1899                         return -ENOMEM;
1900                 }
1901         }
1902
1903         /* confiugre msix for sleep until rx interrupt */
1904         ixgbe_configure_msix(dev);
1905
1906         /* initialize transmission unit */
1907         ixgbe_dev_tx_init(dev);
1908
1909         /* This can fail when allocating mbufs for descriptor rings */
1910         err = ixgbe_dev_rx_init(dev);
1911         if (err) {
1912                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
1913                 goto error;
1914         }
1915
1916         err = ixgbe_dev_rxtx_start(dev);
1917         if (err < 0) {
1918                 PMD_INIT_LOG(ERR, "Unable to start rxtx queues");
1919                 goto error;
1920         }
1921
1922         /* Skip link setup if loopback mode is enabled for 82599. */
1923         if (hw->mac.type == ixgbe_mac_82599EB &&
1924                         dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_82599_TX_RX)
1925                 goto skip_link_setup;
1926
1927         if (ixgbe_is_sfp(hw) && hw->phy.multispeed_fiber) {
1928                 err = hw->mac.ops.setup_sfp(hw);
1929                 if (err)
1930                         goto error;
1931         }
1932
1933         if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
1934                 /* Turn on the copper */
1935                 ixgbe_set_phy_power(hw, true);
1936         } else {
1937                 /* Turn on the laser */
1938                 ixgbe_enable_tx_laser(hw);
1939         }
1940
1941         err = ixgbe_check_link(hw, &speed, &link_up, 0);
1942         if (err)
1943                 goto error;
1944         dev->data->dev_link.link_status = link_up;
1945
1946         err = ixgbe_get_link_capabilities(hw, &speed, &negotiate);
1947         if (err)
1948                 goto error;
1949
1950         switch(dev->data->dev_conf.link_speed) {
1951         case ETH_LINK_SPEED_AUTONEG:
1952                 speed = (hw->mac.type != ixgbe_mac_82598EB) ?
1953                                 IXGBE_LINK_SPEED_82599_AUTONEG :
1954                                 IXGBE_LINK_SPEED_82598_AUTONEG;
1955                 break;
1956         case ETH_LINK_SPEED_100:
1957                 /*
1958                  * Invalid for 82598 but error will be detected by
1959                  * ixgbe_setup_link()
1960                  */
1961                 speed = IXGBE_LINK_SPEED_100_FULL;
1962                 break;
1963         case ETH_LINK_SPEED_1000:
1964                 speed = IXGBE_LINK_SPEED_1GB_FULL;
1965                 break;
1966         case ETH_LINK_SPEED_10000:
1967                 speed = IXGBE_LINK_SPEED_10GB_FULL;
1968                 break;
1969         default:
1970                 PMD_INIT_LOG(ERR, "Invalid link_speed (%hu) for port %hhu",
1971                              dev->data->dev_conf.link_speed,
1972                              dev->data->port_id);
1973                 goto error;
1974         }
1975
1976         err = ixgbe_setup_link(hw, speed, link_up);
1977         if (err)
1978                 goto error;
1979
1980 skip_link_setup:
1981
1982         /* check if lsc interrupt is enabled */
1983         if (dev->data->dev_conf.intr_conf.lsc != 0) {
1984                 if (rte_intr_allow_others(intr_handle)) {
1985                         rte_intr_callback_register(intr_handle,
1986                                                    ixgbe_dev_interrupt_handler,
1987                                                    (void *)dev);
1988                         ixgbe_dev_lsc_interrupt_setup(dev);
1989                 } else
1990                         PMD_INIT_LOG(INFO, "lsc won't enable because of"
1991                                      " no intr multiplex\n");
1992         }
1993
1994         /* check if rxq interrupt is enabled */
1995         if (dev->data->dev_conf.intr_conf.rxq != 0)
1996                 ixgbe_dev_rxq_interrupt_setup(dev);
1997
1998         /* enable uio/vfio intr/eventfd mapping */
1999         rte_intr_enable(intr_handle);
2000
2001         /* resume enabled intr since hw reset */
2002         ixgbe_enable_intr(dev);
2003
2004         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
2005                 ETH_VLAN_EXTEND_MASK;
2006         ixgbe_vlan_offload_set(dev, mask);
2007
2008         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
2009                 /* Enable vlan filtering for VMDq */
2010                 ixgbe_vmdq_vlan_hw_filter_enable(dev);
2011         }
2012
2013         /* Configure DCB hw */
2014         ixgbe_configure_dcb(dev);
2015
2016         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) {
2017                 err = ixgbe_fdir_configure(dev);
2018                 if (err)
2019                         goto error;
2020         }
2021
2022         /* Restore vf rate limit */
2023         if (vfinfo != NULL) {
2024                 for (vf = 0; vf < dev->pci_dev->max_vfs; vf++)
2025                         for (idx = 0; idx < IXGBE_MAX_QUEUE_NUM_PER_VF; idx++)
2026                                 if (vfinfo[vf].tx_rate[idx] != 0)
2027                                         ixgbe_set_vf_rate_limit(dev, vf,
2028                                                 vfinfo[vf].tx_rate[idx],
2029                                                 1 << idx);
2030         }
2031
2032         ixgbe_restore_statistics_mapping(dev);
2033
2034         return (0);
2035
2036 error:
2037         PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err);
2038         ixgbe_dev_clear_queues(dev);
2039         return -EIO;
2040 }
2041
2042 /*
2043  * Stop device: disable rx and tx functions to allow for reconfiguring.
2044  */
2045 static void
2046 ixgbe_dev_stop(struct rte_eth_dev *dev)
2047 {
2048         struct rte_eth_link link;
2049         struct ixgbe_hw *hw =
2050                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2051         struct ixgbe_vf_info *vfinfo =
2052                 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private);
2053         struct ixgbe_filter_info *filter_info =
2054                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
2055         struct ixgbe_5tuple_filter *p_5tuple, *p_5tuple_next;
2056         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
2057         int vf;
2058
2059         PMD_INIT_FUNC_TRACE();
2060
2061         /* disable interrupts */
2062         ixgbe_disable_intr(hw);
2063
2064         /* disable intr eventfd mapping */
2065         rte_intr_disable(intr_handle);
2066
2067         /* reset the NIC */
2068         ixgbe_pf_reset_hw(hw);
2069         hw->adapter_stopped = 0;
2070
2071         /* stop adapter */
2072         ixgbe_stop_adapter(hw);
2073
2074         for (vf = 0; vfinfo != NULL &&
2075                      vf < dev->pci_dev->max_vfs; vf++)
2076                 vfinfo[vf].clear_to_send = false;
2077
2078         if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
2079                 /* Turn off the copper */
2080                 ixgbe_set_phy_power(hw, false);
2081         } else {
2082                 /* Turn off the laser */
2083                 ixgbe_disable_tx_laser(hw);
2084         }
2085
2086         ixgbe_dev_clear_queues(dev);
2087
2088         /* Clear stored conf */
2089         dev->data->scattered_rx = 0;
2090         dev->data->lro = 0;
2091
2092         /* Clear recorded link status */
2093         memset(&link, 0, sizeof(link));
2094         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
2095
2096         /* Remove all ntuple filters of the device */
2097         for (p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list);
2098              p_5tuple != NULL; p_5tuple = p_5tuple_next) {
2099                 p_5tuple_next = TAILQ_NEXT(p_5tuple, entries);
2100                 TAILQ_REMOVE(&filter_info->fivetuple_list,
2101                              p_5tuple, entries);
2102                 rte_free(p_5tuple);
2103         }
2104         memset(filter_info->fivetuple_mask, 0,
2105                 sizeof(uint32_t) * IXGBE_5TUPLE_ARRAY_SIZE);
2106
2107         /* Clean datapath event and queue/vec mapping */
2108         rte_intr_efd_disable(intr_handle);
2109         if (intr_handle->intr_vec != NULL) {
2110                 rte_free(intr_handle->intr_vec);
2111                 intr_handle->intr_vec = NULL;
2112         }
2113 }
2114
2115 /*
2116  * Set device link up: enable tx.
2117  */
2118 static int
2119 ixgbe_dev_set_link_up(struct rte_eth_dev *dev)
2120 {
2121         struct ixgbe_hw *hw =
2122                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2123         if (hw->mac.type == ixgbe_mac_82599EB) {
2124 #ifdef RTE_NIC_BYPASS
2125                 if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) {
2126                         /* Not suported in bypass mode */
2127                         PMD_INIT_LOG(ERR, "Set link up is not supported "
2128                                      "by device id 0x%x", hw->device_id);
2129                         return -ENOTSUP;
2130                 }
2131 #endif
2132         }
2133
2134         if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
2135                 /* Turn on the copper */
2136                 ixgbe_set_phy_power(hw, true);
2137         } else {
2138                 /* Turn on the laser */
2139                 ixgbe_enable_tx_laser(hw);
2140         }
2141
2142         return 0;
2143 }
2144
2145 /*
2146  * Set device link down: disable tx.
2147  */
2148 static int
2149 ixgbe_dev_set_link_down(struct rte_eth_dev *dev)
2150 {
2151         struct ixgbe_hw *hw =
2152                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2153         if (hw->mac.type == ixgbe_mac_82599EB) {
2154 #ifdef RTE_NIC_BYPASS
2155                 if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) {
2156                         /* Not suported in bypass mode */
2157                         PMD_INIT_LOG(ERR, "Set link down is not supported "
2158                                      "by device id 0x%x", hw->device_id);
2159                         return -ENOTSUP;
2160                 }
2161 #endif
2162         }
2163
2164         if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
2165                 /* Turn off the copper */
2166                 ixgbe_set_phy_power(hw, false);
2167         } else {
2168                 /* Turn off the laser */
2169                 ixgbe_disable_tx_laser(hw);
2170         }
2171
2172         return 0;
2173 }
2174
2175 /*
2176  * Reest and stop device.
2177  */
2178 static void
2179 ixgbe_dev_close(struct rte_eth_dev *dev)
2180 {
2181         struct ixgbe_hw *hw =
2182                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2183
2184         PMD_INIT_FUNC_TRACE();
2185
2186         ixgbe_pf_reset_hw(hw);
2187
2188         ixgbe_dev_stop(dev);
2189         hw->adapter_stopped = 1;
2190
2191         ixgbe_dev_free_queues(dev);
2192
2193         ixgbe_disable_pcie_master(hw);
2194
2195         /* reprogram the RAR[0] in case user changed it. */
2196         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
2197 }
2198
2199 static void
2200 ixgbe_read_stats_registers(struct ixgbe_hw *hw, struct ixgbe_hw_stats
2201                                                    *hw_stats, uint64_t *total_missed_rx,
2202                                                    uint64_t *total_qbrc, uint64_t *total_qprc,
2203                                                    uint64_t *total_qprdc)
2204 {
2205         uint32_t bprc, lxon, lxoff, total;
2206         unsigned i;
2207
2208         hw_stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
2209         hw_stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
2210         hw_stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
2211         hw_stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
2212
2213         for (i = 0; i < 8; i++) {
2214                 uint32_t mp;
2215                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
2216                 /* global total per queue */
2217                 hw_stats->mpc[i] += mp;
2218                 /* Running comprehensive total for stats display */
2219                 *total_missed_rx += hw_stats->mpc[i];
2220                 if (hw->mac.type == ixgbe_mac_82598EB) {
2221                         hw_stats->rnbc[i] +=
2222                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
2223                         hw_stats->pxonrxc[i] +=
2224                                 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
2225                         hw_stats->pxoffrxc[i] +=
2226                                 IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
2227                 } else {
2228                         hw_stats->pxonrxc[i] +=
2229                                 IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
2230                         hw_stats->pxoffrxc[i] +=
2231                                 IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i));
2232                         hw_stats->pxon2offc[i] +=
2233                                 IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
2234                 }
2235                 hw_stats->pxontxc[i] +=
2236                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
2237                 hw_stats->pxofftxc[i] +=
2238                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
2239         }
2240         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
2241                 hw_stats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
2242                 hw_stats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
2243                 hw_stats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
2244                 hw_stats->qbrc[i] +=
2245                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32);
2246                 hw_stats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
2247                 hw_stats->qbtc[i] +=
2248                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32);
2249                 *total_qprdc += hw_stats->qprdc[i] +=
2250                                 IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
2251
2252                 *total_qprc += hw_stats->qprc[i];
2253                 *total_qbrc += hw_stats->qbrc[i];
2254         }
2255         hw_stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
2256         hw_stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
2257         hw_stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
2258
2259         /* Note that gprc counts missed packets */
2260         hw_stats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
2261
2262         if (hw->mac.type != ixgbe_mac_82598EB) {
2263                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
2264                 hw_stats->gorc += ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
2265                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
2266                 hw_stats->gotc += ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
2267                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
2268                 hw_stats->tor += ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
2269                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
2270                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
2271         } else {
2272                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
2273                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
2274                 /* 82598 only has a counter in the high register */
2275                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
2276                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
2277                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
2278         }
2279
2280         /*
2281          * Workaround: mprc hardware is incorrectly counting
2282          * broadcasts, so for now we subtract those.
2283          */
2284         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
2285         hw_stats->bprc += bprc;
2286         hw_stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
2287         if (hw->mac.type == ixgbe_mac_82598EB)
2288                 hw_stats->mprc -= bprc;
2289
2290         hw_stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
2291         hw_stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
2292         hw_stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
2293         hw_stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
2294         hw_stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
2295         hw_stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
2296
2297         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
2298         hw_stats->lxontxc += lxon;
2299         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
2300         hw_stats->lxofftxc += lxoff;
2301         total = lxon + lxoff;
2302
2303         hw_stats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
2304         hw_stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
2305         hw_stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
2306         hw_stats->gptc -= total;
2307         hw_stats->mptc -= total;
2308         hw_stats->ptc64 -= total;
2309         hw_stats->gotc -= total * ETHER_MIN_LEN;
2310
2311         hw_stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
2312         hw_stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
2313         hw_stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
2314         hw_stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
2315         hw_stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
2316         hw_stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
2317         hw_stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
2318         hw_stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
2319         hw_stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
2320         hw_stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
2321         hw_stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
2322         hw_stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
2323         hw_stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
2324         hw_stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
2325         hw_stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
2326         hw_stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);
2327         hw_stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
2328         hw_stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
2329         /* Only read FCOE on 82599 */
2330         if (hw->mac.type != ixgbe_mac_82598EB) {
2331                 hw_stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
2332                 hw_stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
2333                 hw_stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
2334                 hw_stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
2335                 hw_stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
2336         }
2337
2338         /* Flow Director Stats registers */
2339         hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
2340         hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
2341 }
2342
2343 /*
2344  * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c
2345  */
2346 static void
2347 ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
2348 {
2349         struct ixgbe_hw *hw =
2350                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2351         struct ixgbe_hw_stats *hw_stats =
2352                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2353         uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc;
2354         unsigned i;
2355
2356         total_missed_rx = 0;
2357         total_qbrc = 0;
2358         total_qprc = 0;
2359         total_qprdc = 0;
2360
2361         ixgbe_read_stats_registers(hw, hw_stats, &total_missed_rx, &total_qbrc,
2362                         &total_qprc, &total_qprdc);
2363
2364         if (stats == NULL)
2365                 return;
2366
2367         /* Fill out the rte_eth_stats statistics structure */
2368         stats->ipackets = total_qprc;
2369         stats->ibytes = total_qbrc;
2370         stats->opackets = hw_stats->gptc;
2371         stats->obytes = hw_stats->gotc;
2372
2373         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
2374                 stats->q_ipackets[i] = hw_stats->qprc[i];
2375                 stats->q_opackets[i] = hw_stats->qptc[i];
2376                 stats->q_ibytes[i] = hw_stats->qbrc[i];
2377                 stats->q_obytes[i] = hw_stats->qbtc[i];
2378                 stats->q_errors[i] = hw_stats->qprdc[i];
2379         }
2380
2381         /* Rx Errors */
2382         stats->ierrors  = hw_stats->crcerrs +
2383                           hw_stats->mspdc +
2384                           hw_stats->rlec +
2385                           hw_stats->ruc +
2386                           hw_stats->roc +
2387                           total_missed_rx +
2388                           hw_stats->illerrc +
2389                           hw_stats->errbc +
2390                           hw_stats->xec +
2391                           hw_stats->mlfc +
2392                           hw_stats->mrfc +
2393                           hw_stats->rfc +
2394                           hw_stats->fccrc +
2395                           hw_stats->fclast;
2396
2397         /* Tx Errors */
2398         stats->oerrors  = 0;
2399 }
2400
2401 static void
2402 ixgbe_dev_stats_reset(struct rte_eth_dev *dev)
2403 {
2404         struct ixgbe_hw_stats *stats =
2405                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2406
2407         /* HW registers are cleared on read */
2408         ixgbe_dev_stats_get(dev, NULL);
2409
2410         /* Reset software totals */
2411         memset(stats, 0, sizeof(*stats));
2412 }
2413
2414 static int
2415 ixgbe_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstats *xstats,
2416                                          unsigned n)
2417 {
2418         struct ixgbe_hw *hw =
2419                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2420         struct ixgbe_hw_stats *hw_stats =
2421                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2422         uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc;
2423         unsigned i, count = IXGBE_NB_XSTATS;
2424
2425         if (n < count)
2426                 return count;
2427
2428         total_missed_rx = 0;
2429         total_qbrc = 0;
2430         total_qprc = 0;
2431         total_qprdc = 0;
2432
2433         ixgbe_read_stats_registers(hw, hw_stats, &total_missed_rx, &total_qbrc,
2434                                                            &total_qprc, &total_qprdc);
2435
2436         /* If this is a reset xstats is NULL, and we have cleared the
2437          * registers by reading them.
2438          */
2439         if (!xstats)
2440                 return 0;
2441
2442         /* Extended stats */
2443         for (i = 0; i < IXGBE_NB_XSTATS; i++) {
2444                 snprintf(xstats[i].name, sizeof(xstats[i].name),
2445                                 "%s", rte_ixgbe_stats_strings[i].name);
2446                 xstats[i].value = *(uint64_t *)(((char *)hw_stats) +
2447                                                         rte_ixgbe_stats_strings[i].offset);
2448         }
2449
2450         return count;
2451 }
2452
2453 static void
2454 ixgbe_dev_xstats_reset(struct rte_eth_dev *dev)
2455 {
2456         struct ixgbe_hw_stats *stats =
2457                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2458
2459         /* HW registers are cleared on read */
2460         ixgbe_dev_xstats_get(dev, NULL, IXGBE_NB_XSTATS);
2461
2462         /* Reset software totals */
2463         memset(stats, 0, sizeof(*stats));
2464 }
2465
2466 static void
2467 ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
2468 {
2469         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2470         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
2471                           IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2472
2473         /* Good Rx packet, include VF loopback */
2474         UPDATE_VF_STAT(IXGBE_VFGPRC,
2475             hw_stats->last_vfgprc, hw_stats->vfgprc);
2476
2477         /* Good Rx octets, include VF loopback */
2478         UPDATE_VF_STAT_36BIT(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
2479             hw_stats->last_vfgorc, hw_stats->vfgorc);
2480
2481         /* Good Tx packet, include VF loopback */
2482         UPDATE_VF_STAT(IXGBE_VFGPTC,
2483             hw_stats->last_vfgptc, hw_stats->vfgptc);
2484
2485         /* Good Tx octets, include VF loopback */
2486         UPDATE_VF_STAT_36BIT(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
2487             hw_stats->last_vfgotc, hw_stats->vfgotc);
2488
2489         /* Rx Multicst Packet */
2490         UPDATE_VF_STAT(IXGBE_VFMPRC,
2491             hw_stats->last_vfmprc, hw_stats->vfmprc);
2492
2493         if (stats == NULL)
2494                 return;
2495
2496         stats->ipackets = hw_stats->vfgprc;
2497         stats->ibytes = hw_stats->vfgorc;
2498         stats->opackets = hw_stats->vfgptc;
2499         stats->obytes = hw_stats->vfgotc;
2500         stats->imcasts = hw_stats->vfmprc;
2501         /* stats->imcasts should be removed as imcasts is deprecated */
2502 }
2503
2504 static void
2505 ixgbevf_dev_stats_reset(struct rte_eth_dev *dev)
2506 {
2507         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
2508                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2509
2510         /* Sync HW register to the last stats */
2511         ixgbevf_dev_stats_get(dev, NULL);
2512
2513         /* reset HW current stats*/
2514         hw_stats->vfgprc = 0;
2515         hw_stats->vfgorc = 0;
2516         hw_stats->vfgptc = 0;
2517         hw_stats->vfgotc = 0;
2518         hw_stats->vfmprc = 0;
2519
2520 }
2521
2522 static void
2523 ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
2524 {
2525         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2526
2527         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
2528         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
2529         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL register */
2530         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */
2531         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
2532         dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC;
2533         dev_info->max_vfs = dev->pci_dev->max_vfs;
2534         if (hw->mac.type == ixgbe_mac_82598EB)
2535                 dev_info->max_vmdq_pools = ETH_16_POOLS;
2536         else
2537                 dev_info->max_vmdq_pools = ETH_64_POOLS;
2538         dev_info->vmdq_queue_num = dev_info->max_rx_queues;
2539         dev_info->rx_offload_capa =
2540                 DEV_RX_OFFLOAD_VLAN_STRIP |
2541                 DEV_RX_OFFLOAD_IPV4_CKSUM |
2542                 DEV_RX_OFFLOAD_UDP_CKSUM  |
2543                 DEV_RX_OFFLOAD_TCP_CKSUM;
2544
2545         /*
2546          * RSC is only supported by 82599 and x540 PF devices in a non-SR-IOV
2547          * mode.
2548          */
2549         if ((hw->mac.type == ixgbe_mac_82599EB ||
2550              hw->mac.type == ixgbe_mac_X540) &&
2551             !RTE_ETH_DEV_SRIOV(dev).active)
2552                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_LRO;
2553
2554         dev_info->tx_offload_capa =
2555                 DEV_TX_OFFLOAD_VLAN_INSERT |
2556                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
2557                 DEV_TX_OFFLOAD_UDP_CKSUM   |
2558                 DEV_TX_OFFLOAD_TCP_CKSUM   |
2559                 DEV_TX_OFFLOAD_SCTP_CKSUM  |
2560                 DEV_TX_OFFLOAD_TCP_TSO;
2561
2562         dev_info->default_rxconf = (struct rte_eth_rxconf) {
2563                 .rx_thresh = {
2564                         .pthresh = IXGBE_DEFAULT_RX_PTHRESH,
2565                         .hthresh = IXGBE_DEFAULT_RX_HTHRESH,
2566                         .wthresh = IXGBE_DEFAULT_RX_WTHRESH,
2567                 },
2568                 .rx_free_thresh = IXGBE_DEFAULT_RX_FREE_THRESH,
2569                 .rx_drop_en = 0,
2570         };
2571
2572         dev_info->default_txconf = (struct rte_eth_txconf) {
2573                 .tx_thresh = {
2574                         .pthresh = IXGBE_DEFAULT_TX_PTHRESH,
2575                         .hthresh = IXGBE_DEFAULT_TX_HTHRESH,
2576                         .wthresh = IXGBE_DEFAULT_TX_WTHRESH,
2577                 },
2578                 .tx_free_thresh = IXGBE_DEFAULT_TX_FREE_THRESH,
2579                 .tx_rs_thresh = IXGBE_DEFAULT_TX_RSBIT_THRESH,
2580                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
2581                                 ETH_TXQ_FLAGS_NOOFFLOADS,
2582         };
2583         dev_info->hash_key_size = IXGBE_HKEY_MAX_INDEX * sizeof(uint32_t);
2584         dev_info->reta_size = ixgbe_reta_size_get(hw->mac.type);
2585         dev_info->flow_type_rss_offloads = IXGBE_RSS_OFFLOAD_ALL;
2586 }
2587
2588 static void
2589 ixgbevf_dev_info_get(struct rte_eth_dev *dev,
2590                      struct rte_eth_dev_info *dev_info)
2591 {
2592         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2593
2594         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
2595         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
2596         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL reg */
2597         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS reg */
2598         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
2599         dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC;
2600         dev_info->max_vfs = dev->pci_dev->max_vfs;
2601         if (hw->mac.type == ixgbe_mac_82598EB)
2602                 dev_info->max_vmdq_pools = ETH_16_POOLS;
2603         else
2604                 dev_info->max_vmdq_pools = ETH_64_POOLS;
2605         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
2606                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
2607                                 DEV_RX_OFFLOAD_UDP_CKSUM  |
2608                                 DEV_RX_OFFLOAD_TCP_CKSUM;
2609         dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
2610                                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
2611                                 DEV_TX_OFFLOAD_UDP_CKSUM   |
2612                                 DEV_TX_OFFLOAD_TCP_CKSUM   |
2613                                 DEV_TX_OFFLOAD_SCTP_CKSUM  |
2614                                 DEV_TX_OFFLOAD_TCP_TSO;
2615
2616         dev_info->default_rxconf = (struct rte_eth_rxconf) {
2617                 .rx_thresh = {
2618                         .pthresh = IXGBE_DEFAULT_RX_PTHRESH,
2619                         .hthresh = IXGBE_DEFAULT_RX_HTHRESH,
2620                         .wthresh = IXGBE_DEFAULT_RX_WTHRESH,
2621                 },
2622                 .rx_free_thresh = IXGBE_DEFAULT_RX_FREE_THRESH,
2623                 .rx_drop_en = 0,
2624         };
2625
2626         dev_info->default_txconf = (struct rte_eth_txconf) {
2627                 .tx_thresh = {
2628                         .pthresh = IXGBE_DEFAULT_TX_PTHRESH,
2629                         .hthresh = IXGBE_DEFAULT_TX_HTHRESH,
2630                         .wthresh = IXGBE_DEFAULT_TX_WTHRESH,
2631                 },
2632                 .tx_free_thresh = IXGBE_DEFAULT_TX_FREE_THRESH,
2633                 .tx_rs_thresh = IXGBE_DEFAULT_TX_RSBIT_THRESH,
2634                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
2635                                 ETH_TXQ_FLAGS_NOOFFLOADS,
2636         };
2637 }
2638
2639 /* return 0 means link status changed, -1 means not changed */
2640 static int
2641 ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
2642 {
2643         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2644         struct rte_eth_link link, old;
2645         ixgbe_link_speed link_speed = IXGBE_LINK_SPEED_UNKNOWN;
2646         int link_up;
2647         int diag;
2648
2649         link.link_status = 0;
2650         link.link_speed = 0;
2651         link.link_duplex = 0;
2652         memset(&old, 0, sizeof(old));
2653         rte_ixgbe_dev_atomic_read_link_status(dev, &old);
2654
2655         hw->mac.get_link_status = true;
2656
2657         /* check if it needs to wait to complete, if lsc interrupt is enabled */
2658         if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
2659                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 0);
2660         else
2661                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 1);
2662
2663         if (diag != 0) {
2664                 link.link_speed = ETH_LINK_SPEED_100;
2665                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
2666                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
2667                 if (link.link_status == old.link_status)
2668                         return -1;
2669                 return 0;
2670         }
2671
2672         if (link_up == 0) {
2673                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
2674                 if (link.link_status == old.link_status)
2675                         return -1;
2676                 return 0;
2677         }
2678         link.link_status = 1;
2679         link.link_duplex = ETH_LINK_FULL_DUPLEX;
2680
2681         switch (link_speed) {
2682         default:
2683         case IXGBE_LINK_SPEED_UNKNOWN:
2684                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
2685                 link.link_speed = ETH_LINK_SPEED_100;
2686                 break;
2687
2688         case IXGBE_LINK_SPEED_100_FULL:
2689                 link.link_speed = ETH_LINK_SPEED_100;
2690                 break;
2691
2692         case IXGBE_LINK_SPEED_1GB_FULL:
2693                 link.link_speed = ETH_LINK_SPEED_1000;
2694                 break;
2695
2696         case IXGBE_LINK_SPEED_10GB_FULL:
2697                 link.link_speed = ETH_LINK_SPEED_10000;
2698                 break;
2699         }
2700         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
2701
2702         if (link.link_status == old.link_status)
2703                 return -1;
2704
2705         return 0;
2706 }
2707
2708 static void
2709 ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev)
2710 {
2711         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2712         uint32_t fctrl;
2713
2714         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2715         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
2716         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2717 }
2718
2719 static void
2720 ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev)
2721 {
2722         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2723         uint32_t fctrl;
2724
2725         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2726         fctrl &= (~IXGBE_FCTRL_UPE);
2727         if (dev->data->all_multicast == 1)
2728                 fctrl |= IXGBE_FCTRL_MPE;
2729         else
2730                 fctrl &= (~IXGBE_FCTRL_MPE);
2731         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2732 }
2733
2734 static void
2735 ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev)
2736 {
2737         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2738         uint32_t fctrl;
2739
2740         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2741         fctrl |= IXGBE_FCTRL_MPE;
2742         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2743 }
2744
2745 static void
2746 ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev)
2747 {
2748         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2749         uint32_t fctrl;
2750
2751         if (dev->data->promiscuous == 1)
2752                 return; /* must remain in all_multicast mode */
2753
2754         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2755         fctrl &= (~IXGBE_FCTRL_MPE);
2756         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2757 }
2758
2759 /**
2760  * It clears the interrupt causes and enables the interrupt.
2761  * It will be called once only during nic initialized.
2762  *
2763  * @param dev
2764  *  Pointer to struct rte_eth_dev.
2765  *
2766  * @return
2767  *  - On success, zero.
2768  *  - On failure, a negative value.
2769  */
2770 static int
2771 ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev)
2772 {
2773         struct ixgbe_interrupt *intr =
2774                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2775
2776         ixgbe_dev_link_status_print(dev);
2777         intr->mask |= IXGBE_EICR_LSC;
2778
2779         return 0;
2780 }
2781
2782 /**
2783  * It clears the interrupt causes and enables the interrupt.
2784  * It will be called once only during nic initialized.
2785  *
2786  * @param dev
2787  *  Pointer to struct rte_eth_dev.
2788  *
2789  * @return
2790  *  - On success, zero.
2791  *  - On failure, a negative value.
2792  */
2793 static int
2794 ixgbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2795 {
2796         struct ixgbe_interrupt *intr =
2797                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2798
2799         intr->mask |= IXGBE_EICR_RTX_QUEUE;
2800
2801         return 0;
2802 }
2803
2804 /*
2805  * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update.
2806  *
2807  * @param dev
2808  *  Pointer to struct rte_eth_dev.
2809  *
2810  * @return
2811  *  - On success, zero.
2812  *  - On failure, a negative value.
2813  */
2814 static int
2815 ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
2816 {
2817         uint32_t eicr;
2818         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2819         struct ixgbe_interrupt *intr =
2820                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2821
2822         /* clear all cause mask */
2823         ixgbe_disable_intr(hw);
2824
2825         /* read-on-clear nic registers here */
2826         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
2827         PMD_DRV_LOG(DEBUG, "eicr %x", eicr);
2828
2829         intr->flags = 0;
2830
2831         /* set flag for async link update */
2832         if (eicr & IXGBE_EICR_LSC)
2833                 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
2834
2835         if (eicr & IXGBE_EICR_MAILBOX)
2836                 intr->flags |= IXGBE_FLAG_MAILBOX;
2837
2838         return 0;
2839 }
2840
2841 static int
2842 ixgbevf_dev_interrupt_get_status(struct rte_eth_dev *dev)
2843 {
2844         uint32_t eicr;
2845         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2846         struct ixgbe_interrupt *intr =
2847                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2848
2849         /* clear all cause mask */
2850         ixgbevf_intr_disable(hw);
2851
2852         /* read-on-clear nic registers here */
2853         eicr = IXGBE_READ_REG(hw, IXGBE_VTEICR);
2854         PMD_DRV_LOG(INFO, "eicr %x", eicr);
2855
2856         intr->flags = 0;
2857
2858         /* set flag for async link update */
2859         if (eicr & IXGBE_EICR_LSC)
2860                 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
2861
2862         return 0;
2863 }
2864
2865 /**
2866  * It gets and then prints the link status.
2867  *
2868  * @param dev
2869  *  Pointer to struct rte_eth_dev.
2870  *
2871  * @return
2872  *  - On success, zero.
2873  *  - On failure, a negative value.
2874  */
2875 static void
2876 ixgbe_dev_link_status_print(struct rte_eth_dev *dev)
2877 {
2878         struct rte_eth_link link;
2879
2880         memset(&link, 0, sizeof(link));
2881         rte_ixgbe_dev_atomic_read_link_status(dev, &link);
2882         if (link.link_status) {
2883                 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
2884                                         (int)(dev->data->port_id),
2885                                         (unsigned)link.link_speed,
2886                         link.link_duplex == ETH_LINK_FULL_DUPLEX ?
2887                                         "full-duplex" : "half-duplex");
2888         } else {
2889                 PMD_INIT_LOG(INFO, " Port %d: Link Down",
2890                                 (int)(dev->data->port_id));
2891         }
2892         PMD_INIT_LOG(DEBUG, "PCI Address: %04d:%02d:%02d:%d",
2893                                 dev->pci_dev->addr.domain,
2894                                 dev->pci_dev->addr.bus,
2895                                 dev->pci_dev->addr.devid,
2896                                 dev->pci_dev->addr.function);
2897 }
2898
2899 /*
2900  * It executes link_update after knowing an interrupt occurred.
2901  *
2902  * @param dev
2903  *  Pointer to struct rte_eth_dev.
2904  *
2905  * @return
2906  *  - On success, zero.
2907  *  - On failure, a negative value.
2908  */
2909 static int
2910 ixgbe_dev_interrupt_action(struct rte_eth_dev *dev)
2911 {
2912         struct ixgbe_interrupt *intr =
2913                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2914         int64_t timeout;
2915         struct rte_eth_link link;
2916         int intr_enable_delay = false;
2917
2918         PMD_DRV_LOG(DEBUG, "intr action type %d", intr->flags);
2919
2920         if (intr->flags & IXGBE_FLAG_MAILBOX) {
2921                 ixgbe_pf_mbx_process(dev);
2922                 intr->flags &= ~IXGBE_FLAG_MAILBOX;
2923         }
2924
2925         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
2926                 /* get the link status before link update, for predicting later */
2927                 memset(&link, 0, sizeof(link));
2928                 rte_ixgbe_dev_atomic_read_link_status(dev, &link);
2929
2930                 ixgbe_dev_link_update(dev, 0);
2931
2932                 /* likely to up */
2933                 if (!link.link_status)
2934                         /* handle it 1 sec later, wait it being stable */
2935                         timeout = IXGBE_LINK_UP_CHECK_TIMEOUT;
2936                 /* likely to down */
2937                 else
2938                         /* handle it 4 sec later, wait it being stable */
2939                         timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT;
2940
2941                 ixgbe_dev_link_status_print(dev);
2942
2943                 intr_enable_delay = true;
2944         }
2945
2946         if (intr_enable_delay) {
2947                 if (rte_eal_alarm_set(timeout * 1000,
2948                                       ixgbe_dev_interrupt_delayed_handler, (void*)dev) < 0)
2949                         PMD_DRV_LOG(ERR, "Error setting alarm");
2950         } else {
2951                 PMD_DRV_LOG(DEBUG, "enable intr immediately");
2952                 ixgbe_enable_intr(dev);
2953                 rte_intr_enable(&(dev->pci_dev->intr_handle));
2954         }
2955
2956
2957         return 0;
2958 }
2959
2960 static int
2961 ixgbevf_dev_interrupt_action(struct rte_eth_dev *dev)
2962 {
2963         struct ixgbe_hw *hw =
2964                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2965
2966         PMD_DRV_LOG(DEBUG, "enable intr immediately");
2967         ixgbevf_intr_enable(hw);
2968         rte_intr_enable(&dev->pci_dev->intr_handle);
2969         return 0;
2970 }
2971
2972 /**
2973  * Interrupt handler which shall be registered for alarm callback for delayed
2974  * handling specific interrupt to wait for the stable nic state. As the
2975  * NIC interrupt state is not stable for ixgbe after link is just down,
2976  * it needs to wait 4 seconds to get the stable status.
2977  *
2978  * @param handle
2979  *  Pointer to interrupt handle.
2980  * @param param
2981  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2982  *
2983  * @return
2984  *  void
2985  */
2986 static void
2987 ixgbe_dev_interrupt_delayed_handler(void *param)
2988 {
2989         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2990         struct ixgbe_interrupt *intr =
2991                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2992         struct ixgbe_hw *hw =
2993                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2994         uint32_t eicr;
2995
2996         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
2997         if (eicr & IXGBE_EICR_MAILBOX)
2998                 ixgbe_pf_mbx_process(dev);
2999
3000         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
3001                 ixgbe_dev_link_update(dev, 0);
3002                 intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
3003                 ixgbe_dev_link_status_print(dev);
3004                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
3005         }
3006
3007         PMD_DRV_LOG(DEBUG, "enable intr in delayed handler S[%08x]", eicr);
3008         ixgbe_enable_intr(dev);
3009         rte_intr_enable(&(dev->pci_dev->intr_handle));
3010 }
3011
3012 /**
3013  * Interrupt handler triggered by NIC  for handling
3014  * specific interrupt.
3015  *
3016  * @param handle
3017  *  Pointer to interrupt handle.
3018  * @param param
3019  *  The address of parameter (struct rte_eth_dev *) regsitered before.
3020  *
3021  * @return
3022  *  void
3023  */
3024 static void
3025 ixgbe_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
3026                             void *param)
3027 {
3028         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
3029
3030         ixgbe_dev_interrupt_get_status(dev);
3031         ixgbe_dev_interrupt_action(dev);
3032 }
3033
3034 static void
3035 ixgbevf_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
3036                               void *param)
3037 {
3038         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
3039
3040         ixgbevf_dev_interrupt_get_status(dev);
3041         ixgbevf_dev_interrupt_action(dev);
3042 }
3043
3044 static int
3045 ixgbe_dev_led_on(struct rte_eth_dev *dev)
3046 {
3047         struct ixgbe_hw *hw;
3048
3049         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3050         return (ixgbe_led_on(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
3051 }
3052
3053 static int
3054 ixgbe_dev_led_off(struct rte_eth_dev *dev)
3055 {
3056         struct ixgbe_hw *hw;
3057
3058         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3059         return (ixgbe_led_off(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
3060 }
3061
3062 static int
3063 ixgbe_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
3064 {
3065         struct ixgbe_hw *hw;
3066         uint32_t mflcn_reg;
3067         uint32_t fccfg_reg;
3068         int rx_pause;
3069         int tx_pause;
3070
3071         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3072
3073         fc_conf->pause_time = hw->fc.pause_time;
3074         fc_conf->high_water = hw->fc.high_water[0];
3075         fc_conf->low_water = hw->fc.low_water[0];
3076         fc_conf->send_xon = hw->fc.send_xon;
3077         fc_conf->autoneg = !hw->fc.disable_fc_autoneg;
3078
3079         /*
3080          * Return rx_pause status according to actual setting of
3081          * MFLCN register.
3082          */
3083         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
3084         if (mflcn_reg & (IXGBE_MFLCN_RPFCE | IXGBE_MFLCN_RFCE))
3085                 rx_pause = 1;
3086         else
3087                 rx_pause = 0;
3088
3089         /*
3090          * Return tx_pause status according to actual setting of
3091          * FCCFG register.
3092          */
3093         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
3094         if (fccfg_reg & (IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY))
3095                 tx_pause = 1;
3096         else
3097                 tx_pause = 0;
3098
3099         if (rx_pause && tx_pause)
3100                 fc_conf->mode = RTE_FC_FULL;
3101         else if (rx_pause)
3102                 fc_conf->mode = RTE_FC_RX_PAUSE;
3103         else if (tx_pause)
3104                 fc_conf->mode = RTE_FC_TX_PAUSE;
3105         else
3106                 fc_conf->mode = RTE_FC_NONE;
3107
3108         return 0;
3109 }
3110
3111 static int
3112 ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
3113 {
3114         struct ixgbe_hw *hw;
3115         int err;
3116         uint32_t rx_buf_size;
3117         uint32_t max_high_water;
3118         uint32_t mflcn;
3119         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
3120                 ixgbe_fc_none,
3121                 ixgbe_fc_rx_pause,
3122                 ixgbe_fc_tx_pause,
3123                 ixgbe_fc_full
3124         };
3125
3126         PMD_INIT_FUNC_TRACE();
3127
3128         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3129         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0));
3130         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
3131
3132         /*
3133          * At least reserve one Ethernet frame for watermark
3134          * high_water/low_water in kilo bytes for ixgbe
3135          */
3136         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
3137         if ((fc_conf->high_water > max_high_water) ||
3138                 (fc_conf->high_water < fc_conf->low_water)) {
3139                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB");
3140                 PMD_INIT_LOG(ERR, "High_water must <= 0x%x", max_high_water);
3141                 return (-EINVAL);
3142         }
3143
3144         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[fc_conf->mode];
3145         hw->fc.pause_time     = fc_conf->pause_time;
3146         hw->fc.high_water[0]  = fc_conf->high_water;
3147         hw->fc.low_water[0]   = fc_conf->low_water;
3148         hw->fc.send_xon       = fc_conf->send_xon;
3149         hw->fc.disable_fc_autoneg = !fc_conf->autoneg;
3150
3151         err = ixgbe_fc_enable(hw);
3152
3153         /* Not negotiated is not an error case */
3154         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) {
3155
3156                 /* check if we want to forward MAC frames - driver doesn't have native
3157                  * capability to do that, so we'll write the registers ourselves */
3158
3159                 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN);
3160
3161                 /* set or clear MFLCN.PMCF bit depending on configuration */
3162                 if (fc_conf->mac_ctrl_frame_fwd != 0)
3163                         mflcn |= IXGBE_MFLCN_PMCF;
3164                 else
3165                         mflcn &= ~IXGBE_MFLCN_PMCF;
3166
3167                 IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn);
3168                 IXGBE_WRITE_FLUSH(hw);
3169
3170                 return 0;
3171         }
3172
3173         PMD_INIT_LOG(ERR, "ixgbe_fc_enable = 0x%x", err);
3174         return -EIO;
3175 }
3176
3177 /**
3178  *  ixgbe_pfc_enable_generic - Enable flow control
3179  *  @hw: pointer to hardware structure
3180  *  @tc_num: traffic class number
3181  *  Enable flow control according to the current settings.
3182  */
3183 static int
3184 ixgbe_dcb_pfc_enable_generic(struct ixgbe_hw *hw,uint8_t tc_num)
3185 {
3186         int ret_val = 0;
3187         uint32_t mflcn_reg, fccfg_reg;
3188         uint32_t reg;
3189         uint32_t fcrtl, fcrth;
3190         uint8_t i;
3191         uint8_t nb_rx_en;
3192
3193         /* Validate the water mark configuration */
3194         if (!hw->fc.pause_time) {
3195                 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
3196                 goto out;
3197         }
3198
3199         /* Low water mark of zero causes XOFF floods */
3200         if (hw->fc.current_mode & ixgbe_fc_tx_pause) {
3201                  /* High/Low water can not be 0 */
3202                 if( (!hw->fc.high_water[tc_num])|| (!hw->fc.low_water[tc_num])) {
3203                         PMD_INIT_LOG(ERR, "Invalid water mark configuration");
3204                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
3205                         goto out;
3206                 }
3207
3208                 if(hw->fc.low_water[tc_num] >= hw->fc.high_water[tc_num]) {
3209                         PMD_INIT_LOG(ERR, "Invalid water mark configuration");
3210                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
3211                         goto out;
3212                 }
3213         }
3214         /* Negotiate the fc mode to use */
3215         ixgbe_fc_autoneg(hw);
3216
3217         /* Disable any previous flow control settings */
3218         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
3219         mflcn_reg &= ~(IXGBE_MFLCN_RPFCE_SHIFT | IXGBE_MFLCN_RFCE|IXGBE_MFLCN_RPFCE);
3220
3221         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
3222         fccfg_reg &= ~(IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY);
3223
3224         switch (hw->fc.current_mode) {
3225         case ixgbe_fc_none:
3226                 /*
3227                  * If the count of enabled RX Priority Flow control >1,
3228                  * and the TX pause can not be disabled
3229                  */
3230                 nb_rx_en = 0;
3231                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
3232                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
3233                         if (reg & IXGBE_FCRTH_FCEN)
3234                                 nb_rx_en++;
3235                 }
3236                 if (nb_rx_en > 1)
3237                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
3238                 break;
3239         case ixgbe_fc_rx_pause:
3240                 /*
3241                  * Rx Flow control is enabled and Tx Flow control is
3242                  * disabled by software override. Since there really
3243                  * isn't a way to advertise that we are capable of RX
3244                  * Pause ONLY, we will advertise that we support both
3245                  * symmetric and asymmetric Rx PAUSE.  Later, we will
3246                  * disable the adapter's ability to send PAUSE frames.
3247                  */
3248                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
3249                 /*
3250                  * If the count of enabled RX Priority Flow control >1,
3251                  * and the TX pause can not be disabled
3252                  */
3253                 nb_rx_en = 0;
3254                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
3255                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
3256                         if (reg & IXGBE_FCRTH_FCEN)
3257                                 nb_rx_en++;
3258                 }
3259                 if (nb_rx_en > 1)
3260                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
3261                 break;
3262         case ixgbe_fc_tx_pause:
3263                 /*
3264                  * Tx Flow control is enabled, and Rx Flow control is
3265                  * disabled by software override.
3266                  */
3267                 fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
3268                 break;
3269         case ixgbe_fc_full:
3270                 /* Flow control (both Rx and Tx) is enabled by SW override. */
3271                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
3272                 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
3273                 break;
3274         default:
3275                 PMD_DRV_LOG(DEBUG, "Flow control param set incorrectly");
3276                 ret_val = IXGBE_ERR_CONFIG;
3277                 goto out;
3278                 break;
3279         }
3280
3281         /* Set 802.3x based flow control settings. */
3282         mflcn_reg |= IXGBE_MFLCN_DPF;
3283         IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg);
3284         IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg);
3285
3286         /* Set up and enable Rx high/low water mark thresholds, enable XON. */
3287         if ((hw->fc.current_mode & ixgbe_fc_tx_pause) &&
3288                 hw->fc.high_water[tc_num]) {
3289                 fcrtl = (hw->fc.low_water[tc_num] << 10) | IXGBE_FCRTL_XONE;
3290                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), fcrtl);
3291                 fcrth = (hw->fc.high_water[tc_num] << 10) | IXGBE_FCRTH_FCEN;
3292         } else {
3293                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), 0);
3294                 /*
3295                  * In order to prevent Tx hangs when the internal Tx
3296                  * switch is enabled we must set the high water mark
3297                  * to the maximum FCRTH value.  This allows the Tx
3298                  * switch to function even under heavy Rx workloads.
3299                  */
3300                 fcrth = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)) - 32;
3301         }
3302         IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(tc_num), fcrth);
3303
3304         /* Configure pause time (2 TCs per register) */
3305         reg = hw->fc.pause_time * 0x00010001;
3306         for (i = 0; i < (IXGBE_DCB_MAX_TRAFFIC_CLASS / 2); i++)
3307                 IXGBE_WRITE_REG(hw, IXGBE_FCTTV(i), reg);
3308
3309         /* Configure flow control refresh threshold value */
3310         IXGBE_WRITE_REG(hw, IXGBE_FCRTV, hw->fc.pause_time / 2);
3311
3312 out:
3313         return ret_val;
3314 }
3315
3316 static int
3317 ixgbe_dcb_pfc_enable(struct rte_eth_dev *dev,uint8_t tc_num)
3318 {
3319         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3320         int32_t ret_val = IXGBE_NOT_IMPLEMENTED;
3321
3322         if(hw->mac.type != ixgbe_mac_82598EB) {
3323                 ret_val = ixgbe_dcb_pfc_enable_generic(hw,tc_num);
3324         }
3325         return ret_val;
3326 }
3327
3328 static int
3329 ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf)
3330 {
3331         int err;
3332         uint32_t rx_buf_size;
3333         uint32_t max_high_water;
3334         uint8_t tc_num;
3335         uint8_t  map[IXGBE_DCB_MAX_USER_PRIORITY] = { 0 };
3336         struct ixgbe_hw *hw =
3337                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3338         struct ixgbe_dcb_config *dcb_config =
3339                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
3340
3341         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
3342                 ixgbe_fc_none,
3343                 ixgbe_fc_rx_pause,
3344                 ixgbe_fc_tx_pause,
3345                 ixgbe_fc_full
3346         };
3347
3348         PMD_INIT_FUNC_TRACE();
3349
3350         ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
3351         tc_num = map[pfc_conf->priority];
3352         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num));
3353         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
3354         /*
3355          * At least reserve one Ethernet frame for watermark
3356          * high_water/low_water in kilo bytes for ixgbe
3357          */
3358         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
3359         if ((pfc_conf->fc.high_water > max_high_water) ||
3360             (pfc_conf->fc.high_water <= pfc_conf->fc.low_water)) {
3361                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB");
3362                 PMD_INIT_LOG(ERR, "High_water must <= 0x%x", max_high_water);
3363                 return (-EINVAL);
3364         }
3365
3366         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[pfc_conf->fc.mode];
3367         hw->fc.pause_time = pfc_conf->fc.pause_time;
3368         hw->fc.send_xon = pfc_conf->fc.send_xon;
3369         hw->fc.low_water[tc_num] =  pfc_conf->fc.low_water;
3370         hw->fc.high_water[tc_num] = pfc_conf->fc.high_water;
3371
3372         err = ixgbe_dcb_pfc_enable(dev,tc_num);
3373
3374         /* Not negotiated is not an error case */
3375         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED))
3376                 return 0;
3377
3378         PMD_INIT_LOG(ERR, "ixgbe_dcb_pfc_enable = 0x%x", err);
3379         return -EIO;
3380 }
3381
3382 static int
3383 ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
3384                           struct rte_eth_rss_reta_entry64 *reta_conf,
3385                           uint16_t reta_size)
3386 {
3387         uint8_t i, j, mask;
3388         uint32_t reta, r;
3389         uint16_t idx, shift;
3390         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3391         uint16_t sp_reta_size;
3392         uint32_t reta_reg;
3393
3394         PMD_INIT_FUNC_TRACE();
3395
3396         if (!ixgbe_rss_update_sp(hw->mac.type)) {
3397                 PMD_DRV_LOG(ERR, "RSS reta update is not supported on this "
3398                         "NIC.");
3399                 return -ENOTSUP;
3400         }
3401
3402         sp_reta_size = ixgbe_reta_size_get(hw->mac.type);
3403         if (reta_size != sp_reta_size) {
3404                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
3405                         "(%d) doesn't match the number hardware can supported "
3406                         "(%d)\n", reta_size, sp_reta_size);
3407                 return -EINVAL;
3408         }
3409
3410         for (i = 0; i < reta_size; i += IXGBE_4_BIT_WIDTH) {
3411                 idx = i / RTE_RETA_GROUP_SIZE;
3412                 shift = i % RTE_RETA_GROUP_SIZE;
3413                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
3414                                                 IXGBE_4_BIT_MASK);
3415                 if (!mask)
3416                         continue;
3417                 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i);
3418                 if (mask == IXGBE_4_BIT_MASK)
3419                         r = 0;
3420                 else
3421                         r = IXGBE_READ_REG(hw, reta_reg);
3422                 for (j = 0, reta = 0; j < IXGBE_4_BIT_WIDTH; j++) {
3423                         if (mask & (0x1 << j))
3424                                 reta |= reta_conf[idx].reta[shift + j] <<
3425                                                         (CHAR_BIT * j);
3426                         else
3427                                 reta |= r & (IXGBE_8_BIT_MASK <<
3428                                                 (CHAR_BIT * j));
3429                 }
3430                 IXGBE_WRITE_REG(hw, reta_reg, reta);
3431         }
3432
3433         return 0;
3434 }
3435
3436 static int
3437 ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
3438                          struct rte_eth_rss_reta_entry64 *reta_conf,
3439                          uint16_t reta_size)
3440 {
3441         uint8_t i, j, mask;
3442         uint32_t reta;
3443         uint16_t idx, shift;
3444         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3445         uint16_t sp_reta_size;
3446         uint32_t reta_reg;
3447
3448         PMD_INIT_FUNC_TRACE();
3449         sp_reta_size = ixgbe_reta_size_get(hw->mac.type);
3450         if (reta_size != sp_reta_size) {
3451                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
3452                         "(%d) doesn't match the number hardware can supported "
3453                         "(%d)\n", reta_size, sp_reta_size);
3454                 return -EINVAL;
3455         }
3456
3457         for (i = 0; i < reta_size; i += IXGBE_4_BIT_WIDTH) {
3458                 idx = i / RTE_RETA_GROUP_SIZE;
3459                 shift = i % RTE_RETA_GROUP_SIZE;
3460                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
3461                                                 IXGBE_4_BIT_MASK);
3462                 if (!mask)
3463                         continue;
3464
3465                 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i);
3466                 reta = IXGBE_READ_REG(hw, reta_reg);
3467                 for (j = 0; j < IXGBE_4_BIT_WIDTH; j++) {
3468                         if (mask & (0x1 << j))
3469                                 reta_conf[idx].reta[shift + j] =
3470                                         ((reta >> (CHAR_BIT * j)) &
3471                                                 IXGBE_8_BIT_MASK);
3472                 }
3473         }
3474
3475         return 0;
3476 }
3477
3478 static void
3479 ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
3480                                 uint32_t index, uint32_t pool)
3481 {
3482         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3483         uint32_t enable_addr = 1;
3484
3485         ixgbe_set_rar(hw, index, mac_addr->addr_bytes, pool, enable_addr);
3486 }
3487
3488 static void
3489 ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
3490 {
3491         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3492
3493         ixgbe_clear_rar(hw, index);
3494 }
3495
3496 static void
3497 ixgbe_set_default_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
3498 {
3499         ixgbe_remove_rar(dev, 0);
3500
3501         ixgbe_add_rar(dev, addr, 0, 0);
3502 }
3503
3504 static int
3505 ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
3506 {
3507         uint32_t hlreg0;
3508         uint32_t maxfrs;
3509         struct ixgbe_hw *hw;
3510         struct rte_eth_dev_info dev_info;
3511         uint32_t frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
3512
3513         ixgbe_dev_info_get(dev, &dev_info);
3514
3515         /* check that mtu is within the allowed range */
3516         if ((mtu < ETHER_MIN_MTU) || (frame_size > dev_info.max_rx_pktlen))
3517                 return -EINVAL;
3518
3519         /* refuse mtu that requires the support of scattered packets when this
3520          * feature has not been enabled before. */
3521         if (!dev->data->scattered_rx &&
3522             (frame_size + 2 * IXGBE_VLAN_TAG_SIZE >
3523              dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM))
3524                 return -EINVAL;
3525
3526         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3527         hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
3528
3529         /* switch to jumbo mode if needed */
3530         if (frame_size > ETHER_MAX_LEN) {
3531                 dev->data->dev_conf.rxmode.jumbo_frame = 1;
3532                 hlreg0 |= IXGBE_HLREG0_JUMBOEN;
3533         } else {
3534                 dev->data->dev_conf.rxmode.jumbo_frame = 0;
3535                 hlreg0 &= ~IXGBE_HLREG0_JUMBOEN;
3536         }
3537         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
3538
3539         /* update max frame size */
3540         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
3541
3542         maxfrs = IXGBE_READ_REG(hw, IXGBE_MAXFRS);
3543         maxfrs &= 0x0000FFFF;
3544         maxfrs |= (dev->data->dev_conf.rxmode.max_rx_pkt_len << 16);
3545         IXGBE_WRITE_REG(hw, IXGBE_MAXFRS, maxfrs);
3546
3547         return 0;
3548 }
3549
3550 /*
3551  * Virtual Function operations
3552  */
3553 static void
3554 ixgbevf_intr_disable(struct ixgbe_hw *hw)
3555 {
3556         PMD_INIT_FUNC_TRACE();
3557
3558         /* Clear interrupt mask to stop from interrupts being generated */
3559         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, IXGBE_VF_IRQ_CLEAR_MASK);
3560
3561         IXGBE_WRITE_FLUSH(hw);
3562 }
3563
3564 static void
3565 ixgbevf_intr_enable(struct ixgbe_hw *hw)
3566 {
3567         PMD_INIT_FUNC_TRACE();
3568
3569         /* VF enable interrupt autoclean */
3570         IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_VF_IRQ_ENABLE_MASK);
3571         IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, IXGBE_VF_IRQ_ENABLE_MASK);
3572         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_VF_IRQ_ENABLE_MASK);
3573
3574         IXGBE_WRITE_FLUSH(hw);
3575 }
3576
3577 static int
3578 ixgbevf_dev_configure(struct rte_eth_dev *dev)
3579 {
3580         struct rte_eth_conf* conf = &dev->data->dev_conf;
3581         struct ixgbe_adapter *adapter =
3582                         (struct ixgbe_adapter *)dev->data->dev_private;
3583
3584         PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d",
3585                      dev->data->port_id);
3586
3587         /*
3588          * VF has no ability to enable/disable HW CRC
3589          * Keep the persistent behavior the same as Host PF
3590          */
3591 #ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC
3592         if (!conf->rxmode.hw_strip_crc) {
3593                 PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip");
3594                 conf->rxmode.hw_strip_crc = 1;
3595         }
3596 #else
3597         if (conf->rxmode.hw_strip_crc) {
3598                 PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip");
3599                 conf->rxmode.hw_strip_crc = 0;
3600         }
3601 #endif
3602
3603         /*
3604          * Initialize to TRUE. If any of Rx queues doesn't meet the bulk
3605          * allocation or vector Rx preconditions we will reset it.
3606          */
3607         adapter->rx_bulk_alloc_allowed = true;
3608         adapter->rx_vec_allowed = true;
3609
3610         return 0;
3611 }
3612
3613 static int
3614 ixgbevf_dev_start(struct rte_eth_dev *dev)
3615 {
3616         struct ixgbe_hw *hw =
3617                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3618         uint32_t intr_vector = 0;
3619         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
3620
3621         int err, mask = 0;
3622
3623         PMD_INIT_FUNC_TRACE();
3624
3625         hw->mac.ops.reset_hw(hw);
3626         hw->mac.get_link_status = true;
3627
3628         /* negotiate mailbox API version to use with the PF. */
3629         ixgbevf_negotiate_api(hw);
3630
3631         ixgbevf_dev_tx_init(dev);
3632
3633         /* This can fail when allocating mbufs for descriptor rings */
3634         err = ixgbevf_dev_rx_init(dev);
3635         if (err) {
3636                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware (%d)", err);
3637                 ixgbe_dev_clear_queues(dev);
3638                 return err;
3639         }
3640
3641         /* Set vfta */
3642         ixgbevf_set_vfta_all(dev,1);
3643
3644         /* Set HW strip */
3645         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
3646                 ETH_VLAN_EXTEND_MASK;
3647         ixgbevf_vlan_offload_set(dev, mask);
3648
3649         ixgbevf_dev_rxtx_start(dev);
3650
3651         /* check and configure queue intr-vector mapping */
3652         if (dev->data->dev_conf.intr_conf.rxq != 0)
3653                 intr_vector = dev->data->nb_rx_queues;
3654
3655         if (rte_intr_efd_enable(intr_handle, intr_vector))
3656                 return -1;
3657
3658         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
3659                 intr_handle->intr_vec =
3660                         rte_zmalloc("intr_vec",
3661                                     dev->data->nb_rx_queues * sizeof(int), 0);
3662                 if (intr_handle->intr_vec == NULL) {
3663                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
3664                                      " intr_vec\n", dev->data->nb_rx_queues);
3665                         return -ENOMEM;
3666                 }
3667         }
3668         ixgbevf_configure_msix(dev);
3669
3670         if (dev->data->dev_conf.intr_conf.lsc != 0) {
3671                 if (rte_intr_allow_others(intr_handle))
3672                         rte_intr_callback_register(intr_handle,
3673                                         ixgbevf_dev_interrupt_handler,
3674                                         (void *)dev);
3675                 else
3676                         PMD_INIT_LOG(INFO, "lsc won't enable because of"
3677                                      " no intr multiplex\n");
3678         }
3679
3680         rte_intr_enable(intr_handle);
3681
3682         /* Re-enable interrupt for VF */
3683         ixgbevf_intr_enable(hw);
3684
3685         return 0;
3686 }
3687
3688 static void
3689 ixgbevf_dev_stop(struct rte_eth_dev *dev)
3690 {
3691         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3692         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
3693
3694         PMD_INIT_FUNC_TRACE();
3695
3696         hw->adapter_stopped = 1;
3697         ixgbe_stop_adapter(hw);
3698
3699         /*
3700           * Clear what we set, but we still keep shadow_vfta to
3701           * restore after device starts
3702           */
3703         ixgbevf_set_vfta_all(dev,0);
3704
3705         /* Clear stored conf */
3706         dev->data->scattered_rx = 0;
3707
3708         ixgbe_dev_clear_queues(dev);
3709
3710         /* disable intr eventfd mapping */
3711         rte_intr_disable(intr_handle);
3712
3713         /* Clean datapath event and queue/vec mapping */
3714         rte_intr_efd_disable(intr_handle);
3715         if (intr_handle->intr_vec != NULL) {
3716                 rte_free(intr_handle->intr_vec);
3717                 intr_handle->intr_vec = NULL;
3718         }
3719 }
3720
3721 static void
3722 ixgbevf_dev_close(struct rte_eth_dev *dev)
3723 {
3724         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3725         struct rte_pci_device *pci_dev;
3726
3727         PMD_INIT_FUNC_TRACE();
3728
3729         ixgbe_reset_hw(hw);
3730
3731         ixgbevf_dev_stop(dev);
3732
3733         ixgbe_dev_free_queues(dev);
3734
3735         /* reprogram the RAR[0] in case user changed it. */
3736         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
3737
3738         pci_dev = dev->pci_dev;
3739         if (pci_dev->intr_handle.intr_vec) {
3740                 rte_free(pci_dev->intr_handle.intr_vec);
3741                 pci_dev->intr_handle.intr_vec = NULL;
3742         }
3743 }
3744
3745 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on)
3746 {
3747         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3748         struct ixgbe_vfta * shadow_vfta =
3749                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
3750         int i = 0, j = 0, vfta = 0, mask = 1;
3751
3752         for (i = 0; i < IXGBE_VFTA_SIZE; i++){
3753                 vfta = shadow_vfta->vfta[i];
3754                 if(vfta){
3755                         mask = 1;
3756                         for (j = 0; j < 32; j++){
3757                                 if(vfta & mask)
3758                                         ixgbe_set_vfta(hw, (i<<5)+j, 0, on);
3759                                 mask<<=1;
3760                         }
3761                 }
3762         }
3763
3764 }
3765
3766 static int
3767 ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
3768 {
3769         struct ixgbe_hw *hw =
3770                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3771         struct ixgbe_vfta * shadow_vfta =
3772                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
3773         uint32_t vid_idx = 0;
3774         uint32_t vid_bit = 0;
3775         int ret = 0;
3776
3777         PMD_INIT_FUNC_TRACE();
3778
3779         /* vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf */
3780         ret = ixgbe_set_vfta(hw, vlan_id, 0, !!on);
3781         if(ret){
3782                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
3783                 return ret;
3784         }
3785         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
3786         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
3787
3788         /* Save what we set and retore it after device reset */
3789         if (on)
3790                 shadow_vfta->vfta[vid_idx] |= vid_bit;
3791         else
3792                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
3793
3794         return 0;
3795 }
3796
3797 static void
3798 ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
3799 {
3800         struct ixgbe_hw *hw =
3801                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3802         uint32_t ctrl;
3803
3804         PMD_INIT_FUNC_TRACE();
3805
3806         if(queue >= hw->mac.max_rx_queues)
3807                 return;
3808
3809         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
3810         if(on)
3811                 ctrl |= IXGBE_RXDCTL_VME;
3812         else
3813                 ctrl &= ~IXGBE_RXDCTL_VME;
3814         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
3815
3816         ixgbe_vlan_hw_strip_bitmap_set( dev, queue, on);
3817 }
3818
3819 static void
3820 ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
3821 {
3822         struct ixgbe_hw *hw =
3823                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3824         uint16_t i;
3825         int on = 0;
3826
3827         /* VF function only support hw strip feature, others are not support */
3828         if(mask & ETH_VLAN_STRIP_MASK){
3829                 on = !!(dev->data->dev_conf.rxmode.hw_vlan_strip);
3830
3831                 for(i=0; i < hw->mac.max_rx_queues; i++)
3832                         ixgbevf_vlan_strip_queue_set(dev,i,on);
3833         }
3834 }
3835
3836 static int
3837 ixgbe_vmdq_mode_check(struct ixgbe_hw *hw)
3838 {
3839         uint32_t reg_val;
3840
3841         /* we only need to do this if VMDq is enabled */
3842         reg_val = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
3843         if (!(reg_val & IXGBE_VT_CTL_VT_ENABLE)) {
3844                 PMD_INIT_LOG(ERR, "VMDq must be enabled for this setting");
3845                 return (-1);
3846         }
3847
3848         return 0;
3849 }
3850
3851 static uint32_t
3852 ixgbe_uta_vector(struct ixgbe_hw *hw, struct ether_addr* uc_addr)
3853 {
3854         uint32_t vector = 0;
3855         switch (hw->mac.mc_filter_type) {
3856         case 0:   /* use bits [47:36] of the address */
3857                 vector = ((uc_addr->addr_bytes[4] >> 4) |
3858                         (((uint16_t)uc_addr->addr_bytes[5]) << 4));
3859                 break;
3860         case 1:   /* use bits [46:35] of the address */
3861                 vector = ((uc_addr->addr_bytes[4] >> 3) |
3862                         (((uint16_t)uc_addr->addr_bytes[5]) << 5));
3863                 break;
3864         case 2:   /* use bits [45:34] of the address */
3865                 vector = ((uc_addr->addr_bytes[4] >> 2) |
3866                         (((uint16_t)uc_addr->addr_bytes[5]) << 6));
3867                 break;
3868         case 3:   /* use bits [43:32] of the address */
3869                 vector = ((uc_addr->addr_bytes[4]) |
3870                         (((uint16_t)uc_addr->addr_bytes[5]) << 8));
3871                 break;
3872         default:  /* Invalid mc_filter_type */
3873                 break;
3874         }
3875
3876         /* vector can only be 12-bits or boundary will be exceeded */
3877         vector &= 0xFFF;
3878         return vector;
3879 }
3880
3881 static int
3882 ixgbe_uc_hash_table_set(struct rte_eth_dev *dev,struct ether_addr* mac_addr,
3883                                uint8_t on)
3884 {
3885         uint32_t vector;
3886         uint32_t uta_idx;
3887         uint32_t reg_val;
3888         uint32_t uta_shift;
3889         uint32_t rc;
3890         const uint32_t ixgbe_uta_idx_mask = 0x7F;
3891         const uint32_t ixgbe_uta_bit_shift = 5;
3892         const uint32_t ixgbe_uta_bit_mask = (0x1 << ixgbe_uta_bit_shift) - 1;
3893         const uint32_t bit1 = 0x1;
3894
3895         struct ixgbe_hw *hw =
3896                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3897         struct ixgbe_uta_info *uta_info =
3898                 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private);
3899
3900         /* The UTA table only exists on 82599 hardware and newer */
3901         if (hw->mac.type < ixgbe_mac_82599EB)
3902                 return (-ENOTSUP);
3903
3904         vector = ixgbe_uta_vector(hw,mac_addr);
3905         uta_idx = (vector >> ixgbe_uta_bit_shift) & ixgbe_uta_idx_mask;
3906         uta_shift = vector & ixgbe_uta_bit_mask;
3907
3908         rc = ((uta_info->uta_shadow[uta_idx] >> uta_shift & bit1) != 0);
3909         if(rc == on)
3910                 return 0;
3911
3912         reg_val = IXGBE_READ_REG(hw, IXGBE_UTA(uta_idx));
3913         if (on) {
3914                 uta_info->uta_in_use++;
3915                 reg_val |= (bit1 << uta_shift);
3916                 uta_info->uta_shadow[uta_idx] |= (bit1 << uta_shift);
3917         } else {
3918                 uta_info->uta_in_use--;
3919                 reg_val &= ~(bit1 << uta_shift);
3920                 uta_info->uta_shadow[uta_idx] &= ~(bit1 << uta_shift);
3921         }
3922
3923         IXGBE_WRITE_REG(hw, IXGBE_UTA(uta_idx), reg_val);
3924
3925         if (uta_info->uta_in_use > 0)
3926                 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL,
3927                                 IXGBE_MCSTCTRL_MFE | hw->mac.mc_filter_type);
3928         else
3929                 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL,hw->mac.mc_filter_type);
3930
3931         return 0;
3932 }
3933
3934 static int
3935 ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on)
3936 {
3937         int i;
3938         struct ixgbe_hw *hw =
3939                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3940         struct ixgbe_uta_info *uta_info =
3941                 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private);
3942
3943         /* The UTA table only exists on 82599 hardware and newer */
3944         if (hw->mac.type < ixgbe_mac_82599EB)
3945                 return (-ENOTSUP);
3946
3947         if(on) {
3948                 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
3949                         uta_info->uta_shadow[i] = ~0;
3950                         IXGBE_WRITE_REG(hw, IXGBE_UTA(i), ~0);
3951                 }
3952         } else {
3953                 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
3954                         uta_info->uta_shadow[i] = 0;
3955                         IXGBE_WRITE_REG(hw, IXGBE_UTA(i), 0);
3956                 }
3957         }
3958         return 0;
3959
3960 }
3961
3962 uint32_t
3963 ixgbe_convert_vm_rx_mask_to_val(uint16_t rx_mask, uint32_t orig_val)
3964 {
3965         uint32_t new_val = orig_val;
3966
3967         if (rx_mask & ETH_VMDQ_ACCEPT_UNTAG)
3968                 new_val |= IXGBE_VMOLR_AUPE;
3969         if (rx_mask & ETH_VMDQ_ACCEPT_HASH_MC)
3970                 new_val |= IXGBE_VMOLR_ROMPE;
3971         if (rx_mask & ETH_VMDQ_ACCEPT_HASH_UC)
3972                 new_val |= IXGBE_VMOLR_ROPE;
3973         if (rx_mask & ETH_VMDQ_ACCEPT_BROADCAST)
3974                 new_val |= IXGBE_VMOLR_BAM;
3975         if (rx_mask & ETH_VMDQ_ACCEPT_MULTICAST)
3976                 new_val |= IXGBE_VMOLR_MPE;
3977
3978         return new_val;
3979 }
3980
3981 static int
3982 ixgbe_set_pool_rx_mode(struct rte_eth_dev *dev, uint16_t pool,
3983                                uint16_t rx_mask, uint8_t on)
3984 {
3985         int val = 0;
3986
3987         struct ixgbe_hw *hw =
3988                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3989         uint32_t vmolr = IXGBE_READ_REG(hw, IXGBE_VMOLR(pool));
3990
3991         if (hw->mac.type == ixgbe_mac_82598EB) {
3992                 PMD_INIT_LOG(ERR, "setting VF receive mode set should be done"
3993                              " on 82599 hardware and newer");
3994                 return (-ENOTSUP);
3995         }
3996         if (ixgbe_vmdq_mode_check(hw) < 0)
3997                 return (-ENOTSUP);
3998
3999         val = ixgbe_convert_vm_rx_mask_to_val(rx_mask, val);
4000
4001         if (on)
4002                 vmolr |= val;
4003         else
4004                 vmolr &= ~val;
4005
4006         IXGBE_WRITE_REG(hw, IXGBE_VMOLR(pool), vmolr);
4007
4008         return 0;
4009 }
4010
4011 static int
4012 ixgbe_set_pool_rx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on)
4013 {
4014         uint32_t reg,addr;
4015         uint32_t val;
4016         const uint8_t bit1 = 0x1;
4017
4018         struct ixgbe_hw *hw =
4019                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4020
4021         if (ixgbe_vmdq_mode_check(hw) < 0)
4022                 return (-ENOTSUP);
4023
4024         addr = IXGBE_VFRE(pool >= ETH_64_POOLS/2);
4025         reg = IXGBE_READ_REG(hw, addr);
4026         val = bit1 << pool;
4027
4028         if (on)
4029                 reg |= val;
4030         else
4031                 reg &= ~val;
4032
4033         IXGBE_WRITE_REG(hw, addr,reg);
4034
4035         return 0;
4036 }
4037
4038 static int
4039 ixgbe_set_pool_tx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on)
4040 {
4041         uint32_t reg,addr;
4042         uint32_t val;
4043         const uint8_t bit1 = 0x1;
4044
4045         struct ixgbe_hw *hw =
4046                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4047
4048         if (ixgbe_vmdq_mode_check(hw) < 0)
4049                 return (-ENOTSUP);
4050
4051         addr = IXGBE_VFTE(pool >= ETH_64_POOLS/2);
4052         reg = IXGBE_READ_REG(hw, addr);
4053         val = bit1 << pool;
4054
4055         if (on)
4056                 reg |= val;
4057         else
4058                 reg &= ~val;
4059
4060         IXGBE_WRITE_REG(hw, addr,reg);
4061
4062         return 0;
4063 }
4064
4065 static int
4066 ixgbe_set_pool_vlan_filter(struct rte_eth_dev *dev, uint16_t vlan,
4067                         uint64_t pool_mask, uint8_t vlan_on)
4068 {
4069         int ret = 0;
4070         uint16_t pool_idx;
4071         struct ixgbe_hw *hw =
4072                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4073
4074         if (ixgbe_vmdq_mode_check(hw) < 0)
4075                 return (-ENOTSUP);
4076         for (pool_idx = 0; pool_idx < ETH_64_POOLS; pool_idx++) {
4077                 if (pool_mask & ((uint64_t)(1ULL << pool_idx)))
4078                         ret = hw->mac.ops.set_vfta(hw,vlan,pool_idx,vlan_on);
4079                         if (ret < 0)
4080                                 return ret;
4081         }
4082
4083         return ret;
4084 }
4085
4086 #define IXGBE_MRCTL_VPME  0x01 /* Virtual Pool Mirroring. */
4087 #define IXGBE_MRCTL_UPME  0x02 /* Uplink Port Mirroring. */
4088 #define IXGBE_MRCTL_DPME  0x04 /* Downlink Port Mirroring. */
4089 #define IXGBE_MRCTL_VLME  0x08 /* VLAN Mirroring. */
4090 #define IXGBE_INVALID_MIRROR_TYPE(mirror_type) \
4091         ((mirror_type) & ~(uint8_t)(ETH_MIRROR_VIRTUAL_POOL_UP | \
4092         ETH_MIRROR_UPLINK_PORT | ETH_MIRROR_DOWNLINK_PORT | ETH_MIRROR_VLAN))
4093
4094 static int
4095 ixgbe_mirror_rule_set(struct rte_eth_dev *dev,
4096                         struct rte_eth_mirror_conf *mirror_conf,
4097                         uint8_t rule_id, uint8_t on)
4098 {
4099         uint32_t mr_ctl,vlvf;
4100         uint32_t mp_lsb = 0;
4101         uint32_t mv_msb = 0;
4102         uint32_t mv_lsb = 0;
4103         uint32_t mp_msb = 0;
4104         uint8_t i = 0;
4105         int reg_index = 0;
4106         uint64_t vlan_mask = 0;
4107
4108         const uint8_t pool_mask_offset = 32;
4109         const uint8_t vlan_mask_offset = 32;
4110         const uint8_t dst_pool_offset = 8;
4111         const uint8_t rule_mr_offset  = 4;
4112         const uint8_t mirror_rule_mask= 0x0F;
4113
4114         struct ixgbe_mirror_info *mr_info =
4115                         (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private));
4116         struct ixgbe_hw *hw =
4117                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4118         uint8_t mirror_type = 0;
4119
4120         if (ixgbe_vmdq_mode_check(hw) < 0)
4121                 return -ENOTSUP;
4122
4123         if (rule_id >= IXGBE_MAX_MIRROR_RULES)
4124                 return -EINVAL;
4125
4126         if (IXGBE_INVALID_MIRROR_TYPE(mirror_conf->rule_type)) {
4127                 PMD_DRV_LOG(ERR, "unsupported mirror type 0x%x.",
4128                         mirror_conf->rule_type);
4129                 return -EINVAL;
4130         }
4131
4132         if (mirror_conf->rule_type & ETH_MIRROR_VLAN) {
4133                 mirror_type |= IXGBE_MRCTL_VLME;
4134                 /* Check if vlan id is valid and find conresponding VLAN ID index in VLVF */
4135                 for (i = 0;i < IXGBE_VLVF_ENTRIES; i++) {
4136                         if (mirror_conf->vlan.vlan_mask & (1ULL << i)) {
4137                                 /* search vlan id related pool vlan filter index */
4138                                 reg_index = ixgbe_find_vlvf_slot(hw,
4139                                                 mirror_conf->vlan.vlan_id[i]);
4140                                 if(reg_index < 0)
4141                                         return -EINVAL;
4142                                 vlvf = IXGBE_READ_REG(hw, IXGBE_VLVF(reg_index));
4143                                 if ((vlvf & IXGBE_VLVF_VIEN) &&
4144                                     ((vlvf & IXGBE_VLVF_VLANID_MASK) ==
4145                                       mirror_conf->vlan.vlan_id[i]))
4146                                         vlan_mask |= (1ULL << reg_index);
4147                                 else
4148                                         return -EINVAL;
4149                         }
4150                 }
4151
4152                 if (on) {
4153                         mv_lsb = vlan_mask & 0xFFFFFFFF;
4154                         mv_msb = vlan_mask >> vlan_mask_offset;
4155
4156                         mr_info->mr_conf[rule_id].vlan.vlan_mask =
4157                                                 mirror_conf->vlan.vlan_mask;
4158                         for(i = 0 ;i < ETH_VMDQ_MAX_VLAN_FILTERS; i++) {
4159                                 if(mirror_conf->vlan.vlan_mask & (1ULL << i))
4160                                         mr_info->mr_conf[rule_id].vlan.vlan_id[i] =
4161                                                 mirror_conf->vlan.vlan_id[i];
4162                         }
4163                 } else {
4164                         mv_lsb = 0;
4165                         mv_msb = 0;
4166                         mr_info->mr_conf[rule_id].vlan.vlan_mask = 0;
4167                         for(i = 0 ;i < ETH_VMDQ_MAX_VLAN_FILTERS; i++)
4168                                 mr_info->mr_conf[rule_id].vlan.vlan_id[i] = 0;
4169                 }
4170         }
4171
4172         /*
4173          * if enable pool mirror, write related pool mask register,if disable
4174          * pool mirror, clear PFMRVM register
4175          */
4176         if (mirror_conf->rule_type & ETH_MIRROR_VIRTUAL_POOL_UP) {
4177                 mirror_type |= IXGBE_MRCTL_VPME;
4178                 if (on) {
4179                         mp_lsb = mirror_conf->pool_mask & 0xFFFFFFFF;
4180                         mp_msb = mirror_conf->pool_mask >> pool_mask_offset;
4181                         mr_info->mr_conf[rule_id].pool_mask =
4182                                         mirror_conf->pool_mask;
4183
4184                 } else {
4185                         mp_lsb = 0;
4186                         mp_msb = 0;
4187                         mr_info->mr_conf[rule_id].pool_mask = 0;
4188                 }
4189         }
4190         if (mirror_conf->rule_type & ETH_MIRROR_UPLINK_PORT)
4191                 mirror_type |= IXGBE_MRCTL_UPME;
4192         if (mirror_conf->rule_type & ETH_MIRROR_DOWNLINK_PORT)
4193                 mirror_type |= IXGBE_MRCTL_DPME;
4194
4195         /* read  mirror control register and recalculate it */
4196         mr_ctl = IXGBE_READ_REG(hw, IXGBE_MRCTL(rule_id));
4197
4198         if (on) {
4199                 mr_ctl |= mirror_type;
4200                 mr_ctl &= mirror_rule_mask;
4201                 mr_ctl |= mirror_conf->dst_pool << dst_pool_offset;
4202         } else
4203                 mr_ctl &= ~(mirror_conf->rule_type & mirror_rule_mask);
4204
4205         mr_info->mr_conf[rule_id].rule_type = mirror_conf->rule_type;
4206         mr_info->mr_conf[rule_id].dst_pool = mirror_conf->dst_pool;
4207
4208         /* write mirrror control  register */
4209         IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl);
4210
4211         /* write pool mirrror control  register */
4212         if (mirror_conf->rule_type == ETH_MIRROR_VIRTUAL_POOL_UP) {
4213                 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), mp_lsb);
4214                 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset),
4215                                 mp_msb);
4216         }
4217         /* write VLAN mirrror control  register */
4218         if (mirror_conf->rule_type == ETH_MIRROR_VLAN) {
4219                 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), mv_lsb);
4220                 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset),
4221                                 mv_msb);
4222         }
4223
4224         return 0;
4225 }
4226
4227 static int
4228 ixgbe_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t rule_id)
4229 {
4230         int mr_ctl = 0;
4231         uint32_t lsb_val = 0;
4232         uint32_t msb_val = 0;
4233         const uint8_t rule_mr_offset = 4;
4234
4235         struct ixgbe_hw *hw =
4236                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4237         struct ixgbe_mirror_info *mr_info =
4238                 (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private));
4239
4240         if (ixgbe_vmdq_mode_check(hw) < 0)
4241                 return (-ENOTSUP);
4242
4243         memset(&mr_info->mr_conf[rule_id], 0,
4244                 sizeof(struct rte_eth_mirror_conf));
4245
4246         /* clear PFVMCTL register */
4247         IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl);
4248
4249         /* clear pool mask register */
4250         IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), lsb_val);
4251         IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset), msb_val);
4252
4253         /* clear vlan mask register */
4254         IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), lsb_val);
4255         IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset), msb_val);
4256
4257         return 0;
4258 }
4259
4260 static int
4261 ixgbevf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
4262 {
4263         uint32_t mask;
4264         struct ixgbe_hw *hw =
4265                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4266
4267         mask = IXGBE_READ_REG(hw, IXGBE_VTEIMS);
4268         mask |= (1 << queue_id);
4269         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
4270
4271         rte_intr_enable(&dev->pci_dev->intr_handle);
4272
4273         return 0;
4274 }
4275
4276 static int
4277 ixgbevf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
4278 {
4279         uint32_t mask;
4280         struct ixgbe_hw *hw =
4281                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4282
4283         mask = IXGBE_READ_REG(hw, IXGBE_VTEIMS);
4284         mask &= ~(1 << queue_id);
4285         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
4286
4287         return 0;
4288 }
4289
4290 static int
4291 ixgbe_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
4292 {
4293         uint32_t mask;
4294         struct ixgbe_hw *hw =
4295                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4296         struct ixgbe_interrupt *intr =
4297                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
4298
4299         if (queue_id < 16) {
4300                 ixgbe_disable_intr(hw);
4301                 intr->mask |= (1 << queue_id);
4302                 ixgbe_enable_intr(dev);
4303         } else if (queue_id < 32) {
4304                 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(0));
4305                 mask &= (1 << queue_id);
4306                 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
4307         } else if (queue_id < 64) {
4308                 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(1));
4309                 mask &= (1 << (queue_id - 32));
4310                 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
4311         }
4312         rte_intr_enable(&dev->pci_dev->intr_handle);
4313
4314         return 0;
4315 }
4316
4317 static int
4318 ixgbe_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
4319 {
4320         uint32_t mask;
4321         struct ixgbe_hw *hw =
4322                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4323         struct ixgbe_interrupt *intr =
4324                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
4325
4326         if (queue_id < 16) {
4327                 ixgbe_disable_intr(hw);
4328                 intr->mask &= ~(1 << queue_id);
4329                 ixgbe_enable_intr(dev);
4330         } else if (queue_id < 32) {
4331                 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(0));
4332                 mask &= ~(1 << queue_id);
4333                 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
4334         } else if (queue_id < 64) {
4335                 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(1));
4336                 mask &= ~(1 << (queue_id - 32));
4337                 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
4338         }
4339
4340         return 0;
4341 }
4342
4343 static void
4344 ixgbevf_set_ivar_map(struct ixgbe_hw *hw, int8_t direction,
4345                      uint8_t queue, uint8_t msix_vector)
4346 {
4347         uint32_t tmp, idx;
4348
4349         if (direction == -1) {
4350                 /* other causes */
4351                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4352                 tmp = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
4353                 tmp &= ~0xFF;
4354                 tmp |= msix_vector;
4355                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, tmp);
4356         } else {
4357                 /* rx or tx cause */
4358                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4359                 idx = ((16 * (queue & 1)) + (8 * direction));
4360                 tmp = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1));
4361                 tmp &= ~(0xFF << idx);
4362                 tmp |= (msix_vector << idx);
4363                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), tmp);
4364         }
4365 }
4366
4367 /**
4368  * set the IVAR registers, mapping interrupt causes to vectors
4369  * @param hw
4370  *  pointer to ixgbe_hw struct
4371  * @direction
4372  *  0 for Rx, 1 for Tx, -1 for other causes
4373  * @queue
4374  *  queue to map the corresponding interrupt to
4375  * @msix_vector
4376  *  the vector to map to the corresponding queue
4377  */
4378 static void
4379 ixgbe_set_ivar_map(struct ixgbe_hw *hw, int8_t direction,
4380                    uint8_t queue, uint8_t msix_vector)
4381 {
4382         uint32_t tmp, idx;
4383
4384         msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4385         if (hw->mac.type == ixgbe_mac_82598EB) {
4386                 if (direction == -1)
4387                         direction = 0;
4388                 idx = (((direction * 64) + queue) >> 2) & 0x1F;
4389                 tmp = IXGBE_READ_REG(hw, IXGBE_IVAR(idx));
4390                 tmp &= ~(0xFF << (8 * (queue & 0x3)));
4391                 tmp |= (msix_vector << (8 * (queue & 0x3)));
4392                 IXGBE_WRITE_REG(hw, IXGBE_IVAR(idx), tmp);
4393         } else if ((hw->mac.type == ixgbe_mac_82599EB) ||
4394                         (hw->mac.type == ixgbe_mac_X540)) {
4395                 if (direction == -1) {
4396                         /* other causes */
4397                         idx = ((queue & 1) * 8);
4398                         tmp = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4399                         tmp &= ~(0xFF << idx);
4400                         tmp |= (msix_vector << idx);
4401                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, tmp);
4402                 } else {
4403                         /* rx or tx causes */
4404                         idx = ((16 * (queue & 1)) + (8 * direction));
4405                         tmp = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1));
4406                         tmp &= ~(0xFF << idx);
4407                         tmp |= (msix_vector << idx);
4408                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), tmp);
4409                 }
4410         }
4411 }
4412
4413 static void
4414 ixgbevf_configure_msix(struct rte_eth_dev *dev)
4415 {
4416         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
4417         struct ixgbe_hw *hw =
4418                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4419         uint32_t q_idx;
4420         uint32_t vector_idx = 0;
4421
4422         /* won't configure msix register if no mapping is done
4423          * between intr vector and event fd.
4424          */
4425         if (!rte_intr_dp_is_en(intr_handle))
4426                 return;
4427
4428         /* Configure all RX queues of VF */
4429         for (q_idx = 0; q_idx < dev->data->nb_rx_queues; q_idx++) {
4430                 /* Force all queue use vector 0,
4431                  * as IXGBE_VF_MAXMSIVECOTR = 1
4432                  */
4433                 ixgbevf_set_ivar_map(hw, 0, q_idx, vector_idx);
4434                 intr_handle->intr_vec[q_idx] = vector_idx;
4435         }
4436
4437         /* Configure VF Rx queue ivar */
4438         ixgbevf_set_ivar_map(hw, -1, 1, vector_idx);
4439 }
4440
4441 /**
4442  * Sets up the hardware to properly generate MSI-X interrupts
4443  * @hw
4444  *  board private structure
4445  */
4446 static void
4447 ixgbe_configure_msix(struct rte_eth_dev *dev)
4448 {
4449         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
4450         struct ixgbe_hw *hw =
4451                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4452         uint32_t queue_id, vec = 0;
4453         uint32_t mask;
4454         uint32_t gpie;
4455
4456         /* won't configure msix register if no mapping is done
4457          * between intr vector and event fd
4458          */
4459         if (!rte_intr_dp_is_en(intr_handle))
4460                 return;
4461
4462         /* setup GPIE for MSI-x mode */
4463         gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
4464         gpie |= IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT |
4465                 IXGBE_GPIE_OCD | IXGBE_GPIE_EIAME;
4466         /* auto clearing and auto setting corresponding bits in EIMS
4467          * when MSI-X interrupt is triggered
4468          */
4469         if (hw->mac.type == ixgbe_mac_82598EB) {
4470                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
4471         } else {
4472                 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
4473                 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
4474         }
4475         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
4476
4477         /* Populate the IVAR table and set the ITR values to the
4478          * corresponding register.
4479          */
4480         for (queue_id = 0; queue_id < dev->data->nb_rx_queues;
4481              queue_id++) {
4482                 /* by default, 1:1 mapping */
4483                 ixgbe_set_ivar_map(hw, 0, queue_id, vec);
4484                 intr_handle->intr_vec[queue_id] = vec;
4485                 if (vec < intr_handle->nb_efd - 1)
4486                         vec++;
4487         }
4488
4489         switch (hw->mac.type) {
4490         case ixgbe_mac_82598EB:
4491                 ixgbe_set_ivar_map(hw, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX,
4492                                    intr_handle->max_intr - 1);
4493                 break;
4494         case ixgbe_mac_82599EB:
4495         case ixgbe_mac_X540:
4496                 ixgbe_set_ivar_map(hw, -1, 1, intr_handle->max_intr - 1);
4497                 break;
4498         default:
4499                 break;
4500         }
4501         IXGBE_WRITE_REG(hw, IXGBE_EITR(queue_id),
4502                         IXGBE_MIN_INTER_INTERRUPT_INTERVAL_DEFAULT & 0xFFF);
4503
4504         /* set up to autoclear timer, and the vectors */
4505         mask = IXGBE_EIMS_ENABLE_MASK;
4506         mask &= ~(IXGBE_EIMS_OTHER |
4507                   IXGBE_EIMS_MAILBOX |
4508                   IXGBE_EIMS_LSC);
4509
4510         IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4511 }
4512
4513 static int ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev,
4514         uint16_t queue_idx, uint16_t tx_rate)
4515 {
4516         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4517         uint32_t rf_dec, rf_int;
4518         uint32_t bcnrc_val;
4519         uint16_t link_speed = dev->data->dev_link.link_speed;
4520
4521         if (queue_idx >= hw->mac.max_tx_queues)
4522                 return -EINVAL;
4523
4524         if (tx_rate != 0) {
4525                 /* Calculate the rate factor values to set */
4526                 rf_int = (uint32_t)link_speed / (uint32_t)tx_rate;
4527                 rf_dec = (uint32_t)link_speed % (uint32_t)tx_rate;
4528                 rf_dec = (rf_dec << IXGBE_RTTBCNRC_RF_INT_SHIFT) / tx_rate;
4529
4530                 bcnrc_val = IXGBE_RTTBCNRC_RS_ENA;
4531                 bcnrc_val |= ((rf_int << IXGBE_RTTBCNRC_RF_INT_SHIFT) &
4532                                 IXGBE_RTTBCNRC_RF_INT_MASK_M);
4533                 bcnrc_val |= (rf_dec & IXGBE_RTTBCNRC_RF_DEC_MASK);
4534         } else {
4535                 bcnrc_val = 0;
4536         }
4537
4538         /*
4539          * Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
4540          * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported, otherwise
4541          * set as 0x4.
4542          */
4543         if ((dev->data->dev_conf.rxmode.jumbo_frame == 1) &&
4544                 (dev->data->dev_conf.rxmode.max_rx_pkt_len >=
4545                                 IXGBE_MAX_JUMBO_FRAME_SIZE))
4546                 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRM,
4547                         IXGBE_MMW_SIZE_JUMBO_FRAME);
4548         else
4549                 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRM,
4550                         IXGBE_MMW_SIZE_DEFAULT);
4551
4552         /* Set RTTBCNRC of queue X */
4553         IXGBE_WRITE_REG(hw, IXGBE_RTTDQSEL, queue_idx);
4554         IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRC, bcnrc_val);
4555         IXGBE_WRITE_FLUSH(hw);
4556
4557         return 0;
4558 }
4559
4560 static int ixgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf,
4561         uint16_t tx_rate, uint64_t q_msk)
4562 {
4563         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4564         struct ixgbe_vf_info *vfinfo =
4565                 *(IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private));
4566         uint8_t  nb_q_per_pool = RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool;
4567         uint32_t queue_stride =
4568                 IXGBE_MAX_RX_QUEUE_NUM / RTE_ETH_DEV_SRIOV(dev).active;
4569         uint32_t queue_idx = vf * queue_stride, idx = 0, vf_idx;
4570         uint32_t queue_end = queue_idx + nb_q_per_pool - 1;
4571         uint16_t total_rate = 0;
4572
4573         if (queue_end >= hw->mac.max_tx_queues)
4574                 return -EINVAL;
4575
4576         if (vfinfo != NULL) {
4577                 for (vf_idx = 0; vf_idx < dev->pci_dev->max_vfs; vf_idx++) {
4578                         if (vf_idx == vf)
4579                                 continue;
4580                         for (idx = 0; idx < RTE_DIM(vfinfo[vf_idx].tx_rate);
4581                                 idx++)
4582                                 total_rate += vfinfo[vf_idx].tx_rate[idx];
4583                 }
4584         } else
4585                 return -EINVAL;
4586
4587         /* Store tx_rate for this vf. */
4588         for (idx = 0; idx < nb_q_per_pool; idx++) {
4589                 if (((uint64_t)0x1 << idx) & q_msk) {
4590                         if (vfinfo[vf].tx_rate[idx] != tx_rate)
4591                                 vfinfo[vf].tx_rate[idx] = tx_rate;
4592                         total_rate += tx_rate;
4593                 }
4594         }
4595
4596         if (total_rate > dev->data->dev_link.link_speed) {
4597                 /*
4598                  * Reset stored TX rate of the VF if it causes exceed
4599                  * link speed.
4600                  */
4601                 memset(vfinfo[vf].tx_rate, 0, sizeof(vfinfo[vf].tx_rate));
4602                 return -EINVAL;
4603         }
4604
4605         /* Set RTTBCNRC of each queue/pool for vf X  */
4606         for (; queue_idx <= queue_end; queue_idx++) {
4607                 if (0x1 & q_msk)
4608                         ixgbe_set_queue_rate_limit(dev, queue_idx, tx_rate);
4609                 q_msk = q_msk >> 1;
4610         }
4611
4612         return 0;
4613 }
4614
4615 static void
4616 ixgbevf_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
4617                      __attribute__((unused)) uint32_t index,
4618                      __attribute__((unused)) uint32_t pool)
4619 {
4620         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4621         int diag;
4622
4623         /*
4624          * On a 82599 VF, adding again the same MAC addr is not an idempotent
4625          * operation. Trap this case to avoid exhausting the [very limited]
4626          * set of PF resources used to store VF MAC addresses.
4627          */
4628         if (memcmp(hw->mac.perm_addr, mac_addr, sizeof(struct ether_addr)) == 0)
4629                 return;
4630         diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes);
4631         if (diag == 0)
4632                 return;
4633         PMD_DRV_LOG(ERR, "Unable to add MAC address - diag=%d", diag);
4634 }
4635
4636 static void
4637 ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index)
4638 {
4639         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4640         struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr;
4641         struct ether_addr *mac_addr;
4642         uint32_t i;
4643         int diag;
4644
4645         /*
4646          * The IXGBE_VF_SET_MACVLAN command of the ixgbe-pf driver does
4647          * not support the deletion of a given MAC address.
4648          * Instead, it imposes to delete all MAC addresses, then to add again
4649          * all MAC addresses with the exception of the one to be deleted.
4650          */
4651         (void) ixgbevf_set_uc_addr_vf(hw, 0, NULL);
4652
4653         /*
4654          * Add again all MAC addresses, with the exception of the deleted one
4655          * and of the permanent MAC address.
4656          */
4657         for (i = 0, mac_addr = dev->data->mac_addrs;
4658              i < hw->mac.num_rar_entries; i++, mac_addr++) {
4659                 /* Skip the deleted MAC address */
4660                 if (i == index)
4661                         continue;
4662                 /* Skip NULL MAC addresses */
4663                 if (is_zero_ether_addr(mac_addr))
4664                         continue;
4665                 /* Skip the permanent MAC address */
4666                 if (memcmp(perm_addr, mac_addr, sizeof(struct ether_addr)) == 0)
4667                         continue;
4668                 diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes);
4669                 if (diag != 0)
4670                         PMD_DRV_LOG(ERR,
4671                                     "Adding again MAC address "
4672                                     "%02x:%02x:%02x:%02x:%02x:%02x failed "
4673                                     "diag=%d",
4674                                     mac_addr->addr_bytes[0],
4675                                     mac_addr->addr_bytes[1],
4676                                     mac_addr->addr_bytes[2],
4677                                     mac_addr->addr_bytes[3],
4678                                     mac_addr->addr_bytes[4],
4679                                     mac_addr->addr_bytes[5],
4680                                     diag);
4681         }
4682 }
4683
4684 static void
4685 ixgbevf_set_default_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
4686 {
4687         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4688
4689         hw->mac.ops.set_rar(hw, 0, (void *)addr, 0, 0);
4690 }
4691
4692 #define MAC_TYPE_FILTER_SUP(type)    do {\
4693         if ((type) != ixgbe_mac_82599EB && (type) != ixgbe_mac_X540 &&\
4694                 (type) != ixgbe_mac_X550)\
4695                 return -ENOTSUP;\
4696 } while (0)
4697
4698 static int
4699 ixgbe_syn_filter_set(struct rte_eth_dev *dev,
4700                         struct rte_eth_syn_filter *filter,
4701                         bool add)
4702 {
4703         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4704         uint32_t synqf;
4705
4706         if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM)
4707                 return -EINVAL;
4708
4709         synqf = IXGBE_READ_REG(hw, IXGBE_SYNQF);
4710
4711         if (add) {
4712                 if (synqf & IXGBE_SYN_FILTER_ENABLE)
4713                         return -EINVAL;
4714                 synqf = (uint32_t)(((filter->queue << IXGBE_SYN_FILTER_QUEUE_SHIFT) &
4715                         IXGBE_SYN_FILTER_QUEUE) | IXGBE_SYN_FILTER_ENABLE);
4716
4717                 if (filter->hig_pri)
4718                         synqf |= IXGBE_SYN_FILTER_SYNQFP;
4719                 else
4720                         synqf &= ~IXGBE_SYN_FILTER_SYNQFP;
4721         } else {
4722                 if (!(synqf & IXGBE_SYN_FILTER_ENABLE))
4723                         return -ENOENT;
4724                 synqf &= ~(IXGBE_SYN_FILTER_QUEUE | IXGBE_SYN_FILTER_ENABLE);
4725         }
4726         IXGBE_WRITE_REG(hw, IXGBE_SYNQF, synqf);
4727         IXGBE_WRITE_FLUSH(hw);
4728         return 0;
4729 }
4730
4731 static int
4732 ixgbe_syn_filter_get(struct rte_eth_dev *dev,
4733                         struct rte_eth_syn_filter *filter)
4734 {
4735         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4736         uint32_t synqf = IXGBE_READ_REG(hw, IXGBE_SYNQF);
4737
4738         if (synqf & IXGBE_SYN_FILTER_ENABLE) {
4739                 filter->hig_pri = (synqf & IXGBE_SYN_FILTER_SYNQFP) ? 1 : 0;
4740                 filter->queue = (uint16_t)((synqf & IXGBE_SYN_FILTER_QUEUE) >> 1);
4741                 return 0;
4742         }
4743         return -ENOENT;
4744 }
4745
4746 static int
4747 ixgbe_syn_filter_handle(struct rte_eth_dev *dev,
4748                         enum rte_filter_op filter_op,
4749                         void *arg)
4750 {
4751         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4752         int ret;
4753
4754         MAC_TYPE_FILTER_SUP(hw->mac.type);
4755
4756         if (filter_op == RTE_ETH_FILTER_NOP)
4757                 return 0;
4758
4759         if (arg == NULL) {
4760                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
4761                             filter_op);
4762                 return -EINVAL;
4763         }
4764
4765         switch (filter_op) {
4766         case RTE_ETH_FILTER_ADD:
4767                 ret = ixgbe_syn_filter_set(dev,
4768                                 (struct rte_eth_syn_filter *)arg,
4769                                 TRUE);
4770                 break;
4771         case RTE_ETH_FILTER_DELETE:
4772                 ret = ixgbe_syn_filter_set(dev,
4773                                 (struct rte_eth_syn_filter *)arg,
4774                                 FALSE);
4775                 break;
4776         case RTE_ETH_FILTER_GET:
4777                 ret = ixgbe_syn_filter_get(dev,
4778                                 (struct rte_eth_syn_filter *)arg);
4779                 break;
4780         default:
4781                 PMD_DRV_LOG(ERR, "unsupported operation %u\n", filter_op);
4782                 ret = -EINVAL;
4783                 break;
4784         }
4785
4786         return ret;
4787 }
4788
4789
4790 static inline enum ixgbe_5tuple_protocol
4791 convert_protocol_type(uint8_t protocol_value)
4792 {
4793         if (protocol_value == IPPROTO_TCP)
4794                 return IXGBE_FILTER_PROTOCOL_TCP;
4795         else if (protocol_value == IPPROTO_UDP)
4796                 return IXGBE_FILTER_PROTOCOL_UDP;
4797         else if (protocol_value == IPPROTO_SCTP)
4798                 return IXGBE_FILTER_PROTOCOL_SCTP;
4799         else
4800                 return IXGBE_FILTER_PROTOCOL_NONE;
4801 }
4802
4803 /*
4804  * add a 5tuple filter
4805  *
4806  * @param
4807  * dev: Pointer to struct rte_eth_dev.
4808  * index: the index the filter allocates.
4809  * filter: ponter to the filter that will be added.
4810  * rx_queue: the queue id the filter assigned to.
4811  *
4812  * @return
4813  *    - On success, zero.
4814  *    - On failure, a negative value.
4815  */
4816 static int
4817 ixgbe_add_5tuple_filter(struct rte_eth_dev *dev,
4818                         struct ixgbe_5tuple_filter *filter)
4819 {
4820         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4821         struct ixgbe_filter_info *filter_info =
4822                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4823         int i, idx, shift;
4824         uint32_t ftqf, sdpqf;
4825         uint32_t l34timir = 0;
4826         uint8_t mask = 0xff;
4827
4828         /*
4829          * look for an unused 5tuple filter index,
4830          * and insert the filter to list.
4831          */
4832         for (i = 0; i < IXGBE_MAX_FTQF_FILTERS; i++) {
4833                 idx = i / (sizeof(uint32_t) * NBBY);
4834                 shift = i % (sizeof(uint32_t) * NBBY);
4835                 if (!(filter_info->fivetuple_mask[idx] & (1 << shift))) {
4836                         filter_info->fivetuple_mask[idx] |= 1 << shift;
4837                         filter->index = i;
4838                         TAILQ_INSERT_TAIL(&filter_info->fivetuple_list,
4839                                           filter,
4840                                           entries);
4841                         break;
4842                 }
4843         }
4844         if (i >= IXGBE_MAX_FTQF_FILTERS) {
4845                 PMD_DRV_LOG(ERR, "5tuple filters are full.");
4846                 return -ENOSYS;
4847         }
4848
4849         sdpqf = (uint32_t)(filter->filter_info.dst_port <<
4850                                 IXGBE_SDPQF_DSTPORT_SHIFT);
4851         sdpqf = sdpqf | (filter->filter_info.src_port & IXGBE_SDPQF_SRCPORT);
4852
4853         ftqf = (uint32_t)(filter->filter_info.proto &
4854                 IXGBE_FTQF_PROTOCOL_MASK);
4855         ftqf |= (uint32_t)((filter->filter_info.priority &
4856                 IXGBE_FTQF_PRIORITY_MASK) << IXGBE_FTQF_PRIORITY_SHIFT);
4857         if (filter->filter_info.src_ip_mask == 0) /* 0 means compare. */
4858                 mask &= IXGBE_FTQF_SOURCE_ADDR_MASK;
4859         if (filter->filter_info.dst_ip_mask == 0)
4860                 mask &= IXGBE_FTQF_DEST_ADDR_MASK;
4861         if (filter->filter_info.src_port_mask == 0)
4862                 mask &= IXGBE_FTQF_SOURCE_PORT_MASK;
4863         if (filter->filter_info.dst_port_mask == 0)
4864                 mask &= IXGBE_FTQF_DEST_PORT_MASK;
4865         if (filter->filter_info.proto_mask == 0)
4866                 mask &= IXGBE_FTQF_PROTOCOL_COMP_MASK;
4867         ftqf |= mask << IXGBE_FTQF_5TUPLE_MASK_SHIFT;
4868         ftqf |= IXGBE_FTQF_POOL_MASK_EN;
4869         ftqf |= IXGBE_FTQF_QUEUE_ENABLE;
4870
4871         IXGBE_WRITE_REG(hw, IXGBE_DAQF(i), filter->filter_info.dst_ip);
4872         IXGBE_WRITE_REG(hw, IXGBE_SAQF(i), filter->filter_info.src_ip);
4873         IXGBE_WRITE_REG(hw, IXGBE_SDPQF(i), sdpqf);
4874         IXGBE_WRITE_REG(hw, IXGBE_FTQF(i), ftqf);
4875
4876         l34timir |= IXGBE_L34T_IMIR_RESERVE;
4877         l34timir |= (uint32_t)(filter->queue <<
4878                                 IXGBE_L34T_IMIR_QUEUE_SHIFT);
4879         IXGBE_WRITE_REG(hw, IXGBE_L34T_IMIR(i), l34timir);
4880         return 0;
4881 }
4882
4883 /*
4884  * remove a 5tuple filter
4885  *
4886  * @param
4887  * dev: Pointer to struct rte_eth_dev.
4888  * filter: the pointer of the filter will be removed.
4889  */
4890 static void
4891 ixgbe_remove_5tuple_filter(struct rte_eth_dev *dev,
4892                         struct ixgbe_5tuple_filter *filter)
4893 {
4894         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4895         struct ixgbe_filter_info *filter_info =
4896                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4897         uint16_t index = filter->index;
4898
4899         filter_info->fivetuple_mask[index / (sizeof(uint32_t) * NBBY)] &=
4900                                 ~(1 << (index % (sizeof(uint32_t) * NBBY)));
4901         TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries);
4902         rte_free(filter);
4903
4904         IXGBE_WRITE_REG(hw, IXGBE_DAQF(index), 0);
4905         IXGBE_WRITE_REG(hw, IXGBE_SAQF(index), 0);
4906         IXGBE_WRITE_REG(hw, IXGBE_SDPQF(index), 0);
4907         IXGBE_WRITE_REG(hw, IXGBE_FTQF(index), 0);
4908         IXGBE_WRITE_REG(hw, IXGBE_L34T_IMIR(index), 0);
4909 }
4910
4911 static int
4912 ixgbevf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
4913 {
4914         struct ixgbe_hw *hw;
4915         uint32_t max_frame = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
4916
4917         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4918
4919         if ((mtu < ETHER_MIN_MTU) || (max_frame > ETHER_MAX_JUMBO_FRAME_LEN))
4920                 return -EINVAL;
4921
4922         /* refuse mtu that requires the support of scattered packets when this
4923          * feature has not been enabled before. */
4924         if (!dev->data->scattered_rx &&
4925             (max_frame + 2 * IXGBE_VLAN_TAG_SIZE >
4926              dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM))
4927                 return -EINVAL;
4928
4929         /*
4930          * When supported by the underlying PF driver, use the IXGBE_VF_SET_MTU
4931          * request of the version 2.0 of the mailbox API.
4932          * For now, use the IXGBE_VF_SET_LPE request of the version 1.0
4933          * of the mailbox API.
4934          * This call to IXGBE_SET_LPE action won't work with ixgbe pf drivers
4935          * prior to 3.11.33 which contains the following change:
4936          * "ixgbe: Enable jumbo frames support w/ SR-IOV"
4937          */
4938         ixgbevf_rlpml_set_vf(hw, max_frame);
4939
4940         /* update max frame size */
4941         dev->data->dev_conf.rxmode.max_rx_pkt_len = max_frame;
4942         return 0;
4943 }
4944
4945 #define MAC_TYPE_FILTER_SUP_EXT(type)    do {\
4946         if ((type) != ixgbe_mac_82599EB && (type) != ixgbe_mac_X540)\
4947                 return -ENOTSUP;\
4948 } while (0)
4949
4950 static inline struct ixgbe_5tuple_filter *
4951 ixgbe_5tuple_filter_lookup(struct ixgbe_5tuple_filter_list *filter_list,
4952                         struct ixgbe_5tuple_filter_info *key)
4953 {
4954         struct ixgbe_5tuple_filter *it;
4955
4956         TAILQ_FOREACH(it, filter_list, entries) {
4957                 if (memcmp(key, &it->filter_info,
4958                         sizeof(struct ixgbe_5tuple_filter_info)) == 0) {
4959                         return it;
4960                 }
4961         }
4962         return NULL;
4963 }
4964
4965 /* translate elements in struct rte_eth_ntuple_filter to struct ixgbe_5tuple_filter_info*/
4966 static inline int
4967 ntuple_filter_to_5tuple(struct rte_eth_ntuple_filter *filter,
4968                         struct ixgbe_5tuple_filter_info *filter_info)
4969 {
4970         if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM ||
4971                 filter->priority > IXGBE_5TUPLE_MAX_PRI ||
4972                 filter->priority < IXGBE_5TUPLE_MIN_PRI)
4973                 return -EINVAL;
4974
4975         switch (filter->dst_ip_mask) {
4976         case UINT32_MAX:
4977                 filter_info->dst_ip_mask = 0;
4978                 filter_info->dst_ip = filter->dst_ip;
4979                 break;
4980         case 0:
4981                 filter_info->dst_ip_mask = 1;
4982                 break;
4983         default:
4984                 PMD_DRV_LOG(ERR, "invalid dst_ip mask.");
4985                 return -EINVAL;
4986         }
4987
4988         switch (filter->src_ip_mask) {
4989         case UINT32_MAX:
4990                 filter_info->src_ip_mask = 0;
4991                 filter_info->src_ip = filter->src_ip;
4992                 break;
4993         case 0:
4994                 filter_info->src_ip_mask = 1;
4995                 break;
4996         default:
4997                 PMD_DRV_LOG(ERR, "invalid src_ip mask.");
4998                 return -EINVAL;
4999         }
5000
5001         switch (filter->dst_port_mask) {
5002         case UINT16_MAX:
5003                 filter_info->dst_port_mask = 0;
5004                 filter_info->dst_port = filter->dst_port;
5005                 break;
5006         case 0:
5007                 filter_info->dst_port_mask = 1;
5008                 break;
5009         default:
5010                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
5011                 return -EINVAL;
5012         }
5013
5014         switch (filter->src_port_mask) {
5015         case UINT16_MAX:
5016                 filter_info->src_port_mask = 0;
5017                 filter_info->src_port = filter->src_port;
5018                 break;
5019         case 0:
5020                 filter_info->src_port_mask = 1;
5021                 break;
5022         default:
5023                 PMD_DRV_LOG(ERR, "invalid src_port mask.");
5024                 return -EINVAL;
5025         }
5026
5027         switch (filter->proto_mask) {
5028         case UINT8_MAX:
5029                 filter_info->proto_mask = 0;
5030                 filter_info->proto =
5031                         convert_protocol_type(filter->proto);
5032                 break;
5033         case 0:
5034                 filter_info->proto_mask = 1;
5035                 break;
5036         default:
5037                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
5038                 return -EINVAL;
5039         }
5040
5041         filter_info->priority = (uint8_t)filter->priority;
5042         return 0;
5043 }
5044
5045 /*
5046  * add or delete a ntuple filter
5047  *
5048  * @param
5049  * dev: Pointer to struct rte_eth_dev.
5050  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
5051  * add: if true, add filter, if false, remove filter
5052  *
5053  * @return
5054  *    - On success, zero.
5055  *    - On failure, a negative value.
5056  */
5057 static int
5058 ixgbe_add_del_ntuple_filter(struct rte_eth_dev *dev,
5059                         struct rte_eth_ntuple_filter *ntuple_filter,
5060                         bool add)
5061 {
5062         struct ixgbe_filter_info *filter_info =
5063                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
5064         struct ixgbe_5tuple_filter_info filter_5tuple;
5065         struct ixgbe_5tuple_filter *filter;
5066         int ret;
5067
5068         if (ntuple_filter->flags != RTE_5TUPLE_FLAGS) {
5069                 PMD_DRV_LOG(ERR, "only 5tuple is supported.");
5070                 return -EINVAL;
5071         }
5072
5073         memset(&filter_5tuple, 0, sizeof(struct ixgbe_5tuple_filter_info));
5074         ret = ntuple_filter_to_5tuple(ntuple_filter, &filter_5tuple);
5075         if (ret < 0)
5076                 return ret;
5077
5078         filter = ixgbe_5tuple_filter_lookup(&filter_info->fivetuple_list,
5079                                          &filter_5tuple);
5080         if (filter != NULL && add) {
5081                 PMD_DRV_LOG(ERR, "filter exists.");
5082                 return -EEXIST;
5083         }
5084         if (filter == NULL && !add) {
5085                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
5086                 return -ENOENT;
5087         }
5088
5089         if (add) {
5090                 filter = rte_zmalloc("ixgbe_5tuple_filter",
5091                                 sizeof(struct ixgbe_5tuple_filter), 0);
5092                 if (filter == NULL)
5093                         return -ENOMEM;
5094                 (void)rte_memcpy(&filter->filter_info,
5095                                  &filter_5tuple,
5096                                  sizeof(struct ixgbe_5tuple_filter_info));
5097                 filter->queue = ntuple_filter->queue;
5098                 ret = ixgbe_add_5tuple_filter(dev, filter);
5099                 if (ret < 0) {
5100                         rte_free(filter);
5101                         return ret;
5102                 }
5103         } else
5104                 ixgbe_remove_5tuple_filter(dev, filter);
5105
5106         return 0;
5107 }
5108
5109 /*
5110  * get a ntuple filter
5111  *
5112  * @param
5113  * dev: Pointer to struct rte_eth_dev.
5114  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
5115  *
5116  * @return
5117  *    - On success, zero.
5118  *    - On failure, a negative value.
5119  */
5120 static int
5121 ixgbe_get_ntuple_filter(struct rte_eth_dev *dev,
5122                         struct rte_eth_ntuple_filter *ntuple_filter)
5123 {
5124         struct ixgbe_filter_info *filter_info =
5125                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
5126         struct ixgbe_5tuple_filter_info filter_5tuple;
5127         struct ixgbe_5tuple_filter *filter;
5128         int ret;
5129
5130         if (ntuple_filter->flags != RTE_5TUPLE_FLAGS) {
5131                 PMD_DRV_LOG(ERR, "only 5tuple is supported.");
5132                 return -EINVAL;
5133         }
5134
5135         memset(&filter_5tuple, 0, sizeof(struct ixgbe_5tuple_filter_info));
5136         ret = ntuple_filter_to_5tuple(ntuple_filter, &filter_5tuple);
5137         if (ret < 0)
5138                 return ret;
5139
5140         filter = ixgbe_5tuple_filter_lookup(&filter_info->fivetuple_list,
5141                                          &filter_5tuple);
5142         if (filter == NULL) {
5143                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
5144                 return -ENOENT;
5145         }
5146         ntuple_filter->queue = filter->queue;
5147         return 0;
5148 }
5149
5150 /*
5151  * ixgbe_ntuple_filter_handle - Handle operations for ntuple filter.
5152  * @dev: pointer to rte_eth_dev structure
5153  * @filter_op:operation will be taken.
5154  * @arg: a pointer to specific structure corresponding to the filter_op
5155  *
5156  * @return
5157  *    - On success, zero.
5158  *    - On failure, a negative value.
5159  */
5160 static int
5161 ixgbe_ntuple_filter_handle(struct rte_eth_dev *dev,
5162                                 enum rte_filter_op filter_op,
5163                                 void *arg)
5164 {
5165         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5166         int ret;
5167
5168         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
5169
5170         if (filter_op == RTE_ETH_FILTER_NOP)
5171                 return 0;
5172
5173         if (arg == NULL) {
5174                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
5175                             filter_op);
5176                 return -EINVAL;
5177         }
5178
5179         switch (filter_op) {
5180         case RTE_ETH_FILTER_ADD:
5181                 ret = ixgbe_add_del_ntuple_filter(dev,
5182                         (struct rte_eth_ntuple_filter *)arg,
5183                         TRUE);
5184                 break;
5185         case RTE_ETH_FILTER_DELETE:
5186                 ret = ixgbe_add_del_ntuple_filter(dev,
5187                         (struct rte_eth_ntuple_filter *)arg,
5188                         FALSE);
5189                 break;
5190         case RTE_ETH_FILTER_GET:
5191                 ret = ixgbe_get_ntuple_filter(dev,
5192                         (struct rte_eth_ntuple_filter *)arg);
5193                 break;
5194         default:
5195                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
5196                 ret = -EINVAL;
5197                 break;
5198         }
5199         return ret;
5200 }
5201
5202 static inline int
5203 ixgbe_ethertype_filter_lookup(struct ixgbe_filter_info *filter_info,
5204                         uint16_t ethertype)
5205 {
5206         int i;
5207
5208         for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) {
5209                 if (filter_info->ethertype_filters[i] == ethertype &&
5210                     (filter_info->ethertype_mask & (1 << i)))
5211                         return i;
5212         }
5213         return -1;
5214 }
5215
5216 static inline int
5217 ixgbe_ethertype_filter_insert(struct ixgbe_filter_info *filter_info,
5218                         uint16_t ethertype)
5219 {
5220         int i;
5221
5222         for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) {
5223                 if (!(filter_info->ethertype_mask & (1 << i))) {
5224                         filter_info->ethertype_mask |= 1 << i;
5225                         filter_info->ethertype_filters[i] = ethertype;
5226                         return i;
5227                 }
5228         }
5229         return -1;
5230 }
5231
5232 static inline int
5233 ixgbe_ethertype_filter_remove(struct ixgbe_filter_info *filter_info,
5234                         uint8_t idx)
5235 {
5236         if (idx >= IXGBE_MAX_ETQF_FILTERS)
5237                 return -1;
5238         filter_info->ethertype_mask &= ~(1 << idx);
5239         filter_info->ethertype_filters[idx] = 0;
5240         return idx;
5241 }
5242
5243 static int
5244 ixgbe_add_del_ethertype_filter(struct rte_eth_dev *dev,
5245                         struct rte_eth_ethertype_filter *filter,
5246                         bool add)
5247 {
5248         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5249         struct ixgbe_filter_info *filter_info =
5250                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
5251         uint32_t etqf = 0;
5252         uint32_t etqs = 0;
5253         int ret;
5254
5255         if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM)
5256                 return -EINVAL;
5257
5258         if (filter->ether_type == ETHER_TYPE_IPv4 ||
5259                 filter->ether_type == ETHER_TYPE_IPv6) {
5260                 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in"
5261                         " ethertype filter.", filter->ether_type);
5262                 return -EINVAL;
5263         }
5264
5265         if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) {
5266                 PMD_DRV_LOG(ERR, "mac compare is unsupported.");
5267                 return -EINVAL;
5268         }
5269         if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) {
5270                 PMD_DRV_LOG(ERR, "drop option is unsupported.");
5271                 return -EINVAL;
5272         }
5273
5274         ret = ixgbe_ethertype_filter_lookup(filter_info, filter->ether_type);
5275         if (ret >= 0 && add) {
5276                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.",
5277                             filter->ether_type);
5278                 return -EEXIST;
5279         }
5280         if (ret < 0 && !add) {
5281                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
5282                             filter->ether_type);
5283                 return -ENOENT;
5284         }
5285
5286         if (add) {
5287                 ret = ixgbe_ethertype_filter_insert(filter_info,
5288                         filter->ether_type);
5289                 if (ret < 0) {
5290                         PMD_DRV_LOG(ERR, "ethertype filters are full.");
5291                         return -ENOSYS;
5292                 }
5293                 etqf = IXGBE_ETQF_FILTER_EN;
5294                 etqf |= (uint32_t)filter->ether_type;
5295                 etqs |= (uint32_t)((filter->queue <<
5296                                     IXGBE_ETQS_RX_QUEUE_SHIFT) &
5297                                     IXGBE_ETQS_RX_QUEUE);
5298                 etqs |= IXGBE_ETQS_QUEUE_EN;
5299         } else {
5300                 ret = ixgbe_ethertype_filter_remove(filter_info, (uint8_t)ret);
5301                 if (ret < 0)
5302                         return -ENOSYS;
5303         }
5304         IXGBE_WRITE_REG(hw, IXGBE_ETQF(ret), etqf);
5305         IXGBE_WRITE_REG(hw, IXGBE_ETQS(ret), etqs);
5306         IXGBE_WRITE_FLUSH(hw);
5307
5308         return 0;
5309 }
5310
5311 static int
5312 ixgbe_get_ethertype_filter(struct rte_eth_dev *dev,
5313                         struct rte_eth_ethertype_filter *filter)
5314 {
5315         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5316         struct ixgbe_filter_info *filter_info =
5317                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
5318         uint32_t etqf, etqs;
5319         int ret;
5320
5321         ret = ixgbe_ethertype_filter_lookup(filter_info, filter->ether_type);
5322         if (ret < 0) {
5323                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
5324                             filter->ether_type);
5325                 return -ENOENT;
5326         }
5327
5328         etqf = IXGBE_READ_REG(hw, IXGBE_ETQF(ret));
5329         if (etqf & IXGBE_ETQF_FILTER_EN) {
5330                 etqs = IXGBE_READ_REG(hw, IXGBE_ETQS(ret));
5331                 filter->ether_type = etqf & IXGBE_ETQF_ETHERTYPE;
5332                 filter->flags = 0;
5333                 filter->queue = (etqs & IXGBE_ETQS_RX_QUEUE) >>
5334                                IXGBE_ETQS_RX_QUEUE_SHIFT;
5335                 return 0;
5336         }
5337         return -ENOENT;
5338 }
5339
5340 /*
5341  * ixgbe_ethertype_filter_handle - Handle operations for ethertype filter.
5342  * @dev: pointer to rte_eth_dev structure
5343  * @filter_op:operation will be taken.
5344  * @arg: a pointer to specific structure corresponding to the filter_op
5345  */
5346 static int
5347 ixgbe_ethertype_filter_handle(struct rte_eth_dev *dev,
5348                                 enum rte_filter_op filter_op,
5349                                 void *arg)
5350 {
5351         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5352         int ret;
5353
5354         MAC_TYPE_FILTER_SUP(hw->mac.type);
5355
5356         if (filter_op == RTE_ETH_FILTER_NOP)
5357                 return 0;
5358
5359         if (arg == NULL) {
5360                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
5361                             filter_op);
5362                 return -EINVAL;
5363         }
5364
5365         switch (filter_op) {
5366         case RTE_ETH_FILTER_ADD:
5367                 ret = ixgbe_add_del_ethertype_filter(dev,
5368                         (struct rte_eth_ethertype_filter *)arg,
5369                         TRUE);
5370                 break;
5371         case RTE_ETH_FILTER_DELETE:
5372                 ret = ixgbe_add_del_ethertype_filter(dev,
5373                         (struct rte_eth_ethertype_filter *)arg,
5374                         FALSE);
5375                 break;
5376         case RTE_ETH_FILTER_GET:
5377                 ret = ixgbe_get_ethertype_filter(dev,
5378                         (struct rte_eth_ethertype_filter *)arg);
5379                 break;
5380         default:
5381                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
5382                 ret = -EINVAL;
5383                 break;
5384         }
5385         return ret;
5386 }
5387
5388 static int
5389 ixgbe_dev_filter_ctrl(struct rte_eth_dev *dev,
5390                      enum rte_filter_type filter_type,
5391                      enum rte_filter_op filter_op,
5392                      void *arg)
5393 {
5394         int ret = -EINVAL;
5395
5396         switch (filter_type) {
5397         case RTE_ETH_FILTER_NTUPLE:
5398                 ret = ixgbe_ntuple_filter_handle(dev, filter_op, arg);
5399                 break;
5400         case RTE_ETH_FILTER_ETHERTYPE:
5401                 ret = ixgbe_ethertype_filter_handle(dev, filter_op, arg);
5402                 break;
5403         case RTE_ETH_FILTER_SYN:
5404                 ret = ixgbe_syn_filter_handle(dev, filter_op, arg);
5405                 break;
5406         case RTE_ETH_FILTER_FDIR:
5407                 ret = ixgbe_fdir_ctrl_func(dev, filter_op, arg);
5408                 break;
5409         default:
5410                 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported",
5411                                                         filter_type);
5412                 break;
5413         }
5414
5415         return ret;
5416 }
5417
5418 static u8 *
5419 ixgbe_dev_addr_list_itr(__attribute__((unused)) struct ixgbe_hw *hw,
5420                         u8 **mc_addr_ptr, u32 *vmdq)
5421 {
5422         u8 *mc_addr;
5423
5424         *vmdq = 0;
5425         mc_addr = *mc_addr_ptr;
5426         *mc_addr_ptr = (mc_addr + sizeof(struct ether_addr));
5427         return mc_addr;
5428 }
5429
5430 static int
5431 ixgbe_dev_set_mc_addr_list(struct rte_eth_dev *dev,
5432                           struct ether_addr *mc_addr_set,
5433                           uint32_t nb_mc_addr)
5434 {
5435         struct ixgbe_hw *hw;
5436         u8 *mc_addr_list;
5437
5438         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5439         mc_addr_list = (u8 *)mc_addr_set;
5440         return ixgbe_update_mc_addr_list(hw, mc_addr_list, nb_mc_addr,
5441                                          ixgbe_dev_addr_list_itr, TRUE);
5442 }
5443
5444 static int
5445 ixgbe_timesync_enable(struct rte_eth_dev *dev)
5446 {
5447         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5448         uint32_t tsync_ctl;
5449         uint32_t tsauxc;
5450
5451         /* Enable system time for platforms where it isn't on by default. */
5452         tsauxc = IXGBE_READ_REG(hw, IXGBE_TSAUXC);
5453         tsauxc &= ~IXGBE_TSAUXC_DISABLE_SYSTIME;
5454         IXGBE_WRITE_REG(hw, IXGBE_TSAUXC, tsauxc);
5455
5456         /* Start incrementing the register used to timestamp PTP packets. */
5457         IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, IXGBE_TIMINCA_INIT);
5458
5459         /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
5460         IXGBE_WRITE_REG(hw, IXGBE_ETQF(IXGBE_ETQF_FILTER_1588),
5461                         (ETHER_TYPE_1588 |
5462                          IXGBE_ETQF_FILTER_EN |
5463                          IXGBE_ETQF_1588));
5464
5465         /* Enable timestamping of received PTP packets. */
5466         tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL);
5467         tsync_ctl |= IXGBE_TSYNCRXCTL_ENABLED;
5468         IXGBE_WRITE_REG(hw, IXGBE_TSYNCRXCTL, tsync_ctl);
5469
5470         /* Enable timestamping of transmitted PTP packets. */
5471         tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL);
5472         tsync_ctl |= IXGBE_TSYNCTXCTL_ENABLED;
5473         IXGBE_WRITE_REG(hw, IXGBE_TSYNCTXCTL, tsync_ctl);
5474
5475         return 0;
5476 }
5477
5478 static int
5479 ixgbe_timesync_disable(struct rte_eth_dev *dev)
5480 {
5481         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5482         uint32_t tsync_ctl;
5483
5484         /* Disable timestamping of transmitted PTP packets. */
5485         tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL);
5486         tsync_ctl &= ~IXGBE_TSYNCTXCTL_ENABLED;
5487         IXGBE_WRITE_REG(hw, IXGBE_TSYNCTXCTL, tsync_ctl);
5488
5489         /* Disable timestamping of received PTP packets. */
5490         tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL);
5491         tsync_ctl &= ~IXGBE_TSYNCRXCTL_ENABLED;
5492         IXGBE_WRITE_REG(hw, IXGBE_TSYNCRXCTL, tsync_ctl);
5493
5494         /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
5495         IXGBE_WRITE_REG(hw, IXGBE_ETQF(IXGBE_ETQF_FILTER_1588), 0);
5496
5497         /* Stop incrementating the System Time registers. */
5498         IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, 0);
5499
5500         return 0;
5501 }
5502
5503 static int
5504 ixgbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
5505                                  struct timespec *timestamp,
5506                                  uint32_t flags __rte_unused)
5507 {
5508         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5509         uint32_t tsync_rxctl;
5510         uint32_t rx_stmpl;
5511         uint32_t rx_stmph;
5512
5513         tsync_rxctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL);
5514         if ((tsync_rxctl & IXGBE_TSYNCRXCTL_VALID) == 0)
5515                 return -EINVAL;
5516
5517         rx_stmpl = IXGBE_READ_REG(hw, IXGBE_RXSTMPL);
5518         rx_stmph = IXGBE_READ_REG(hw, IXGBE_RXSTMPH);
5519
5520         timestamp->tv_sec = (uint64_t)(((uint64_t)rx_stmph << 32) | rx_stmpl);
5521         timestamp->tv_nsec = 0;
5522
5523         return  0;
5524 }
5525
5526 static int
5527 ixgbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
5528                                  struct timespec *timestamp)
5529 {
5530         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5531         uint32_t tsync_txctl;
5532         uint32_t tx_stmpl;
5533         uint32_t tx_stmph;
5534
5535         tsync_txctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL);
5536         if ((tsync_txctl & IXGBE_TSYNCTXCTL_VALID) == 0)
5537                 return -EINVAL;
5538
5539         tx_stmpl = IXGBE_READ_REG(hw, IXGBE_TXSTMPL);
5540         tx_stmph = IXGBE_READ_REG(hw, IXGBE_TXSTMPH);
5541
5542         timestamp->tv_sec = (uint64_t)(((uint64_t)tx_stmph << 32) | tx_stmpl);
5543         timestamp->tv_nsec = 0;
5544
5545         return  0;
5546 }
5547
5548 static int
5549 ixgbe_get_reg_length(struct rte_eth_dev *dev)
5550 {
5551         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5552         int count = 0;
5553         int g_ind = 0;
5554         const struct reg_info *reg_group;
5555         const struct reg_info **reg_set = (hw->mac.type == ixgbe_mac_82598EB) ?
5556                                     ixgbe_regs_mac_82598EB : ixgbe_regs_others;
5557
5558         while ((reg_group = reg_set[g_ind++]))
5559                 count += ixgbe_regs_group_count(reg_group);
5560
5561         return count;
5562 }
5563
5564 static int
5565 ixgbevf_get_reg_length(struct rte_eth_dev *dev __rte_unused)
5566 {
5567         int count = 0;
5568         int g_ind = 0;
5569         const struct reg_info *reg_group;
5570
5571         while ((reg_group = ixgbevf_regs[g_ind++]))
5572                 count += ixgbe_regs_group_count(reg_group);
5573
5574         return count;
5575 }
5576
5577 static int
5578 ixgbe_get_regs(struct rte_eth_dev *dev,
5579               struct rte_dev_reg_info *regs)
5580 {
5581         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5582         uint32_t *data = regs->data;
5583         int g_ind = 0;
5584         int count = 0;
5585         const struct reg_info *reg_group;
5586         const struct reg_info **reg_set = (hw->mac.type == ixgbe_mac_82598EB) ?
5587                                     ixgbe_regs_mac_82598EB : ixgbe_regs_others;
5588
5589         /* Support only full register dump */
5590         if ((regs->length == 0) ||
5591             (regs->length == (uint32_t)ixgbe_get_reg_length(dev))) {
5592                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
5593                         hw->device_id;
5594                 while ((reg_group = reg_set[g_ind++]))
5595                         count += ixgbe_read_regs_group(dev, &data[count],
5596                                 reg_group);
5597                 return 0;
5598         }
5599
5600         return -ENOTSUP;
5601 }
5602
5603 static int
5604 ixgbevf_get_regs(struct rte_eth_dev *dev,
5605                 struct rte_dev_reg_info *regs)
5606 {
5607         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5608         uint32_t *data = regs->data;
5609         int g_ind = 0;
5610         int count = 0;
5611         const struct reg_info *reg_group;
5612
5613         /* Support only full register dump */
5614         if ((regs->length == 0) ||
5615             (regs->length == (uint32_t)ixgbevf_get_reg_length(dev))) {
5616                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
5617                         hw->device_id;
5618                 while ((reg_group = ixgbevf_regs[g_ind++]))
5619                         count += ixgbe_read_regs_group(dev, &data[count],
5620                                                       reg_group);
5621                 return 0;
5622         }
5623
5624         return -ENOTSUP;
5625 }
5626
5627 static int
5628 ixgbe_get_eeprom_length(struct rte_eth_dev *dev)
5629 {
5630         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5631
5632         /* Return unit is byte count */
5633         return hw->eeprom.word_size * 2;
5634 }
5635
5636 static int
5637 ixgbe_get_eeprom(struct rte_eth_dev *dev,
5638                 struct rte_dev_eeprom_info *in_eeprom)
5639 {
5640         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5641         struct ixgbe_eeprom_info *eeprom = &hw->eeprom;
5642         uint16_t *data = in_eeprom->data;
5643         int first, length;
5644
5645         first = in_eeprom->offset >> 1;
5646         length = in_eeprom->length >> 1;
5647         if ((first > hw->eeprom.word_size) ||
5648             ((first + length) > hw->eeprom.word_size))
5649                 return -EINVAL;
5650
5651         in_eeprom->magic = hw->vendor_id | (hw->device_id << 16);
5652
5653         return eeprom->ops.read_buffer(hw, first, length, data);
5654 }
5655
5656 static int
5657 ixgbe_set_eeprom(struct rte_eth_dev *dev,
5658                 struct rte_dev_eeprom_info *in_eeprom)
5659 {
5660         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5661         struct ixgbe_eeprom_info *eeprom = &hw->eeprom;
5662         uint16_t *data = in_eeprom->data;
5663         int first, length;
5664
5665         first = in_eeprom->offset >> 1;
5666         length = in_eeprom->length >> 1;
5667         if ((first > hw->eeprom.word_size) ||
5668             ((first + length) > hw->eeprom.word_size))
5669                 return -EINVAL;
5670
5671         in_eeprom->magic = hw->vendor_id | (hw->device_id << 16);
5672
5673         return eeprom->ops.write_buffer(hw,  first, length, data);
5674 }
5675
5676 uint16_t
5677 ixgbe_reta_size_get(enum ixgbe_mac_type mac_type) {
5678         switch (mac_type) {
5679         case ixgbe_mac_X550:
5680         case ixgbe_mac_X550EM_x:
5681                 return ETH_RSS_RETA_SIZE_512;
5682         case ixgbe_mac_X550_vf:
5683         case ixgbe_mac_X550EM_x_vf:
5684                 return ETH_RSS_RETA_SIZE_64;
5685         default:
5686                 return ETH_RSS_RETA_SIZE_128;
5687         }
5688 }
5689
5690 uint32_t
5691 ixgbe_reta_reg_get(enum ixgbe_mac_type mac_type, uint16_t reta_idx) {
5692         switch (mac_type) {
5693         case ixgbe_mac_X550:
5694         case ixgbe_mac_X550EM_x:
5695                 if (reta_idx < ETH_RSS_RETA_SIZE_128)
5696                         return IXGBE_RETA(reta_idx >> 2);
5697                 else
5698                         return IXGBE_ERETA((reta_idx - ETH_RSS_RETA_SIZE_128) >> 2);
5699         case ixgbe_mac_X550_vf:
5700         case ixgbe_mac_X550EM_x_vf:
5701                 return IXGBE_VFRETA(reta_idx >> 2);
5702         default:
5703                 return IXGBE_RETA(reta_idx >> 2);
5704         }
5705 }
5706
5707 uint32_t
5708 ixgbe_mrqc_reg_get(enum ixgbe_mac_type mac_type) {
5709         switch (mac_type) {
5710         case ixgbe_mac_X550_vf:
5711         case ixgbe_mac_X550EM_x_vf:
5712                 return IXGBE_VFMRQC;
5713         default:
5714                 return IXGBE_MRQC;
5715         }
5716 }
5717
5718 uint32_t
5719 ixgbe_rssrk_reg_get(enum ixgbe_mac_type mac_type, uint8_t i) {
5720         switch (mac_type) {
5721         case ixgbe_mac_X550_vf:
5722         case ixgbe_mac_X550EM_x_vf:
5723                 return IXGBE_VFRSSRK(i);
5724         default:
5725                 return IXGBE_RSSRK(i);
5726         }
5727 }
5728
5729 bool
5730 ixgbe_rss_update_sp(enum ixgbe_mac_type mac_type) {
5731         switch (mac_type) {
5732         case ixgbe_mac_82599_vf:
5733         case ixgbe_mac_X540_vf:
5734                 return 0;
5735         default:
5736                 return 1;
5737         }
5738 }
5739
5740 static int
5741 ixgbe_dev_get_dcb_info(struct rte_eth_dev *dev,
5742                         struct rte_eth_dcb_info *dcb_info)
5743 {
5744         struct ixgbe_dcb_config *dcb_config =
5745                         IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
5746         struct ixgbe_dcb_tc_config *tc;
5747         uint8_t i, j;
5748
5749         if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_DCB_FLAG)
5750                 dcb_info->nb_tcs = dcb_config->num_tcs.pg_tcs;
5751         else
5752                 dcb_info->nb_tcs = 1;
5753
5754         if (dcb_config->vt_mode) { /* vt is enabled*/
5755                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
5756                                 &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf;
5757                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++)
5758                         dcb_info->prio_tc[i] = vmdq_rx_conf->dcb_tc[i];
5759                 for (i = 0; i < vmdq_rx_conf->nb_queue_pools; i++) {
5760                         for (j = 0; j < dcb_info->nb_tcs; j++) {
5761                                 dcb_info->tc_queue.tc_rxq[i][j].base =
5762                                                 i * dcb_info->nb_tcs + j;
5763                                 dcb_info->tc_queue.tc_rxq[i][j].nb_queue = 1;
5764                                 dcb_info->tc_queue.tc_txq[i][j].base =
5765                                                 i * dcb_info->nb_tcs + j;
5766                                 dcb_info->tc_queue.tc_txq[i][j].nb_queue = 1;
5767                         }
5768                 }
5769         } else { /* vt is disabled*/
5770                 struct rte_eth_dcb_rx_conf *rx_conf =
5771                                 &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf;
5772                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++)
5773                         dcb_info->prio_tc[i] = rx_conf->dcb_tc[i];
5774                 if (dcb_info->nb_tcs == ETH_4_TCS) {
5775                         for (i = 0; i < dcb_info->nb_tcs; i++) {
5776                                 dcb_info->tc_queue.tc_rxq[0][i].base = i * 32;
5777                                 dcb_info->tc_queue.tc_rxq[0][i].nb_queue = 16;
5778                         }
5779                         dcb_info->tc_queue.tc_txq[0][0].base = 0;
5780                         dcb_info->tc_queue.tc_txq[0][1].base = 64;
5781                         dcb_info->tc_queue.tc_txq[0][2].base = 96;
5782                         dcb_info->tc_queue.tc_txq[0][3].base = 112;
5783                         dcb_info->tc_queue.tc_txq[0][0].nb_queue = 64;
5784                         dcb_info->tc_queue.tc_txq[0][1].nb_queue = 32;
5785                         dcb_info->tc_queue.tc_txq[0][2].nb_queue = 16;
5786                         dcb_info->tc_queue.tc_txq[0][3].nb_queue = 16;
5787                 } else if (dcb_info->nb_tcs == ETH_8_TCS) {
5788                         for (i = 0; i < dcb_info->nb_tcs; i++) {
5789                                 dcb_info->tc_queue.tc_rxq[0][i].base = i * 16;
5790                                 dcb_info->tc_queue.tc_rxq[0][i].nb_queue = 16;
5791                         }
5792                         dcb_info->tc_queue.tc_txq[0][0].base = 0;
5793                         dcb_info->tc_queue.tc_txq[0][1].base = 32;
5794                         dcb_info->tc_queue.tc_txq[0][2].base = 64;
5795                         dcb_info->tc_queue.tc_txq[0][3].base = 80;
5796                         dcb_info->tc_queue.tc_txq[0][4].base = 96;
5797                         dcb_info->tc_queue.tc_txq[0][5].base = 104;
5798                         dcb_info->tc_queue.tc_txq[0][6].base = 112;
5799                         dcb_info->tc_queue.tc_txq[0][7].base = 120;
5800                         dcb_info->tc_queue.tc_txq[0][0].nb_queue = 32;
5801                         dcb_info->tc_queue.tc_txq[0][1].nb_queue = 32;
5802                         dcb_info->tc_queue.tc_txq[0][2].nb_queue = 16;
5803                         dcb_info->tc_queue.tc_txq[0][3].nb_queue = 16;
5804                         dcb_info->tc_queue.tc_txq[0][4].nb_queue = 8;
5805                         dcb_info->tc_queue.tc_txq[0][5].nb_queue = 8;
5806                         dcb_info->tc_queue.tc_txq[0][6].nb_queue = 8;
5807                         dcb_info->tc_queue.tc_txq[0][7].nb_queue = 8;
5808                 }
5809         }
5810         for (i = 0; i < dcb_info->nb_tcs; i++) {
5811                 tc = &dcb_config->tc_config[i];
5812                 dcb_info->tc_bws[i] = tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent;
5813         }
5814         return 0;
5815 }
5816
5817 static struct rte_driver rte_ixgbe_driver = {
5818         .type = PMD_PDEV,
5819         .init = rte_ixgbe_pmd_init,
5820 };
5821
5822 static struct rte_driver rte_ixgbevf_driver = {
5823         .type = PMD_PDEV,
5824         .init = rte_ixgbevf_pmd_init,
5825 };
5826
5827 PMD_REGISTER_DRIVER(rte_ixgbe_driver);
5828 PMD_REGISTER_DRIVER(rte_ixgbevf_driver);