drivers/net/ixgbe/ixgbe_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <inttypes.h>
  42 #include <netinet/in.h>
  43 #include <rte_byteorder.h>
  44 #include <rte_common.h>
  45 #include <rte_cycles.h>
  46
  47 #include <rte_interrupts.h>
  48 #include <rte_log.h>
  49 #include <rte_debug.h>
  50 #include <rte_pci.h>
  51 #include <rte_atomic.h>
  52 #include <rte_branch_prediction.h>
  53 #include <rte_memory.h>
  54 #include <rte_memzone.h>
  55 #include <rte_eal.h>
  56 #include <rte_alarm.h>
  57 #include <rte_ether.h>
  58 #include <rte_ethdev.h>
  59 #include <rte_atomic.h>
  60 #include <rte_malloc.h>
  61 #include <rte_random.h>
  62 #include <rte_dev.h>
  63
  64 #include "ixgbe_logs.h"
  65 #include "base/ixgbe_api.h"
  66 #include "base/ixgbe_vf.h"
  67 #include "base/ixgbe_common.h"
  68 #include "ixgbe_ethdev.h"
  69 #include "ixgbe_bypass.h"
  70 #include "ixgbe_rxtx.h"
  71 #include "base/ixgbe_type.h"
  72 #include "base/ixgbe_phy.h"
  73 #include "ixgbe_regs.h"
  74
  75 /*
  76  * High threshold controlling when to start sending XOFF frames. Must be at
  77  * least 8 bytes less than receive packet buffer size. This value is in units
  78  * of 1024 bytes.
  79  */
  80 #define IXGBE_FC_HI    0x80
  81
  82 /*
  83  * Low threshold controlling when to start sending XON frames. This value is
  84  * in units of 1024 bytes.
  85  */
  86 #define IXGBE_FC_LO    0x40
  87
  88 /* Default minimum inter-interrupt interval for EITR configuration */
  89 #define IXGBE_MIN_INTER_INTERRUPT_INTERVAL_DEFAULT    0x79E
  90
  91 /* Timer value included in XOFF frames. */
  92 #define IXGBE_FC_PAUSE 0x680
  93
  94 #define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */
  95 #define IXGBE_LINK_UP_CHECK_TIMEOUT   1000 /* ms */
  96 #define IXGBE_VMDQ_NUM_UC_MAC         4096 /* Maximum nb. of UC MAC addr. */
  97
  98 #define IXGBE_MMW_SIZE_DEFAULT        0x4
  99 #define IXGBE_MMW_SIZE_JUMBO_FRAME    0x14
 100 #define IXGBE_MAX_RING_DESC           4096 /* replicate define from rxtx */
 101
 102 /*
 103  *  Default values for RX/TX configuration
 104  */
 105 #define IXGBE_DEFAULT_RX_FREE_THRESH  32
 106 #define IXGBE_DEFAULT_RX_PTHRESH      8
 107 #define IXGBE_DEFAULT_RX_HTHRESH      8
 108 #define IXGBE_DEFAULT_RX_WTHRESH      0
 109
 110 #define IXGBE_DEFAULT_TX_FREE_THRESH  32
 111 #define IXGBE_DEFAULT_TX_PTHRESH      32
 112 #define IXGBE_DEFAULT_TX_HTHRESH      0
 113 #define IXGBE_DEFAULT_TX_WTHRESH      0
 114 #define IXGBE_DEFAULT_TX_RSBIT_THRESH 32
 115
 116 /* Bit shift and mask */
 117 #define IXGBE_4_BIT_WIDTH  (CHAR_BIT / 2)
 118 #define IXGBE_4_BIT_MASK   RTE_LEN2MASK(IXGBE_4_BIT_WIDTH, uint8_t)
 119 #define IXGBE_8_BIT_WIDTH  CHAR_BIT
 120 #define IXGBE_8_BIT_MASK   UINT8_MAX
 121
 122 #define IXGBEVF_PMD_NAME "rte_ixgbevf_pmd" /* PMD name */
 123
 124 #define IXGBE_QUEUE_STAT_COUNTERS (sizeof(hw_stats->qprc) / sizeof(hw_stats->qprc[0]))
 125
 126 #define IXGBE_HKEY_MAX_INDEX 10
 127
 128 /* Additional timesync values. */
 129 #define IXGBE_TIMINCA_16NS_SHIFT 24
 130 #define IXGBE_TIMINCA_INCVALUE   16000000
 131 #define IXGBE_TIMINCA_INIT       ((0x02 << IXGBE_TIMINCA_16NS_SHIFT) \
 132                                   | IXGBE_TIMINCA_INCVALUE)
 133
 134 static int eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev);
 135 static int eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev);
 136 static int  ixgbe_dev_configure(struct rte_eth_dev *dev);
 137 static int  ixgbe_dev_start(struct rte_eth_dev *dev);
 138 static void ixgbe_dev_stop(struct rte_eth_dev *dev);
 139 static int  ixgbe_dev_set_link_up(struct rte_eth_dev *dev);
 140 static int  ixgbe_dev_set_link_down(struct rte_eth_dev *dev);
 141 static void ixgbe_dev_close(struct rte_eth_dev *dev);
 142 static void ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev);
 143 static void ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev);
 144 static void ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev);
 145 static void ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev);
 146 static int ixgbe_dev_link_update(struct rte_eth_dev *dev,
 147                                 int wait_to_complete);
 148 static void ixgbe_dev_stats_get(struct rte_eth_dev *dev,
 149                                 struct rte_eth_stats *stats);
 150 static int ixgbe_dev_xstats_get(struct rte_eth_dev *dev,
 151                                 struct rte_eth_xstats *xstats, unsigned n);
 152 static int ixgbevf_dev_xstats_get(struct rte_eth_dev *dev,
 153                                   struct rte_eth_xstats *xstats, unsigned n);
 154 static void ixgbe_dev_stats_reset(struct rte_eth_dev *dev);
 155 static void ixgbe_dev_xstats_reset(struct rte_eth_dev *dev);
 156 static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 157                                              uint16_t queue_id,
 158                                              uint8_t stat_idx,
 159                                              uint8_t is_rx);
 160 static void ixgbe_dev_info_get(struct rte_eth_dev *dev,
 161                                struct rte_eth_dev_info *dev_info);
 162 static void ixgbevf_dev_info_get(struct rte_eth_dev *dev,
 163                                  struct rte_eth_dev_info *dev_info);
 164 static int ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 165
 166 static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev,
 167                 uint16_t vlan_id, int on);
 168 static void ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 169 static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev,
 170                 uint16_t queue, bool on);
 171 static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue,
 172                 int on);
 173 static void ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 174 static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue);
 175 static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue);
 176 static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 177 static void ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 178
 179 static int ixgbe_dev_led_on(struct rte_eth_dev *dev);
 180 static int ixgbe_dev_led_off(struct rte_eth_dev *dev);
 181 static int ixgbe_flow_ctrl_get(struct rte_eth_dev *dev,
 182                                struct rte_eth_fc_conf *fc_conf);
 183 static int ixgbe_flow_ctrl_set(struct rte_eth_dev *dev,
 184                                struct rte_eth_fc_conf *fc_conf);
 185 static int ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 186                 struct rte_eth_pfc_conf *pfc_conf);
 187 static int ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
 188                         struct rte_eth_rss_reta_entry64 *reta_conf,
 189                         uint16_t reta_size);
 190 static int ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
 191                         struct rte_eth_rss_reta_entry64 *reta_conf,
 192                         uint16_t reta_size);
 193 static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev);
 194 static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev);
 195 static int ixgbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 196 static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev);
 197 static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev);
 198 static void ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle,
 199                 void *param);
 200 static void ixgbe_dev_interrupt_delayed_handler(void *param);
 201 static void ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 202                 uint32_t index, uint32_t pool);
 203 static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index);
 204 static void ixgbe_set_default_mac_addr(struct rte_eth_dev *dev,
 205                                            struct ether_addr *mac_addr);
 206 static void ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config);
 207
 208 /* For Virtual Function support */
 209 static int eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev);
 210 static int eth_ixgbevf_dev_uninit(struct rte_eth_dev *eth_dev);
 211 static int  ixgbevf_dev_configure(struct rte_eth_dev *dev);
 212 static int  ixgbevf_dev_start(struct rte_eth_dev *dev);
 213 static void ixgbevf_dev_stop(struct rte_eth_dev *dev);
 214 static void ixgbevf_dev_close(struct rte_eth_dev *dev);
 215 static void ixgbevf_intr_disable(struct ixgbe_hw *hw);
 216 static void ixgbevf_intr_enable(struct ixgbe_hw *hw);
 217 static void ixgbevf_dev_stats_get(struct rte_eth_dev *dev,
 218                 struct rte_eth_stats *stats);
 219 static void ixgbevf_dev_stats_reset(struct rte_eth_dev *dev);
 220 static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev,
 221                 uint16_t vlan_id, int on);
 222 static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev,
 223                 uint16_t queue, int on);
 224 static void ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 225 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 226 static int ixgbevf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev,
 227                                             uint16_t queue_id);
 228 static int ixgbevf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev,
 229                                              uint16_t queue_id);
 230 static void ixgbevf_set_ivar_map(struct ixgbe_hw *hw, int8_t direction,
 231                                  uint8_t queue, uint8_t msix_vector);
 232 static void ixgbevf_configure_msix(struct rte_eth_dev *dev);
 233
 234 /* For Eth VMDQ APIs support */
 235 static int ixgbe_uc_hash_table_set(struct rte_eth_dev *dev, struct
 236                 ether_addr* mac_addr,uint8_t on);
 237 static int ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev,uint8_t on);
 238 static int  ixgbe_set_pool_rx_mode(struct rte_eth_dev *dev,  uint16_t pool,
 239                 uint16_t rx_mask, uint8_t on);
 240 static int ixgbe_set_pool_rx(struct rte_eth_dev *dev,uint16_t pool,uint8_t on);
 241 static int ixgbe_set_pool_tx(struct rte_eth_dev *dev,uint16_t pool,uint8_t on);
 242 static int ixgbe_set_pool_vlan_filter(struct rte_eth_dev *dev, uint16_t vlan,
 243                 uint64_t pool_mask,uint8_t vlan_on);
 244 static int ixgbe_mirror_rule_set(struct rte_eth_dev *dev,
 245                 struct rte_eth_mirror_conf *mirror_conf,
 246                 uint8_t rule_id, uint8_t on);
 247 static int ixgbe_mirror_rule_reset(struct rte_eth_dev *dev,
 248                 uint8_t rule_id);
 249 static int ixgbe_dev_rx_queue_intr_enable(struct rte_eth_dev *dev,
 250                                           uint16_t queue_id);
 251 static int ixgbe_dev_rx_queue_intr_disable(struct rte_eth_dev *dev,
 252                                            uint16_t queue_id);
 253 static void ixgbe_set_ivar_map(struct ixgbe_hw *hw, int8_t direction,
 254                                uint8_t queue, uint8_t msix_vector);
 255 static void ixgbe_configure_msix(struct rte_eth_dev *dev);
 256
 257 static int ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev,
 258                 uint16_t queue_idx, uint16_t tx_rate);
 259 static int ixgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf,
 260                 uint16_t tx_rate, uint64_t q_msk);
 261
 262 static void ixgbevf_add_mac_addr(struct rte_eth_dev *dev,
 263                                  struct ether_addr *mac_addr,
 264                                  uint32_t index, uint32_t pool);
 265 static void ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index);
 266 static void ixgbevf_set_default_mac_addr(struct rte_eth_dev *dev,
 267                                              struct ether_addr *mac_addr);
 268 static int ixgbe_syn_filter_set(struct rte_eth_dev *dev,
 269                         struct rte_eth_syn_filter *filter,
 270                         bool add);
 271 static int ixgbe_syn_filter_get(struct rte_eth_dev *dev,
 272                         struct rte_eth_syn_filter *filter);
 273 static int ixgbe_syn_filter_handle(struct rte_eth_dev *dev,
 274                         enum rte_filter_op filter_op,
 275                         void *arg);
 276 static int ixgbe_add_5tuple_filter(struct rte_eth_dev *dev,
 277                         struct ixgbe_5tuple_filter *filter);
 278 static void ixgbe_remove_5tuple_filter(struct rte_eth_dev *dev,
 279                         struct ixgbe_5tuple_filter *filter);
 280 static int ixgbe_add_del_ntuple_filter(struct rte_eth_dev *dev,
 281                         struct rte_eth_ntuple_filter *filter,
 282                         bool add);
 283 static int ixgbe_ntuple_filter_handle(struct rte_eth_dev *dev,
 284                                 enum rte_filter_op filter_op,
 285                                 void *arg);
 286 static int ixgbe_get_ntuple_filter(struct rte_eth_dev *dev,
 287                         struct rte_eth_ntuple_filter *filter);
 288 static int ixgbe_add_del_ethertype_filter(struct rte_eth_dev *dev,
 289                         struct rte_eth_ethertype_filter *filter,
 290                         bool add);
 291 static int ixgbe_ethertype_filter_handle(struct rte_eth_dev *dev,
 292                                 enum rte_filter_op filter_op,
 293                                 void *arg);
 294 static int ixgbe_get_ethertype_filter(struct rte_eth_dev *dev,
 295                         struct rte_eth_ethertype_filter *filter);
 296 static int ixgbe_dev_filter_ctrl(struct rte_eth_dev *dev,
 297                      enum rte_filter_type filter_type,
 298                      enum rte_filter_op filter_op,
 299                      void *arg);
 300 static int ixgbevf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu);
 301
 302 static int ixgbe_dev_set_mc_addr_list(struct rte_eth_dev *dev,
 303                                       struct ether_addr *mc_addr_set,
 304                                       uint32_t nb_mc_addr);
 305 static int ixgbe_dev_get_dcb_info(struct rte_eth_dev *dev,
 306                                    struct rte_eth_dcb_info *dcb_info);
 307
 308 static int ixgbe_get_reg_length(struct rte_eth_dev *dev);
 309 static int ixgbe_get_regs(struct rte_eth_dev *dev,
 310                             struct rte_dev_reg_info *regs);
 311 static int ixgbe_get_eeprom_length(struct rte_eth_dev *dev);
 312 static int ixgbe_get_eeprom(struct rte_eth_dev *dev,
 313                                 struct rte_dev_eeprom_info *eeprom);
 314 static int ixgbe_set_eeprom(struct rte_eth_dev *dev,
 315                                 struct rte_dev_eeprom_info *eeprom);
 316
 317 static int ixgbevf_get_reg_length(struct rte_eth_dev *dev);
 318 static int ixgbevf_get_regs(struct rte_eth_dev *dev,
 319                                 struct rte_dev_reg_info *regs);
 320
 321 static int ixgbe_timesync_enable(struct rte_eth_dev *dev);
 322 static int ixgbe_timesync_disable(struct rte_eth_dev *dev);
 323 static int ixgbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
 324                                             struct timespec *timestamp,
 325                                             uint32_t flags);
 326 static int ixgbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
 327                                             struct timespec *timestamp);
 328
 329 /*
 330  * Define VF Stats MACRO for Non "cleared on read" register
 331  */
 332 #define UPDATE_VF_STAT(reg, last, cur)                          \
 333 {                                                               \
 334         uint32_t latest = IXGBE_READ_REG(hw, reg);              \
 335         cur += (latest - last) & UINT_MAX;                      \
 336         last = latest;                                          \
 337 }
 338
 339 #define UPDATE_VF_STAT_36BIT(lsb, msb, last, cur)                \
 340 {                                                                \
 341         u64 new_lsb = IXGBE_READ_REG(hw, lsb);                   \
 342         u64 new_msb = IXGBE_READ_REG(hw, msb);                   \
 343         u64 latest = ((new_msb << 32) | new_lsb);                \
 344         cur += (0x1000000000LL + latest - last) & 0xFFFFFFFFFLL; \
 345         last = latest;                                           \
 346 }
 347
 348 #define IXGBE_SET_HWSTRIP(h, q) do{\
 349                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 350                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 351                 (h)->bitmap[idx] |= 1 << bit;\
 352         }while(0)
 353
 354 #define IXGBE_CLEAR_HWSTRIP(h, q) do{\
 355                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 356                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 357                 (h)->bitmap[idx] &= ~(1 << bit);\
 358         }while(0)
 359
 360 #define IXGBE_GET_HWSTRIP(h, q, r) do{\
 361                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 362                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 363                 (r) = (h)->bitmap[idx] >> bit & 1;\
 364         }while(0)
 365
 366 /*
 367  * The set of PCI devices this driver supports
 368  */
 369 static const struct rte_pci_id pci_id_ixgbe_map[] = {
 370
 371 #define RTE_PCI_DEV_ID_DECL_IXGBE(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 372 #include "rte_pci_dev_ids.h"
 373
 374 { .vendor_id = 0, /* sentinel */ },
 375 };
 376
 377
 378 /*
 379  * The set of PCI devices this driver supports (for 82599 VF)
 380  */
 381 static const struct rte_pci_id pci_id_ixgbevf_map[] = {
 382
 383 #define RTE_PCI_DEV_ID_DECL_IXGBEVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 384 #include "rte_pci_dev_ids.h"
 385 { .vendor_id = 0, /* sentinel */ },
 386
 387 };
 388
 389 static const struct rte_eth_desc_lim rx_desc_lim = {
 390         .nb_max = IXGBE_MAX_RING_DESC,
 391         .nb_min = IXGBE_MIN_RING_DESC,
 392         .nb_align = IXGBE_RXD_ALIGN,
 393 };
 394
 395 static const struct rte_eth_desc_lim tx_desc_lim = {
 396         .nb_max = IXGBE_MAX_RING_DESC,
 397         .nb_min = IXGBE_MIN_RING_DESC,
 398         .nb_align = IXGBE_TXD_ALIGN,
 399 };
 400
 401 static const struct eth_dev_ops ixgbe_eth_dev_ops = {
 402         .dev_configure        = ixgbe_dev_configure,
 403         .dev_start            = ixgbe_dev_start,
 404         .dev_stop             = ixgbe_dev_stop,
 405         .dev_set_link_up    = ixgbe_dev_set_link_up,
 406         .dev_set_link_down  = ixgbe_dev_set_link_down,
 407         .dev_close            = ixgbe_dev_close,
 408         .promiscuous_enable   = ixgbe_dev_promiscuous_enable,
 409         .promiscuous_disable  = ixgbe_dev_promiscuous_disable,
 410         .allmulticast_enable  = ixgbe_dev_allmulticast_enable,
 411         .allmulticast_disable = ixgbe_dev_allmulticast_disable,
 412         .link_update          = ixgbe_dev_link_update,
 413         .stats_get            = ixgbe_dev_stats_get,
 414         .xstats_get           = ixgbe_dev_xstats_get,
 415         .stats_reset          = ixgbe_dev_stats_reset,
 416         .xstats_reset         = ixgbe_dev_xstats_reset,
 417         .queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set,
 418         .dev_infos_get        = ixgbe_dev_info_get,
 419         .mtu_set              = ixgbe_dev_mtu_set,
 420         .vlan_filter_set      = ixgbe_vlan_filter_set,
 421         .vlan_tpid_set        = ixgbe_vlan_tpid_set,
 422         .vlan_offload_set     = ixgbe_vlan_offload_set,
 423         .vlan_strip_queue_set = ixgbe_vlan_strip_queue_set,
 424         .rx_queue_start       = ixgbe_dev_rx_queue_start,
 425         .rx_queue_stop        = ixgbe_dev_rx_queue_stop,
 426         .tx_queue_start       = ixgbe_dev_tx_queue_start,
 427         .tx_queue_stop        = ixgbe_dev_tx_queue_stop,
 428         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 429         .rx_queue_intr_enable = ixgbe_dev_rx_queue_intr_enable,
 430         .rx_queue_intr_disable = ixgbe_dev_rx_queue_intr_disable,
 431         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 432         .rx_queue_count       = ixgbe_dev_rx_queue_count,
 433         .rx_descriptor_done   = ixgbe_dev_rx_descriptor_done,
 434         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 435         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 436         .dev_led_on           = ixgbe_dev_led_on,
 437         .dev_led_off          = ixgbe_dev_led_off,
 438         .flow_ctrl_get        = ixgbe_flow_ctrl_get,
 439         .flow_ctrl_set        = ixgbe_flow_ctrl_set,
 440         .priority_flow_ctrl_set = ixgbe_priority_flow_ctrl_set,
 441         .mac_addr_add         = ixgbe_add_rar,
 442         .mac_addr_remove      = ixgbe_remove_rar,
 443         .mac_addr_set         = ixgbe_set_default_mac_addr,
 444         .uc_hash_table_set    = ixgbe_uc_hash_table_set,
 445         .uc_all_hash_table_set  = ixgbe_uc_all_hash_table_set,
 446         .mirror_rule_set      = ixgbe_mirror_rule_set,
 447         .mirror_rule_reset    = ixgbe_mirror_rule_reset,
 448         .set_vf_rx_mode       = ixgbe_set_pool_rx_mode,
 449         .set_vf_rx            = ixgbe_set_pool_rx,
 450         .set_vf_tx            = ixgbe_set_pool_tx,
 451         .set_vf_vlan_filter   = ixgbe_set_pool_vlan_filter,
 452         .set_queue_rate_limit = ixgbe_set_queue_rate_limit,
 453         .set_vf_rate_limit    = ixgbe_set_vf_rate_limit,
 454         .reta_update          = ixgbe_dev_rss_reta_update,
 455         .reta_query           = ixgbe_dev_rss_reta_query,
 456 #ifdef RTE_NIC_BYPASS
 457         .bypass_init          = ixgbe_bypass_init,
 458         .bypass_state_set     = ixgbe_bypass_state_store,
 459         .bypass_state_show    = ixgbe_bypass_state_show,
 460         .bypass_event_set     = ixgbe_bypass_event_store,
 461         .bypass_event_show    = ixgbe_bypass_event_show,
 462         .bypass_wd_timeout_set  = ixgbe_bypass_wd_timeout_store,
 463         .bypass_wd_timeout_show = ixgbe_bypass_wd_timeout_show,
 464         .bypass_ver_show      = ixgbe_bypass_ver_show,
 465         .bypass_wd_reset      = ixgbe_bypass_wd_reset,
 466 #endif /* RTE_NIC_BYPASS */
 467         .rss_hash_update      = ixgbe_dev_rss_hash_update,
 468         .rss_hash_conf_get    = ixgbe_dev_rss_hash_conf_get,
 469         .filter_ctrl          = ixgbe_dev_filter_ctrl,
 470         .set_mc_addr_list     = ixgbe_dev_set_mc_addr_list,
 471         .rxq_info_get         = ixgbe_rxq_info_get,
 472         .txq_info_get         = ixgbe_txq_info_get,
 473         .timesync_enable      = ixgbe_timesync_enable,
 474         .timesync_disable     = ixgbe_timesync_disable,
 475         .timesync_read_rx_timestamp = ixgbe_timesync_read_rx_timestamp,
 476         .timesync_read_tx_timestamp = ixgbe_timesync_read_tx_timestamp,
 477         .get_reg_length       = ixgbe_get_reg_length,
 478         .get_reg              = ixgbe_get_regs,
 479         .get_eeprom_length    = ixgbe_get_eeprom_length,
 480         .get_eeprom           = ixgbe_get_eeprom,
 481         .set_eeprom           = ixgbe_set_eeprom,
 482         .get_dcb_info         = ixgbe_dev_get_dcb_info,
 483 };
 484
 485 /*
 486  * dev_ops for virtual function, bare necessities for basic vf
 487  * operation have been implemented
 488  */
 489 static const struct eth_dev_ops ixgbevf_eth_dev_ops = {
 490         .dev_configure        = ixgbevf_dev_configure,
 491         .dev_start            = ixgbevf_dev_start,
 492         .dev_stop             = ixgbevf_dev_stop,
 493         .link_update          = ixgbe_dev_link_update,
 494         .stats_get            = ixgbevf_dev_stats_get,
 495         .xstats_get           = ixgbevf_dev_xstats_get,
 496         .stats_reset          = ixgbevf_dev_stats_reset,
 497         .xstats_reset         = ixgbevf_dev_stats_reset,
 498         .dev_close            = ixgbevf_dev_close,
 499         .dev_infos_get        = ixgbevf_dev_info_get,
 500         .mtu_set              = ixgbevf_dev_set_mtu,
 501         .vlan_filter_set      = ixgbevf_vlan_filter_set,
 502         .vlan_strip_queue_set = ixgbevf_vlan_strip_queue_set,
 503         .vlan_offload_set     = ixgbevf_vlan_offload_set,
 504         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 505         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 506         .rx_descriptor_done   = ixgbe_dev_rx_descriptor_done,
 507         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 508         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 509         .rx_queue_intr_enable = ixgbevf_dev_rx_queue_intr_enable,
 510         .rx_queue_intr_disable = ixgbevf_dev_rx_queue_intr_disable,
 511         .mac_addr_add         = ixgbevf_add_mac_addr,
 512         .mac_addr_remove      = ixgbevf_remove_mac_addr,
 513         .set_mc_addr_list     = ixgbe_dev_set_mc_addr_list,
 514         .rxq_info_get         = ixgbe_rxq_info_get,
 515         .txq_info_get         = ixgbe_txq_info_get,
 516         .mac_addr_set         = ixgbevf_set_default_mac_addr,
 517         .get_reg_length       = ixgbevf_get_reg_length,
 518         .get_reg              = ixgbevf_get_regs,
 519         .reta_update          = ixgbe_dev_rss_reta_update,
 520         .reta_query           = ixgbe_dev_rss_reta_query,
 521         .rss_hash_update      = ixgbe_dev_rss_hash_update,
 522         .rss_hash_conf_get    = ixgbe_dev_rss_hash_conf_get,
 523 };
 524
 525 /* store statistics names and its offset in stats structure */
 526 struct rte_ixgbe_xstats_name_off {
 527         char name[RTE_ETH_XSTATS_NAME_SIZE];
 528         unsigned offset;
 529 };
 530
 531 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_stats_strings[] = {
 532         {"rx_crc_errors", offsetof(struct ixgbe_hw_stats, crcerrs)},
 533         {"rx_illegal_byte_errors", offsetof(struct ixgbe_hw_stats, illerrc)},
 534         {"rx_error_bytes", offsetof(struct ixgbe_hw_stats, errbc)},
 535         {"mac_local_errors", offsetof(struct ixgbe_hw_stats, mlfc)},
 536         {"mac_remote_errors", offsetof(struct ixgbe_hw_stats, mrfc)},
 537         {"rx_length_errors", offsetof(struct ixgbe_hw_stats, rlec)},
 538         {"tx_xon_packets", offsetof(struct ixgbe_hw_stats, lxontxc)},
 539         {"rx_xon_packets", offsetof(struct ixgbe_hw_stats, lxonrxc)},
 540         {"tx_xoff_packets", offsetof(struct ixgbe_hw_stats, lxofftxc)},
 541         {"rx_xoff_packets", offsetof(struct ixgbe_hw_stats, lxoffrxc)},
 542         {"rx_size_64_packets", offsetof(struct ixgbe_hw_stats, prc64)},
 543         {"rx_size_65_to_127_packets", offsetof(struct ixgbe_hw_stats, prc127)},
 544         {"rx_size_128_to_255_packets", offsetof(struct ixgbe_hw_stats, prc255)},
 545         {"rx_size_256_to_511_packets", offsetof(struct ixgbe_hw_stats, prc511)},
 546         {"rx_size_512_to_1023_packets", offsetof(struct ixgbe_hw_stats,
 547                 prc1023)},
 548         {"rx_size_1024_to_max_packets", offsetof(struct ixgbe_hw_stats,
 549                 prc1522)},
 550         {"rx_broadcast_packets", offsetof(struct ixgbe_hw_stats, bprc)},
 551         {"rx_multicast_packets", offsetof(struct ixgbe_hw_stats, mprc)},
 552         {"rx_fragment_errors", offsetof(struct ixgbe_hw_stats, rfc)},
 553         {"rx_undersize_errors", offsetof(struct ixgbe_hw_stats, ruc)},
 554         {"rx_oversize_errors", offsetof(struct ixgbe_hw_stats, roc)},
 555         {"rx_jabber_errors", offsetof(struct ixgbe_hw_stats, rjc)},
 556         {"rx_management_packets", offsetof(struct ixgbe_hw_stats, mngprc)},
 557         {"rx_management_dropped", offsetof(struct ixgbe_hw_stats, mngpdc)},
 558         {"tx_management_packets", offsetof(struct ixgbe_hw_stats, mngptc)},
 559         {"rx_total_packets", offsetof(struct ixgbe_hw_stats, tpr)},
 560         {"rx_total_bytes", offsetof(struct ixgbe_hw_stats, tor)},
 561         {"tx_total_packets", offsetof(struct ixgbe_hw_stats, tpt)},
 562         {"tx_size_64_packets", offsetof(struct ixgbe_hw_stats, ptc64)},
 563         {"tx_size_65_to_127_packets", offsetof(struct ixgbe_hw_stats, ptc127)},
 564         {"tx_size_128_to_255_packets", offsetof(struct ixgbe_hw_stats, ptc255)},
 565         {"tx_size_256_to_511_packets", offsetof(struct ixgbe_hw_stats, ptc511)},
 566         {"tx_size_512_to_1023_packets", offsetof(struct ixgbe_hw_stats,
 567                 ptc1023)},
 568         {"tx_size_1024_to_max_packets", offsetof(struct ixgbe_hw_stats,
 569                 ptc1522)},
 570         {"tx_multicast_packets", offsetof(struct ixgbe_hw_stats, mptc)},
 571         {"tx_broadcast_packets", offsetof(struct ixgbe_hw_stats, bptc)},
 572         {"rx_mac_short_packet_dropped", offsetof(struct ixgbe_hw_stats, mspdc)},
 573         {"rx_l3_l4_xsum_error", offsetof(struct ixgbe_hw_stats, xec)},
 574
 575         {"flow_director_added_filters", offsetof(struct ixgbe_hw_stats,
 576                 fdirustat_add)},
 577         {"flow_director_removed_filters", offsetof(struct ixgbe_hw_stats,
 578                 fdirustat_remove)},
 579         {"flow_director_filter_add_errors", offsetof(struct ixgbe_hw_stats,
 580                 fdirfstat_fadd)},
 581         {"flow_director_filter_remove_errors", offsetof(struct ixgbe_hw_stats,
 582                 fdirfstat_fremove)},
 583         {"flow_director_matched_filters", offsetof(struct ixgbe_hw_stats,
 584                 fdirmatch)},
 585         {"flow_director_missed_filters", offsetof(struct ixgbe_hw_stats,
 586                 fdirmiss)},
 587
 588         {"rx_fcoe_crc_errors", offsetof(struct ixgbe_hw_stats, fccrc)},
 589         {"rx_fcoe_dropped", offsetof(struct ixgbe_hw_stats, fcoerpdc)},
 590         {"rx_fcoe_mbuf_allocation_errors", offsetof(struct ixgbe_hw_stats,
 591                 fclast)},
 592         {"rx_fcoe_packets", offsetof(struct ixgbe_hw_stats, fcoeprc)},
 593         {"tx_fcoe_packets", offsetof(struct ixgbe_hw_stats, fcoeptc)},
 594         {"rx_fcoe_bytes", offsetof(struct ixgbe_hw_stats, fcoedwrc)},
 595         {"tx_fcoe_bytes", offsetof(struct ixgbe_hw_stats, fcoedwtc)},
 596         {"rx_fcoe_no_direct_data_placement", offsetof(struct ixgbe_hw_stats,
 597                 fcoe_noddp)},
 598         {"rx_fcoe_no_direct_data_placement_ext_buff",
 599                 offsetof(struct ixgbe_hw_stats, fcoe_noddp_ext_buff)},
 600
 601         {"tx_flow_control_xon_packets", offsetof(struct ixgbe_hw_stats,
 602                 lxontxc)},
 603         {"rx_flow_control_xon_packets", offsetof(struct ixgbe_hw_stats,
 604                 lxonrxc)},
 605         {"tx_flow_control_xoff_packets", offsetof(struct ixgbe_hw_stats,
 606                 lxofftxc)},
 607         {"rx_flow_control_xoff_packets", offsetof(struct ixgbe_hw_stats,
 608                 lxoffrxc)},
 609         {"rx_total_missed_packets", offsetof(struct ixgbe_hw_stats, mpctotal)},
 610 };
 611
 612 #define IXGBE_NB_HW_STATS (sizeof(rte_ixgbe_stats_strings) / \
 613                            sizeof(rte_ixgbe_stats_strings[0]))
 614
 615 /* Per-queue statistics */
 616 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_rxq_strings[] = {
 617         {"mbuf_allocation_errors", offsetof(struct ixgbe_hw_stats, rnbc)},
 618         {"dropped", offsetof(struct ixgbe_hw_stats, mpc)},
 619         {"xon_packets", offsetof(struct ixgbe_hw_stats, pxonrxc)},
 620         {"xoff_packets", offsetof(struct ixgbe_hw_stats, pxoffrxc)},
 621 };
 622
 623 #define IXGBE_NB_RXQ_PRIO_STATS (sizeof(rte_ixgbe_rxq_strings) / \
 624                            sizeof(rte_ixgbe_rxq_strings[0]))
 625
 626 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_txq_strings[] = {
 627         {"xon_packets", offsetof(struct ixgbe_hw_stats, pxontxc)},
 628         {"xoff_packets", offsetof(struct ixgbe_hw_stats, pxofftxc)},
 629         {"xon_to_xoff_packets", offsetof(struct ixgbe_hw_stats,
 630                 pxon2offc)},
 631 };
 632
 633 #define IXGBE_NB_TXQ_PRIO_STATS (sizeof(rte_ixgbe_txq_strings) / \
 634                            sizeof(rte_ixgbe_txq_strings[0]))
 635
 636 static const struct rte_ixgbe_xstats_name_off rte_ixgbevf_stats_strings[] = {
 637         {"rx_multicast_packets", offsetof(struct ixgbevf_hw_stats, vfmprc)},
 638 };
 639
 640 #define IXGBEVF_NB_XSTATS (sizeof(rte_ixgbevf_stats_strings) /  \
 641                 sizeof(rte_ixgbevf_stats_strings[0]))
 642
 643 /**
 644  * Atomically reads the link status information from global
 645  * structure rte_eth_dev.
 646  *
 647  * @param dev
 648  *   - Pointer to the structure rte_eth_dev to read from.
 649  *   - Pointer to the buffer to be saved with the link status.
 650  *
 651  * @return
 652  *   - On success, zero.
 653  *   - On failure, negative value.
 654  */
 655 static inline int
 656 rte_ixgbe_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 657                                 struct rte_eth_link *link)
 658 {
 659         struct rte_eth_link *dst = link;
 660         struct rte_eth_link *src = &(dev->data->dev_link);
 661
 662         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 663                                         *(uint64_t *)src) == 0)
 664                 return -1;
 665
 666         return 0;
 667 }
 668
 669 /**
 670  * Atomically writes the link status information into global
 671  * structure rte_eth_dev.
 672  *
 673  * @param dev
 674  *   - Pointer to the structure rte_eth_dev to read from.
 675  *   - Pointer to the buffer to be saved with the link status.
 676  *
 677  * @return
 678  *   - On success, zero.
 679  *   - On failure, negative value.
 680  */
 681 static inline int
 682 rte_ixgbe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 683                                 struct rte_eth_link *link)
 684 {
 685         struct rte_eth_link *dst = &(dev->data->dev_link);
 686         struct rte_eth_link *src = link;
 687
 688         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 689                                         *(uint64_t *)src) == 0)
 690                 return -1;
 691
 692         return 0;
 693 }
 694
 695 /*
 696  * This function is the same as ixgbe_is_sfp() in base/ixgbe.h.
 697  */
 698 static inline int
 699 ixgbe_is_sfp(struct ixgbe_hw *hw)
 700 {
 701         switch (hw->phy.type) {
 702         case ixgbe_phy_sfp_avago:
 703         case ixgbe_phy_sfp_ftl:
 704         case ixgbe_phy_sfp_intel:
 705         case ixgbe_phy_sfp_unknown:
 706         case ixgbe_phy_sfp_passive_tyco:
 707         case ixgbe_phy_sfp_passive_unknown:
 708                 return 1;
 709         default:
 710                 return 0;
 711         }
 712 }
 713
 714 static inline int32_t
 715 ixgbe_pf_reset_hw(struct ixgbe_hw *hw)
 716 {
 717         uint32_t ctrl_ext;
 718         int32_t status;
 719
 720         status = ixgbe_reset_hw(hw);
 721
 722         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 723         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 724         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
 725         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 726         IXGBE_WRITE_FLUSH(hw);
 727
 728         return status;
 729 }
 730
 731 static inline void
 732 ixgbe_enable_intr(struct rte_eth_dev *dev)
 733 {
 734         struct ixgbe_interrupt *intr =
 735                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 736         struct ixgbe_hw *hw =
 737                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 738
 739         IXGBE_WRITE_REG(hw, IXGBE_EIMS, intr->mask);
 740         IXGBE_WRITE_FLUSH(hw);
 741 }
 742
 743 /*
 744  * This function is based on ixgbe_disable_intr() in base/ixgbe.h.
 745  */
 746 static void
 747 ixgbe_disable_intr(struct ixgbe_hw *hw)
 748 {
 749         PMD_INIT_FUNC_TRACE();
 750
 751         if (hw->mac.type == ixgbe_mac_82598EB) {
 752                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ~0);
 753         } else {
 754                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xFFFF0000);
 755                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), ~0);
 756                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), ~0);
 757         }
 758         IXGBE_WRITE_FLUSH(hw);
 759 }
 760
 761 /*
 762  * This function resets queue statistics mapping registers.
 763  * From Niantic datasheet, Initialization of Statistics section:
 764  * "...if software requires the queue counters, the RQSMR and TQSM registers
 765  * must be re-programmed following a device reset.
 766  */
 767 static void
 768 ixgbe_reset_qstat_mappings(struct ixgbe_hw *hw)
 769 {
 770         uint32_t i;
 771
 772         for(i = 0; i != IXGBE_NB_STAT_MAPPING_REGS; i++) {
 773                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), 0);
 774                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), 0);
 775         }
 776 }
 777
 778
 779 static int
 780 ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 781                                   uint16_t queue_id,
 782                                   uint8_t stat_idx,
 783                                   uint8_t is_rx)
 784 {
 785 #define QSM_REG_NB_BITS_PER_QMAP_FIELD 8
 786 #define NB_QMAP_FIELDS_PER_QSM_REG 4
 787 #define QMAP_FIELD_RESERVED_BITS_MASK 0x0f
 788
 789         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 790         struct ixgbe_stat_mapping_registers *stat_mappings =
 791                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(eth_dev->data->dev_private);
 792         uint32_t qsmr_mask = 0;
 793         uint32_t clearing_mask = QMAP_FIELD_RESERVED_BITS_MASK;
 794         uint32_t q_map;
 795         uint8_t n, offset;
 796
 797         if ((hw->mac.type != ixgbe_mac_82599EB) &&
 798                 (hw->mac.type != ixgbe_mac_X540) &&
 799                 (hw->mac.type != ixgbe_mac_X550) &&
 800                 (hw->mac.type != ixgbe_mac_X550EM_x))
 801                 return -ENOSYS;
 802
 803         PMD_INIT_LOG(DEBUG, "Setting port %d, %s queue_id %d to stat index %d",
 804                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX",
 805                      queue_id, stat_idx);
 806
 807         n = (uint8_t)(queue_id / NB_QMAP_FIELDS_PER_QSM_REG);
 808         if (n >= IXGBE_NB_STAT_MAPPING_REGS) {
 809                 PMD_INIT_LOG(ERR, "Nb of stat mapping registers exceeded");
 810                 return -EIO;
 811         }
 812         offset = (uint8_t)(queue_id % NB_QMAP_FIELDS_PER_QSM_REG);
 813
 814         /* Now clear any previous stat_idx set */
 815         clearing_mask <<= (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 816         if (!is_rx)
 817                 stat_mappings->tqsm[n] &= ~clearing_mask;
 818         else
 819                 stat_mappings->rqsmr[n] &= ~clearing_mask;
 820
 821         q_map = (uint32_t)stat_idx;
 822         q_map &= QMAP_FIELD_RESERVED_BITS_MASK;
 823         qsmr_mask = q_map << (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 824         if (!is_rx)
 825                 stat_mappings->tqsm[n] |= qsmr_mask;
 826         else
 827                 stat_mappings->rqsmr[n] |= qsmr_mask;
 828
 829         PMD_INIT_LOG(DEBUG, "Set port %d, %s queue_id %d to stat index %d",
 830                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX",
 831                      queue_id, stat_idx);
 832         PMD_INIT_LOG(DEBUG, "%s[%d] = 0x%08x", is_rx ? "RQSMR" : "TQSM", n,
 833                      is_rx ? stat_mappings->rqsmr[n] : stat_mappings->tqsm[n]);
 834
 835         /* Now write the mapping in the appropriate register */
 836         if (is_rx) {
 837                 PMD_INIT_LOG(DEBUG, "Write 0x%x to RX IXGBE stat mapping reg:%d",
 838                              stat_mappings->rqsmr[n], n);
 839                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(n), stat_mappings->rqsmr[n]);
 840         }
 841         else {
 842                 PMD_INIT_LOG(DEBUG, "Write 0x%x to TX IXGBE stat mapping reg:%d",
 843                              stat_mappings->tqsm[n], n);
 844                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(n), stat_mappings->tqsm[n]);
 845         }
 846         return 0;
 847 }
 848
 849 static void
 850 ixgbe_restore_statistics_mapping(struct rte_eth_dev * dev)
 851 {
 852         struct ixgbe_stat_mapping_registers *stat_mappings =
 853                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(dev->data->dev_private);
 854         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 855         int i;
 856
 857         /* write whatever was in stat mapping table to the NIC */
 858         for (i = 0; i < IXGBE_NB_STAT_MAPPING_REGS; i++) {
 859                 /* rx */
 860                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), stat_mappings->rqsmr[i]);
 861
 862                 /* tx */
 863                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), stat_mappings->tqsm[i]);
 864         }
 865 }
 866
 867 static void
 868 ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config)
 869 {
 870         uint8_t i;
 871         struct ixgbe_dcb_tc_config *tc;
 872         uint8_t dcb_max_tc = IXGBE_DCB_MAX_TRAFFIC_CLASS;
 873
 874         dcb_config->num_tcs.pg_tcs = dcb_max_tc;
 875         dcb_config->num_tcs.pfc_tcs = dcb_max_tc;
 876         for (i = 0; i < dcb_max_tc; i++) {
 877                 tc = &dcb_config->tc_config[i];
 878                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_id = i;
 879                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
 880                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 881                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_id = i;
 882                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
 883                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 884                 tc->pfc = ixgbe_dcb_pfc_disabled;
 885         }
 886
 887         /* Initialize default user to priority mapping, UPx->TC0 */
 888         tc = &dcb_config->tc_config[0];
 889         tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
 890         tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
 891         for (i = 0; i< IXGBE_DCB_MAX_BW_GROUP; i++) {
 892                 dcb_config->bw_percentage[IXGBE_DCB_TX_CONFIG][i] = 100;
 893                 dcb_config->bw_percentage[IXGBE_DCB_RX_CONFIG][i] = 100;
 894         }
 895         dcb_config->rx_pba_cfg = ixgbe_dcb_pba_equal;
 896         dcb_config->pfc_mode_enable = false;
 897         dcb_config->vt_mode = true;
 898         dcb_config->round_robin_enable = false;
 899         /* support all DCB capabilities in 82599 */
 900         dcb_config->support.capabilities = 0xFF;
 901
 902         /*we only support 4 Tcs for X540, X550 */
 903         if (hw->mac.type == ixgbe_mac_X540 ||
 904                 hw->mac.type == ixgbe_mac_X550 ||
 905                 hw->mac.type == ixgbe_mac_X550EM_x) {
 906                 dcb_config->num_tcs.pg_tcs = 4;
 907                 dcb_config->num_tcs.pfc_tcs = 4;
 908         }
 909 }
 910
 911 /*
 912  * Ensure that all locks are released before first NVM or PHY access
 913  */
 914 static void
 915 ixgbe_swfw_lock_reset(struct ixgbe_hw *hw)
 916 {
 917         uint16_t mask;
 918
 919         /*
 920          * Phy lock should not fail in this early stage. If this is the case,
 921          * it is due to an improper exit of the application.
 922          * So force the release of the faulty lock. Release of common lock
 923          * is done automatically by swfw_sync function.
 924          */
 925         mask = IXGBE_GSSR_PHY0_SM << hw->bus.func;
 926         if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) {
 927                 PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released", hw->bus.func);
 928         }
 929         ixgbe_release_swfw_semaphore(hw, mask);
 930
 931         /*
 932          * These ones are more tricky since they are common to all ports; but
 933          * swfw_sync retries last long enough (1s) to be almost sure that if
 934          * lock can not be taken it is due to an improper lock of the
 935          * semaphore.
 936          */
 937         mask = IXGBE_GSSR_EEP_SM | IXGBE_GSSR_MAC_CSR_SM | IXGBE_GSSR_SW_MNG_SM;
 938         if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) {
 939                 PMD_DRV_LOG(DEBUG, "SWFW common locks released");
 940         }
 941         ixgbe_release_swfw_semaphore(hw, mask);
 942 }
 943
 944 /*
 945  * This function is based on code in ixgbe_attach() in base/ixgbe.c.
 946  * It returns 0 on success.
 947  */
 948 static int
 949 eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev)
 950 {
 951         struct rte_pci_device *pci_dev;
 952         struct ixgbe_hw *hw =
 953                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 954         struct ixgbe_vfta * shadow_vfta =
 955                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 956         struct ixgbe_hwstrip *hwstrip =
 957                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 958         struct ixgbe_dcb_config *dcb_config =
 959                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private);
 960         struct ixgbe_filter_info *filter_info =
 961                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private);
 962         uint32_t ctrl_ext;
 963         uint16_t csum;
 964         int diag, i;
 965
 966         PMD_INIT_FUNC_TRACE();
 967
 968         eth_dev->dev_ops = &ixgbe_eth_dev_ops;
 969         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
 970         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
 971
 972         /*
 973          * For secondary processes, we don't initialise any further as primary
 974          * has already done this work. Only check we don't need a different
 975          * RX and TX function.
 976          */
 977         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 978                 struct ixgbe_tx_queue *txq;
 979                 /* TX queue function in primary, set by last queue initialized
 980                  * Tx queue may not initialized by primary process */
 981                 if (eth_dev->data->tx_queues) {
 982                         txq = eth_dev->data->tx_queues[eth_dev->data->nb_tx_queues-1];
 983                         ixgbe_set_tx_function(eth_dev, txq);
 984                 } else {
 985                         /* Use default TX function if we get here */
 986                         PMD_INIT_LOG(NOTICE, "No TX queues configured yet. "
 987                                              "Using default TX function.");
 988                 }
 989
 990                 ixgbe_set_rx_function(eth_dev);
 991
 992                 return 0;
 993         }
 994         pci_dev = eth_dev->pci_dev;
 995
 996         rte_eth_copy_pci_info(eth_dev, pci_dev);
 997
 998         /* Vendor and Device ID need to be set before init of shared code */
 999         hw->device_id = pci_dev->id.device_id;
1000         hw->vendor_id = pci_dev->id.vendor_id;
1001         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
1002         hw->allow_unsupported_sfp = 1;
1003
1004         /* Initialize the shared code (base driver) */
1005 #ifdef RTE_NIC_BYPASS
1006         diag = ixgbe_bypass_init_shared_code(hw);
1007 #else
1008         diag = ixgbe_init_shared_code(hw);
1009 #endif /* RTE_NIC_BYPASS */
1010
1011         if (diag != IXGBE_SUCCESS) {
1012                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
1013                 return -EIO;
1014         }
1015
1016         /* pick up the PCI bus settings for reporting later */
1017         ixgbe_get_bus_info(hw);
1018
1019         /* Unlock any pending hardware semaphore */
1020         ixgbe_swfw_lock_reset(hw);
1021
1022         /* Initialize DCB configuration*/
1023         memset(dcb_config, 0, sizeof(struct ixgbe_dcb_config));
1024         ixgbe_dcb_init(hw,dcb_config);
1025         /* Get Hardware Flow Control setting */
1026         hw->fc.requested_mode = ixgbe_fc_full;
1027         hw->fc.current_mode = ixgbe_fc_full;
1028         hw->fc.pause_time = IXGBE_FC_PAUSE;
1029         for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
1030                 hw->fc.low_water[i] = IXGBE_FC_LO;
1031                 hw->fc.high_water[i] = IXGBE_FC_HI;
1032         }
1033         hw->fc.send_xon = 1;
1034
1035         /* Make sure we have a good EEPROM before we read from it */
1036         diag = ixgbe_validate_eeprom_checksum(hw, &csum);
1037         if (diag != IXGBE_SUCCESS) {
1038                 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag);
1039                 return -EIO;
1040         }
1041
1042 #ifdef RTE_NIC_BYPASS
1043         diag = ixgbe_bypass_init_hw(hw);
1044 #else
1045         diag = ixgbe_init_hw(hw);
1046 #endif /* RTE_NIC_BYPASS */
1047
1048         /*
1049          * Devices with copper phys will fail to initialise if ixgbe_init_hw()
1050          * is called too soon after the kernel driver unbinding/binding occurs.
1051          * The failure occurs in ixgbe_identify_phy_generic() for all devices,
1052          * but for non-copper devies, ixgbe_identify_sfp_module_generic() is
1053          * also called. See ixgbe_identify_phy_82599(). The reason for the
1054          * failure is not known, and only occuts when virtualisation features
1055          * are disabled in the bios. A delay of 100ms  was found to be enough by
1056          * trial-and-error, and is doubled to be safe.
1057          */
1058         if (diag && (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper)) {
1059                 rte_delay_ms(200);
1060                 diag = ixgbe_init_hw(hw);
1061         }
1062
1063         if (diag == IXGBE_ERR_EEPROM_VERSION) {
1064                 PMD_INIT_LOG(ERR, "This device is a pre-production adapter/"
1065                     "LOM.  Please be aware there may be issues associated "
1066                     "with your hardware.");
1067                 PMD_INIT_LOG(ERR, "If you are experiencing problems "
1068                     "please contact your Intel or hardware representative "
1069                     "who provided you with this hardware.");
1070         } else if (diag == IXGBE_ERR_SFP_NOT_SUPPORTED)
1071                 PMD_INIT_LOG(ERR, "Unsupported SFP+ Module");
1072         if (diag) {
1073                 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag);
1074                 return -EIO;
1075         }
1076
1077         /* Reset the hw statistics */
1078         ixgbe_dev_stats_reset(eth_dev);
1079
1080         /* disable interrupt */
1081         ixgbe_disable_intr(hw);
1082
1083         /* reset mappings for queue statistics hw counters*/
1084         ixgbe_reset_qstat_mappings(hw);
1085
1086         /* Allocate memory for storing MAC addresses */
1087         eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
1088                         hw->mac.num_rar_entries, 0);
1089         if (eth_dev->data->mac_addrs == NULL) {
1090                 PMD_INIT_LOG(ERR,
1091                         "Failed to allocate %u bytes needed to store "
1092                         "MAC addresses",
1093                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
1094                 return -ENOMEM;
1095         }
1096         /* Copy the permanent MAC address */
1097         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
1098                         &eth_dev->data->mac_addrs[0]);
1099
1100         /* Allocate memory for storing hash filter MAC addresses */
1101         eth_dev->data->hash_mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
1102                         IXGBE_VMDQ_NUM_UC_MAC, 0);
1103         if (eth_dev->data->hash_mac_addrs == NULL) {
1104                 PMD_INIT_LOG(ERR,
1105                         "Failed to allocate %d bytes needed to store MAC addresses",
1106                         ETHER_ADDR_LEN * IXGBE_VMDQ_NUM_UC_MAC);
1107                 return -ENOMEM;
1108         }
1109
1110         /* initialize the vfta */
1111         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
1112
1113         /* initialize the hw strip bitmap*/
1114         memset(hwstrip, 0, sizeof(*hwstrip));
1115
1116         /* initialize PF if max_vfs not zero */
1117         ixgbe_pf_host_init(eth_dev);
1118
1119         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1120         /* let hardware know driver is loaded */
1121         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
1122         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
1123         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
1124         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
1125         IXGBE_WRITE_FLUSH(hw);
1126
1127         if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
1128                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d, SFP+: %d",
1129                              (int) hw->mac.type, (int) hw->phy.type,
1130                              (int) hw->phy.sfp_type);
1131         else
1132                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d",
1133                              (int) hw->mac.type, (int) hw->phy.type);
1134
1135         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
1136                         eth_dev->data->port_id, pci_dev->id.vendor_id,
1137                         pci_dev->id.device_id);
1138
1139         rte_intr_callback_register(&pci_dev->intr_handle,
1140                                    ixgbe_dev_interrupt_handler,
1141                                    (void *)eth_dev);
1142
1143         /* enable uio/vfio intr/eventfd mapping */
1144         rte_intr_enable(&pci_dev->intr_handle);
1145
1146         /* enable support intr */
1147         ixgbe_enable_intr(eth_dev);
1148
1149         /* initialize 5tuple filter list */
1150         TAILQ_INIT(&filter_info->fivetuple_list);
1151         memset(filter_info->fivetuple_mask, 0,
1152                 sizeof(uint32_t) * IXGBE_5TUPLE_ARRAY_SIZE);
1153
1154         return 0;
1155 }
1156
1157 static int
1158 eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev)
1159 {
1160         struct rte_pci_device *pci_dev;
1161         struct ixgbe_hw *hw;
1162
1163         PMD_INIT_FUNC_TRACE();
1164
1165         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1166                 return -EPERM;
1167
1168         hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1169         pci_dev = eth_dev->pci_dev;
1170
1171         if (hw->adapter_stopped == 0)
1172                 ixgbe_dev_close(eth_dev);
1173
1174         eth_dev->dev_ops = NULL;
1175         eth_dev->rx_pkt_burst = NULL;
1176         eth_dev->tx_pkt_burst = NULL;
1177
1178         /* Unlock any pending hardware semaphore */
1179         ixgbe_swfw_lock_reset(hw);
1180
1181         /* disable uio intr before callback unregister */
1182         rte_intr_disable(&(pci_dev->intr_handle));
1183         rte_intr_callback_unregister(&(pci_dev->intr_handle),
1184                 ixgbe_dev_interrupt_handler, (void *)eth_dev);
1185
1186         /* uninitialize PF if max_vfs not zero */
1187         ixgbe_pf_host_uninit(eth_dev);
1188
1189         rte_free(eth_dev->data->mac_addrs);
1190         eth_dev->data->mac_addrs = NULL;
1191
1192         rte_free(eth_dev->data->hash_mac_addrs);
1193         eth_dev->data->hash_mac_addrs = NULL;
1194
1195         return 0;
1196 }
1197
1198 /*
1199  * Negotiate mailbox API version with the PF.
1200  * After reset API version is always set to the basic one (ixgbe_mbox_api_10).
1201  * Then we try to negotiate starting with the most recent one.
1202  * If all negotiation attempts fail, then we will proceed with
1203  * the default one (ixgbe_mbox_api_10).
1204  */
1205 static void
1206 ixgbevf_negotiate_api(struct ixgbe_hw *hw)
1207 {
1208         int32_t i;
1209
1210         /* start with highest supported, proceed down */
1211         static const enum ixgbe_pfvf_api_rev sup_ver[] = {
1212                 ixgbe_mbox_api_11,
1213                 ixgbe_mbox_api_10,
1214         };
1215
1216         for (i = 0;
1217                         i != RTE_DIM(sup_ver) &&
1218                         ixgbevf_negotiate_api_version(hw, sup_ver[i]) != 0;
1219                         i++)
1220                 ;
1221 }
1222
1223 static void
1224 generate_random_mac_addr(struct ether_addr *mac_addr)
1225 {
1226         uint64_t random;
1227
1228         /* Set Organizationally Unique Identifier (OUI) prefix. */
1229         mac_addr->addr_bytes[0] = 0x00;
1230         mac_addr->addr_bytes[1] = 0x09;
1231         mac_addr->addr_bytes[2] = 0xC0;
1232         /* Force indication of locally assigned MAC address. */
1233         mac_addr->addr_bytes[0] |= ETHER_LOCAL_ADMIN_ADDR;
1234         /* Generate the last 3 bytes of the MAC address with a random number. */
1235         random = rte_rand();
1236         memcpy(&mac_addr->addr_bytes[3], &random, 3);
1237 }
1238
1239 /*
1240  * Virtual Function device init
1241  */
1242 static int
1243 eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev)
1244 {
1245         int diag;
1246         uint32_t tc, tcs;
1247         struct rte_pci_device *pci_dev;
1248         struct ixgbe_hw *hw =
1249                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1250         struct ixgbe_vfta * shadow_vfta =
1251                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
1252         struct ixgbe_hwstrip *hwstrip =
1253                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
1254         struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr;
1255
1256         PMD_INIT_FUNC_TRACE();
1257
1258         eth_dev->dev_ops = &ixgbevf_eth_dev_ops;
1259         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
1260         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
1261
1262         /* for secondary processes, we don't initialise any further as primary
1263          * has already done this work. Only check we don't need a different
1264          * RX function */
1265         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
1266                 if (eth_dev->data->scattered_rx)
1267                         eth_dev->rx_pkt_burst = ixgbe_recv_pkts_lro_single_alloc;
1268                 return 0;
1269         }
1270
1271         pci_dev = eth_dev->pci_dev;
1272
1273         rte_eth_copy_pci_info(eth_dev, pci_dev);
1274
1275         hw->device_id = pci_dev->id.device_id;
1276         hw->vendor_id = pci_dev->id.vendor_id;
1277         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
1278
1279         /* initialize the vfta */
1280         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
1281
1282         /* initialize the hw strip bitmap*/
1283         memset(hwstrip, 0, sizeof(*hwstrip));
1284
1285         /* Initialize the shared code (base driver) */
1286         diag = ixgbe_init_shared_code(hw);
1287         if (diag != IXGBE_SUCCESS) {
1288                 PMD_INIT_LOG(ERR, "Shared code init failed for ixgbevf: %d", diag);
1289                 return -EIO;
1290         }
1291
1292         /* init_mailbox_params */
1293         hw->mbx.ops.init_params(hw);
1294
1295         /* Reset the hw statistics */
1296         ixgbevf_dev_stats_reset(eth_dev);
1297
1298         /* Disable the interrupts for VF */
1299         ixgbevf_intr_disable(hw);
1300
1301         hw->mac.num_rar_entries = 128; /* The MAX of the underlying PF */
1302         diag = hw->mac.ops.reset_hw(hw);
1303
1304         /*
1305          * The VF reset operation returns the IXGBE_ERR_INVALID_MAC_ADDR when
1306          * the underlying PF driver has not assigned a MAC address to the VF.
1307          * In this case, assign a random MAC address.
1308          */
1309         if ((diag != IXGBE_SUCCESS) && (diag != IXGBE_ERR_INVALID_MAC_ADDR)) {
1310                 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
1311                 return (diag);
1312         }
1313
1314         /* negotiate mailbox API version to use with the PF. */
1315         ixgbevf_negotiate_api(hw);
1316
1317         /* Get Rx/Tx queue count via mailbox, which is ready after reset_hw */
1318         ixgbevf_get_queues(hw, &tcs, &tc);
1319
1320         /* Allocate memory for storing MAC addresses */
1321         eth_dev->data->mac_addrs = rte_zmalloc("ixgbevf", ETHER_ADDR_LEN *
1322                         hw->mac.num_rar_entries, 0);
1323         if (eth_dev->data->mac_addrs == NULL) {
1324                 PMD_INIT_LOG(ERR,
1325                         "Failed to allocate %u bytes needed to store "
1326                         "MAC addresses",
1327                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
1328                 return -ENOMEM;
1329         }
1330
1331         /* Generate a random MAC address, if none was assigned by PF. */
1332         if (is_zero_ether_addr(perm_addr)) {
1333                 generate_random_mac_addr(perm_addr);
1334                 diag = ixgbe_set_rar_vf(hw, 1, perm_addr->addr_bytes, 0, 1);
1335                 if (diag) {
1336                         rte_free(eth_dev->data->mac_addrs);
1337                         eth_dev->data->mac_addrs = NULL;
1338                         return diag;
1339                 }
1340                 PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF");
1341                 PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address "
1342                              "%02x:%02x:%02x:%02x:%02x:%02x",
1343                              perm_addr->addr_bytes[0],
1344                              perm_addr->addr_bytes[1],
1345                              perm_addr->addr_bytes[2],
1346                              perm_addr->addr_bytes[3],
1347                              perm_addr->addr_bytes[4],
1348                              perm_addr->addr_bytes[5]);
1349         }
1350
1351         /* Copy the permanent MAC address */
1352         ether_addr_copy(perm_addr, &eth_dev->data->mac_addrs[0]);
1353
1354         /* reset the hardware with the new settings */
1355         diag = hw->mac.ops.start_hw(hw);
1356         switch (diag) {
1357                 case  0:
1358                         break;
1359
1360                 default:
1361                         PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
1362                         return (-EIO);
1363         }
1364
1365         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x mac.type=%s",
1366                      eth_dev->data->port_id, pci_dev->id.vendor_id,
1367                      pci_dev->id.device_id, "ixgbe_mac_82599_vf");
1368
1369         return 0;
1370 }
1371
1372 /* Virtual Function device uninit */
1373
1374 static int
1375 eth_ixgbevf_dev_uninit(struct rte_eth_dev *eth_dev)
1376 {
1377         struct ixgbe_hw *hw;
1378         unsigned i;
1379
1380         PMD_INIT_FUNC_TRACE();
1381
1382         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1383                 return -EPERM;
1384
1385         hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1386
1387         if (hw->adapter_stopped == 0)
1388                 ixgbevf_dev_close(eth_dev);
1389
1390         eth_dev->dev_ops = NULL;
1391         eth_dev->rx_pkt_burst = NULL;
1392         eth_dev->tx_pkt_burst = NULL;
1393
1394         /* Disable the interrupts for VF */
1395         ixgbevf_intr_disable(hw);
1396
1397         for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
1398                 ixgbe_dev_rx_queue_release(eth_dev->data->rx_queues[i]);
1399                 eth_dev->data->rx_queues[i] = NULL;
1400         }
1401         eth_dev->data->nb_rx_queues = 0;
1402
1403         for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
1404                 ixgbe_dev_tx_queue_release(eth_dev->data->tx_queues[i]);
1405                 eth_dev->data->tx_queues[i] = NULL;
1406         }
1407         eth_dev->data->nb_tx_queues = 0;
1408
1409         rte_free(eth_dev->data->mac_addrs);
1410         eth_dev->data->mac_addrs = NULL;
1411
1412         return 0;
1413 }
1414
1415 static struct eth_driver rte_ixgbe_pmd = {
1416         .pci_drv = {
1417                 .name = "rte_ixgbe_pmd",
1418                 .id_table = pci_id_ixgbe_map,
1419                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
1420                         RTE_PCI_DRV_DETACHABLE,
1421         },
1422         .eth_dev_init = eth_ixgbe_dev_init,
1423         .eth_dev_uninit = eth_ixgbe_dev_uninit,
1424         .dev_private_size = sizeof(struct ixgbe_adapter),
1425 };
1426
1427 /*
1428  * virtual function driver struct
1429  */
1430 static struct eth_driver rte_ixgbevf_pmd = {
1431         .pci_drv = {
1432                 .name = "rte_ixgbevf_pmd",
1433                 .id_table = pci_id_ixgbevf_map,
1434                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
1435         },
1436         .eth_dev_init = eth_ixgbevf_dev_init,
1437         .eth_dev_uninit = eth_ixgbevf_dev_uninit,
1438         .dev_private_size = sizeof(struct ixgbe_adapter),
1439 };
1440
1441 /*
1442  * Driver initialization routine.
1443  * Invoked once at EAL init time.
1444  * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
1445  */
1446 static int
1447 rte_ixgbe_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
1448 {
1449         PMD_INIT_FUNC_TRACE();
1450
1451         rte_eth_driver_register(&rte_ixgbe_pmd);
1452         return 0;
1453 }
1454
1455 /*
1456  * VF Driver initialization routine.
1457  * Invoked one at EAL init time.
1458  * Register itself as the [Virtual Poll Mode] Driver of PCI niantic devices.
1459  */
1460 static int
1461 rte_ixgbevf_pmd_init(const char *name __rte_unused, const char *param __rte_unused)
1462 {
1463         PMD_INIT_FUNC_TRACE();
1464
1465         rte_eth_driver_register(&rte_ixgbevf_pmd);
1466         return (0);
1467 }
1468
1469 static int
1470 ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1471 {
1472         struct ixgbe_hw *hw =
1473                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1474         struct ixgbe_vfta * shadow_vfta =
1475                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1476         uint32_t vfta;
1477         uint32_t vid_idx;
1478         uint32_t vid_bit;
1479
1480         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
1481         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
1482         vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(vid_idx));
1483         if (on)
1484                 vfta |= vid_bit;
1485         else
1486                 vfta &= ~vid_bit;
1487         IXGBE_WRITE_REG(hw, IXGBE_VFTA(vid_idx), vfta);
1488
1489         /* update local VFTA copy */
1490         shadow_vfta->vfta[vid_idx] = vfta;
1491
1492         return 0;
1493 }
1494
1495 static void
1496 ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
1497 {
1498         if (on)
1499                 ixgbe_vlan_hw_strip_enable(dev, queue);
1500         else
1501                 ixgbe_vlan_hw_strip_disable(dev, queue);
1502 }
1503
1504 static void
1505 ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
1506 {
1507         struct ixgbe_hw *hw =
1508                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1509
1510         /* Only the high 16-bits is valid */
1511         IXGBE_WRITE_REG(hw, IXGBE_EXVET, tpid << 16);
1512 }
1513
1514 void
1515 ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev)
1516 {
1517         struct ixgbe_hw *hw =
1518                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1519         uint32_t vlnctrl;
1520
1521         PMD_INIT_FUNC_TRACE();
1522
1523         /* Filter Table Disable */
1524         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1525         vlnctrl &= ~IXGBE_VLNCTRL_VFE;
1526
1527         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
1528 }
1529
1530 void
1531 ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1532 {
1533         struct ixgbe_hw *hw =
1534                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1535         struct ixgbe_vfta * shadow_vfta =
1536                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1537         uint32_t vlnctrl;
1538         uint16_t i;
1539
1540         PMD_INIT_FUNC_TRACE();
1541
1542         /* Filter Table Enable */
1543         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1544         vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
1545         vlnctrl |= IXGBE_VLNCTRL_VFE;
1546
1547         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
1548
1549         /* write whatever is in local vfta copy */
1550         for (i = 0; i < IXGBE_VFTA_SIZE; i++)
1551                 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), shadow_vfta->vfta[i]);
1552 }
1553
1554 static void
1555 ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on)
1556 {
1557         struct ixgbe_hwstrip *hwstrip =
1558                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(dev->data->dev_private);
1559
1560         if(queue >= IXGBE_MAX_RX_QUEUE_NUM)
1561                 return;
1562
1563         if (on)
1564                 IXGBE_SET_HWSTRIP(hwstrip, queue);
1565         else
1566                 IXGBE_CLEAR_HWSTRIP(hwstrip, queue);
1567 }
1568
1569 static void
1570 ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue)
1571 {
1572         struct ixgbe_hw *hw =
1573                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1574         uint32_t ctrl;
1575
1576         PMD_INIT_FUNC_TRACE();
1577
1578         if (hw->mac.type == ixgbe_mac_82598EB) {
1579                 /* No queue level support */
1580                 PMD_INIT_LOG(NOTICE, "82598EB not support queue level hw strip");
1581                 return;
1582         }
1583         else {
1584                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1585                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1586                 ctrl &= ~IXGBE_RXDCTL_VME;
1587                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
1588         }
1589         /* record those setting for HW strip per queue */
1590         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 0);
1591 }
1592
1593 static void
1594 ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue)
1595 {
1596         struct ixgbe_hw *hw =
1597                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1598         uint32_t ctrl;
1599
1600         PMD_INIT_FUNC_TRACE();
1601
1602         if (hw->mac.type == ixgbe_mac_82598EB) {
1603                 /* No queue level supported */
1604                 PMD_INIT_LOG(NOTICE, "82598EB not support queue level hw strip");
1605                 return;
1606         }
1607         else {
1608                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1609                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1610                 ctrl |= IXGBE_RXDCTL_VME;
1611                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
1612         }
1613         /* record those setting for HW strip per queue */
1614         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 1);
1615 }
1616
1617 void
1618 ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev)
1619 {
1620         struct ixgbe_hw *hw =
1621                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1622         uint32_t ctrl;
1623         uint16_t i;
1624
1625         PMD_INIT_FUNC_TRACE();
1626
1627         if (hw->mac.type == ixgbe_mac_82598EB) {
1628                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1629                 ctrl &= ~IXGBE_VLNCTRL_VME;
1630                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1631         }
1632         else {
1633                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1634                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1635                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1636                         ctrl &= ~IXGBE_RXDCTL_VME;
1637                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1638
1639                         /* record those setting for HW strip per queue */
1640                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 0);
1641                 }
1642         }
1643 }
1644
1645 void
1646 ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev)
1647 {
1648         struct ixgbe_hw *hw =
1649                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1650         uint32_t ctrl;
1651         uint16_t i;
1652
1653         PMD_INIT_FUNC_TRACE();
1654
1655         if (hw->mac.type == ixgbe_mac_82598EB) {
1656                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1657                 ctrl |= IXGBE_VLNCTRL_VME;
1658                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1659         }
1660         else {
1661                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1662                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1663                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1664                         ctrl |= IXGBE_RXDCTL_VME;
1665                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1666
1667                         /* record those setting for HW strip per queue */
1668                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 1);
1669                 }
1670         }
1671 }
1672
1673 static void
1674 ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1675 {
1676         struct ixgbe_hw *hw =
1677                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1678         uint32_t ctrl;
1679
1680         PMD_INIT_FUNC_TRACE();
1681
1682         /* DMATXCTRL: Geric Double VLAN Disable */
1683         ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1684         ctrl &= ~IXGBE_DMATXCTL_GDV;
1685         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1686
1687         /* CTRL_EXT: Global Double VLAN Disable */
1688         ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1689         ctrl &= ~IXGBE_EXTENDED_VLAN;
1690         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1691
1692 }
1693
1694 static void
1695 ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1696 {
1697         struct ixgbe_hw *hw =
1698                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1699         uint32_t ctrl;
1700
1701         PMD_INIT_FUNC_TRACE();
1702
1703         /* DMATXCTRL: Geric Double VLAN Enable */
1704         ctrl  = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1705         ctrl |= IXGBE_DMATXCTL_GDV;
1706         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1707
1708         /* CTRL_EXT: Global Double VLAN Enable */
1709         ctrl  = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1710         ctrl |= IXGBE_EXTENDED_VLAN;
1711         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1712
1713         /*
1714          * VET EXT field in the EXVET register = 0x8100 by default
1715          * So no need to change. Same to VT field of DMATXCTL register
1716          */
1717 }
1718
1719 static void
1720 ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1721 {
1722         if(mask & ETH_VLAN_STRIP_MASK){
1723                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1724                         ixgbe_vlan_hw_strip_enable_all(dev);
1725                 else
1726                         ixgbe_vlan_hw_strip_disable_all(dev);
1727         }
1728
1729         if(mask & ETH_VLAN_FILTER_MASK){
1730                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1731                         ixgbe_vlan_hw_filter_enable(dev);
1732                 else
1733                         ixgbe_vlan_hw_filter_disable(dev);
1734         }
1735
1736         if(mask & ETH_VLAN_EXTEND_MASK){
1737                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1738                         ixgbe_vlan_hw_extend_enable(dev);
1739                 else
1740                         ixgbe_vlan_hw_extend_disable(dev);
1741         }
1742 }
1743
1744 static void
1745 ixgbe_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1746 {
1747         struct ixgbe_hw *hw =
1748                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1749         /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
1750         uint32_t vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1751         vlanctrl |= IXGBE_VLNCTRL_VFE ; /* enable vlan filters */
1752         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
1753 }
1754
1755 static int
1756 ixgbe_check_vf_rss_rxq_num(struct rte_eth_dev *dev, uint16_t nb_rx_q)
1757 {
1758         switch (nb_rx_q) {
1759         case 1:
1760         case 2:
1761                 RTE_ETH_DEV_SRIOV(dev).active = ETH_64_POOLS;
1762                 break;
1763         case 4:
1764                 RTE_ETH_DEV_SRIOV(dev).active = ETH_32_POOLS;
1765                 break;
1766         default:
1767                 return -EINVAL;
1768         }
1769
1770         RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = nb_rx_q;
1771         RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx = dev->pci_dev->max_vfs * nb_rx_q;
1772
1773         return 0;
1774 }
1775
1776 static int
1777 ixgbe_check_mq_mode(struct rte_eth_dev *dev)
1778 {
1779         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
1780         uint16_t nb_rx_q = dev->data->nb_rx_queues;
1781         uint16_t nb_tx_q = dev->data->nb_rx_queues;
1782
1783         if (RTE_ETH_DEV_SRIOV(dev).active != 0) {
1784                 /* check multi-queue mode */
1785                 switch (dev_conf->rxmode.mq_mode) {
1786                 case ETH_MQ_RX_VMDQ_DCB:
1787                 case ETH_MQ_RX_VMDQ_DCB_RSS:
1788                         /* DCB/RSS VMDQ in SRIOV mode, not implement yet */
1789                         PMD_INIT_LOG(ERR, "SRIOV active,"
1790                                         " unsupported mq_mode rx %d.",
1791                                         dev_conf->rxmode.mq_mode);
1792                         return -EINVAL;
1793                 case ETH_MQ_RX_RSS:
1794                 case ETH_MQ_RX_VMDQ_RSS:
1795                         dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_RSS;
1796                         if (nb_rx_q <= RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool)
1797                                 if (ixgbe_check_vf_rss_rxq_num(dev, nb_rx_q)) {
1798                                         PMD_INIT_LOG(ERR, "SRIOV is active,"
1799                                                 " invalid queue number"
1800                                                 " for VMDQ RSS, allowed"
1801                                                 " value are 1, 2 or 4.");
1802                                         return -EINVAL;
1803                                 }
1804                         break;
1805                 case ETH_MQ_RX_VMDQ_ONLY:
1806                 case ETH_MQ_RX_NONE:
1807                         /* if nothing mq mode configure, use default scheme */
1808                         dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY;
1809                         if (RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool > 1)
1810                                 RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 1;
1811                         break;
1812                 default: /* ETH_MQ_RX_DCB, ETH_MQ_RX_DCB_RSS or ETH_MQ_TX_DCB*/
1813                         /* SRIOV only works in VMDq enable mode */
1814                         PMD_INIT_LOG(ERR, "SRIOV is active,"
1815                                         " wrong mq_mode rx %d.",
1816                                         dev_conf->rxmode.mq_mode);
1817                         return -EINVAL;
1818                 }
1819
1820                 switch (dev_conf->txmode.mq_mode) {
1821                 case ETH_MQ_TX_VMDQ_DCB:
1822                         /* DCB VMDQ in SRIOV mode, not implement yet */
1823                         PMD_INIT_LOG(ERR, "SRIOV is active,"
1824                                         " unsupported VMDQ mq_mode tx %d.",
1825                                         dev_conf->txmode.mq_mode);
1826                         return -EINVAL;
1827                 default: /* ETH_MQ_TX_VMDQ_ONLY or ETH_MQ_TX_NONE */
1828                         dev->data->dev_conf.txmode.mq_mode = ETH_MQ_TX_VMDQ_ONLY;
1829                         break;
1830                 }
1831
1832                 /* check valid queue number */
1833                 if ((nb_rx_q > RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool) ||
1834                     (nb_tx_q > RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool)) {
1835                         PMD_INIT_LOG(ERR, "SRIOV is active,"
1836                                         " queue number must less equal to %d.",
1837                                         RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool);
1838                         return -EINVAL;
1839                 }
1840         } else {
1841                 /* check configuration for vmdb+dcb mode */
1842                 if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_VMDQ_DCB) {
1843                         const struct rte_eth_vmdq_dcb_conf *conf;
1844
1845                         if (nb_rx_q != IXGBE_VMDQ_DCB_NB_QUEUES) {
1846                                 PMD_INIT_LOG(ERR, "VMDQ+DCB, nb_rx_q != %d.",
1847                                                 IXGBE_VMDQ_DCB_NB_QUEUES);
1848                                 return -EINVAL;
1849                         }
1850                         conf = &dev_conf->rx_adv_conf.vmdq_dcb_conf;
1851                         if (!(conf->nb_queue_pools == ETH_16_POOLS ||
1852                                conf->nb_queue_pools == ETH_32_POOLS)) {
1853                                 PMD_INIT_LOG(ERR, "VMDQ+DCB selected,"
1854                                                 " nb_queue_pools must be %d or %d.",
1855                                                 ETH_16_POOLS, ETH_32_POOLS);
1856                                 return -EINVAL;
1857                         }
1858                 }
1859                 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_VMDQ_DCB) {
1860                         const struct rte_eth_vmdq_dcb_tx_conf *conf;
1861
1862                         if (nb_tx_q != IXGBE_VMDQ_DCB_NB_QUEUES) {
1863                                 PMD_INIT_LOG(ERR, "VMDQ+DCB, nb_tx_q != %d",
1864                                                  IXGBE_VMDQ_DCB_NB_QUEUES);
1865                                 return -EINVAL;
1866                         }
1867                         conf = &dev_conf->tx_adv_conf.vmdq_dcb_tx_conf;
1868                         if (!(conf->nb_queue_pools == ETH_16_POOLS ||
1869                                conf->nb_queue_pools == ETH_32_POOLS)) {
1870                                 PMD_INIT_LOG(ERR, "VMDQ+DCB selected,"
1871                                                 " nb_queue_pools != %d and"
1872                                                 " nb_queue_pools != %d.",
1873                                                 ETH_16_POOLS, ETH_32_POOLS);
1874                                 return -EINVAL;
1875                         }
1876                 }
1877
1878                 /* For DCB mode check our configuration before we go further */
1879                 if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_DCB) {
1880                         const struct rte_eth_dcb_rx_conf *conf;
1881
1882                         if (nb_rx_q != IXGBE_DCB_NB_QUEUES) {
1883                                 PMD_INIT_LOG(ERR, "DCB selected, nb_rx_q != %d.",
1884                                                  IXGBE_DCB_NB_QUEUES);
1885                                 return -EINVAL;
1886                         }
1887                         conf = &dev_conf->rx_adv_conf.dcb_rx_conf;
1888                         if (!(conf->nb_tcs == ETH_4_TCS ||
1889                                conf->nb_tcs == ETH_8_TCS)) {
1890                                 PMD_INIT_LOG(ERR, "DCB selected, nb_tcs != %d"
1891                                                 " and nb_tcs != %d.",
1892                                                 ETH_4_TCS, ETH_8_TCS);
1893                                 return -EINVAL;
1894                         }
1895                 }
1896
1897                 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_DCB) {
1898                         const struct rte_eth_dcb_tx_conf *conf;
1899
1900                         if (nb_tx_q != IXGBE_DCB_NB_QUEUES) {
1901                                 PMD_INIT_LOG(ERR, "DCB, nb_tx_q != %d.",
1902                                                  IXGBE_DCB_NB_QUEUES);
1903                                 return -EINVAL;
1904                         }
1905                         conf = &dev_conf->tx_adv_conf.dcb_tx_conf;
1906                         if (!(conf->nb_tcs == ETH_4_TCS ||
1907                                conf->nb_tcs == ETH_8_TCS)) {
1908                                 PMD_INIT_LOG(ERR, "DCB selected, nb_tcs != %d"
1909                                                 " and nb_tcs != %d.",
1910                                                 ETH_4_TCS, ETH_8_TCS);
1911                                 return -EINVAL;
1912                         }
1913                 }
1914         }
1915         return 0;
1916 }
1917
1918 static int
1919 ixgbe_dev_configure(struct rte_eth_dev *dev)
1920 {
1921         struct ixgbe_interrupt *intr =
1922                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1923         struct ixgbe_adapter *adapter =
1924                 (struct ixgbe_adapter *)dev->data->dev_private;
1925         int ret;
1926
1927         PMD_INIT_FUNC_TRACE();
1928         /* multipe queue mode checking */
1929         ret  = ixgbe_check_mq_mode(dev);
1930         if (ret != 0) {
1931                 PMD_DRV_LOG(ERR, "ixgbe_check_mq_mode fails with %d.",
1932                             ret);
1933                 return ret;
1934         }
1935
1936         /* set flag to update link status after init */
1937         intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1938
1939         /*
1940          * Initialize to TRUE. If any of Rx queues doesn't meet the bulk
1941          * allocation or vector Rx preconditions we will reset it.
1942          */
1943         adapter->rx_bulk_alloc_allowed = true;
1944         adapter->rx_vec_allowed = true;
1945
1946         return 0;
1947 }
1948
1949 /*
1950  * Configure device link speed and setup link.
1951  * It returns 0 on success.
1952  */
1953 static int
1954 ixgbe_dev_start(struct rte_eth_dev *dev)
1955 {
1956         struct ixgbe_hw *hw =
1957                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1958         struct ixgbe_vf_info *vfinfo =
1959                 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private);
1960         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1961         uint32_t intr_vector = 0;
1962         int err, link_up = 0, negotiate = 0;
1963         uint32_t speed = 0;
1964         int mask = 0;
1965         int status;
1966         uint16_t vf, idx;
1967
1968         PMD_INIT_FUNC_TRACE();
1969
1970         /* IXGBE devices don't support half duplex */
1971         if ((dev->data->dev_conf.link_duplex != ETH_LINK_AUTONEG_DUPLEX) &&
1972                         (dev->data->dev_conf.link_duplex != ETH_LINK_FULL_DUPLEX)) {
1973                 PMD_INIT_LOG(ERR, "Invalid link_duplex (%hu) for port %hhu",
1974                              dev->data->dev_conf.link_duplex,
1975                              dev->data->port_id);
1976                 return -EINVAL;
1977         }
1978
1979         /* stop adapter */
1980         hw->adapter_stopped = 0;
1981         ixgbe_stop_adapter(hw);
1982
1983         /* reinitialize adapter
1984          * this calls reset and start */
1985         status = ixgbe_pf_reset_hw(hw);
1986         if (status != 0)
1987                 return -1;
1988         hw->mac.ops.start_hw(hw);
1989         hw->mac.get_link_status = true;
1990
1991         /* configure PF module if SRIOV enabled */
1992         ixgbe_pf_host_configure(dev);
1993
1994         /* check and configure queue intr-vector mapping */
1995         if ((rte_intr_cap_multiple(intr_handle) ||
1996              !RTE_ETH_DEV_SRIOV(dev).active) &&
1997             dev->data->dev_conf.intr_conf.rxq != 0) {
1998                 intr_vector = dev->data->nb_rx_queues;
1999                 if (rte_intr_efd_enable(intr_handle, intr_vector))
2000                         return -1;
2001         }
2002
2003         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
2004                 intr_handle->intr_vec =
2005                         rte_zmalloc("intr_vec",
2006                                     dev->data->nb_rx_queues * sizeof(int), 0);
2007                 if (intr_handle->intr_vec == NULL) {
2008                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2009                                      " intr_vec\n", dev->data->nb_rx_queues);
2010                         return -ENOMEM;
2011                 }
2012         }
2013
2014         /* confiugre msix for sleep until rx interrupt */
2015         ixgbe_configure_msix(dev);
2016
2017         /* initialize transmission unit */
2018         ixgbe_dev_tx_init(dev);
2019
2020         /* This can fail when allocating mbufs for descriptor rings */
2021         err = ixgbe_dev_rx_init(dev);
2022         if (err) {
2023                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
2024                 goto error;
2025         }
2026
2027         err = ixgbe_dev_rxtx_start(dev);
2028         if (err < 0) {
2029                 PMD_INIT_LOG(ERR, "Unable to start rxtx queues");
2030                 goto error;
2031         }
2032
2033         /* Skip link setup if loopback mode is enabled for 82599. */
2034         if (hw->mac.type == ixgbe_mac_82599EB &&
2035                         dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_82599_TX_RX)
2036                 goto skip_link_setup;
2037
2038         if (ixgbe_is_sfp(hw) && hw->phy.multispeed_fiber) {
2039                 err = hw->mac.ops.setup_sfp(hw);
2040                 if (err)
2041                         goto error;
2042         }
2043
2044         if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
2045                 /* Turn on the copper */
2046                 ixgbe_set_phy_power(hw, true);
2047         } else {
2048                 /* Turn on the laser */
2049                 ixgbe_enable_tx_laser(hw);
2050         }
2051
2052         err = ixgbe_check_link(hw, &speed, &link_up, 0);
2053         if (err)
2054                 goto error;
2055         dev->data->dev_link.link_status = link_up;
2056
2057         err = ixgbe_get_link_capabilities(hw, &speed, &negotiate);
2058         if (err)
2059                 goto error;
2060
2061         switch(dev->data->dev_conf.link_speed) {
2062         case ETH_LINK_SPEED_AUTONEG:
2063                 speed = (hw->mac.type != ixgbe_mac_82598EB) ?
2064                                 IXGBE_LINK_SPEED_82599_AUTONEG :
2065                                 IXGBE_LINK_SPEED_82598_AUTONEG;
2066                 break;
2067         case ETH_LINK_SPEED_100:
2068                 /*
2069                  * Invalid for 82598 but error will be detected by
2070                  * ixgbe_setup_link()
2071                  */
2072                 speed = IXGBE_LINK_SPEED_100_FULL;
2073                 break;
2074         case ETH_LINK_SPEED_1000:
2075                 speed = IXGBE_LINK_SPEED_1GB_FULL;
2076                 break;
2077         case ETH_LINK_SPEED_10000:
2078                 speed = IXGBE_LINK_SPEED_10GB_FULL;
2079                 break;
2080         default:
2081                 PMD_INIT_LOG(ERR, "Invalid link_speed (%hu) for port %hhu",
2082                              dev->data->dev_conf.link_speed,
2083                              dev->data->port_id);
2084                 goto error;
2085         }
2086
2087         err = ixgbe_setup_link(hw, speed, link_up);
2088         if (err)
2089                 goto error;
2090
2091 skip_link_setup:
2092
2093         if (rte_intr_allow_others(intr_handle)) {
2094                 /* check if lsc interrupt is enabled */
2095                 if (dev->data->dev_conf.intr_conf.lsc != 0)
2096                         ixgbe_dev_lsc_interrupt_setup(dev);
2097         } else {
2098                 rte_intr_callback_unregister(intr_handle,
2099                                              ixgbe_dev_interrupt_handler,
2100                                              (void *)dev);
2101                 if (dev->data->dev_conf.intr_conf.lsc != 0)
2102                         PMD_INIT_LOG(INFO, "lsc won't enable because of"
2103                                      " no intr multiplex\n");
2104         }
2105
2106         /* check if rxq interrupt is enabled */
2107         if (dev->data->dev_conf.intr_conf.rxq != 0 &&
2108             rte_intr_dp_is_en(intr_handle))
2109                 ixgbe_dev_rxq_interrupt_setup(dev);
2110
2111         /* enable uio/vfio intr/eventfd mapping */
2112         rte_intr_enable(intr_handle);
2113
2114         /* resume enabled intr since hw reset */
2115         ixgbe_enable_intr(dev);
2116
2117         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
2118                 ETH_VLAN_EXTEND_MASK;
2119         ixgbe_vlan_offload_set(dev, mask);
2120
2121         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
2122                 /* Enable vlan filtering for VMDq */
2123                 ixgbe_vmdq_vlan_hw_filter_enable(dev);
2124         }
2125
2126         /* Configure DCB hw */
2127         ixgbe_configure_dcb(dev);
2128
2129         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) {
2130                 err = ixgbe_fdir_configure(dev);
2131                 if (err)
2132                         goto error;
2133         }
2134
2135         /* Restore vf rate limit */
2136         if (vfinfo != NULL) {
2137                 for (vf = 0; vf < dev->pci_dev->max_vfs; vf++)
2138                         for (idx = 0; idx < IXGBE_MAX_QUEUE_NUM_PER_VF; idx++)
2139                                 if (vfinfo[vf].tx_rate[idx] != 0)
2140                                         ixgbe_set_vf_rate_limit(dev, vf,
2141                                                 vfinfo[vf].tx_rate[idx],
2142                                                 1 << idx);
2143         }
2144
2145         ixgbe_restore_statistics_mapping(dev);
2146
2147         return (0);
2148
2149 error:
2150         PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err);
2151         ixgbe_dev_clear_queues(dev);
2152         return -EIO;
2153 }
2154
2155 /*
2156  * Stop device: disable rx and tx functions to allow for reconfiguring.
2157  */
2158 static void
2159 ixgbe_dev_stop(struct rte_eth_dev *dev)
2160 {
2161         struct rte_eth_link link;
2162         struct ixgbe_hw *hw =
2163                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2164         struct ixgbe_vf_info *vfinfo =
2165                 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private);
2166         struct ixgbe_filter_info *filter_info =
2167                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
2168         struct ixgbe_5tuple_filter *p_5tuple, *p_5tuple_next;
2169         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
2170         int vf;
2171
2172         PMD_INIT_FUNC_TRACE();
2173
2174         /* disable interrupts */
2175         ixgbe_disable_intr(hw);
2176
2177         /* disable intr eventfd mapping */
2178         rte_intr_disable(intr_handle);
2179
2180         /* reset the NIC */
2181         ixgbe_pf_reset_hw(hw);
2182         hw->adapter_stopped = 0;
2183
2184         /* stop adapter */
2185         ixgbe_stop_adapter(hw);
2186
2187         for (vf = 0; vfinfo != NULL &&
2188                      vf < dev->pci_dev->max_vfs; vf++)
2189                 vfinfo[vf].clear_to_send = false;
2190
2191         if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
2192                 /* Turn off the copper */
2193                 ixgbe_set_phy_power(hw, false);
2194         } else {
2195                 /* Turn off the laser */
2196                 ixgbe_disable_tx_laser(hw);
2197         }
2198
2199         ixgbe_dev_clear_queues(dev);
2200
2201         /* Clear stored conf */
2202         dev->data->scattered_rx = 0;
2203         dev->data->lro = 0;
2204
2205         /* Clear recorded link status */
2206         memset(&link, 0, sizeof(link));
2207         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
2208
2209         /* Remove all ntuple filters of the device */
2210         for (p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list);
2211              p_5tuple != NULL; p_5tuple = p_5tuple_next) {
2212                 p_5tuple_next = TAILQ_NEXT(p_5tuple, entries);
2213                 TAILQ_REMOVE(&filter_info->fivetuple_list,
2214                              p_5tuple, entries);
2215                 rte_free(p_5tuple);
2216         }
2217         memset(filter_info->fivetuple_mask, 0,
2218                 sizeof(uint32_t) * IXGBE_5TUPLE_ARRAY_SIZE);
2219
2220         if (!rte_intr_allow_others(intr_handle))
2221                 /* resume to the default handler */
2222                 rte_intr_callback_register(intr_handle,
2223                                            ixgbe_dev_interrupt_handler,
2224                                            (void *)dev);
2225
2226         /* Clean datapath event and queue/vec mapping */
2227         rte_intr_efd_disable(intr_handle);
2228         if (intr_handle->intr_vec != NULL) {
2229                 rte_free(intr_handle->intr_vec);
2230                 intr_handle->intr_vec = NULL;
2231         }
2232 }
2233
2234 /*
2235  * Set device link up: enable tx.
2236  */
2237 static int
2238 ixgbe_dev_set_link_up(struct rte_eth_dev *dev)
2239 {
2240         struct ixgbe_hw *hw =
2241                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2242         if (hw->mac.type == ixgbe_mac_82599EB) {
2243 #ifdef RTE_NIC_BYPASS
2244                 if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) {
2245                         /* Not suported in bypass mode */
2246                         PMD_INIT_LOG(ERR, "Set link up is not supported "
2247                                      "by device id 0x%x", hw->device_id);
2248                         return -ENOTSUP;
2249                 }
2250 #endif
2251         }
2252
2253         if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
2254                 /* Turn on the copper */
2255                 ixgbe_set_phy_power(hw, true);
2256         } else {
2257                 /* Turn on the laser */
2258                 ixgbe_enable_tx_laser(hw);
2259         }
2260
2261         return 0;
2262 }
2263
2264 /*
2265  * Set device link down: disable tx.
2266  */
2267 static int
2268 ixgbe_dev_set_link_down(struct rte_eth_dev *dev)
2269 {
2270         struct ixgbe_hw *hw =
2271                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2272         if (hw->mac.type == ixgbe_mac_82599EB) {
2273 #ifdef RTE_NIC_BYPASS
2274                 if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) {
2275                         /* Not suported in bypass mode */
2276                         PMD_INIT_LOG(ERR, "Set link down is not supported "
2277                                      "by device id 0x%x", hw->device_id);
2278                         return -ENOTSUP;
2279                 }
2280 #endif
2281         }
2282
2283         if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
2284                 /* Turn off the copper */
2285                 ixgbe_set_phy_power(hw, false);
2286         } else {
2287                 /* Turn off the laser */
2288                 ixgbe_disable_tx_laser(hw);
2289         }
2290
2291         return 0;
2292 }
2293
2294 /*
2295  * Reest and stop device.
2296  */
2297 static void
2298 ixgbe_dev_close(struct rte_eth_dev *dev)
2299 {
2300         struct ixgbe_hw *hw =
2301                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2302
2303         PMD_INIT_FUNC_TRACE();
2304
2305         ixgbe_pf_reset_hw(hw);
2306
2307         ixgbe_dev_stop(dev);
2308         hw->adapter_stopped = 1;
2309
2310         ixgbe_dev_free_queues(dev);
2311
2312         ixgbe_disable_pcie_master(hw);
2313
2314         /* reprogram the RAR[0] in case user changed it. */
2315         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
2316 }
2317
2318 static void
2319 ixgbe_read_stats_registers(struct ixgbe_hw *hw, struct ixgbe_hw_stats
2320                                                    *hw_stats, uint64_t *total_missed_rx,
2321                                                    uint64_t *total_qbrc, uint64_t *total_qprc,
2322                                                    uint64_t *total_qprdc)
2323 {
2324         uint32_t bprc, lxon, lxoff, total;
2325         unsigned i;
2326
2327         hw_stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
2328         hw_stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
2329         hw_stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
2330         hw_stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
2331
2332         for (i = 0; i < 8; i++) {
2333                 uint32_t mp;
2334                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
2335                 /* global total per queue */
2336                 hw_stats->mpc[i] += mp;
2337                 /* Running comprehensive total for stats display */
2338                 *total_missed_rx += hw_stats->mpc[i];
2339                 if (hw->mac.type == ixgbe_mac_82598EB) {
2340                         hw_stats->rnbc[i] +=
2341                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
2342                         hw_stats->pxonrxc[i] +=
2343                                 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
2344                         hw_stats->pxoffrxc[i] +=
2345                                 IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
2346                 } else {
2347                         hw_stats->pxonrxc[i] +=
2348                                 IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
2349                         hw_stats->pxoffrxc[i] +=
2350                                 IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i));
2351                         hw_stats->pxon2offc[i] +=
2352                                 IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
2353                 }
2354                 hw_stats->pxontxc[i] +=
2355                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
2356                 hw_stats->pxofftxc[i] +=
2357                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
2358         }
2359         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
2360                 hw_stats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
2361                 hw_stats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
2362                 hw_stats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
2363                 hw_stats->qbrc[i] +=
2364                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32);
2365                 hw_stats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
2366                 hw_stats->qbtc[i] +=
2367                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32);
2368                 *total_qprdc += hw_stats->qprdc[i] +=
2369                                 IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
2370
2371                 *total_qprc += hw_stats->qprc[i];
2372                 *total_qbrc += hw_stats->qbrc[i];
2373         }
2374         hw_stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
2375         hw_stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
2376         hw_stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
2377
2378         /* Note that gprc counts missed packets */
2379         hw_stats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
2380
2381         if (hw->mac.type != ixgbe_mac_82598EB) {
2382                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
2383                 hw_stats->gorc += ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
2384                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
2385                 hw_stats->gotc += ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
2386                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
2387                 hw_stats->tor += ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
2388                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
2389                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
2390         } else {
2391                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
2392                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
2393                 /* 82598 only has a counter in the high register */
2394                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
2395                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
2396                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
2397         }
2398
2399         /*
2400          * Workaround: mprc hardware is incorrectly counting
2401          * broadcasts, so for now we subtract those.
2402          */
2403         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
2404         hw_stats->bprc += bprc;
2405         hw_stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
2406         if (hw->mac.type == ixgbe_mac_82598EB)
2407                 hw_stats->mprc -= bprc;
2408
2409         hw_stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
2410         hw_stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
2411         hw_stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
2412         hw_stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
2413         hw_stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
2414         hw_stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
2415
2416         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
2417         hw_stats->lxontxc += lxon;
2418         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
2419         hw_stats->lxofftxc += lxoff;
2420         total = lxon + lxoff;
2421
2422         hw_stats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
2423         hw_stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
2424         hw_stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
2425         hw_stats->gptc -= total;
2426         hw_stats->mptc -= total;
2427         hw_stats->ptc64 -= total;
2428         hw_stats->gotc -= total * ETHER_MIN_LEN;
2429
2430         hw_stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
2431         hw_stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
2432         hw_stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
2433         hw_stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
2434         hw_stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
2435         hw_stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
2436         hw_stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
2437         hw_stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
2438         hw_stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
2439         hw_stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
2440         hw_stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
2441         hw_stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
2442         hw_stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
2443         hw_stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
2444         hw_stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
2445         hw_stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);
2446         hw_stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
2447         hw_stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
2448         /* Only read FCOE on 82599 */
2449         if (hw->mac.type != ixgbe_mac_82598EB) {
2450                 hw_stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
2451                 hw_stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
2452                 hw_stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
2453                 hw_stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
2454                 hw_stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
2455         }
2456
2457         /* Flow Director Stats registers */
2458         hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
2459         hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
2460 }
2461
2462 /*
2463  * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c
2464  */
2465 static void
2466 ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
2467 {
2468         struct ixgbe_hw *hw =
2469                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2470         struct ixgbe_hw_stats *hw_stats =
2471                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2472         uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc;
2473         unsigned i;
2474
2475         total_missed_rx = 0;
2476         total_qbrc = 0;
2477         total_qprc = 0;
2478         total_qprdc = 0;
2479
2480         ixgbe_read_stats_registers(hw, hw_stats, &total_missed_rx, &total_qbrc,
2481                         &total_qprc, &total_qprdc);
2482
2483         if (stats == NULL)
2484                 return;
2485
2486         /* Fill out the rte_eth_stats statistics structure */
2487         stats->ipackets = total_qprc;
2488         stats->ibytes = total_qbrc;
2489         stats->opackets = hw_stats->gptc;
2490         stats->obytes = hw_stats->gotc;
2491
2492         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
2493                 stats->q_ipackets[i] = hw_stats->qprc[i];
2494                 stats->q_opackets[i] = hw_stats->qptc[i];
2495                 stats->q_ibytes[i] = hw_stats->qbrc[i];
2496                 stats->q_obytes[i] = hw_stats->qbtc[i];
2497                 stats->q_errors[i] = hw_stats->qprdc[i];
2498         }
2499
2500         /* Rx Errors */
2501         stats->ierrors  = hw_stats->crcerrs +
2502                           hw_stats->mspdc +
2503                           hw_stats->rlec +
2504                           hw_stats->ruc +
2505                           hw_stats->roc +
2506                           total_missed_rx +
2507                           hw_stats->illerrc +
2508                           hw_stats->errbc +
2509                           hw_stats->rfc +
2510                           hw_stats->fccrc +
2511                           hw_stats->fclast;
2512
2513         /* Tx Errors */
2514         stats->oerrors  = 0;
2515 }
2516
2517 static void
2518 ixgbe_dev_stats_reset(struct rte_eth_dev *dev)
2519 {
2520         struct ixgbe_hw_stats *stats =
2521                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2522
2523         /* HW registers are cleared on read */
2524         ixgbe_dev_stats_get(dev, NULL);
2525
2526         /* Reset software totals */
2527         memset(stats, 0, sizeof(*stats));
2528 }
2529
2530 /* This function calculates the number of xstats based on the current config */
2531 static unsigned
2532 ixgbe_xstats_calc_num(void) {
2533         return IXGBE_NB_HW_STATS + (IXGBE_NB_RXQ_PRIO_STATS * 8) +
2534                 (IXGBE_NB_TXQ_PRIO_STATS * 8);
2535 }
2536
2537 static int
2538 ixgbe_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstats *xstats,
2539                                          unsigned n)
2540 {
2541         struct ixgbe_hw *hw =
2542                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2543         struct ixgbe_hw_stats *hw_stats =
2544                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2545         uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc;
2546         unsigned i, stat, count = 0;
2547
2548         count = ixgbe_xstats_calc_num();
2549
2550         if (n < count)
2551                 return count;
2552
2553         total_missed_rx = 0;
2554         total_qbrc = 0;
2555         total_qprc = 0;
2556         total_qprdc = 0;
2557
2558         ixgbe_read_stats_registers(hw, hw_stats, &total_missed_rx, &total_qbrc,
2559                                    &total_qprc, &total_qprdc);
2560
2561         /* If this is a reset xstats is NULL, and we have cleared the
2562          * registers by reading them.
2563          */
2564         if (!xstats)
2565                 return 0;
2566
2567         /* Extended stats from ixgbe_hw_stats */
2568         count = 0;
2569         for (i = 0; i < IXGBE_NB_HW_STATS; i++) {
2570                 snprintf(xstats[count].name, sizeof(xstats[count].name), "%s",
2571                          rte_ixgbe_stats_strings[i].name);
2572                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
2573                                 rte_ixgbe_stats_strings[i].offset);
2574                 count++;
2575         }
2576
2577         /* RX Priority Stats */
2578         for (stat = 0; stat < IXGBE_NB_RXQ_PRIO_STATS; stat++) {
2579                 for (i = 0; i < 8; i++) {
2580                         snprintf(xstats[count].name, sizeof(xstats[count].name),
2581                                  "rx_priority%u_%s", i,
2582                                  rte_ixgbe_rxq_strings[stat].name);
2583                         xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
2584                                         rte_ixgbe_rxq_strings[stat].offset +
2585                                         (sizeof(uint64_t) * i));
2586                         count++;
2587                 }
2588         }
2589
2590         /* TX Priority Stats */
2591         for (stat = 0; stat < IXGBE_NB_TXQ_PRIO_STATS; stat++) {
2592                 for (i = 0; i < 8; i++) {
2593                         snprintf(xstats[count].name, sizeof(xstats[count].name),
2594                                  "tx_priority%u_%s", i,
2595                                  rte_ixgbe_txq_strings[stat].name);
2596                         xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
2597                                         rte_ixgbe_txq_strings[stat].offset +
2598                                         (sizeof(uint64_t) * i));
2599                         count++;
2600                 }
2601         }
2602
2603         return count;
2604 }
2605
2606 static void
2607 ixgbe_dev_xstats_reset(struct rte_eth_dev *dev)
2608 {
2609         struct ixgbe_hw_stats *stats =
2610                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2611
2612         unsigned count = ixgbe_xstats_calc_num();
2613
2614         /* HW registers are cleared on read */
2615         ixgbe_dev_xstats_get(dev, NULL, count);
2616
2617         /* Reset software totals */
2618         memset(stats, 0, sizeof(*stats));
2619 }
2620
2621 static void
2622 ixgbevf_update_stats(struct rte_eth_dev *dev)
2623 {
2624         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2625         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
2626                           IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2627
2628         /* Good Rx packet, include VF loopback */
2629         UPDATE_VF_STAT(IXGBE_VFGPRC,
2630             hw_stats->last_vfgprc, hw_stats->vfgprc);
2631
2632         /* Good Rx octets, include VF loopback */
2633         UPDATE_VF_STAT_36BIT(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
2634             hw_stats->last_vfgorc, hw_stats->vfgorc);
2635
2636         /* Good Tx packet, include VF loopback */
2637         UPDATE_VF_STAT(IXGBE_VFGPTC,
2638             hw_stats->last_vfgptc, hw_stats->vfgptc);
2639
2640         /* Good Tx octets, include VF loopback */
2641         UPDATE_VF_STAT_36BIT(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
2642             hw_stats->last_vfgotc, hw_stats->vfgotc);
2643
2644         /* Rx Multicst Packet */
2645         UPDATE_VF_STAT(IXGBE_VFMPRC,
2646             hw_stats->last_vfmprc, hw_stats->vfmprc);
2647 }
2648
2649 static int
2650 ixgbevf_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstats *xstats,
2651                        unsigned n)
2652 {
2653         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *)
2654                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2655         unsigned i;
2656
2657         if (n < IXGBEVF_NB_XSTATS)
2658                 return IXGBEVF_NB_XSTATS;
2659
2660         ixgbevf_update_stats(dev);
2661
2662         if (!xstats)
2663                 return 0;
2664
2665         /* Extended stats */
2666         for (i = 0; i < IXGBEVF_NB_XSTATS; i++) {
2667                 snprintf(xstats[i].name, sizeof(xstats[i].name),
2668                          "%s", rte_ixgbevf_stats_strings[i].name);
2669                 xstats[i].value = *(uint64_t *)(((char *)hw_stats) +
2670                         rte_ixgbevf_stats_strings[i].offset);
2671         }
2672
2673         return IXGBEVF_NB_XSTATS;
2674 }
2675
2676 static void
2677 ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
2678 {
2679         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *)
2680                           IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2681
2682         ixgbevf_update_stats(dev);
2683
2684         if (stats == NULL)
2685                 return;
2686
2687         stats->ipackets = hw_stats->vfgprc;
2688         stats->ibytes = hw_stats->vfgorc;
2689         stats->opackets = hw_stats->vfgptc;
2690         stats->obytes = hw_stats->vfgotc;
2691         stats->imcasts = hw_stats->vfmprc;
2692         /* stats->imcasts should be removed as imcasts is deprecated */
2693 }
2694
2695 static void
2696 ixgbevf_dev_stats_reset(struct rte_eth_dev *dev)
2697 {
2698         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
2699                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2700
2701         /* Sync HW register to the last stats */
2702         ixgbevf_dev_stats_get(dev, NULL);
2703
2704         /* reset HW current stats*/
2705         hw_stats->vfgprc = 0;
2706         hw_stats->vfgorc = 0;
2707         hw_stats->vfgptc = 0;
2708         hw_stats->vfgotc = 0;
2709         hw_stats->vfmprc = 0;
2710
2711 }
2712
2713 static void
2714 ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
2715 {
2716         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2717
2718         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
2719         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
2720         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL register */
2721         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */
2722         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
2723         dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC;
2724         dev_info->max_vfs = dev->pci_dev->max_vfs;
2725         if (hw->mac.type == ixgbe_mac_82598EB)
2726                 dev_info->max_vmdq_pools = ETH_16_POOLS;
2727         else
2728                 dev_info->max_vmdq_pools = ETH_64_POOLS;
2729         dev_info->vmdq_queue_num = dev_info->max_rx_queues;
2730         dev_info->rx_offload_capa =
2731                 DEV_RX_OFFLOAD_VLAN_STRIP |
2732                 DEV_RX_OFFLOAD_IPV4_CKSUM |
2733                 DEV_RX_OFFLOAD_UDP_CKSUM  |
2734                 DEV_RX_OFFLOAD_TCP_CKSUM;
2735
2736         /*
2737          * RSC is only supported by 82599 and x540 PF devices in a non-SR-IOV
2738          * mode.
2739          */
2740         if ((hw->mac.type == ixgbe_mac_82599EB ||
2741              hw->mac.type == ixgbe_mac_X540) &&
2742             !RTE_ETH_DEV_SRIOV(dev).active)
2743                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_LRO;
2744
2745         dev_info->tx_offload_capa =
2746                 DEV_TX_OFFLOAD_VLAN_INSERT |
2747                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
2748                 DEV_TX_OFFLOAD_UDP_CKSUM   |
2749                 DEV_TX_OFFLOAD_TCP_CKSUM   |
2750                 DEV_TX_OFFLOAD_SCTP_CKSUM  |
2751                 DEV_TX_OFFLOAD_TCP_TSO;
2752
2753         dev_info->default_rxconf = (struct rte_eth_rxconf) {
2754                 .rx_thresh = {
2755                         .pthresh = IXGBE_DEFAULT_RX_PTHRESH,
2756                         .hthresh = IXGBE_DEFAULT_RX_HTHRESH,
2757                         .wthresh = IXGBE_DEFAULT_RX_WTHRESH,
2758                 },
2759                 .rx_free_thresh = IXGBE_DEFAULT_RX_FREE_THRESH,
2760                 .rx_drop_en = 0,
2761         };
2762
2763         dev_info->default_txconf = (struct rte_eth_txconf) {
2764                 .tx_thresh = {
2765                         .pthresh = IXGBE_DEFAULT_TX_PTHRESH,
2766                         .hthresh = IXGBE_DEFAULT_TX_HTHRESH,
2767                         .wthresh = IXGBE_DEFAULT_TX_WTHRESH,
2768                 },
2769                 .tx_free_thresh = IXGBE_DEFAULT_TX_FREE_THRESH,
2770                 .tx_rs_thresh = IXGBE_DEFAULT_TX_RSBIT_THRESH,
2771                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
2772                                 ETH_TXQ_FLAGS_NOOFFLOADS,
2773         };
2774
2775         dev_info->rx_desc_lim = rx_desc_lim;
2776         dev_info->tx_desc_lim = tx_desc_lim;
2777
2778         dev_info->hash_key_size = IXGBE_HKEY_MAX_INDEX * sizeof(uint32_t);
2779         dev_info->reta_size = ixgbe_reta_size_get(hw->mac.type);
2780         dev_info->flow_type_rss_offloads = IXGBE_RSS_OFFLOAD_ALL;
2781 }
2782
2783 static void
2784 ixgbevf_dev_info_get(struct rte_eth_dev *dev,
2785                      struct rte_eth_dev_info *dev_info)
2786 {
2787         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2788
2789         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
2790         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
2791         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL reg */
2792         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS reg */
2793         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
2794         dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC;
2795         dev_info->max_vfs = dev->pci_dev->max_vfs;
2796         if (hw->mac.type == ixgbe_mac_82598EB)
2797                 dev_info->max_vmdq_pools = ETH_16_POOLS;
2798         else
2799                 dev_info->max_vmdq_pools = ETH_64_POOLS;
2800         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
2801                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
2802                                 DEV_RX_OFFLOAD_UDP_CKSUM  |
2803                                 DEV_RX_OFFLOAD_TCP_CKSUM;
2804         dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
2805                                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
2806                                 DEV_TX_OFFLOAD_UDP_CKSUM   |
2807                                 DEV_TX_OFFLOAD_TCP_CKSUM   |
2808                                 DEV_TX_OFFLOAD_SCTP_CKSUM  |
2809                                 DEV_TX_OFFLOAD_TCP_TSO;
2810
2811         dev_info->default_rxconf = (struct rte_eth_rxconf) {
2812                 .rx_thresh = {
2813                         .pthresh = IXGBE_DEFAULT_RX_PTHRESH,
2814                         .hthresh = IXGBE_DEFAULT_RX_HTHRESH,
2815                         .wthresh = IXGBE_DEFAULT_RX_WTHRESH,
2816                 },
2817                 .rx_free_thresh = IXGBE_DEFAULT_RX_FREE_THRESH,
2818                 .rx_drop_en = 0,
2819         };
2820
2821         dev_info->default_txconf = (struct rte_eth_txconf) {
2822                 .tx_thresh = {
2823                         .pthresh = IXGBE_DEFAULT_TX_PTHRESH,
2824                         .hthresh = IXGBE_DEFAULT_TX_HTHRESH,
2825                         .wthresh = IXGBE_DEFAULT_TX_WTHRESH,
2826                 },
2827                 .tx_free_thresh = IXGBE_DEFAULT_TX_FREE_THRESH,
2828                 .tx_rs_thresh = IXGBE_DEFAULT_TX_RSBIT_THRESH,
2829                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
2830                                 ETH_TXQ_FLAGS_NOOFFLOADS,
2831         };
2832
2833         dev_info->rx_desc_lim = rx_desc_lim;
2834         dev_info->tx_desc_lim = tx_desc_lim;
2835 }
2836
2837 /* return 0 means link status changed, -1 means not changed */
2838 static int
2839 ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
2840 {
2841         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2842         struct rte_eth_link link, old;
2843         ixgbe_link_speed link_speed = IXGBE_LINK_SPEED_UNKNOWN;
2844         int link_up;
2845         int diag;
2846
2847         link.link_status = 0;
2848         link.link_speed = 0;
2849         link.link_duplex = 0;
2850         memset(&old, 0, sizeof(old));
2851         rte_ixgbe_dev_atomic_read_link_status(dev, &old);
2852
2853         hw->mac.get_link_status = true;
2854
2855         /* check if it needs to wait to complete, if lsc interrupt is enabled */
2856         if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
2857                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 0);
2858         else
2859                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 1);
2860
2861         if (diag != 0) {
2862                 link.link_speed = ETH_LINK_SPEED_100;
2863                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
2864                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
2865                 if (link.link_status == old.link_status)
2866                         return -1;
2867                 return 0;
2868         }
2869
2870         if (link_up == 0) {
2871                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
2872                 if (link.link_status == old.link_status)
2873                         return -1;
2874                 return 0;
2875         }
2876         link.link_status = 1;
2877         link.link_duplex = ETH_LINK_FULL_DUPLEX;
2878
2879         switch (link_speed) {
2880         default:
2881         case IXGBE_LINK_SPEED_UNKNOWN:
2882                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
2883                 link.link_speed = ETH_LINK_SPEED_100;
2884                 break;
2885
2886         case IXGBE_LINK_SPEED_100_FULL:
2887                 link.link_speed = ETH_LINK_SPEED_100;
2888                 break;
2889
2890         case IXGBE_LINK_SPEED_1GB_FULL:
2891                 link.link_speed = ETH_LINK_SPEED_1000;
2892                 break;
2893
2894         case IXGBE_LINK_SPEED_10GB_FULL:
2895                 link.link_speed = ETH_LINK_SPEED_10000;
2896                 break;
2897         }
2898         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
2899
2900         if (link.link_status == old.link_status)
2901                 return -1;
2902
2903         return 0;
2904 }
2905
2906 static void
2907 ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev)
2908 {
2909         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2910         uint32_t fctrl;
2911
2912         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2913         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
2914         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2915 }
2916
2917 static void
2918 ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev)
2919 {
2920         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2921         uint32_t fctrl;
2922
2923         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2924         fctrl &= (~IXGBE_FCTRL_UPE);
2925         if (dev->data->all_multicast == 1)
2926                 fctrl |= IXGBE_FCTRL_MPE;
2927         else
2928                 fctrl &= (~IXGBE_FCTRL_MPE);
2929         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2930 }
2931
2932 static void
2933 ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev)
2934 {
2935         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2936         uint32_t fctrl;
2937
2938         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2939         fctrl |= IXGBE_FCTRL_MPE;
2940         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2941 }
2942
2943 static void
2944 ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev)
2945 {
2946         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2947         uint32_t fctrl;
2948
2949         if (dev->data->promiscuous == 1)
2950                 return; /* must remain in all_multicast mode */
2951
2952         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2953         fctrl &= (~IXGBE_FCTRL_MPE);
2954         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2955 }
2956
2957 /**
2958  * It clears the interrupt causes and enables the interrupt.
2959  * It will be called once only during nic initialized.
2960  *
2961  * @param dev
2962  *  Pointer to struct rte_eth_dev.
2963  *
2964  * @return
2965  *  - On success, zero.
2966  *  - On failure, a negative value.
2967  */
2968 static int
2969 ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev)
2970 {
2971         struct ixgbe_interrupt *intr =
2972                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2973
2974         ixgbe_dev_link_status_print(dev);
2975         intr->mask |= IXGBE_EICR_LSC;
2976
2977         return 0;
2978 }
2979
2980 /**
2981  * It clears the interrupt causes and enables the interrupt.
2982  * It will be called once only during nic initialized.
2983  *
2984  * @param dev
2985  *  Pointer to struct rte_eth_dev.
2986  *
2987  * @return
2988  *  - On success, zero.
2989  *  - On failure, a negative value.
2990  */
2991 static int
2992 ixgbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2993 {
2994         struct ixgbe_interrupt *intr =
2995                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2996
2997         intr->mask |= IXGBE_EICR_RTX_QUEUE;
2998
2999         return 0;
3000 }
3001
3002 /*
3003  * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update.
3004  *
3005  * @param dev
3006  *  Pointer to struct rte_eth_dev.
3007  *
3008  * @return
3009  *  - On success, zero.
3010  *  - On failure, a negative value.
3011  */
3012 static int
3013 ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
3014 {
3015         uint32_t eicr;
3016         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3017         struct ixgbe_interrupt *intr =
3018                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
3019
3020         /* clear all cause mask */
3021         ixgbe_disable_intr(hw);
3022
3023         /* read-on-clear nic registers here */
3024         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
3025         PMD_DRV_LOG(DEBUG, "eicr %x", eicr);
3026
3027         intr->flags = 0;
3028
3029         /* set flag for async link update */
3030         if (eicr & IXGBE_EICR_LSC)
3031                 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
3032
3033         if (eicr & IXGBE_EICR_MAILBOX)
3034                 intr->flags |= IXGBE_FLAG_MAILBOX;
3035
3036         return 0;
3037 }
3038
3039 /**
3040  * It gets and then prints the link status.
3041  *
3042  * @param dev
3043  *  Pointer to struct rte_eth_dev.
3044  *
3045  * @return
3046  *  - On success, zero.
3047  *  - On failure, a negative value.
3048  */
3049 static void
3050 ixgbe_dev_link_status_print(struct rte_eth_dev *dev)
3051 {
3052         struct rte_eth_link link;
3053
3054         memset(&link, 0, sizeof(link));
3055         rte_ixgbe_dev_atomic_read_link_status(dev, &link);
3056         if (link.link_status) {
3057                 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
3058                                         (int)(dev->data->port_id),
3059                                         (unsigned)link.link_speed,
3060                         link.link_duplex == ETH_LINK_FULL_DUPLEX ?
3061                                         "full-duplex" : "half-duplex");
3062         } else {
3063                 PMD_INIT_LOG(INFO, " Port %d: Link Down",
3064                                 (int)(dev->data->port_id));
3065         }
3066         PMD_INIT_LOG(DEBUG, "PCI Address: %04d:%02d:%02d:%d",
3067                                 dev->pci_dev->addr.domain,
3068                                 dev->pci_dev->addr.bus,
3069                                 dev->pci_dev->addr.devid,
3070                                 dev->pci_dev->addr.function);
3071 }
3072
3073 /*
3074  * It executes link_update after knowing an interrupt occurred.
3075  *
3076  * @param dev
3077  *  Pointer to struct rte_eth_dev.
3078  *
3079  * @return
3080  *  - On success, zero.
3081  *  - On failure, a negative value.
3082  */
3083 static int
3084 ixgbe_dev_interrupt_action(struct rte_eth_dev *dev)
3085 {
3086         struct ixgbe_interrupt *intr =
3087                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
3088         int64_t timeout;
3089         struct rte_eth_link link;
3090         int intr_enable_delay = false;
3091
3092         PMD_DRV_LOG(DEBUG, "intr action type %d", intr->flags);
3093
3094         if (intr->flags & IXGBE_FLAG_MAILBOX) {
3095                 ixgbe_pf_mbx_process(dev);
3096                 intr->flags &= ~IXGBE_FLAG_MAILBOX;
3097         }
3098
3099         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
3100                 /* get the link status before link update, for predicting later */
3101                 memset(&link, 0, sizeof(link));
3102                 rte_ixgbe_dev_atomic_read_link_status(dev, &link);
3103
3104                 ixgbe_dev_link_update(dev, 0);
3105
3106                 /* likely to up */
3107                 if (!link.link_status)
3108                         /* handle it 1 sec later, wait it being stable */
3109                         timeout = IXGBE_LINK_UP_CHECK_TIMEOUT;
3110                 /* likely to down */
3111                 else
3112                         /* handle it 4 sec later, wait it being stable */
3113                         timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT;
3114
3115                 ixgbe_dev_link_status_print(dev);
3116
3117                 intr_enable_delay = true;
3118         }
3119
3120         if (intr_enable_delay) {
3121                 if (rte_eal_alarm_set(timeout * 1000,
3122                                       ixgbe_dev_interrupt_delayed_handler, (void*)dev) < 0)
3123                         PMD_DRV_LOG(ERR, "Error setting alarm");
3124         } else {
3125                 PMD_DRV_LOG(DEBUG, "enable intr immediately");
3126                 ixgbe_enable_intr(dev);
3127                 rte_intr_enable(&(dev->pci_dev->intr_handle));
3128         }
3129
3130
3131         return 0;
3132 }
3133
3134 /**
3135  * Interrupt handler which shall be registered for alarm callback for delayed
3136  * handling specific interrupt to wait for the stable nic state. As the
3137  * NIC interrupt state is not stable for ixgbe after link is just down,
3138  * it needs to wait 4 seconds to get the stable status.
3139  *
3140  * @param handle
3141  *  Pointer to interrupt handle.
3142  * @param param
3143  *  The address of parameter (struct rte_eth_dev *) regsitered before.
3144  *
3145  * @return
3146  *  void
3147  */
3148 static void
3149 ixgbe_dev_interrupt_delayed_handler(void *param)
3150 {
3151         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
3152         struct ixgbe_interrupt *intr =
3153                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
3154         struct ixgbe_hw *hw =
3155                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3156         uint32_t eicr;
3157
3158         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
3159         if (eicr & IXGBE_EICR_MAILBOX)
3160                 ixgbe_pf_mbx_process(dev);
3161
3162         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
3163                 ixgbe_dev_link_update(dev, 0);
3164                 intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
3165                 ixgbe_dev_link_status_print(dev);
3166                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
3167         }
3168
3169         PMD_DRV_LOG(DEBUG, "enable intr in delayed handler S[%08x]", eicr);
3170         ixgbe_enable_intr(dev);
3171         rte_intr_enable(&(dev->pci_dev->intr_handle));
3172 }
3173
3174 /**
3175  * Interrupt handler triggered by NIC  for handling
3176  * specific interrupt.
3177  *
3178  * @param handle
3179  *  Pointer to interrupt handle.
3180  * @param param
3181  *  The address of parameter (struct rte_eth_dev *) regsitered before.
3182  *
3183  * @return
3184  *  void
3185  */
3186 static void
3187 ixgbe_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
3188                             void *param)
3189 {
3190         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
3191
3192         ixgbe_dev_interrupt_get_status(dev);
3193         ixgbe_dev_interrupt_action(dev);
3194 }
3195
3196 static int
3197 ixgbe_dev_led_on(struct rte_eth_dev *dev)
3198 {
3199         struct ixgbe_hw *hw;
3200
3201         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3202         return (ixgbe_led_on(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
3203 }
3204
3205 static int
3206 ixgbe_dev_led_off(struct rte_eth_dev *dev)
3207 {
3208         struct ixgbe_hw *hw;
3209
3210         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3211         return (ixgbe_led_off(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
3212 }
3213
3214 static int
3215 ixgbe_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
3216 {
3217         struct ixgbe_hw *hw;
3218         uint32_t mflcn_reg;
3219         uint32_t fccfg_reg;
3220         int rx_pause;
3221         int tx_pause;
3222
3223         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3224
3225         fc_conf->pause_time = hw->fc.pause_time;
3226         fc_conf->high_water = hw->fc.high_water[0];
3227         fc_conf->low_water = hw->fc.low_water[0];
3228         fc_conf->send_xon = hw->fc.send_xon;
3229         fc_conf->autoneg = !hw->fc.disable_fc_autoneg;
3230
3231         /*
3232          * Return rx_pause status according to actual setting of
3233          * MFLCN register.
3234          */
3235         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
3236         if (mflcn_reg & (IXGBE_MFLCN_RPFCE | IXGBE_MFLCN_RFCE))
3237                 rx_pause = 1;
3238         else
3239                 rx_pause = 0;
3240
3241         /*
3242          * Return tx_pause status according to actual setting of
3243          * FCCFG register.
3244          */
3245         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
3246         if (fccfg_reg & (IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY))
3247                 tx_pause = 1;
3248         else
3249                 tx_pause = 0;
3250
3251         if (rx_pause && tx_pause)
3252                 fc_conf->mode = RTE_FC_FULL;
3253         else if (rx_pause)
3254                 fc_conf->mode = RTE_FC_RX_PAUSE;
3255         else if (tx_pause)
3256                 fc_conf->mode = RTE_FC_TX_PAUSE;
3257         else
3258                 fc_conf->mode = RTE_FC_NONE;
3259
3260         return 0;
3261 }
3262
3263 static int
3264 ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
3265 {
3266         struct ixgbe_hw *hw;
3267         int err;
3268         uint32_t rx_buf_size;
3269         uint32_t max_high_water;
3270         uint32_t mflcn;
3271         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
3272                 ixgbe_fc_none,
3273                 ixgbe_fc_rx_pause,
3274                 ixgbe_fc_tx_pause,
3275                 ixgbe_fc_full
3276         };
3277
3278         PMD_INIT_FUNC_TRACE();
3279
3280         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3281         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0));
3282         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
3283
3284         /*
3285          * At least reserve one Ethernet frame for watermark
3286          * high_water/low_water in kilo bytes for ixgbe
3287          */
3288         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
3289         if ((fc_conf->high_water > max_high_water) ||
3290                 (fc_conf->high_water < fc_conf->low_water)) {
3291                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB");
3292                 PMD_INIT_LOG(ERR, "High_water must <= 0x%x", max_high_water);
3293                 return (-EINVAL);
3294         }
3295
3296         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[fc_conf->mode];
3297         hw->fc.pause_time     = fc_conf->pause_time;
3298         hw->fc.high_water[0]  = fc_conf->high_water;
3299         hw->fc.low_water[0]   = fc_conf->low_water;
3300         hw->fc.send_xon       = fc_conf->send_xon;
3301         hw->fc.disable_fc_autoneg = !fc_conf->autoneg;
3302
3303         err = ixgbe_fc_enable(hw);
3304
3305         /* Not negotiated is not an error case */
3306         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) {
3307
3308                 /* check if we want to forward MAC frames - driver doesn't have native
3309                  * capability to do that, so we'll write the registers ourselves */
3310
3311                 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN);
3312
3313                 /* set or clear MFLCN.PMCF bit depending on configuration */
3314                 if (fc_conf->mac_ctrl_frame_fwd != 0)
3315                         mflcn |= IXGBE_MFLCN_PMCF;
3316                 else
3317                         mflcn &= ~IXGBE_MFLCN_PMCF;
3318
3319                 IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn);
3320                 IXGBE_WRITE_FLUSH(hw);
3321
3322                 return 0;
3323         }
3324
3325         PMD_INIT_LOG(ERR, "ixgbe_fc_enable = 0x%x", err);
3326         return -EIO;
3327 }
3328
3329 /**
3330  *  ixgbe_pfc_enable_generic - Enable flow control
3331  *  @hw: pointer to hardware structure
3332  *  @tc_num: traffic class number
3333  *  Enable flow control according to the current settings.
3334  */
3335 static int
3336 ixgbe_dcb_pfc_enable_generic(struct ixgbe_hw *hw,uint8_t tc_num)
3337 {
3338         int ret_val = 0;
3339         uint32_t mflcn_reg, fccfg_reg;
3340         uint32_t reg;
3341         uint32_t fcrtl, fcrth;
3342         uint8_t i;
3343         uint8_t nb_rx_en;
3344
3345         /* Validate the water mark configuration */
3346         if (!hw->fc.pause_time) {
3347                 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
3348                 goto out;
3349         }
3350
3351         /* Low water mark of zero causes XOFF floods */
3352         if (hw->fc.current_mode & ixgbe_fc_tx_pause) {
3353                  /* High/Low water can not be 0 */
3354                 if( (!hw->fc.high_water[tc_num])|| (!hw->fc.low_water[tc_num])) {
3355                         PMD_INIT_LOG(ERR, "Invalid water mark configuration");
3356                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
3357                         goto out;
3358                 }
3359
3360                 if(hw->fc.low_water[tc_num] >= hw->fc.high_water[tc_num]) {
3361                         PMD_INIT_LOG(ERR, "Invalid water mark configuration");
3362                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
3363                         goto out;
3364                 }
3365         }
3366         /* Negotiate the fc mode to use */
3367         ixgbe_fc_autoneg(hw);
3368
3369         /* Disable any previous flow control settings */
3370         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
3371         mflcn_reg &= ~(IXGBE_MFLCN_RPFCE_SHIFT | IXGBE_MFLCN_RFCE|IXGBE_MFLCN_RPFCE);
3372
3373         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
3374         fccfg_reg &= ~(IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY);
3375
3376         switch (hw->fc.current_mode) {
3377         case ixgbe_fc_none:
3378                 /*
3379                  * If the count of enabled RX Priority Flow control >1,
3380                  * and the TX pause can not be disabled
3381                  */
3382                 nb_rx_en = 0;
3383                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
3384                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
3385                         if (reg & IXGBE_FCRTH_FCEN)
3386                                 nb_rx_en++;
3387                 }
3388                 if (nb_rx_en > 1)
3389                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
3390                 break;
3391         case ixgbe_fc_rx_pause:
3392                 /*
3393                  * Rx Flow control is enabled and Tx Flow control is
3394                  * disabled by software override. Since there really
3395                  * isn't a way to advertise that we are capable of RX
3396                  * Pause ONLY, we will advertise that we support both
3397                  * symmetric and asymmetric Rx PAUSE.  Later, we will
3398                  * disable the adapter's ability to send PAUSE frames.
3399                  */
3400                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
3401                 /*
3402                  * If the count of enabled RX Priority Flow control >1,
3403                  * and the TX pause can not be disabled
3404                  */
3405                 nb_rx_en = 0;
3406                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
3407                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
3408                         if (reg & IXGBE_FCRTH_FCEN)
3409                                 nb_rx_en++;
3410                 }
3411                 if (nb_rx_en > 1)
3412                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
3413                 break;
3414         case ixgbe_fc_tx_pause:
3415                 /*
3416                  * Tx Flow control is enabled, and Rx Flow control is
3417                  * disabled by software override.
3418                  */
3419                 fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
3420                 break;
3421         case ixgbe_fc_full:
3422                 /* Flow control (both Rx and Tx) is enabled by SW override. */
3423                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
3424                 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
3425                 break;
3426         default:
3427                 PMD_DRV_LOG(DEBUG, "Flow control param set incorrectly");
3428                 ret_val = IXGBE_ERR_CONFIG;
3429                 goto out;
3430                 break;
3431         }
3432
3433         /* Set 802.3x based flow control settings. */
3434         mflcn_reg |= IXGBE_MFLCN_DPF;
3435         IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg);
3436         IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg);
3437
3438         /* Set up and enable Rx high/low water mark thresholds, enable XON. */
3439         if ((hw->fc.current_mode & ixgbe_fc_tx_pause) &&
3440                 hw->fc.high_water[tc_num]) {
3441                 fcrtl = (hw->fc.low_water[tc_num] << 10) | IXGBE_FCRTL_XONE;
3442                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), fcrtl);
3443                 fcrth = (hw->fc.high_water[tc_num] << 10) | IXGBE_FCRTH_FCEN;
3444         } else {
3445                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), 0);
3446                 /*
3447                  * In order to prevent Tx hangs when the internal Tx
3448                  * switch is enabled we must set the high water mark
3449                  * to the maximum FCRTH value.  This allows the Tx
3450                  * switch to function even under heavy Rx workloads.
3451                  */
3452                 fcrth = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)) - 32;
3453         }
3454         IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(tc_num), fcrth);
3455
3456         /* Configure pause time (2 TCs per register) */
3457         reg = hw->fc.pause_time * 0x00010001;
3458         for (i = 0; i < (IXGBE_DCB_MAX_TRAFFIC_CLASS / 2); i++)
3459                 IXGBE_WRITE_REG(hw, IXGBE_FCTTV(i), reg);
3460
3461         /* Configure flow control refresh threshold value */
3462         IXGBE_WRITE_REG(hw, IXGBE_FCRTV, hw->fc.pause_time / 2);
3463
3464 out:
3465         return ret_val;
3466 }
3467
3468 static int
3469 ixgbe_dcb_pfc_enable(struct rte_eth_dev *dev,uint8_t tc_num)
3470 {
3471         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3472         int32_t ret_val = IXGBE_NOT_IMPLEMENTED;
3473
3474         if(hw->mac.type != ixgbe_mac_82598EB) {
3475                 ret_val = ixgbe_dcb_pfc_enable_generic(hw,tc_num);
3476         }
3477         return ret_val;
3478 }
3479
3480 static int
3481 ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf)
3482 {
3483         int err;
3484         uint32_t rx_buf_size;
3485         uint32_t max_high_water;
3486         uint8_t tc_num;
3487         uint8_t  map[IXGBE_DCB_MAX_USER_PRIORITY] = { 0 };
3488         struct ixgbe_hw *hw =
3489                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3490         struct ixgbe_dcb_config *dcb_config =
3491                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
3492
3493         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
3494                 ixgbe_fc_none,
3495                 ixgbe_fc_rx_pause,
3496                 ixgbe_fc_tx_pause,
3497                 ixgbe_fc_full
3498         };
3499
3500         PMD_INIT_FUNC_TRACE();
3501
3502         ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
3503         tc_num = map[pfc_conf->priority];
3504         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num));
3505         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
3506         /*
3507          * At least reserve one Ethernet frame for watermark
3508          * high_water/low_water in kilo bytes for ixgbe
3509          */
3510         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
3511         if ((pfc_conf->fc.high_water > max_high_water) ||
3512             (pfc_conf->fc.high_water <= pfc_conf->fc.low_water)) {
3513                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB");
3514                 PMD_INIT_LOG(ERR, "High_water must <= 0x%x", max_high_water);
3515                 return (-EINVAL);
3516         }
3517
3518         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[pfc_conf->fc.mode];
3519         hw->fc.pause_time = pfc_conf->fc.pause_time;
3520         hw->fc.send_xon = pfc_conf->fc.send_xon;
3521         hw->fc.low_water[tc_num] =  pfc_conf->fc.low_water;
3522         hw->fc.high_water[tc_num] = pfc_conf->fc.high_water;
3523
3524         err = ixgbe_dcb_pfc_enable(dev,tc_num);
3525
3526         /* Not negotiated is not an error case */
3527         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED))
3528                 return 0;
3529
3530         PMD_INIT_LOG(ERR, "ixgbe_dcb_pfc_enable = 0x%x", err);
3531         return -EIO;
3532 }
3533
3534 static int
3535 ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
3536                           struct rte_eth_rss_reta_entry64 *reta_conf,
3537                           uint16_t reta_size)
3538 {
3539         uint8_t i, j, mask;
3540         uint32_t reta, r;
3541         uint16_t idx, shift;
3542         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3543         uint16_t sp_reta_size;
3544         uint32_t reta_reg;
3545
3546         PMD_INIT_FUNC_TRACE();
3547
3548         if (!ixgbe_rss_update_sp(hw->mac.type)) {
3549                 PMD_DRV_LOG(ERR, "RSS reta update is not supported on this "
3550                         "NIC.");
3551                 return -ENOTSUP;
3552         }
3553
3554         sp_reta_size = ixgbe_reta_size_get(hw->mac.type);
3555         if (reta_size != sp_reta_size) {
3556                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
3557                         "(%d) doesn't match the number hardware can supported "
3558                         "(%d)\n", reta_size, sp_reta_size);
3559                 return -EINVAL;
3560         }
3561
3562         for (i = 0; i < reta_size; i += IXGBE_4_BIT_WIDTH) {
3563                 idx = i / RTE_RETA_GROUP_SIZE;
3564                 shift = i % RTE_RETA_GROUP_SIZE;
3565                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
3566                                                 IXGBE_4_BIT_MASK);
3567                 if (!mask)
3568                         continue;
3569                 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i);
3570                 if (mask == IXGBE_4_BIT_MASK)
3571                         r = 0;
3572                 else
3573                         r = IXGBE_READ_REG(hw, reta_reg);
3574                 for (j = 0, reta = 0; j < IXGBE_4_BIT_WIDTH; j++) {
3575                         if (mask & (0x1 << j))
3576                                 reta |= reta_conf[idx].reta[shift + j] <<
3577                                                         (CHAR_BIT * j);
3578                         else
3579                                 reta |= r & (IXGBE_8_BIT_MASK <<
3580                                                 (CHAR_BIT * j));
3581                 }
3582                 IXGBE_WRITE_REG(hw, reta_reg, reta);
3583         }
3584
3585         return 0;
3586 }
3587
3588 static int
3589 ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
3590                          struct rte_eth_rss_reta_entry64 *reta_conf,
3591                          uint16_t reta_size)
3592 {
3593         uint8_t i, j, mask;
3594         uint32_t reta;
3595         uint16_t idx, shift;
3596         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3597         uint16_t sp_reta_size;
3598         uint32_t reta_reg;
3599
3600         PMD_INIT_FUNC_TRACE();
3601         sp_reta_size = ixgbe_reta_size_get(hw->mac.type);
3602         if (reta_size != sp_reta_size) {
3603                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
3604                         "(%d) doesn't match the number hardware can supported "
3605                         "(%d)\n", reta_size, sp_reta_size);
3606                 return -EINVAL;
3607         }
3608
3609         for (i = 0; i < reta_size; i += IXGBE_4_BIT_WIDTH) {
3610                 idx = i / RTE_RETA_GROUP_SIZE;
3611                 shift = i % RTE_RETA_GROUP_SIZE;
3612                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
3613                                                 IXGBE_4_BIT_MASK);
3614                 if (!mask)
3615                         continue;
3616
3617                 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i);
3618                 reta = IXGBE_READ_REG(hw, reta_reg);
3619                 for (j = 0; j < IXGBE_4_BIT_WIDTH; j++) {
3620                         if (mask & (0x1 << j))
3621                                 reta_conf[idx].reta[shift + j] =
3622                                         ((reta >> (CHAR_BIT * j)) &
3623                                                 IXGBE_8_BIT_MASK);
3624                 }
3625         }
3626
3627         return 0;
3628 }
3629
3630 static void
3631 ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
3632                                 uint32_t index, uint32_t pool)
3633 {
3634         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3635         uint32_t enable_addr = 1;
3636
3637         ixgbe_set_rar(hw, index, mac_addr->addr_bytes, pool, enable_addr);
3638 }
3639
3640 static void
3641 ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
3642 {
3643         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3644
3645         ixgbe_clear_rar(hw, index);
3646 }
3647
3648 static void
3649 ixgbe_set_default_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
3650 {
3651         ixgbe_remove_rar(dev, 0);
3652
3653         ixgbe_add_rar(dev, addr, 0, 0);
3654 }
3655
3656 static int
3657 ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
3658 {
3659         uint32_t hlreg0;
3660         uint32_t maxfrs;
3661         struct ixgbe_hw *hw;
3662         struct rte_eth_dev_info dev_info;
3663         uint32_t frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
3664
3665         ixgbe_dev_info_get(dev, &dev_info);
3666
3667         /* check that mtu is within the allowed range */
3668         if ((mtu < ETHER_MIN_MTU) || (frame_size > dev_info.max_rx_pktlen))
3669                 return -EINVAL;
3670
3671         /* refuse mtu that requires the support of scattered packets when this
3672          * feature has not been enabled before. */
3673         if (!dev->data->scattered_rx &&
3674             (frame_size + 2 * IXGBE_VLAN_TAG_SIZE >
3675              dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM))
3676                 return -EINVAL;
3677
3678         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3679         hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
3680
3681         /* switch to jumbo mode if needed */
3682         if (frame_size > ETHER_MAX_LEN) {
3683                 dev->data->dev_conf.rxmode.jumbo_frame = 1;
3684                 hlreg0 |= IXGBE_HLREG0_JUMBOEN;
3685         } else {
3686                 dev->data->dev_conf.rxmode.jumbo_frame = 0;
3687                 hlreg0 &= ~IXGBE_HLREG0_JUMBOEN;
3688         }
3689         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
3690
3691         /* update max frame size */
3692         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
3693
3694         maxfrs = IXGBE_READ_REG(hw, IXGBE_MAXFRS);
3695         maxfrs &= 0x0000FFFF;
3696         maxfrs |= (dev->data->dev_conf.rxmode.max_rx_pkt_len << 16);
3697         IXGBE_WRITE_REG(hw, IXGBE_MAXFRS, maxfrs);
3698
3699         return 0;
3700 }
3701
3702 /*
3703  * Virtual Function operations
3704  */
3705 static void
3706 ixgbevf_intr_disable(struct ixgbe_hw *hw)
3707 {
3708         PMD_INIT_FUNC_TRACE();
3709
3710         /* Clear interrupt mask to stop from interrupts being generated */
3711         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, IXGBE_VF_IRQ_CLEAR_MASK);
3712
3713         IXGBE_WRITE_FLUSH(hw);
3714 }
3715
3716 static void
3717 ixgbevf_intr_enable(struct ixgbe_hw *hw)
3718 {
3719         PMD_INIT_FUNC_TRACE();
3720
3721         /* VF enable interrupt autoclean */
3722         IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_VF_IRQ_ENABLE_MASK);
3723         IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, IXGBE_VF_IRQ_ENABLE_MASK);
3724         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_VF_IRQ_ENABLE_MASK);
3725
3726         IXGBE_WRITE_FLUSH(hw);
3727 }
3728
3729 static int
3730 ixgbevf_dev_configure(struct rte_eth_dev *dev)
3731 {
3732         struct rte_eth_conf* conf = &dev->data->dev_conf;
3733         struct ixgbe_adapter *adapter =
3734                         (struct ixgbe_adapter *)dev->data->dev_private;
3735
3736         PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d",
3737                      dev->data->port_id);
3738
3739         /*
3740          * VF has no ability to enable/disable HW CRC
3741          * Keep the persistent behavior the same as Host PF
3742          */
3743 #ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC
3744         if (!conf->rxmode.hw_strip_crc) {
3745                 PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip");
3746                 conf->rxmode.hw_strip_crc = 1;
3747         }
3748 #else
3749         if (conf->rxmode.hw_strip_crc) {
3750                 PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip");
3751                 conf->rxmode.hw_strip_crc = 0;
3752         }
3753 #endif
3754
3755         /*
3756          * Initialize to TRUE. If any of Rx queues doesn't meet the bulk
3757          * allocation or vector Rx preconditions we will reset it.
3758          */
3759         adapter->rx_bulk_alloc_allowed = true;
3760         adapter->rx_vec_allowed = true;
3761
3762         return 0;
3763 }
3764
3765 static int
3766 ixgbevf_dev_start(struct rte_eth_dev *dev)
3767 {
3768         struct ixgbe_hw *hw =
3769                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3770         uint32_t intr_vector = 0;
3771         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
3772
3773         int err, mask = 0;
3774
3775         PMD_INIT_FUNC_TRACE();
3776
3777         hw->mac.ops.reset_hw(hw);
3778         hw->mac.get_link_status = true;
3779
3780         /* negotiate mailbox API version to use with the PF. */
3781         ixgbevf_negotiate_api(hw);
3782
3783         ixgbevf_dev_tx_init(dev);
3784
3785         /* This can fail when allocating mbufs for descriptor rings */
3786         err = ixgbevf_dev_rx_init(dev);
3787         if (err) {
3788                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware (%d)", err);
3789                 ixgbe_dev_clear_queues(dev);
3790                 return err;
3791         }
3792
3793         /* Set vfta */
3794         ixgbevf_set_vfta_all(dev,1);
3795
3796         /* Set HW strip */
3797         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
3798                 ETH_VLAN_EXTEND_MASK;
3799         ixgbevf_vlan_offload_set(dev, mask);
3800
3801         ixgbevf_dev_rxtx_start(dev);
3802
3803         /* check and configure queue intr-vector mapping */
3804         if (dev->data->dev_conf.intr_conf.rxq != 0) {
3805                 intr_vector = dev->data->nb_rx_queues;
3806                 if (rte_intr_efd_enable(intr_handle, intr_vector))
3807                         return -1;
3808         }
3809
3810         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
3811                 intr_handle->intr_vec =
3812                         rte_zmalloc("intr_vec",
3813                                     dev->data->nb_rx_queues * sizeof(int), 0);
3814                 if (intr_handle->intr_vec == NULL) {
3815                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
3816                                      " intr_vec\n", dev->data->nb_rx_queues);
3817                         return -ENOMEM;
3818                 }
3819         }
3820         ixgbevf_configure_msix(dev);
3821
3822         rte_intr_enable(intr_handle);
3823
3824         /* Re-enable interrupt for VF */
3825         ixgbevf_intr_enable(hw);
3826
3827         return 0;
3828 }
3829
3830 static void
3831 ixgbevf_dev_stop(struct rte_eth_dev *dev)
3832 {
3833         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3834         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
3835
3836         PMD_INIT_FUNC_TRACE();
3837
3838         hw->adapter_stopped = 1;
3839         ixgbe_stop_adapter(hw);
3840
3841         /*
3842           * Clear what we set, but we still keep shadow_vfta to
3843           * restore after device starts
3844           */
3845         ixgbevf_set_vfta_all(dev,0);
3846
3847         /* Clear stored conf */
3848         dev->data->scattered_rx = 0;
3849
3850         ixgbe_dev_clear_queues(dev);
3851
3852         /* disable intr eventfd mapping */
3853         rte_intr_disable(intr_handle);
3854
3855         /* Clean datapath event and queue/vec mapping */
3856         rte_intr_efd_disable(intr_handle);
3857         if (intr_handle->intr_vec != NULL) {
3858                 rte_free(intr_handle->intr_vec);
3859                 intr_handle->intr_vec = NULL;
3860         }
3861 }
3862
3863 static void
3864 ixgbevf_dev_close(struct rte_eth_dev *dev)
3865 {
3866         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3867
3868         PMD_INIT_FUNC_TRACE();
3869
3870         ixgbe_reset_hw(hw);
3871
3872         ixgbevf_dev_stop(dev);
3873
3874         ixgbe_dev_free_queues(dev);
3875
3876         /* reprogram the RAR[0] in case user changed it. */
3877         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
3878 }
3879
3880 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on)
3881 {
3882         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3883         struct ixgbe_vfta * shadow_vfta =
3884                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
3885         int i = 0, j = 0, vfta = 0, mask = 1;
3886
3887         for (i = 0; i < IXGBE_VFTA_SIZE; i++){
3888                 vfta = shadow_vfta->vfta[i];
3889                 if(vfta){
3890                         mask = 1;
3891                         for (j = 0; j < 32; j++){
3892                                 if(vfta & mask)
3893                                         ixgbe_set_vfta(hw, (i<<5)+j, 0, on);
3894                                 mask<<=1;
3895                         }
3896                 }
3897         }
3898
3899 }
3900
3901 static int
3902 ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
3903 {
3904         struct ixgbe_hw *hw =
3905                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3906         struct ixgbe_vfta * shadow_vfta =
3907                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
3908         uint32_t vid_idx = 0;
3909         uint32_t vid_bit = 0;
3910         int ret = 0;
3911
3912         PMD_INIT_FUNC_TRACE();
3913
3914         /* vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf */
3915         ret = ixgbe_set_vfta(hw, vlan_id, 0, !!on);
3916         if(ret){
3917                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
3918                 return ret;
3919         }
3920         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
3921         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
3922
3923         /* Save what we set and retore it after device reset */
3924         if (on)
3925                 shadow_vfta->vfta[vid_idx] |= vid_bit;
3926         else
3927                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
3928
3929         return 0;
3930 }
3931
3932 static void
3933 ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
3934 {
3935         struct ixgbe_hw *hw =
3936                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3937         uint32_t ctrl;
3938
3939         PMD_INIT_FUNC_TRACE();
3940
3941         if(queue >= hw->mac.max_rx_queues)
3942                 return;
3943
3944         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
3945         if(on)
3946                 ctrl |= IXGBE_RXDCTL_VME;
3947         else
3948                 ctrl &= ~IXGBE_RXDCTL_VME;
3949         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
3950
3951         ixgbe_vlan_hw_strip_bitmap_set( dev, queue, on);
3952 }
3953
3954 static void
3955 ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
3956 {
3957         struct ixgbe_hw *hw =
3958                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3959         uint16_t i;
3960         int on = 0;
3961
3962         /* VF function only support hw strip feature, others are not support */
3963         if(mask & ETH_VLAN_STRIP_MASK){
3964                 on = !!(dev->data->dev_conf.rxmode.hw_vlan_strip);
3965
3966                 for(i=0; i < hw->mac.max_rx_queues; i++)
3967                         ixgbevf_vlan_strip_queue_set(dev,i,on);
3968         }
3969 }
3970
3971 static int
3972 ixgbe_vmdq_mode_check(struct ixgbe_hw *hw)
3973 {
3974         uint32_t reg_val;
3975
3976         /* we only need to do this if VMDq is enabled */
3977         reg_val = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
3978         if (!(reg_val & IXGBE_VT_CTL_VT_ENABLE)) {
3979                 PMD_INIT_LOG(ERR, "VMDq must be enabled for this setting");
3980                 return (-1);
3981         }
3982
3983         return 0;
3984 }
3985
3986 static uint32_t
3987 ixgbe_uta_vector(struct ixgbe_hw *hw, struct ether_addr* uc_addr)
3988 {
3989         uint32_t vector = 0;
3990         switch (hw->mac.mc_filter_type) {
3991         case 0:   /* use bits [47:36] of the address */
3992                 vector = ((uc_addr->addr_bytes[4] >> 4) |
3993                         (((uint16_t)uc_addr->addr_bytes[5]) << 4));
3994                 break;
3995         case 1:   /* use bits [46:35] of the address */
3996                 vector = ((uc_addr->addr_bytes[4] >> 3) |
3997                         (((uint16_t)uc_addr->addr_bytes[5]) << 5));
3998                 break;
3999         case 2:   /* use bits [45:34] of the address */
4000                 vector = ((uc_addr->addr_bytes[4] >> 2) |
4001                         (((uint16_t)uc_addr->addr_bytes[5]) << 6));
4002                 break;
4003         case 3:   /* use bits [43:32] of the address */
4004                 vector = ((uc_addr->addr_bytes[4]) |
4005                         (((uint16_t)uc_addr->addr_bytes[5]) << 8));
4006                 break;
4007         default:  /* Invalid mc_filter_type */
4008                 break;
4009         }
4010
4011         /* vector can only be 12-bits or boundary will be exceeded */
4012         vector &= 0xFFF;
4013         return vector;
4014 }
4015
4016 static int
4017 ixgbe_uc_hash_table_set(struct rte_eth_dev *dev,struct ether_addr* mac_addr,
4018                                uint8_t on)
4019 {
4020         uint32_t vector;
4021         uint32_t uta_idx;
4022         uint32_t reg_val;
4023         uint32_t uta_shift;
4024         uint32_t rc;
4025         const uint32_t ixgbe_uta_idx_mask = 0x7F;
4026         const uint32_t ixgbe_uta_bit_shift = 5;
4027         const uint32_t ixgbe_uta_bit_mask = (0x1 << ixgbe_uta_bit_shift) - 1;
4028         const uint32_t bit1 = 0x1;
4029
4030         struct ixgbe_hw *hw =
4031                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4032         struct ixgbe_uta_info *uta_info =
4033                 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private);
4034
4035         /* The UTA table only exists on 82599 hardware and newer */
4036         if (hw->mac.type < ixgbe_mac_82599EB)
4037                 return (-ENOTSUP);
4038
4039         vector = ixgbe_uta_vector(hw,mac_addr);
4040         uta_idx = (vector >> ixgbe_uta_bit_shift) & ixgbe_uta_idx_mask;
4041         uta_shift = vector & ixgbe_uta_bit_mask;
4042
4043         rc = ((uta_info->uta_shadow[uta_idx] >> uta_shift & bit1) != 0);
4044         if(rc == on)
4045                 return 0;
4046
4047         reg_val = IXGBE_READ_REG(hw, IXGBE_UTA(uta_idx));
4048         if (on) {
4049                 uta_info->uta_in_use++;
4050                 reg_val |= (bit1 << uta_shift);
4051                 uta_info->uta_shadow[uta_idx] |= (bit1 << uta_shift);
4052         } else {
4053                 uta_info->uta_in_use--;
4054                 reg_val &= ~(bit1 << uta_shift);
4055                 uta_info->uta_shadow[uta_idx] &= ~(bit1 << uta_shift);
4056         }
4057
4058         IXGBE_WRITE_REG(hw, IXGBE_UTA(uta_idx), reg_val);
4059
4060         if (uta_info->uta_in_use > 0)
4061                 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL,
4062                                 IXGBE_MCSTCTRL_MFE | hw->mac.mc_filter_type);
4063         else
4064                 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL,hw->mac.mc_filter_type);
4065
4066         return 0;
4067 }
4068
4069 static int
4070 ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on)
4071 {
4072         int i;
4073         struct ixgbe_hw *hw =
4074                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4075         struct ixgbe_uta_info *uta_info =
4076                 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private);
4077
4078         /* The UTA table only exists on 82599 hardware and newer */
4079         if (hw->mac.type < ixgbe_mac_82599EB)
4080                 return (-ENOTSUP);
4081
4082         if(on) {
4083                 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
4084                         uta_info->uta_shadow[i] = ~0;
4085                         IXGBE_WRITE_REG(hw, IXGBE_UTA(i), ~0);
4086                 }
4087         } else {
4088                 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
4089                         uta_info->uta_shadow[i] = 0;
4090                         IXGBE_WRITE_REG(hw, IXGBE_UTA(i), 0);
4091                 }
4092         }
4093         return 0;
4094
4095 }
4096
4097 uint32_t
4098 ixgbe_convert_vm_rx_mask_to_val(uint16_t rx_mask, uint32_t orig_val)
4099 {
4100         uint32_t new_val = orig_val;
4101
4102         if (rx_mask & ETH_VMDQ_ACCEPT_UNTAG)
4103                 new_val |= IXGBE_VMOLR_AUPE;
4104         if (rx_mask & ETH_VMDQ_ACCEPT_HASH_MC)
4105                 new_val |= IXGBE_VMOLR_ROMPE;
4106         if (rx_mask & ETH_VMDQ_ACCEPT_HASH_UC)
4107                 new_val |= IXGBE_VMOLR_ROPE;
4108         if (rx_mask & ETH_VMDQ_ACCEPT_BROADCAST)
4109                 new_val |= IXGBE_VMOLR_BAM;
4110         if (rx_mask & ETH_VMDQ_ACCEPT_MULTICAST)
4111                 new_val |= IXGBE_VMOLR_MPE;
4112
4113         return new_val;
4114 }
4115
4116 static int
4117 ixgbe_set_pool_rx_mode(struct rte_eth_dev *dev, uint16_t pool,
4118                                uint16_t rx_mask, uint8_t on)
4119 {
4120         int val = 0;
4121
4122         struct ixgbe_hw *hw =
4123                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4124         uint32_t vmolr = IXGBE_READ_REG(hw, IXGBE_VMOLR(pool));
4125
4126         if (hw->mac.type == ixgbe_mac_82598EB) {
4127                 PMD_INIT_LOG(ERR, "setting VF receive mode set should be done"
4128                              " on 82599 hardware and newer");
4129                 return (-ENOTSUP);
4130         }
4131         if (ixgbe_vmdq_mode_check(hw) < 0)
4132                 return (-ENOTSUP);
4133
4134         val = ixgbe_convert_vm_rx_mask_to_val(rx_mask, val);
4135
4136         if (on)
4137                 vmolr |= val;
4138         else
4139                 vmolr &= ~val;
4140
4141         IXGBE_WRITE_REG(hw, IXGBE_VMOLR(pool), vmolr);
4142
4143         return 0;
4144 }
4145
4146 static int
4147 ixgbe_set_pool_rx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on)
4148 {
4149         uint32_t reg,addr;
4150         uint32_t val;
4151         const uint8_t bit1 = 0x1;
4152
4153         struct ixgbe_hw *hw =
4154                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4155
4156         if (ixgbe_vmdq_mode_check(hw) < 0)
4157                 return (-ENOTSUP);
4158
4159         addr = IXGBE_VFRE(pool >= ETH_64_POOLS/2);
4160         reg = IXGBE_READ_REG(hw, addr);
4161         val = bit1 << pool;
4162
4163         if (on)
4164                 reg |= val;
4165         else
4166                 reg &= ~val;
4167
4168         IXGBE_WRITE_REG(hw, addr,reg);
4169
4170         return 0;
4171 }
4172
4173 static int
4174 ixgbe_set_pool_tx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on)
4175 {
4176         uint32_t reg,addr;
4177         uint32_t val;
4178         const uint8_t bit1 = 0x1;
4179
4180         struct ixgbe_hw *hw =
4181                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4182
4183         if (ixgbe_vmdq_mode_check(hw) < 0)
4184                 return (-ENOTSUP);
4185
4186         addr = IXGBE_VFTE(pool >= ETH_64_POOLS/2);
4187         reg = IXGBE_READ_REG(hw, addr);
4188         val = bit1 << pool;
4189
4190         if (on)
4191                 reg |= val;
4192         else
4193                 reg &= ~val;
4194
4195         IXGBE_WRITE_REG(hw, addr,reg);
4196
4197         return 0;
4198 }
4199
4200 static int
4201 ixgbe_set_pool_vlan_filter(struct rte_eth_dev *dev, uint16_t vlan,
4202                         uint64_t pool_mask, uint8_t vlan_on)
4203 {
4204         int ret = 0;
4205         uint16_t pool_idx;
4206         struct ixgbe_hw *hw =
4207                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4208
4209         if (ixgbe_vmdq_mode_check(hw) < 0)
4210                 return (-ENOTSUP);
4211         for (pool_idx = 0; pool_idx < ETH_64_POOLS; pool_idx++) {
4212                 if (pool_mask & ((uint64_t)(1ULL << pool_idx)))
4213                         ret = hw->mac.ops.set_vfta(hw,vlan,pool_idx,vlan_on);
4214                         if (ret < 0)
4215                                 return ret;
4216         }
4217
4218         return ret;
4219 }
4220
4221 #define IXGBE_MRCTL_VPME  0x01 /* Virtual Pool Mirroring. */
4222 #define IXGBE_MRCTL_UPME  0x02 /* Uplink Port Mirroring. */
4223 #define IXGBE_MRCTL_DPME  0x04 /* Downlink Port Mirroring. */
4224 #define IXGBE_MRCTL_VLME  0x08 /* VLAN Mirroring. */
4225 #define IXGBE_INVALID_MIRROR_TYPE(mirror_type) \
4226         ((mirror_type) & ~(uint8_t)(ETH_MIRROR_VIRTUAL_POOL_UP | \
4227         ETH_MIRROR_UPLINK_PORT | ETH_MIRROR_DOWNLINK_PORT | ETH_MIRROR_VLAN))
4228
4229 static int
4230 ixgbe_mirror_rule_set(struct rte_eth_dev *dev,
4231                         struct rte_eth_mirror_conf *mirror_conf,
4232                         uint8_t rule_id, uint8_t on)
4233 {
4234         uint32_t mr_ctl,vlvf;
4235         uint32_t mp_lsb = 0;
4236         uint32_t mv_msb = 0;
4237         uint32_t mv_lsb = 0;
4238         uint32_t mp_msb = 0;
4239         uint8_t i = 0;
4240         int reg_index = 0;
4241         uint64_t vlan_mask = 0;
4242
4243         const uint8_t pool_mask_offset = 32;
4244         const uint8_t vlan_mask_offset = 32;
4245         const uint8_t dst_pool_offset = 8;
4246         const uint8_t rule_mr_offset  = 4;
4247         const uint8_t mirror_rule_mask= 0x0F;
4248
4249         struct ixgbe_mirror_info *mr_info =
4250                         (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private));
4251         struct ixgbe_hw *hw =
4252                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4253         uint8_t mirror_type = 0;
4254
4255         if (ixgbe_vmdq_mode_check(hw) < 0)
4256                 return -ENOTSUP;
4257
4258         if (rule_id >= IXGBE_MAX_MIRROR_RULES)
4259                 return -EINVAL;
4260
4261         if (IXGBE_INVALID_MIRROR_TYPE(mirror_conf->rule_type)) {
4262                 PMD_DRV_LOG(ERR, "unsupported mirror type 0x%x.",
4263                         mirror_conf->rule_type);
4264                 return -EINVAL;
4265         }
4266
4267         if (mirror_conf->rule_type & ETH_MIRROR_VLAN) {
4268                 mirror_type |= IXGBE_MRCTL_VLME;
4269                 /* Check if vlan id is valid and find conresponding VLAN ID index in VLVF */
4270                 for (i = 0;i < IXGBE_VLVF_ENTRIES; i++) {
4271                         if (mirror_conf->vlan.vlan_mask & (1ULL << i)) {
4272                                 /* search vlan id related pool vlan filter index */
4273                                 reg_index = ixgbe_find_vlvf_slot(hw,
4274                                                 mirror_conf->vlan.vlan_id[i]);
4275                                 if(reg_index < 0)
4276                                         return -EINVAL;
4277                                 vlvf = IXGBE_READ_REG(hw, IXGBE_VLVF(reg_index));
4278                                 if ((vlvf & IXGBE_VLVF_VIEN) &&
4279                                     ((vlvf & IXGBE_VLVF_VLANID_MASK) ==
4280                                       mirror_conf->vlan.vlan_id[i]))
4281                                         vlan_mask |= (1ULL << reg_index);
4282                                 else
4283                                         return -EINVAL;
4284                         }
4285                 }
4286
4287                 if (on) {
4288                         mv_lsb = vlan_mask & 0xFFFFFFFF;
4289                         mv_msb = vlan_mask >> vlan_mask_offset;
4290
4291                         mr_info->mr_conf[rule_id].vlan.vlan_mask =
4292                                                 mirror_conf->vlan.vlan_mask;
4293                         for(i = 0 ;i < ETH_VMDQ_MAX_VLAN_FILTERS; i++) {
4294                                 if(mirror_conf->vlan.vlan_mask & (1ULL << i))
4295                                         mr_info->mr_conf[rule_id].vlan.vlan_id[i] =
4296                                                 mirror_conf->vlan.vlan_id[i];
4297                         }
4298                 } else {
4299                         mv_lsb = 0;
4300                         mv_msb = 0;
4301                         mr_info->mr_conf[rule_id].vlan.vlan_mask = 0;
4302                         for(i = 0 ;i < ETH_VMDQ_MAX_VLAN_FILTERS; i++)
4303                                 mr_info->mr_conf[rule_id].vlan.vlan_id[i] = 0;
4304                 }
4305         }
4306
4307         /*
4308          * if enable pool mirror, write related pool mask register,if disable
4309          * pool mirror, clear PFMRVM register
4310          */
4311         if (mirror_conf->rule_type & ETH_MIRROR_VIRTUAL_POOL_UP) {
4312                 mirror_type |= IXGBE_MRCTL_VPME;
4313                 if (on) {
4314                         mp_lsb = mirror_conf->pool_mask & 0xFFFFFFFF;
4315                         mp_msb = mirror_conf->pool_mask >> pool_mask_offset;
4316                         mr_info->mr_conf[rule_id].pool_mask =
4317                                         mirror_conf->pool_mask;
4318
4319                 } else {
4320                         mp_lsb = 0;
4321                         mp_msb = 0;
4322                         mr_info->mr_conf[rule_id].pool_mask = 0;
4323                 }
4324         }
4325         if (mirror_conf->rule_type & ETH_MIRROR_UPLINK_PORT)
4326                 mirror_type |= IXGBE_MRCTL_UPME;
4327         if (mirror_conf->rule_type & ETH_MIRROR_DOWNLINK_PORT)
4328                 mirror_type |= IXGBE_MRCTL_DPME;
4329
4330         /* read  mirror control register and recalculate it */
4331         mr_ctl = IXGBE_READ_REG(hw, IXGBE_MRCTL(rule_id));
4332
4333         if (on) {
4334                 mr_ctl |= mirror_type;
4335                 mr_ctl &= mirror_rule_mask;
4336                 mr_ctl |= mirror_conf->dst_pool << dst_pool_offset;
4337         } else
4338                 mr_ctl &= ~(mirror_conf->rule_type & mirror_rule_mask);
4339
4340         mr_info->mr_conf[rule_id].rule_type = mirror_conf->rule_type;
4341         mr_info->mr_conf[rule_id].dst_pool = mirror_conf->dst_pool;
4342
4343         /* write mirrror control  register */
4344         IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl);
4345
4346         /* write pool mirrror control  register */
4347         if (mirror_conf->rule_type == ETH_MIRROR_VIRTUAL_POOL_UP) {
4348                 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), mp_lsb);
4349                 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset),
4350                                 mp_msb);
4351         }
4352         /* write VLAN mirrror control  register */
4353         if (mirror_conf->rule_type == ETH_MIRROR_VLAN) {
4354                 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), mv_lsb);
4355                 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset),
4356                                 mv_msb);
4357         }
4358
4359         return 0;
4360 }
4361
4362 static int
4363 ixgbe_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t rule_id)
4364 {
4365         int mr_ctl = 0;
4366         uint32_t lsb_val = 0;
4367         uint32_t msb_val = 0;
4368         const uint8_t rule_mr_offset = 4;
4369
4370         struct ixgbe_hw *hw =
4371                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4372         struct ixgbe_mirror_info *mr_info =
4373                 (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private));
4374
4375         if (ixgbe_vmdq_mode_check(hw) < 0)
4376                 return (-ENOTSUP);
4377
4378         memset(&mr_info->mr_conf[rule_id], 0,
4379                 sizeof(struct rte_eth_mirror_conf));
4380
4381         /* clear PFVMCTL register */
4382         IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl);
4383
4384         /* clear pool mask register */
4385         IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), lsb_val);
4386         IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset), msb_val);
4387
4388         /* clear vlan mask register */
4389         IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), lsb_val);
4390         IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset), msb_val);
4391
4392         return 0;
4393 }
4394
4395 static int
4396 ixgbevf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
4397 {
4398         uint32_t mask;
4399         struct ixgbe_hw *hw =
4400                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4401
4402         mask = IXGBE_READ_REG(hw, IXGBE_VTEIMS);
4403         mask |= (1 << IXGBE_MISC_VEC_ID);
4404         RTE_SET_USED(queue_id);
4405         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
4406
4407         rte_intr_enable(&dev->pci_dev->intr_handle);
4408
4409         return 0;
4410 }
4411
4412 static int
4413 ixgbevf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
4414 {
4415         uint32_t mask;
4416         struct ixgbe_hw *hw =
4417                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4418
4419         mask = IXGBE_READ_REG(hw, IXGBE_VTEIMS);
4420         mask &= ~(1 << IXGBE_MISC_VEC_ID);
4421         RTE_SET_USED(queue_id);
4422         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
4423
4424         return 0;
4425 }
4426
4427 static int
4428 ixgbe_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
4429 {
4430         uint32_t mask;
4431         struct ixgbe_hw *hw =
4432                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4433         struct ixgbe_interrupt *intr =
4434                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
4435
4436         if (queue_id < 16) {
4437                 ixgbe_disable_intr(hw);
4438                 intr->mask |= (1 << queue_id);
4439                 ixgbe_enable_intr(dev);
4440         } else if (queue_id < 32) {
4441                 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(0));
4442                 mask &= (1 << queue_id);
4443                 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
4444         } else if (queue_id < 64) {
4445                 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(1));
4446                 mask &= (1 << (queue_id - 32));
4447                 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
4448         }
4449         rte_intr_enable(&dev->pci_dev->intr_handle);
4450
4451         return 0;
4452 }
4453
4454 static int
4455 ixgbe_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
4456 {
4457         uint32_t mask;
4458         struct ixgbe_hw *hw =
4459                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4460         struct ixgbe_interrupt *intr =
4461                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
4462
4463         if (queue_id < 16) {
4464                 ixgbe_disable_intr(hw);
4465                 intr->mask &= ~(1 << queue_id);
4466                 ixgbe_enable_intr(dev);
4467         } else if (queue_id < 32) {
4468                 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(0));
4469                 mask &= ~(1 << queue_id);
4470                 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
4471         } else if (queue_id < 64) {
4472                 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(1));
4473                 mask &= ~(1 << (queue_id - 32));
4474                 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
4475         }
4476
4477         return 0;
4478 }
4479
4480 static void
4481 ixgbevf_set_ivar_map(struct ixgbe_hw *hw, int8_t direction,
4482                      uint8_t queue, uint8_t msix_vector)
4483 {
4484         uint32_t tmp, idx;
4485
4486         if (direction == -1) {
4487                 /* other causes */
4488                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4489                 tmp = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
4490                 tmp &= ~0xFF;
4491                 tmp |= msix_vector;
4492                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, tmp);
4493         } else {
4494                 /* rx or tx cause */
4495                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4496                 idx = ((16 * (queue & 1)) + (8 * direction));
4497                 tmp = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1));
4498                 tmp &= ~(0xFF << idx);
4499                 tmp |= (msix_vector << idx);
4500                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), tmp);
4501         }
4502 }
4503
4504 /**
4505  * set the IVAR registers, mapping interrupt causes to vectors
4506  * @param hw
4507  *  pointer to ixgbe_hw struct
4508  * @direction
4509  *  0 for Rx, 1 for Tx, -1 for other causes
4510  * @queue
4511  *  queue to map the corresponding interrupt to
4512  * @msix_vector
4513  *  the vector to map to the corresponding queue
4514  */
4515 static void
4516 ixgbe_set_ivar_map(struct ixgbe_hw *hw, int8_t direction,
4517                    uint8_t queue, uint8_t msix_vector)
4518 {
4519         uint32_t tmp, idx;
4520
4521         msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4522         if (hw->mac.type == ixgbe_mac_82598EB) {
4523                 if (direction == -1)
4524                         direction = 0;
4525                 idx = (((direction * 64) + queue) >> 2) & 0x1F;
4526                 tmp = IXGBE_READ_REG(hw, IXGBE_IVAR(idx));
4527                 tmp &= ~(0xFF << (8 * (queue & 0x3)));
4528                 tmp |= (msix_vector << (8 * (queue & 0x3)));
4529                 IXGBE_WRITE_REG(hw, IXGBE_IVAR(idx), tmp);
4530         } else if ((hw->mac.type == ixgbe_mac_82599EB) ||
4531                         (hw->mac.type == ixgbe_mac_X540)) {
4532                 if (direction == -1) {
4533                         /* other causes */
4534                         idx = ((queue & 1) * 8);
4535                         tmp = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4536                         tmp &= ~(0xFF << idx);
4537                         tmp |= (msix_vector << idx);
4538                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, tmp);
4539                 } else {
4540                         /* rx or tx causes */
4541                         idx = ((16 * (queue & 1)) + (8 * direction));
4542                         tmp = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1));
4543                         tmp &= ~(0xFF << idx);
4544                         tmp |= (msix_vector << idx);
4545                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), tmp);
4546                 }
4547         }
4548 }
4549
4550 static void
4551 ixgbevf_configure_msix(struct rte_eth_dev *dev)
4552 {
4553         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
4554         struct ixgbe_hw *hw =
4555                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4556         uint32_t q_idx;
4557         uint32_t vector_idx = IXGBE_MISC_VEC_ID;
4558
4559         /* won't configure msix register if no mapping is done
4560          * between intr vector and event fd.
4561          */
4562         if (!rte_intr_dp_is_en(intr_handle))
4563                 return;
4564
4565         /* Configure all RX queues of VF */
4566         for (q_idx = 0; q_idx < dev->data->nb_rx_queues; q_idx++) {
4567                 /* Force all queue use vector 0,
4568                  * as IXGBE_VF_MAXMSIVECOTR = 1
4569                  */
4570                 ixgbevf_set_ivar_map(hw, 0, q_idx, vector_idx);
4571                 intr_handle->intr_vec[q_idx] = vector_idx;
4572         }
4573
4574         /* Configure VF other cause ivar */
4575         ixgbevf_set_ivar_map(hw, -1, 1, vector_idx);
4576 }
4577
4578 /**
4579  * Sets up the hardware to properly generate MSI-X interrupts
4580  * @hw
4581  *  board private structure
4582  */
4583 static void
4584 ixgbe_configure_msix(struct rte_eth_dev *dev)
4585 {
4586         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
4587         struct ixgbe_hw *hw =
4588                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4589         uint32_t queue_id, base = IXGBE_MISC_VEC_ID;
4590         uint32_t vec = IXGBE_MISC_VEC_ID;
4591         uint32_t mask;
4592         uint32_t gpie;
4593
4594         /* won't configure msix register if no mapping is done
4595          * between intr vector and event fd
4596          */
4597         if (!rte_intr_dp_is_en(intr_handle))
4598                 return;
4599
4600         if (rte_intr_allow_others(intr_handle))
4601                 vec = base = IXGBE_RX_VEC_START;
4602
4603         /* setup GPIE for MSI-x mode */
4604         gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
4605         gpie |= IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT |
4606                 IXGBE_GPIE_OCD | IXGBE_GPIE_EIAME;
4607         /* auto clearing and auto setting corresponding bits in EIMS
4608          * when MSI-X interrupt is triggered
4609          */
4610         if (hw->mac.type == ixgbe_mac_82598EB) {
4611                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
4612         } else {
4613                 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
4614                 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
4615         }
4616         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
4617
4618         /* Populate the IVAR table and set the ITR values to the
4619          * corresponding register.
4620          */
4621         for (queue_id = 0; queue_id < dev->data->nb_rx_queues;
4622              queue_id++) {
4623                 /* by default, 1:1 mapping */
4624                 ixgbe_set_ivar_map(hw, 0, queue_id, vec);
4625                 intr_handle->intr_vec[queue_id] = vec;
4626                 if (vec < base + intr_handle->nb_efd - 1)
4627                         vec++;
4628         }
4629
4630         switch (hw->mac.type) {
4631         case ixgbe_mac_82598EB:
4632                 ixgbe_set_ivar_map(hw, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX,
4633                                    IXGBE_MISC_VEC_ID);
4634                 break;
4635         case ixgbe_mac_82599EB:
4636         case ixgbe_mac_X540:
4637                 ixgbe_set_ivar_map(hw, -1, 1, IXGBE_MISC_VEC_ID);
4638                 break;
4639         default:
4640                 break;
4641         }
4642         IXGBE_WRITE_REG(hw, IXGBE_EITR(IXGBE_MISC_VEC_ID),
4643                         IXGBE_MIN_INTER_INTERRUPT_INTERVAL_DEFAULT & 0xFFF);
4644
4645         /* set up to autoclear timer, and the vectors */
4646         mask = IXGBE_EIMS_ENABLE_MASK;
4647         mask &= ~(IXGBE_EIMS_OTHER |
4648                   IXGBE_EIMS_MAILBOX |
4649                   IXGBE_EIMS_LSC);
4650
4651         IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4652 }
4653
4654 static int ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev,
4655         uint16_t queue_idx, uint16_t tx_rate)
4656 {
4657         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4658         uint32_t rf_dec, rf_int;
4659         uint32_t bcnrc_val;
4660         uint16_t link_speed = dev->data->dev_link.link_speed;
4661
4662         if (queue_idx >= hw->mac.max_tx_queues)
4663                 return -EINVAL;
4664
4665         if (tx_rate != 0) {
4666                 /* Calculate the rate factor values to set */
4667                 rf_int = (uint32_t)link_speed / (uint32_t)tx_rate;
4668                 rf_dec = (uint32_t)link_speed % (uint32_t)tx_rate;
4669                 rf_dec = (rf_dec << IXGBE_RTTBCNRC_RF_INT_SHIFT) / tx_rate;
4670
4671                 bcnrc_val = IXGBE_RTTBCNRC_RS_ENA;
4672                 bcnrc_val |= ((rf_int << IXGBE_RTTBCNRC_RF_INT_SHIFT) &
4673                                 IXGBE_RTTBCNRC_RF_INT_MASK_M);
4674                 bcnrc_val |= (rf_dec & IXGBE_RTTBCNRC_RF_DEC_MASK);
4675         } else {
4676                 bcnrc_val = 0;
4677         }
4678
4679         /*
4680          * Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
4681          * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported, otherwise
4682          * set as 0x4.
4683          */
4684         if ((dev->data->dev_conf.rxmode.jumbo_frame == 1) &&
4685                 (dev->data->dev_conf.rxmode.max_rx_pkt_len >=
4686                                 IXGBE_MAX_JUMBO_FRAME_SIZE))
4687                 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRM,
4688                         IXGBE_MMW_SIZE_JUMBO_FRAME);
4689         else
4690                 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRM,
4691                         IXGBE_MMW_SIZE_DEFAULT);
4692
4693         /* Set RTTBCNRC of queue X */
4694         IXGBE_WRITE_REG(hw, IXGBE_RTTDQSEL, queue_idx);
4695         IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRC, bcnrc_val);
4696         IXGBE_WRITE_FLUSH(hw);
4697
4698         return 0;
4699 }
4700
4701 static int ixgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf,
4702         uint16_t tx_rate, uint64_t q_msk)
4703 {
4704         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4705         struct ixgbe_vf_info *vfinfo =
4706                 *(IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private));
4707         uint8_t  nb_q_per_pool = RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool;
4708         uint32_t queue_stride =
4709                 IXGBE_MAX_RX_QUEUE_NUM / RTE_ETH_DEV_SRIOV(dev).active;
4710         uint32_t queue_idx = vf * queue_stride, idx = 0, vf_idx;
4711         uint32_t queue_end = queue_idx + nb_q_per_pool - 1;
4712         uint16_t total_rate = 0;
4713
4714         if (queue_end >= hw->mac.max_tx_queues)
4715                 return -EINVAL;
4716
4717         if (vfinfo != NULL) {
4718                 for (vf_idx = 0; vf_idx < dev->pci_dev->max_vfs; vf_idx++) {
4719                         if (vf_idx == vf)
4720                                 continue;
4721                         for (idx = 0; idx < RTE_DIM(vfinfo[vf_idx].tx_rate);
4722                                 idx++)
4723                                 total_rate += vfinfo[vf_idx].tx_rate[idx];
4724                 }
4725         } else
4726                 return -EINVAL;
4727
4728         /* Store tx_rate for this vf. */
4729         for (idx = 0; idx < nb_q_per_pool; idx++) {
4730                 if (((uint64_t)0x1 << idx) & q_msk) {
4731                         if (vfinfo[vf].tx_rate[idx] != tx_rate)
4732                                 vfinfo[vf].tx_rate[idx] = tx_rate;
4733                         total_rate += tx_rate;
4734                 }
4735         }
4736
4737         if (total_rate > dev->data->dev_link.link_speed) {
4738                 /*
4739                  * Reset stored TX rate of the VF if it causes exceed
4740                  * link speed.
4741                  */
4742                 memset(vfinfo[vf].tx_rate, 0, sizeof(vfinfo[vf].tx_rate));
4743                 return -EINVAL;
4744         }
4745
4746         /* Set RTTBCNRC of each queue/pool for vf X  */
4747         for (; queue_idx <= queue_end; queue_idx++) {
4748                 if (0x1 & q_msk)
4749                         ixgbe_set_queue_rate_limit(dev, queue_idx, tx_rate);
4750                 q_msk = q_msk >> 1;
4751         }
4752
4753         return 0;
4754 }
4755
4756 static void
4757 ixgbevf_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
4758                      __attribute__((unused)) uint32_t index,
4759                      __attribute__((unused)) uint32_t pool)
4760 {
4761         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4762         int diag;
4763
4764         /*
4765          * On a 82599 VF, adding again the same MAC addr is not an idempotent
4766          * operation. Trap this case to avoid exhausting the [very limited]
4767          * set of PF resources used to store VF MAC addresses.
4768          */
4769         if (memcmp(hw->mac.perm_addr, mac_addr, sizeof(struct ether_addr)) == 0)
4770                 return;
4771         diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes);
4772         if (diag == 0)
4773                 return;
4774         PMD_DRV_LOG(ERR, "Unable to add MAC address - diag=%d", diag);
4775 }
4776
4777 static void
4778 ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index)
4779 {
4780         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4781         struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr;
4782         struct ether_addr *mac_addr;
4783         uint32_t i;
4784         int diag;
4785
4786         /*
4787          * The IXGBE_VF_SET_MACVLAN command of the ixgbe-pf driver does
4788          * not support the deletion of a given MAC address.
4789          * Instead, it imposes to delete all MAC addresses, then to add again
4790          * all MAC addresses with the exception of the one to be deleted.
4791          */
4792         (void) ixgbevf_set_uc_addr_vf(hw, 0, NULL);
4793
4794         /*
4795          * Add again all MAC addresses, with the exception of the deleted one
4796          * and of the permanent MAC address.
4797          */
4798         for (i = 0, mac_addr = dev->data->mac_addrs;
4799              i < hw->mac.num_rar_entries; i++, mac_addr++) {
4800                 /* Skip the deleted MAC address */
4801                 if (i == index)
4802                         continue;
4803                 /* Skip NULL MAC addresses */
4804                 if (is_zero_ether_addr(mac_addr))
4805                         continue;
4806                 /* Skip the permanent MAC address */
4807                 if (memcmp(perm_addr, mac_addr, sizeof(struct ether_addr)) == 0)
4808                         continue;
4809                 diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes);
4810                 if (diag != 0)
4811                         PMD_DRV_LOG(ERR,
4812                                     "Adding again MAC address "
4813                                     "%02x:%02x:%02x:%02x:%02x:%02x failed "
4814                                     "diag=%d",
4815                                     mac_addr->addr_bytes[0],
4816                                     mac_addr->addr_bytes[1],
4817                                     mac_addr->addr_bytes[2],
4818                                     mac_addr->addr_bytes[3],
4819                                     mac_addr->addr_bytes[4],
4820                                     mac_addr->addr_bytes[5],
4821                                     diag);
4822         }
4823 }
4824
4825 static void
4826 ixgbevf_set_default_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
4827 {
4828         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4829
4830         hw->mac.ops.set_rar(hw, 0, (void *)addr, 0, 0);
4831 }
4832
4833 #define MAC_TYPE_FILTER_SUP(type)    do {\
4834         if ((type) != ixgbe_mac_82599EB && (type) != ixgbe_mac_X540 &&\
4835                 (type) != ixgbe_mac_X550)\
4836                 return -ENOTSUP;\
4837 } while (0)
4838
4839 static int
4840 ixgbe_syn_filter_set(struct rte_eth_dev *dev,
4841                         struct rte_eth_syn_filter *filter,
4842                         bool add)
4843 {
4844         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4845         uint32_t synqf;
4846
4847         if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM)
4848                 return -EINVAL;
4849
4850         synqf = IXGBE_READ_REG(hw, IXGBE_SYNQF);
4851
4852         if (add) {
4853                 if (synqf & IXGBE_SYN_FILTER_ENABLE)
4854                         return -EINVAL;
4855                 synqf = (uint32_t)(((filter->queue << IXGBE_SYN_FILTER_QUEUE_SHIFT) &
4856                         IXGBE_SYN_FILTER_QUEUE) | IXGBE_SYN_FILTER_ENABLE);
4857
4858                 if (filter->hig_pri)
4859                         synqf |= IXGBE_SYN_FILTER_SYNQFP;
4860                 else
4861                         synqf &= ~IXGBE_SYN_FILTER_SYNQFP;
4862         } else {
4863                 if (!(synqf & IXGBE_SYN_FILTER_ENABLE))
4864                         return -ENOENT;
4865                 synqf &= ~(IXGBE_SYN_FILTER_QUEUE | IXGBE_SYN_FILTER_ENABLE);
4866         }
4867         IXGBE_WRITE_REG(hw, IXGBE_SYNQF, synqf);
4868         IXGBE_WRITE_FLUSH(hw);
4869         return 0;
4870 }
4871
4872 static int
4873 ixgbe_syn_filter_get(struct rte_eth_dev *dev,
4874                         struct rte_eth_syn_filter *filter)
4875 {
4876         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4877         uint32_t synqf = IXGBE_READ_REG(hw, IXGBE_SYNQF);
4878
4879         if (synqf & IXGBE_SYN_FILTER_ENABLE) {
4880                 filter->hig_pri = (synqf & IXGBE_SYN_FILTER_SYNQFP) ? 1 : 0;
4881                 filter->queue = (uint16_t)((synqf & IXGBE_SYN_FILTER_QUEUE) >> 1);
4882                 return 0;
4883         }
4884         return -ENOENT;
4885 }
4886
4887 static int
4888 ixgbe_syn_filter_handle(struct rte_eth_dev *dev,
4889                         enum rte_filter_op filter_op,
4890                         void *arg)
4891 {
4892         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4893         int ret;
4894
4895         MAC_TYPE_FILTER_SUP(hw->mac.type);
4896
4897         if (filter_op == RTE_ETH_FILTER_NOP)
4898                 return 0;
4899
4900         if (arg == NULL) {
4901                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
4902                             filter_op);
4903                 return -EINVAL;
4904         }
4905
4906         switch (filter_op) {
4907         case RTE_ETH_FILTER_ADD:
4908                 ret = ixgbe_syn_filter_set(dev,
4909                                 (struct rte_eth_syn_filter *)arg,
4910                                 TRUE);
4911                 break;
4912         case RTE_ETH_FILTER_DELETE:
4913                 ret = ixgbe_syn_filter_set(dev,
4914                                 (struct rte_eth_syn_filter *)arg,
4915                                 FALSE);
4916                 break;
4917         case RTE_ETH_FILTER_GET:
4918                 ret = ixgbe_syn_filter_get(dev,
4919                                 (struct rte_eth_syn_filter *)arg);
4920                 break;
4921         default:
4922                 PMD_DRV_LOG(ERR, "unsupported operation %u\n", filter_op);
4923                 ret = -EINVAL;
4924                 break;
4925         }
4926
4927         return ret;
4928 }
4929
4930
4931 static inline enum ixgbe_5tuple_protocol
4932 convert_protocol_type(uint8_t protocol_value)
4933 {
4934         if (protocol_value == IPPROTO_TCP)
4935                 return IXGBE_FILTER_PROTOCOL_TCP;
4936         else if (protocol_value == IPPROTO_UDP)
4937                 return IXGBE_FILTER_PROTOCOL_UDP;
4938         else if (protocol_value == IPPROTO_SCTP)
4939                 return IXGBE_FILTER_PROTOCOL_SCTP;
4940         else
4941                 return IXGBE_FILTER_PROTOCOL_NONE;
4942 }
4943
4944 /*
4945  * add a 5tuple filter
4946  *
4947  * @param
4948  * dev: Pointer to struct rte_eth_dev.
4949  * index: the index the filter allocates.
4950  * filter: ponter to the filter that will be added.
4951  * rx_queue: the queue id the filter assigned to.
4952  *
4953  * @return
4954  *    - On success, zero.
4955  *    - On failure, a negative value.
4956  */
4957 static int
4958 ixgbe_add_5tuple_filter(struct rte_eth_dev *dev,
4959                         struct ixgbe_5tuple_filter *filter)
4960 {
4961         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4962         struct ixgbe_filter_info *filter_info =
4963                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4964         int i, idx, shift;
4965         uint32_t ftqf, sdpqf;
4966         uint32_t l34timir = 0;
4967         uint8_t mask = 0xff;
4968
4969         /*
4970          * look for an unused 5tuple filter index,
4971          * and insert the filter to list.
4972          */
4973         for (i = 0; i < IXGBE_MAX_FTQF_FILTERS; i++) {
4974                 idx = i / (sizeof(uint32_t) * NBBY);
4975                 shift = i % (sizeof(uint32_t) * NBBY);
4976                 if (!(filter_info->fivetuple_mask[idx] & (1 << shift))) {
4977                         filter_info->fivetuple_mask[idx] |= 1 << shift;
4978                         filter->index = i;
4979                         TAILQ_INSERT_TAIL(&filter_info->fivetuple_list,
4980                                           filter,
4981                                           entries);
4982                         break;
4983                 }
4984         }
4985         if (i >= IXGBE_MAX_FTQF_FILTERS) {
4986                 PMD_DRV_LOG(ERR, "5tuple filters are full.");
4987                 return -ENOSYS;
4988         }
4989
4990         sdpqf = (uint32_t)(filter->filter_info.dst_port <<
4991                                 IXGBE_SDPQF_DSTPORT_SHIFT);
4992         sdpqf = sdpqf | (filter->filter_info.src_port & IXGBE_SDPQF_SRCPORT);
4993
4994         ftqf = (uint32_t)(filter->filter_info.proto &
4995                 IXGBE_FTQF_PROTOCOL_MASK);
4996         ftqf |= (uint32_t)((filter->filter_info.priority &
4997                 IXGBE_FTQF_PRIORITY_MASK) << IXGBE_FTQF_PRIORITY_SHIFT);
4998         if (filter->filter_info.src_ip_mask == 0) /* 0 means compare. */
4999                 mask &= IXGBE_FTQF_SOURCE_ADDR_MASK;
5000         if (filter->filter_info.dst_ip_mask == 0)
5001                 mask &= IXGBE_FTQF_DEST_ADDR_MASK;
5002         if (filter->filter_info.src_port_mask == 0)
5003                 mask &= IXGBE_FTQF_SOURCE_PORT_MASK;
5004         if (filter->filter_info.dst_port_mask == 0)
5005                 mask &= IXGBE_FTQF_DEST_PORT_MASK;
5006         if (filter->filter_info.proto_mask == 0)
5007                 mask &= IXGBE_FTQF_PROTOCOL_COMP_MASK;
5008         ftqf |= mask << IXGBE_FTQF_5TUPLE_MASK_SHIFT;
5009         ftqf |= IXGBE_FTQF_POOL_MASK_EN;
5010         ftqf |= IXGBE_FTQF_QUEUE_ENABLE;
5011
5012         IXGBE_WRITE_REG(hw, IXGBE_DAQF(i), filter->filter_info.dst_ip);
5013         IXGBE_WRITE_REG(hw, IXGBE_SAQF(i), filter->filter_info.src_ip);
5014         IXGBE_WRITE_REG(hw, IXGBE_SDPQF(i), sdpqf);
5015         IXGBE_WRITE_REG(hw, IXGBE_FTQF(i), ftqf);
5016
5017         l34timir |= IXGBE_L34T_IMIR_RESERVE;
5018         l34timir |= (uint32_t)(filter->queue <<
5019                                 IXGBE_L34T_IMIR_QUEUE_SHIFT);
5020         IXGBE_WRITE_REG(hw, IXGBE_L34T_IMIR(i), l34timir);
5021         return 0;
5022 }
5023
5024 /*
5025  * remove a 5tuple filter
5026  *
5027  * @param
5028  * dev: Pointer to struct rte_eth_dev.
5029  * filter: the pointer of the filter will be removed.
5030  */
5031 static void
5032 ixgbe_remove_5tuple_filter(struct rte_eth_dev *dev,
5033                         struct ixgbe_5tuple_filter *filter)
5034 {
5035         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5036         struct ixgbe_filter_info *filter_info =
5037                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
5038         uint16_t index = filter->index;
5039
5040         filter_info->fivetuple_mask[index / (sizeof(uint32_t) * NBBY)] &=
5041                                 ~(1 << (index % (sizeof(uint32_t) * NBBY)));
5042         TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries);
5043         rte_free(filter);
5044
5045         IXGBE_WRITE_REG(hw, IXGBE_DAQF(index), 0);
5046         IXGBE_WRITE_REG(hw, IXGBE_SAQF(index), 0);
5047         IXGBE_WRITE_REG(hw, IXGBE_SDPQF(index), 0);
5048         IXGBE_WRITE_REG(hw, IXGBE_FTQF(index), 0);
5049         IXGBE_WRITE_REG(hw, IXGBE_L34T_IMIR(index), 0);
5050 }
5051
5052 static int
5053 ixgbevf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
5054 {
5055         struct ixgbe_hw *hw;
5056         uint32_t max_frame = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
5057
5058         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5059
5060         if ((mtu < ETHER_MIN_MTU) || (max_frame > ETHER_MAX_JUMBO_FRAME_LEN))
5061                 return -EINVAL;
5062
5063         /* refuse mtu that requires the support of scattered packets when this
5064          * feature has not been enabled before. */
5065         if (!dev->data->scattered_rx &&
5066             (max_frame + 2 * IXGBE_VLAN_TAG_SIZE >
5067              dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM))
5068                 return -EINVAL;
5069
5070         /*
5071          * When supported by the underlying PF driver, use the IXGBE_VF_SET_MTU
5072          * request of the version 2.0 of the mailbox API.
5073          * For now, use the IXGBE_VF_SET_LPE request of the version 1.0
5074          * of the mailbox API.
5075          * This call to IXGBE_SET_LPE action won't work with ixgbe pf drivers
5076          * prior to 3.11.33 which contains the following change:
5077          * "ixgbe: Enable jumbo frames support w/ SR-IOV"
5078          */
5079         ixgbevf_rlpml_set_vf(hw, max_frame);
5080
5081         /* update max frame size */
5082         dev->data->dev_conf.rxmode.max_rx_pkt_len = max_frame;
5083         return 0;
5084 }
5085
5086 #define MAC_TYPE_FILTER_SUP_EXT(type)    do {\
5087         if ((type) != ixgbe_mac_82599EB && (type) != ixgbe_mac_X540)\
5088                 return -ENOTSUP;\
5089 } while (0)
5090
5091 static inline struct ixgbe_5tuple_filter *
5092 ixgbe_5tuple_filter_lookup(struct ixgbe_5tuple_filter_list *filter_list,
5093                         struct ixgbe_5tuple_filter_info *key)
5094 {
5095         struct ixgbe_5tuple_filter *it;
5096
5097         TAILQ_FOREACH(it, filter_list, entries) {
5098                 if (memcmp(key, &it->filter_info,
5099                         sizeof(struct ixgbe_5tuple_filter_info)) == 0) {
5100                         return it;
5101                 }
5102         }
5103         return NULL;
5104 }
5105
5106 /* translate elements in struct rte_eth_ntuple_filter to struct ixgbe_5tuple_filter_info*/
5107 static inline int
5108 ntuple_filter_to_5tuple(struct rte_eth_ntuple_filter *filter,
5109                         struct ixgbe_5tuple_filter_info *filter_info)
5110 {
5111         if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM ||
5112                 filter->priority > IXGBE_5TUPLE_MAX_PRI ||
5113                 filter->priority < IXGBE_5TUPLE_MIN_PRI)
5114                 return -EINVAL;
5115
5116         switch (filter->dst_ip_mask) {
5117         case UINT32_MAX:
5118                 filter_info->dst_ip_mask = 0;
5119                 filter_info->dst_ip = filter->dst_ip;
5120                 break;
5121         case 0:
5122                 filter_info->dst_ip_mask = 1;
5123                 break;
5124         default:
5125                 PMD_DRV_LOG(ERR, "invalid dst_ip mask.");
5126                 return -EINVAL;
5127         }
5128
5129         switch (filter->src_ip_mask) {
5130         case UINT32_MAX:
5131                 filter_info->src_ip_mask = 0;
5132                 filter_info->src_ip = filter->src_ip;
5133                 break;
5134         case 0:
5135                 filter_info->src_ip_mask = 1;
5136                 break;
5137         default:
5138                 PMD_DRV_LOG(ERR, "invalid src_ip mask.");
5139                 return -EINVAL;
5140         }
5141
5142         switch (filter->dst_port_mask) {
5143         case UINT16_MAX:
5144                 filter_info->dst_port_mask = 0;
5145                 filter_info->dst_port = filter->dst_port;
5146                 break;
5147         case 0:
5148                 filter_info->dst_port_mask = 1;
5149                 break;
5150         default:
5151                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
5152                 return -EINVAL;
5153         }
5154
5155         switch (filter->src_port_mask) {
5156         case UINT16_MAX:
5157                 filter_info->src_port_mask = 0;
5158                 filter_info->src_port = filter->src_port;
5159                 break;
5160         case 0:
5161                 filter_info->src_port_mask = 1;
5162                 break;
5163         default:
5164                 PMD_DRV_LOG(ERR, "invalid src_port mask.");
5165                 return -EINVAL;
5166         }
5167
5168         switch (filter->proto_mask) {
5169         case UINT8_MAX:
5170                 filter_info->proto_mask = 0;
5171                 filter_info->proto =
5172                         convert_protocol_type(filter->proto);
5173                 break;
5174         case 0:
5175                 filter_info->proto_mask = 1;
5176                 break;
5177         default:
5178                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
5179                 return -EINVAL;
5180         }
5181
5182         filter_info->priority = (uint8_t)filter->priority;
5183         return 0;
5184 }
5185
5186 /*
5187  * add or delete a ntuple filter
5188  *
5189  * @param
5190  * dev: Pointer to struct rte_eth_dev.
5191  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
5192  * add: if true, add filter, if false, remove filter
5193  *
5194  * @return
5195  *    - On success, zero.
5196  *    - On failure, a negative value.
5197  */
5198 static int
5199 ixgbe_add_del_ntuple_filter(struct rte_eth_dev *dev,
5200                         struct rte_eth_ntuple_filter *ntuple_filter,
5201                         bool add)
5202 {
5203         struct ixgbe_filter_info *filter_info =
5204                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
5205         struct ixgbe_5tuple_filter_info filter_5tuple;
5206         struct ixgbe_5tuple_filter *filter;
5207         int ret;
5208
5209         if (ntuple_filter->flags != RTE_5TUPLE_FLAGS) {
5210                 PMD_DRV_LOG(ERR, "only 5tuple is supported.");
5211                 return -EINVAL;
5212         }
5213
5214         memset(&filter_5tuple, 0, sizeof(struct ixgbe_5tuple_filter_info));
5215         ret = ntuple_filter_to_5tuple(ntuple_filter, &filter_5tuple);
5216         if (ret < 0)
5217                 return ret;
5218
5219         filter = ixgbe_5tuple_filter_lookup(&filter_info->fivetuple_list,
5220                                          &filter_5tuple);
5221         if (filter != NULL && add) {
5222                 PMD_DRV_LOG(ERR, "filter exists.");
5223                 return -EEXIST;
5224         }
5225         if (filter == NULL && !add) {
5226                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
5227                 return -ENOENT;
5228         }
5229
5230         if (add) {
5231                 filter = rte_zmalloc("ixgbe_5tuple_filter",
5232                                 sizeof(struct ixgbe_5tuple_filter), 0);
5233                 if (filter == NULL)
5234                         return -ENOMEM;
5235                 (void)rte_memcpy(&filter->filter_info,
5236                                  &filter_5tuple,
5237                                  sizeof(struct ixgbe_5tuple_filter_info));
5238                 filter->queue = ntuple_filter->queue;
5239                 ret = ixgbe_add_5tuple_filter(dev, filter);
5240                 if (ret < 0) {
5241                         rte_free(filter);
5242                         return ret;
5243                 }
5244         } else
5245                 ixgbe_remove_5tuple_filter(dev, filter);
5246
5247         return 0;
5248 }
5249
5250 /*
5251  * get a ntuple filter
5252  *
5253  * @param
5254  * dev: Pointer to struct rte_eth_dev.
5255  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
5256  *
5257  * @return
5258  *    - On success, zero.
5259  *    - On failure, a negative value.
5260  */
5261 static int
5262 ixgbe_get_ntuple_filter(struct rte_eth_dev *dev,
5263                         struct rte_eth_ntuple_filter *ntuple_filter)
5264 {
5265         struct ixgbe_filter_info *filter_info =
5266                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
5267         struct ixgbe_5tuple_filter_info filter_5tuple;
5268         struct ixgbe_5tuple_filter *filter;
5269         int ret;
5270
5271         if (ntuple_filter->flags != RTE_5TUPLE_FLAGS) {
5272                 PMD_DRV_LOG(ERR, "only 5tuple is supported.");
5273                 return -EINVAL;
5274         }
5275
5276         memset(&filter_5tuple, 0, sizeof(struct ixgbe_5tuple_filter_info));
5277         ret = ntuple_filter_to_5tuple(ntuple_filter, &filter_5tuple);
5278         if (ret < 0)
5279                 return ret;
5280
5281         filter = ixgbe_5tuple_filter_lookup(&filter_info->fivetuple_list,
5282                                          &filter_5tuple);
5283         if (filter == NULL) {
5284                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
5285                 return -ENOENT;
5286         }
5287         ntuple_filter->queue = filter->queue;
5288         return 0;
5289 }
5290
5291 /*
5292  * ixgbe_ntuple_filter_handle - Handle operations for ntuple filter.
5293  * @dev: pointer to rte_eth_dev structure
5294  * @filter_op:operation will be taken.
5295  * @arg: a pointer to specific structure corresponding to the filter_op
5296  *
5297  * @return
5298  *    - On success, zero.
5299  *    - On failure, a negative value.
5300  */
5301 static int
5302 ixgbe_ntuple_filter_handle(struct rte_eth_dev *dev,
5303                                 enum rte_filter_op filter_op,
5304                                 void *arg)
5305 {
5306         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5307         int ret;
5308
5309         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
5310
5311         if (filter_op == RTE_ETH_FILTER_NOP)
5312                 return 0;
5313
5314         if (arg == NULL) {
5315                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
5316                             filter_op);
5317                 return -EINVAL;
5318         }
5319
5320         switch (filter_op) {
5321         case RTE_ETH_FILTER_ADD:
5322                 ret = ixgbe_add_del_ntuple_filter(dev,
5323                         (struct rte_eth_ntuple_filter *)arg,
5324                         TRUE);
5325                 break;
5326         case RTE_ETH_FILTER_DELETE:
5327                 ret = ixgbe_add_del_ntuple_filter(dev,
5328                         (struct rte_eth_ntuple_filter *)arg,
5329                         FALSE);
5330                 break;
5331         case RTE_ETH_FILTER_GET:
5332                 ret = ixgbe_get_ntuple_filter(dev,
5333                         (struct rte_eth_ntuple_filter *)arg);
5334                 break;
5335         default:
5336                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
5337                 ret = -EINVAL;
5338                 break;
5339         }
5340         return ret;
5341 }
5342
5343 static inline int
5344 ixgbe_ethertype_filter_lookup(struct ixgbe_filter_info *filter_info,
5345                         uint16_t ethertype)
5346 {
5347         int i;
5348
5349         for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) {
5350                 if (filter_info->ethertype_filters[i] == ethertype &&
5351                     (filter_info->ethertype_mask & (1 << i)))
5352                         return i;
5353         }
5354         return -1;
5355 }
5356
5357 static inline int
5358 ixgbe_ethertype_filter_insert(struct ixgbe_filter_info *filter_info,
5359                         uint16_t ethertype)
5360 {
5361         int i;
5362
5363         for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) {
5364                 if (!(filter_info->ethertype_mask & (1 << i))) {
5365                         filter_info->ethertype_mask |= 1 << i;
5366                         filter_info->ethertype_filters[i] = ethertype;
5367                         return i;
5368                 }
5369         }
5370         return -1;
5371 }
5372
5373 static inline int
5374 ixgbe_ethertype_filter_remove(struct ixgbe_filter_info *filter_info,
5375                         uint8_t idx)
5376 {
5377         if (idx >= IXGBE_MAX_ETQF_FILTERS)
5378                 return -1;
5379         filter_info->ethertype_mask &= ~(1 << idx);
5380         filter_info->ethertype_filters[idx] = 0;
5381         return idx;
5382 }
5383
5384 static int
5385 ixgbe_add_del_ethertype_filter(struct rte_eth_dev *dev,
5386                         struct rte_eth_ethertype_filter *filter,
5387                         bool add)
5388 {
5389         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5390         struct ixgbe_filter_info *filter_info =
5391                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
5392         uint32_t etqf = 0;
5393         uint32_t etqs = 0;
5394         int ret;
5395
5396         if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM)
5397                 return -EINVAL;
5398
5399         if (filter->ether_type == ETHER_TYPE_IPv4 ||
5400                 filter->ether_type == ETHER_TYPE_IPv6) {
5401                 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in"
5402                         " ethertype filter.", filter->ether_type);
5403                 return -EINVAL;
5404         }
5405
5406         if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) {
5407                 PMD_DRV_LOG(ERR, "mac compare is unsupported.");
5408                 return -EINVAL;
5409         }
5410         if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) {
5411                 PMD_DRV_LOG(ERR, "drop option is unsupported.");
5412                 return -EINVAL;
5413         }
5414
5415         ret = ixgbe_ethertype_filter_lookup(filter_info, filter->ether_type);
5416         if (ret >= 0 && add) {
5417                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.",
5418                             filter->ether_type);
5419                 return -EEXIST;
5420         }
5421         if (ret < 0 && !add) {
5422                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
5423                             filter->ether_type);
5424                 return -ENOENT;
5425         }
5426
5427         if (add) {
5428                 ret = ixgbe_ethertype_filter_insert(filter_info,
5429                         filter->ether_type);
5430                 if (ret < 0) {
5431                         PMD_DRV_LOG(ERR, "ethertype filters are full.");
5432                         return -ENOSYS;
5433                 }
5434                 etqf = IXGBE_ETQF_FILTER_EN;
5435                 etqf |= (uint32_t)filter->ether_type;
5436                 etqs |= (uint32_t)((filter->queue <<
5437                                     IXGBE_ETQS_RX_QUEUE_SHIFT) &
5438                                     IXGBE_ETQS_RX_QUEUE);
5439                 etqs |= IXGBE_ETQS_QUEUE_EN;
5440         } else {
5441                 ret = ixgbe_ethertype_filter_remove(filter_info, (uint8_t)ret);
5442                 if (ret < 0)
5443                         return -ENOSYS;
5444         }
5445         IXGBE_WRITE_REG(hw, IXGBE_ETQF(ret), etqf);
5446         IXGBE_WRITE_REG(hw, IXGBE_ETQS(ret), etqs);
5447         IXGBE_WRITE_FLUSH(hw);
5448
5449         return 0;
5450 }
5451
5452 static int
5453 ixgbe_get_ethertype_filter(struct rte_eth_dev *dev,
5454                         struct rte_eth_ethertype_filter *filter)
5455 {
5456         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5457         struct ixgbe_filter_info *filter_info =
5458                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
5459         uint32_t etqf, etqs;
5460         int ret;
5461
5462         ret = ixgbe_ethertype_filter_lookup(filter_info, filter->ether_type);
5463         if (ret < 0) {
5464                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
5465                             filter->ether_type);
5466                 return -ENOENT;
5467         }
5468
5469         etqf = IXGBE_READ_REG(hw, IXGBE_ETQF(ret));
5470         if (etqf & IXGBE_ETQF_FILTER_EN) {
5471                 etqs = IXGBE_READ_REG(hw, IXGBE_ETQS(ret));
5472                 filter->ether_type = etqf & IXGBE_ETQF_ETHERTYPE;
5473                 filter->flags = 0;
5474                 filter->queue = (etqs & IXGBE_ETQS_RX_QUEUE) >>
5475                                IXGBE_ETQS_RX_QUEUE_SHIFT;
5476                 return 0;
5477         }
5478         return -ENOENT;
5479 }
5480
5481 /*
5482  * ixgbe_ethertype_filter_handle - Handle operations for ethertype filter.
5483  * @dev: pointer to rte_eth_dev structure
5484  * @filter_op:operation will be taken.
5485  * @arg: a pointer to specific structure corresponding to the filter_op
5486  */
5487 static int
5488 ixgbe_ethertype_filter_handle(struct rte_eth_dev *dev,
5489                                 enum rte_filter_op filter_op,
5490                                 void *arg)
5491 {
5492         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5493         int ret;
5494
5495         MAC_TYPE_FILTER_SUP(hw->mac.type);
5496
5497         if (filter_op == RTE_ETH_FILTER_NOP)
5498                 return 0;
5499
5500         if (arg == NULL) {
5501                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
5502                             filter_op);
5503                 return -EINVAL;
5504         }
5505
5506         switch (filter_op) {
5507         case RTE_ETH_FILTER_ADD:
5508                 ret = ixgbe_add_del_ethertype_filter(dev,
5509                         (struct rte_eth_ethertype_filter *)arg,
5510                         TRUE);
5511                 break;
5512         case RTE_ETH_FILTER_DELETE:
5513                 ret = ixgbe_add_del_ethertype_filter(dev,
5514                         (struct rte_eth_ethertype_filter *)arg,
5515                         FALSE);
5516                 break;
5517         case RTE_ETH_FILTER_GET:
5518                 ret = ixgbe_get_ethertype_filter(dev,
5519                         (struct rte_eth_ethertype_filter *)arg);
5520                 break;
5521         default:
5522                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
5523                 ret = -EINVAL;
5524                 break;
5525         }
5526         return ret;
5527 }
5528
5529 static int
5530 ixgbe_dev_filter_ctrl(struct rte_eth_dev *dev,
5531                      enum rte_filter_type filter_type,
5532                      enum rte_filter_op filter_op,
5533                      void *arg)
5534 {
5535         int ret = -EINVAL;
5536
5537         switch (filter_type) {
5538         case RTE_ETH_FILTER_NTUPLE:
5539                 ret = ixgbe_ntuple_filter_handle(dev, filter_op, arg);
5540                 break;
5541         case RTE_ETH_FILTER_ETHERTYPE:
5542                 ret = ixgbe_ethertype_filter_handle(dev, filter_op, arg);
5543                 break;
5544         case RTE_ETH_FILTER_SYN:
5545                 ret = ixgbe_syn_filter_handle(dev, filter_op, arg);
5546                 break;
5547         case RTE_ETH_FILTER_FDIR:
5548                 ret = ixgbe_fdir_ctrl_func(dev, filter_op, arg);
5549                 break;
5550         default:
5551                 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported",
5552                                                         filter_type);
5553                 break;
5554         }
5555
5556         return ret;
5557 }
5558
5559 static u8 *
5560 ixgbe_dev_addr_list_itr(__attribute__((unused)) struct ixgbe_hw *hw,
5561                         u8 **mc_addr_ptr, u32 *vmdq)
5562 {
5563         u8 *mc_addr;
5564
5565         *vmdq = 0;
5566         mc_addr = *mc_addr_ptr;
5567         *mc_addr_ptr = (mc_addr + sizeof(struct ether_addr));
5568         return mc_addr;
5569 }
5570
5571 static int
5572 ixgbe_dev_set_mc_addr_list(struct rte_eth_dev *dev,
5573                           struct ether_addr *mc_addr_set,
5574                           uint32_t nb_mc_addr)
5575 {
5576         struct ixgbe_hw *hw;
5577         u8 *mc_addr_list;
5578
5579         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5580         mc_addr_list = (u8 *)mc_addr_set;
5581         return ixgbe_update_mc_addr_list(hw, mc_addr_list, nb_mc_addr,
5582                                          ixgbe_dev_addr_list_itr, TRUE);
5583 }
5584
5585 static int
5586 ixgbe_timesync_enable(struct rte_eth_dev *dev)
5587 {
5588         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5589         uint32_t tsync_ctl;
5590         uint32_t tsauxc;
5591
5592         /* Enable system time for platforms where it isn't on by default. */
5593         tsauxc = IXGBE_READ_REG(hw, IXGBE_TSAUXC);
5594         tsauxc &= ~IXGBE_TSAUXC_DISABLE_SYSTIME;
5595         IXGBE_WRITE_REG(hw, IXGBE_TSAUXC, tsauxc);
5596
5597         /* Start incrementing the register used to timestamp PTP packets. */
5598         IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, IXGBE_TIMINCA_INIT);
5599
5600         /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
5601         IXGBE_WRITE_REG(hw, IXGBE_ETQF(IXGBE_ETQF_FILTER_1588),
5602                         (ETHER_TYPE_1588 |
5603                          IXGBE_ETQF_FILTER_EN |
5604                          IXGBE_ETQF_1588));
5605
5606         /* Enable timestamping of received PTP packets. */
5607         tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL);
5608         tsync_ctl |= IXGBE_TSYNCRXCTL_ENABLED;
5609         IXGBE_WRITE_REG(hw, IXGBE_TSYNCRXCTL, tsync_ctl);
5610
5611         /* Enable timestamping of transmitted PTP packets. */
5612         tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL);
5613         tsync_ctl |= IXGBE_TSYNCTXCTL_ENABLED;
5614         IXGBE_WRITE_REG(hw, IXGBE_TSYNCTXCTL, tsync_ctl);
5615
5616         return 0;
5617 }
5618
5619 static int
5620 ixgbe_timesync_disable(struct rte_eth_dev *dev)
5621 {
5622         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5623         uint32_t tsync_ctl;
5624
5625         /* Disable timestamping of transmitted PTP packets. */
5626         tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL);
5627         tsync_ctl &= ~IXGBE_TSYNCTXCTL_ENABLED;
5628         IXGBE_WRITE_REG(hw, IXGBE_TSYNCTXCTL, tsync_ctl);
5629
5630         /* Disable timestamping of received PTP packets. */
5631         tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL);
5632         tsync_ctl &= ~IXGBE_TSYNCRXCTL_ENABLED;
5633         IXGBE_WRITE_REG(hw, IXGBE_TSYNCRXCTL, tsync_ctl);
5634
5635         /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
5636         IXGBE_WRITE_REG(hw, IXGBE_ETQF(IXGBE_ETQF_FILTER_1588), 0);
5637
5638         /* Stop incrementating the System Time registers. */
5639         IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, 0);
5640
5641         return 0;
5642 }
5643
5644 static int
5645 ixgbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
5646                                  struct timespec *timestamp,
5647                                  uint32_t flags __rte_unused)
5648 {
5649         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5650         uint32_t tsync_rxctl;
5651         uint32_t rx_stmpl;
5652         uint32_t rx_stmph;
5653
5654         tsync_rxctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL);
5655         if ((tsync_rxctl & IXGBE_TSYNCRXCTL_VALID) == 0)
5656                 return -EINVAL;
5657
5658         rx_stmpl = IXGBE_READ_REG(hw, IXGBE_RXSTMPL);
5659         rx_stmph = IXGBE_READ_REG(hw, IXGBE_RXSTMPH);
5660
5661         timestamp->tv_sec = (uint64_t)(((uint64_t)rx_stmph << 32) | rx_stmpl);
5662         timestamp->tv_nsec = 0;
5663
5664         return  0;
5665 }
5666
5667 static int
5668 ixgbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
5669                                  struct timespec *timestamp)
5670 {
5671         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5672         uint32_t tsync_txctl;
5673         uint32_t tx_stmpl;
5674         uint32_t tx_stmph;
5675
5676         tsync_txctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL);
5677         if ((tsync_txctl & IXGBE_TSYNCTXCTL_VALID) == 0)
5678                 return -EINVAL;
5679
5680         tx_stmpl = IXGBE_READ_REG(hw, IXGBE_TXSTMPL);
5681         tx_stmph = IXGBE_READ_REG(hw, IXGBE_TXSTMPH);
5682
5683         timestamp->tv_sec = (uint64_t)(((uint64_t)tx_stmph << 32) | tx_stmpl);
5684         timestamp->tv_nsec = 0;
5685
5686         return  0;
5687 }
5688
5689 static int
5690 ixgbe_get_reg_length(struct rte_eth_dev *dev)
5691 {
5692         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5693         int count = 0;
5694         int g_ind = 0;
5695         const struct reg_info *reg_group;
5696         const struct reg_info **reg_set = (hw->mac.type == ixgbe_mac_82598EB) ?
5697                                     ixgbe_regs_mac_82598EB : ixgbe_regs_others;
5698
5699         while ((reg_group = reg_set[g_ind++]))
5700                 count += ixgbe_regs_group_count(reg_group);
5701
5702         return count;
5703 }
5704
5705 static int
5706 ixgbevf_get_reg_length(struct rte_eth_dev *dev __rte_unused)
5707 {
5708         int count = 0;
5709         int g_ind = 0;
5710         const struct reg_info *reg_group;
5711
5712         while ((reg_group = ixgbevf_regs[g_ind++]))
5713                 count += ixgbe_regs_group_count(reg_group);
5714
5715         return count;
5716 }
5717
5718 static int
5719 ixgbe_get_regs(struct rte_eth_dev *dev,
5720               struct rte_dev_reg_info *regs)
5721 {
5722         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5723         uint32_t *data = regs->data;
5724         int g_ind = 0;
5725         int count = 0;
5726         const struct reg_info *reg_group;
5727         const struct reg_info **reg_set = (hw->mac.type == ixgbe_mac_82598EB) ?
5728                                     ixgbe_regs_mac_82598EB : ixgbe_regs_others;
5729
5730         /* Support only full register dump */
5731         if ((regs->length == 0) ||
5732             (regs->length == (uint32_t)ixgbe_get_reg_length(dev))) {
5733                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
5734                         hw->device_id;
5735                 while ((reg_group = reg_set[g_ind++]))
5736                         count += ixgbe_read_regs_group(dev, &data[count],
5737                                 reg_group);
5738                 return 0;
5739         }
5740
5741         return -ENOTSUP;
5742 }
5743
5744 static int
5745 ixgbevf_get_regs(struct rte_eth_dev *dev,
5746                 struct rte_dev_reg_info *regs)
5747 {
5748         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5749         uint32_t *data = regs->data;
5750         int g_ind = 0;
5751         int count = 0;
5752         const struct reg_info *reg_group;
5753
5754         /* Support only full register dump */
5755         if ((regs->length == 0) ||
5756             (regs->length == (uint32_t)ixgbevf_get_reg_length(dev))) {
5757                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
5758                         hw->device_id;
5759                 while ((reg_group = ixgbevf_regs[g_ind++]))
5760                         count += ixgbe_read_regs_group(dev, &data[count],
5761                                                       reg_group);
5762                 return 0;
5763         }
5764
5765         return -ENOTSUP;
5766 }
5767
5768 static int
5769 ixgbe_get_eeprom_length(struct rte_eth_dev *dev)
5770 {
5771         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5772
5773         /* Return unit is byte count */
5774         return hw->eeprom.word_size * 2;
5775 }
5776
5777 static int
5778 ixgbe_get_eeprom(struct rte_eth_dev *dev,
5779                 struct rte_dev_eeprom_info *in_eeprom)
5780 {
5781         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5782         struct ixgbe_eeprom_info *eeprom = &hw->eeprom;
5783         uint16_t *data = in_eeprom->data;
5784         int first, length;
5785
5786         first = in_eeprom->offset >> 1;
5787         length = in_eeprom->length >> 1;
5788         if ((first > hw->eeprom.word_size) ||
5789             ((first + length) > hw->eeprom.word_size))
5790                 return -EINVAL;
5791
5792         in_eeprom->magic = hw->vendor_id | (hw->device_id << 16);
5793
5794         return eeprom->ops.read_buffer(hw, first, length, data);
5795 }
5796
5797 static int
5798 ixgbe_set_eeprom(struct rte_eth_dev *dev,
5799                 struct rte_dev_eeprom_info *in_eeprom)
5800 {
5801         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5802         struct ixgbe_eeprom_info *eeprom = &hw->eeprom;
5803         uint16_t *data = in_eeprom->data;
5804         int first, length;
5805
5806         first = in_eeprom->offset >> 1;
5807         length = in_eeprom->length >> 1;
5808         if ((first > hw->eeprom.word_size) ||
5809             ((first + length) > hw->eeprom.word_size))
5810                 return -EINVAL;
5811
5812         in_eeprom->magic = hw->vendor_id | (hw->device_id << 16);
5813
5814         return eeprom->ops.write_buffer(hw,  first, length, data);
5815 }
5816
5817 uint16_t
5818 ixgbe_reta_size_get(enum ixgbe_mac_type mac_type) {
5819         switch (mac_type) {
5820         case ixgbe_mac_X550:
5821         case ixgbe_mac_X550EM_x:
5822                 return ETH_RSS_RETA_SIZE_512;
5823         case ixgbe_mac_X550_vf:
5824         case ixgbe_mac_X550EM_x_vf:
5825                 return ETH_RSS_RETA_SIZE_64;
5826         default:
5827                 return ETH_RSS_RETA_SIZE_128;
5828         }
5829 }
5830
5831 uint32_t
5832 ixgbe_reta_reg_get(enum ixgbe_mac_type mac_type, uint16_t reta_idx) {
5833         switch (mac_type) {
5834         case ixgbe_mac_X550:
5835         case ixgbe_mac_X550EM_x:
5836                 if (reta_idx < ETH_RSS_RETA_SIZE_128)
5837                         return IXGBE_RETA(reta_idx >> 2);
5838                 else
5839                         return IXGBE_ERETA((reta_idx - ETH_RSS_RETA_SIZE_128) >> 2);
5840         case ixgbe_mac_X550_vf:
5841         case ixgbe_mac_X550EM_x_vf:
5842                 return IXGBE_VFRETA(reta_idx >> 2);
5843         default:
5844                 return IXGBE_RETA(reta_idx >> 2);
5845         }
5846 }
5847
5848 uint32_t
5849 ixgbe_mrqc_reg_get(enum ixgbe_mac_type mac_type) {
5850         switch (mac_type) {
5851         case ixgbe_mac_X550_vf:
5852         case ixgbe_mac_X550EM_x_vf:
5853                 return IXGBE_VFMRQC;
5854         default:
5855                 return IXGBE_MRQC;
5856         }
5857 }
5858
5859 uint32_t
5860 ixgbe_rssrk_reg_get(enum ixgbe_mac_type mac_type, uint8_t i) {
5861         switch (mac_type) {
5862         case ixgbe_mac_X550_vf:
5863         case ixgbe_mac_X550EM_x_vf:
5864                 return IXGBE_VFRSSRK(i);
5865         default:
5866                 return IXGBE_RSSRK(i);
5867         }
5868 }
5869
5870 bool
5871 ixgbe_rss_update_sp(enum ixgbe_mac_type mac_type) {
5872         switch (mac_type) {
5873         case ixgbe_mac_82599_vf:
5874         case ixgbe_mac_X540_vf:
5875                 return 0;
5876         default:
5877                 return 1;
5878         }
5879 }
5880
5881 static int
5882 ixgbe_dev_get_dcb_info(struct rte_eth_dev *dev,
5883                         struct rte_eth_dcb_info *dcb_info)
5884 {
5885         struct ixgbe_dcb_config *dcb_config =
5886                         IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
5887         struct ixgbe_dcb_tc_config *tc;
5888         uint8_t i, j;
5889
5890         if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_DCB_FLAG)
5891                 dcb_info->nb_tcs = dcb_config->num_tcs.pg_tcs;
5892         else
5893                 dcb_info->nb_tcs = 1;
5894
5895         if (dcb_config->vt_mode) { /* vt is enabled*/
5896                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
5897                                 &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf;
5898                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++)
5899                         dcb_info->prio_tc[i] = vmdq_rx_conf->dcb_tc[i];
5900                 for (i = 0; i < vmdq_rx_conf->nb_queue_pools; i++) {
5901                         for (j = 0; j < dcb_info->nb_tcs; j++) {
5902                                 dcb_info->tc_queue.tc_rxq[i][j].base =
5903                                                 i * dcb_info->nb_tcs + j;
5904                                 dcb_info->tc_queue.tc_rxq[i][j].nb_queue = 1;
5905                                 dcb_info->tc_queue.tc_txq[i][j].base =
5906                                                 i * dcb_info->nb_tcs + j;
5907                                 dcb_info->tc_queue.tc_txq[i][j].nb_queue = 1;
5908                         }
5909                 }
5910         } else { /* vt is disabled*/
5911                 struct rte_eth_dcb_rx_conf *rx_conf =
5912                                 &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf;
5913                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++)
5914                         dcb_info->prio_tc[i] = rx_conf->dcb_tc[i];
5915                 if (dcb_info->nb_tcs == ETH_4_TCS) {
5916                         for (i = 0; i < dcb_info->nb_tcs; i++) {
5917                                 dcb_info->tc_queue.tc_rxq[0][i].base = i * 32;
5918                                 dcb_info->tc_queue.tc_rxq[0][i].nb_queue = 16;
5919                         }
5920                         dcb_info->tc_queue.tc_txq[0][0].base = 0;
5921                         dcb_info->tc_queue.tc_txq[0][1].base = 64;
5922                         dcb_info->tc_queue.tc_txq[0][2].base = 96;
5923                         dcb_info->tc_queue.tc_txq[0][3].base = 112;
5924                         dcb_info->tc_queue.tc_txq[0][0].nb_queue = 64;
5925                         dcb_info->tc_queue.tc_txq[0][1].nb_queue = 32;
5926                         dcb_info->tc_queue.tc_txq[0][2].nb_queue = 16;
5927                         dcb_info->tc_queue.tc_txq[0][3].nb_queue = 16;
5928                 } else if (dcb_info->nb_tcs == ETH_8_TCS) {
5929                         for (i = 0; i < dcb_info->nb_tcs; i++) {
5930                                 dcb_info->tc_queue.tc_rxq[0][i].base = i * 16;
5931                                 dcb_info->tc_queue.tc_rxq[0][i].nb_queue = 16;
5932                         }
5933                         dcb_info->tc_queue.tc_txq[0][0].base = 0;
5934                         dcb_info->tc_queue.tc_txq[0][1].base = 32;
5935                         dcb_info->tc_queue.tc_txq[0][2].base = 64;
5936                         dcb_info->tc_queue.tc_txq[0][3].base = 80;
5937                         dcb_info->tc_queue.tc_txq[0][4].base = 96;
5938                         dcb_info->tc_queue.tc_txq[0][5].base = 104;
5939                         dcb_info->tc_queue.tc_txq[0][6].base = 112;
5940                         dcb_info->tc_queue.tc_txq[0][7].base = 120;
5941                         dcb_info->tc_queue.tc_txq[0][0].nb_queue = 32;
5942                         dcb_info->tc_queue.tc_txq[0][1].nb_queue = 32;
5943                         dcb_info->tc_queue.tc_txq[0][2].nb_queue = 16;
5944                         dcb_info->tc_queue.tc_txq[0][3].nb_queue = 16;
5945                         dcb_info->tc_queue.tc_txq[0][4].nb_queue = 8;
5946                         dcb_info->tc_queue.tc_txq[0][5].nb_queue = 8;
5947                         dcb_info->tc_queue.tc_txq[0][6].nb_queue = 8;
5948                         dcb_info->tc_queue.tc_txq[0][7].nb_queue = 8;
5949                 }
5950         }
5951         for (i = 0; i < dcb_info->nb_tcs; i++) {
5952                 tc = &dcb_config->tc_config[i];
5953                 dcb_info->tc_bws[i] = tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent;
5954         }
5955         return 0;
5956 }
5957
5958 static struct rte_driver rte_ixgbe_driver = {
5959         .type = PMD_PDEV,
5960         .init = rte_ixgbe_pmd_init,
5961 };
5962
5963 static struct rte_driver rte_ixgbevf_driver = {
5964         .type = PMD_PDEV,
5965         .init = rte_ixgbevf_pmd_init,
5966 };
5967
5968 PMD_REGISTER_DRIVER(rte_ixgbe_driver);
5969 PMD_REGISTER_DRIVER(rte_ixgbevf_driver);