drivers/net/ixgbe/ixgbe_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <inttypes.h>
  42 #include <netinet/in.h>
  43 #include <rte_byteorder.h>
  44 #include <rte_common.h>
  45 #include <rte_cycles.h>
  46
  47 #include <rte_interrupts.h>
  48 #include <rte_log.h>
  49 #include <rte_debug.h>
  50 #include <rte_pci.h>
  51 #include <rte_atomic.h>
  52 #include <rte_branch_prediction.h>
  53 #include <rte_memory.h>
  54 #include <rte_memzone.h>
  55 #include <rte_eal.h>
  56 #include <rte_alarm.h>
  57 #include <rte_ether.h>
  58 #include <rte_ethdev.h>
  59 #include <rte_atomic.h>
  60 #include <rte_malloc.h>
  61 #include <rte_random.h>
  62 #include <rte_dev.h>
  63
  64 #include "ixgbe_logs.h"
  65 #include "base/ixgbe_api.h"
  66 #include "base/ixgbe_vf.h"
  67 #include "base/ixgbe_common.h"
  68 #include "ixgbe_ethdev.h"
  69 #include "ixgbe_bypass.h"
  70 #include "ixgbe_rxtx.h"
  71 #include "base/ixgbe_type.h"
  72 #include "base/ixgbe_phy.h"
  73 #include "ixgbe_regs.h"
  74
  75 /*
  76  * High threshold controlling when to start sending XOFF frames. Must be at
  77  * least 8 bytes less than receive packet buffer size. This value is in units
  78  * of 1024 bytes.
  79  */
  80 #define IXGBE_FC_HI    0x80
  81
  82 /*
  83  * Low threshold controlling when to start sending XON frames. This value is
  84  * in units of 1024 bytes.
  85  */
  86 #define IXGBE_FC_LO    0x40
  87
  88 /* Timer value included in XOFF frames. */
  89 #define IXGBE_FC_PAUSE 0x680
  90
  91 #define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */
  92 #define IXGBE_LINK_UP_CHECK_TIMEOUT   1000 /* ms */
  93 #define IXGBE_VMDQ_NUM_UC_MAC         4096 /* Maximum nb. of UC MAC addr. */
  94
  95 #define IXGBE_MMW_SIZE_DEFAULT        0x4
  96 #define IXGBE_MMW_SIZE_JUMBO_FRAME    0x14
  97 #define IXGBE_MAX_RING_DESC           4096 /* replicate define from rxtx */
  98
  99 /*
 100  *  Default values for RX/TX configuration
 101  */
 102 #define IXGBE_DEFAULT_RX_FREE_THRESH  32
 103 #define IXGBE_DEFAULT_RX_PTHRESH      8
 104 #define IXGBE_DEFAULT_RX_HTHRESH      8
 105 #define IXGBE_DEFAULT_RX_WTHRESH      0
 106
 107 #define IXGBE_DEFAULT_TX_FREE_THRESH  32
 108 #define IXGBE_DEFAULT_TX_PTHRESH      32
 109 #define IXGBE_DEFAULT_TX_HTHRESH      0
 110 #define IXGBE_DEFAULT_TX_WTHRESH      0
 111 #define IXGBE_DEFAULT_TX_RSBIT_THRESH 32
 112
 113 /* Bit shift and mask */
 114 #define IXGBE_4_BIT_WIDTH  (CHAR_BIT / 2)
 115 #define IXGBE_4_BIT_MASK   RTE_LEN2MASK(IXGBE_4_BIT_WIDTH, uint8_t)
 116 #define IXGBE_8_BIT_WIDTH  CHAR_BIT
 117 #define IXGBE_8_BIT_MASK   UINT8_MAX
 118
 119 #define IXGBEVF_PMD_NAME "rte_ixgbevf_pmd" /* PMD name */
 120
 121 #define IXGBE_QUEUE_STAT_COUNTERS (sizeof(hw_stats->qprc) / sizeof(hw_stats->qprc[0]))
 122
 123 #define IXGBE_HKEY_MAX_INDEX 10
 124
 125 /* Additional timesync values. */
 126 #define IXGBE_TIMINCA_16NS_SHIFT 24
 127 #define IXGBE_TIMINCA_INCVALUE   16000000
 128 #define IXGBE_TIMINCA_INIT       ((0x02 << IXGBE_TIMINCA_16NS_SHIFT) \
 129                                   | IXGBE_TIMINCA_INCVALUE)
 130
 131 static int eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev);
 132 static int  ixgbe_dev_configure(struct rte_eth_dev *dev);
 133 static int  ixgbe_dev_start(struct rte_eth_dev *dev);
 134 static void ixgbe_dev_stop(struct rte_eth_dev *dev);
 135 static int  ixgbe_dev_set_link_up(struct rte_eth_dev *dev);
 136 static int  ixgbe_dev_set_link_down(struct rte_eth_dev *dev);
 137 static void ixgbe_dev_close(struct rte_eth_dev *dev);
 138 static void ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev);
 139 static void ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev);
 140 static void ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev);
 141 static void ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev);
 142 static int ixgbe_dev_link_update(struct rte_eth_dev *dev,
 143                                 int wait_to_complete);
 144 static void ixgbe_dev_stats_get(struct rte_eth_dev *dev,
 145                                 struct rte_eth_stats *stats);
 146 static int ixgbe_dev_xstats_get(struct rte_eth_dev *dev,
 147                                 struct rte_eth_xstats *xstats, unsigned n);
 148 static void ixgbe_dev_stats_reset(struct rte_eth_dev *dev);
 149 static void ixgbe_dev_xstats_reset(struct rte_eth_dev *dev);
 150 static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 151                                              uint16_t queue_id,
 152                                              uint8_t stat_idx,
 153                                              uint8_t is_rx);
 154 static void ixgbe_dev_info_get(struct rte_eth_dev *dev,
 155                                struct rte_eth_dev_info *dev_info);
 156 static void ixgbevf_dev_info_get(struct rte_eth_dev *dev,
 157                                  struct rte_eth_dev_info *dev_info);
 158 static int ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 159
 160 static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev,
 161                 uint16_t vlan_id, int on);
 162 static void ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 163 static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev,
 164                 uint16_t queue, bool on);
 165 static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue,
 166                 int on);
 167 static void ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 168 static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue);
 169 static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue);
 170 static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 171 static void ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 172
 173 static int ixgbe_dev_led_on(struct rte_eth_dev *dev);
 174 static int ixgbe_dev_led_off(struct rte_eth_dev *dev);
 175 static int ixgbe_flow_ctrl_get(struct rte_eth_dev *dev,
 176                                struct rte_eth_fc_conf *fc_conf);
 177 static int ixgbe_flow_ctrl_set(struct rte_eth_dev *dev,
 178                                struct rte_eth_fc_conf *fc_conf);
 179 static int ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 180                 struct rte_eth_pfc_conf *pfc_conf);
 181 static int ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
 182                         struct rte_eth_rss_reta_entry64 *reta_conf,
 183                         uint16_t reta_size);
 184 static int ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
 185                         struct rte_eth_rss_reta_entry64 *reta_conf,
 186                         uint16_t reta_size);
 187 static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev);
 188 static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev);
 189 static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev);
 190 static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev);
 191 static void ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle,
 192                 void *param);
 193 static void ixgbe_dev_interrupt_delayed_handler(void *param);
 194 static void ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 195                 uint32_t index, uint32_t pool);
 196 static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index);
 197 static void ixgbe_set_default_mac_addr(struct rte_eth_dev *dev,
 198                                            struct ether_addr *mac_addr);
 199 static void ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config);
 200
 201 /* For Virtual Function support */
 202 static int eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev);
 203 static int  ixgbevf_dev_configure(struct rte_eth_dev *dev);
 204 static int  ixgbevf_dev_start(struct rte_eth_dev *dev);
 205 static void ixgbevf_dev_stop(struct rte_eth_dev *dev);
 206 static void ixgbevf_dev_close(struct rte_eth_dev *dev);
 207 static void ixgbevf_intr_disable(struct ixgbe_hw *hw);
 208 static void ixgbevf_dev_stats_get(struct rte_eth_dev *dev,
 209                 struct rte_eth_stats *stats);
 210 static void ixgbevf_dev_stats_reset(struct rte_eth_dev *dev);
 211 static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev,
 212                 uint16_t vlan_id, int on);
 213 static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev,
 214                 uint16_t queue, int on);
 215 static void ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 216 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 217
 218 /* For Eth VMDQ APIs support */
 219 static int ixgbe_uc_hash_table_set(struct rte_eth_dev *dev, struct
 220                 ether_addr* mac_addr,uint8_t on);
 221 static int ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev,uint8_t on);
 222 static int  ixgbe_set_pool_rx_mode(struct rte_eth_dev *dev,  uint16_t pool,
 223                 uint16_t rx_mask, uint8_t on);
 224 static int ixgbe_set_pool_rx(struct rte_eth_dev *dev,uint16_t pool,uint8_t on);
 225 static int ixgbe_set_pool_tx(struct rte_eth_dev *dev,uint16_t pool,uint8_t on);
 226 static int ixgbe_set_pool_vlan_filter(struct rte_eth_dev *dev, uint16_t vlan,
 227                 uint64_t pool_mask,uint8_t vlan_on);
 228 static int ixgbe_mirror_rule_set(struct rte_eth_dev *dev,
 229                 struct rte_eth_mirror_conf *mirror_conf,
 230                 uint8_t rule_id, uint8_t on);
 231 static int ixgbe_mirror_rule_reset(struct rte_eth_dev *dev,
 232                 uint8_t rule_id);
 233
 234 static int ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev,
 235                 uint16_t queue_idx, uint16_t tx_rate);
 236 static int ixgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf,
 237                 uint16_t tx_rate, uint64_t q_msk);
 238
 239 static void ixgbevf_add_mac_addr(struct rte_eth_dev *dev,
 240                                  struct ether_addr *mac_addr,
 241                                  uint32_t index, uint32_t pool);
 242 static void ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index);
 243 static void ixgbevf_set_default_mac_addr(struct rte_eth_dev *dev,
 244                                              struct ether_addr *mac_addr);
 245 static int ixgbe_syn_filter_set(struct rte_eth_dev *dev,
 246                         struct rte_eth_syn_filter *filter,
 247                         bool add);
 248 static int ixgbe_syn_filter_get(struct rte_eth_dev *dev,
 249                         struct rte_eth_syn_filter *filter);
 250 static int ixgbe_syn_filter_handle(struct rte_eth_dev *dev,
 251                         enum rte_filter_op filter_op,
 252                         void *arg);
 253 static int ixgbe_add_5tuple_filter(struct rte_eth_dev *dev,
 254                         struct ixgbe_5tuple_filter *filter);
 255 static void ixgbe_remove_5tuple_filter(struct rte_eth_dev *dev,
 256                         struct ixgbe_5tuple_filter *filter);
 257 static int ixgbe_add_del_ntuple_filter(struct rte_eth_dev *dev,
 258                         struct rte_eth_ntuple_filter *filter,
 259                         bool add);
 260 static int ixgbe_ntuple_filter_handle(struct rte_eth_dev *dev,
 261                                 enum rte_filter_op filter_op,
 262                                 void *arg);
 263 static int ixgbe_get_ntuple_filter(struct rte_eth_dev *dev,
 264                         struct rte_eth_ntuple_filter *filter);
 265 static int ixgbe_add_del_ethertype_filter(struct rte_eth_dev *dev,
 266                         struct rte_eth_ethertype_filter *filter,
 267                         bool add);
 268 static int ixgbe_ethertype_filter_handle(struct rte_eth_dev *dev,
 269                                 enum rte_filter_op filter_op,
 270                                 void *arg);
 271 static int ixgbe_get_ethertype_filter(struct rte_eth_dev *dev,
 272                         struct rte_eth_ethertype_filter *filter);
 273 static int ixgbe_dev_filter_ctrl(struct rte_eth_dev *dev,
 274                      enum rte_filter_type filter_type,
 275                      enum rte_filter_op filter_op,
 276                      void *arg);
 277 static int ixgbevf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu);
 278
 279 static int ixgbe_dev_set_mc_addr_list(struct rte_eth_dev *dev,
 280                                       struct ether_addr *mc_addr_set,
 281                                       uint32_t nb_mc_addr);
 282
 283 static int ixgbe_get_reg_length(struct rte_eth_dev *dev);
 284 static int ixgbe_get_regs(struct rte_eth_dev *dev,
 285                             struct rte_dev_reg_info *regs);
 286 static int ixgbe_get_eeprom_length(struct rte_eth_dev *dev);
 287 static int ixgbe_get_eeprom(struct rte_eth_dev *dev,
 288                                 struct rte_dev_eeprom_info *eeprom);
 289 static int ixgbe_set_eeprom(struct rte_eth_dev *dev,
 290                                 struct rte_dev_eeprom_info *eeprom);
 291
 292 static int ixgbevf_get_reg_length(struct rte_eth_dev *dev);
 293 static int ixgbevf_get_regs(struct rte_eth_dev *dev,
 294                                 struct rte_dev_reg_info *regs);
 295
 296 static int ixgbe_timesync_enable(struct rte_eth_dev *dev);
 297 static int ixgbe_timesync_disable(struct rte_eth_dev *dev);
 298 static int ixgbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
 299                                             struct timespec *timestamp,
 300                                             uint32_t flags);
 301 static int ixgbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
 302                                             struct timespec *timestamp);
 303
 304 /*
 305  * Define VF Stats MACRO for Non "cleared on read" register
 306  */
 307 #define UPDATE_VF_STAT(reg, last, cur)                          \
 308 {                                                               \
 309         u32 latest = IXGBE_READ_REG(hw, reg);                   \
 310         cur += latest - last;                                   \
 311         last = latest;                                          \
 312 }
 313
 314 #define UPDATE_VF_STAT_36BIT(lsb, msb, last, cur)                \
 315 {                                                                \
 316         u64 new_lsb = IXGBE_READ_REG(hw, lsb);                   \
 317         u64 new_msb = IXGBE_READ_REG(hw, msb);                   \
 318         u64 latest = ((new_msb << 32) | new_lsb);                \
 319         cur += (0x1000000000LL + latest - last) & 0xFFFFFFFFFLL; \
 320         last = latest;                                           \
 321 }
 322
 323 #define IXGBE_SET_HWSTRIP(h, q) do{\
 324                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 325                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 326                 (h)->bitmap[idx] |= 1 << bit;\
 327         }while(0)
 328
 329 #define IXGBE_CLEAR_HWSTRIP(h, q) do{\
 330                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 331                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 332                 (h)->bitmap[idx] &= ~(1 << bit);\
 333         }while(0)
 334
 335 #define IXGBE_GET_HWSTRIP(h, q, r) do{\
 336                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 337                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 338                 (r) = (h)->bitmap[idx] >> bit & 1;\
 339         }while(0)
 340
 341 /*
 342  * The set of PCI devices this driver supports
 343  */
 344 static const struct rte_pci_id pci_id_ixgbe_map[] = {
 345
 346 #define RTE_PCI_DEV_ID_DECL_IXGBE(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 347 #include "rte_pci_dev_ids.h"
 348
 349 { .vendor_id = 0, /* sentinel */ },
 350 };
 351
 352
 353 /*
 354  * The set of PCI devices this driver supports (for 82599 VF)
 355  */
 356 static const struct rte_pci_id pci_id_ixgbevf_map[] = {
 357
 358 #define RTE_PCI_DEV_ID_DECL_IXGBEVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 359 #include "rte_pci_dev_ids.h"
 360 { .vendor_id = 0, /* sentinel */ },
 361
 362 };
 363
 364 static const struct eth_dev_ops ixgbe_eth_dev_ops = {
 365         .dev_configure        = ixgbe_dev_configure,
 366         .dev_start            = ixgbe_dev_start,
 367         .dev_stop             = ixgbe_dev_stop,
 368         .dev_set_link_up    = ixgbe_dev_set_link_up,
 369         .dev_set_link_down  = ixgbe_dev_set_link_down,
 370         .dev_close            = ixgbe_dev_close,
 371         .promiscuous_enable   = ixgbe_dev_promiscuous_enable,
 372         .promiscuous_disable  = ixgbe_dev_promiscuous_disable,
 373         .allmulticast_enable  = ixgbe_dev_allmulticast_enable,
 374         .allmulticast_disable = ixgbe_dev_allmulticast_disable,
 375         .link_update          = ixgbe_dev_link_update,
 376         .stats_get            = ixgbe_dev_stats_get,
 377         .xstats_get           = ixgbe_dev_xstats_get,
 378         .stats_reset          = ixgbe_dev_stats_reset,
 379         .xstats_reset         = ixgbe_dev_xstats_reset,
 380         .queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set,
 381         .dev_infos_get        = ixgbe_dev_info_get,
 382         .mtu_set              = ixgbe_dev_mtu_set,
 383         .vlan_filter_set      = ixgbe_vlan_filter_set,
 384         .vlan_tpid_set        = ixgbe_vlan_tpid_set,
 385         .vlan_offload_set     = ixgbe_vlan_offload_set,
 386         .vlan_strip_queue_set = ixgbe_vlan_strip_queue_set,
 387         .rx_queue_start       = ixgbe_dev_rx_queue_start,
 388         .rx_queue_stop        = ixgbe_dev_rx_queue_stop,
 389         .tx_queue_start       = ixgbe_dev_tx_queue_start,
 390         .tx_queue_stop        = ixgbe_dev_tx_queue_stop,
 391         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 392         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 393         .rx_queue_count       = ixgbe_dev_rx_queue_count,
 394         .rx_descriptor_done   = ixgbe_dev_rx_descriptor_done,
 395         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 396         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 397         .dev_led_on           = ixgbe_dev_led_on,
 398         .dev_led_off          = ixgbe_dev_led_off,
 399         .flow_ctrl_get        = ixgbe_flow_ctrl_get,
 400         .flow_ctrl_set        = ixgbe_flow_ctrl_set,
 401         .priority_flow_ctrl_set = ixgbe_priority_flow_ctrl_set,
 402         .mac_addr_add         = ixgbe_add_rar,
 403         .mac_addr_remove      = ixgbe_remove_rar,
 404         .mac_addr_set         = ixgbe_set_default_mac_addr,
 405         .uc_hash_table_set    = ixgbe_uc_hash_table_set,
 406         .uc_all_hash_table_set  = ixgbe_uc_all_hash_table_set,
 407         .mirror_rule_set      = ixgbe_mirror_rule_set,
 408         .mirror_rule_reset    = ixgbe_mirror_rule_reset,
 409         .set_vf_rx_mode       = ixgbe_set_pool_rx_mode,
 410         .set_vf_rx            = ixgbe_set_pool_rx,
 411         .set_vf_tx            = ixgbe_set_pool_tx,
 412         .set_vf_vlan_filter   = ixgbe_set_pool_vlan_filter,
 413         .set_queue_rate_limit = ixgbe_set_queue_rate_limit,
 414         .set_vf_rate_limit    = ixgbe_set_vf_rate_limit,
 415         .reta_update          = ixgbe_dev_rss_reta_update,
 416         .reta_query           = ixgbe_dev_rss_reta_query,
 417 #ifdef RTE_NIC_BYPASS
 418         .bypass_init          = ixgbe_bypass_init,
 419         .bypass_state_set     = ixgbe_bypass_state_store,
 420         .bypass_state_show    = ixgbe_bypass_state_show,
 421         .bypass_event_set     = ixgbe_bypass_event_store,
 422         .bypass_event_show    = ixgbe_bypass_event_show,
 423         .bypass_wd_timeout_set  = ixgbe_bypass_wd_timeout_store,
 424         .bypass_wd_timeout_show = ixgbe_bypass_wd_timeout_show,
 425         .bypass_ver_show      = ixgbe_bypass_ver_show,
 426         .bypass_wd_reset      = ixgbe_bypass_wd_reset,
 427 #endif /* RTE_NIC_BYPASS */
 428         .rss_hash_update      = ixgbe_dev_rss_hash_update,
 429         .rss_hash_conf_get    = ixgbe_dev_rss_hash_conf_get,
 430         .filter_ctrl          = ixgbe_dev_filter_ctrl,
 431         .set_mc_addr_list     = ixgbe_dev_set_mc_addr_list,
 432         .timesync_enable      = ixgbe_timesync_enable,
 433         .timesync_disable     = ixgbe_timesync_disable,
 434         .timesync_read_rx_timestamp = ixgbe_timesync_read_rx_timestamp,
 435         .timesync_read_tx_timestamp = ixgbe_timesync_read_tx_timestamp,
 436         .get_reg_length       = ixgbe_get_reg_length,
 437         .get_reg              = ixgbe_get_regs,
 438         .get_eeprom_length    = ixgbe_get_eeprom_length,
 439         .get_eeprom           = ixgbe_get_eeprom,
 440         .set_eeprom           = ixgbe_set_eeprom,
 441 };
 442
 443 /*
 444  * dev_ops for virtual function, bare necessities for basic vf
 445  * operation have been implemented
 446  */
 447 static const struct eth_dev_ops ixgbevf_eth_dev_ops = {
 448         .dev_configure        = ixgbevf_dev_configure,
 449         .dev_start            = ixgbevf_dev_start,
 450         .dev_stop             = ixgbevf_dev_stop,
 451         .link_update          = ixgbe_dev_link_update,
 452         .stats_get            = ixgbevf_dev_stats_get,
 453         .stats_reset          = ixgbevf_dev_stats_reset,
 454         .dev_close            = ixgbevf_dev_close,
 455         .dev_infos_get        = ixgbevf_dev_info_get,
 456         .mtu_set              = ixgbevf_dev_set_mtu,
 457         .vlan_filter_set      = ixgbevf_vlan_filter_set,
 458         .vlan_strip_queue_set = ixgbevf_vlan_strip_queue_set,
 459         .vlan_offload_set     = ixgbevf_vlan_offload_set,
 460         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 461         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 462         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 463         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 464         .mac_addr_add         = ixgbevf_add_mac_addr,
 465         .mac_addr_remove      = ixgbevf_remove_mac_addr,
 466         .set_mc_addr_list     = ixgbe_dev_set_mc_addr_list,
 467         .mac_addr_set         = ixgbevf_set_default_mac_addr,
 468         .get_reg_length       = ixgbevf_get_reg_length,
 469         .get_reg              = ixgbevf_get_regs,
 470 };
 471
 472 /* store statistics names and its offset in stats structure */
 473 struct rte_ixgbe_xstats_name_off {
 474         char name[RTE_ETH_XSTATS_NAME_SIZE];
 475         unsigned offset;
 476 };
 477
 478 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_stats_strings[] = {
 479         {"rx_illegal_byte_err", offsetof(struct ixgbe_hw_stats, errbc)},
 480         {"rx_len_err", offsetof(struct ixgbe_hw_stats, rlec)},
 481         {"rx_undersize_count", offsetof(struct ixgbe_hw_stats, ruc)},
 482         {"rx_oversize_count", offsetof(struct ixgbe_hw_stats, roc)},
 483         {"rx_fragment_count", offsetof(struct ixgbe_hw_stats, rfc)},
 484         {"rx_jabber_count", offsetof(struct ixgbe_hw_stats, rjc)},
 485         {"l3_l4_xsum_error", offsetof(struct ixgbe_hw_stats, xec)},
 486         {"mac_local_fault", offsetof(struct ixgbe_hw_stats, mlfc)},
 487         {"mac_remote_fault", offsetof(struct ixgbe_hw_stats, mrfc)},
 488         {"mac_short_pkt_discard", offsetof(struct ixgbe_hw_stats, mspdc)},
 489         {"fccrc_error", offsetof(struct ixgbe_hw_stats, fccrc)},
 490         {"fcoe_drop", offsetof(struct ixgbe_hw_stats, fcoerpdc)},
 491         {"fc_last_error", offsetof(struct ixgbe_hw_stats, fclast)},
 492         {"rx_broadcast_packets", offsetof(struct ixgbe_hw_stats, bprc)},
 493         {"rx_phy_multicast_packets", offsetof(struct ixgbe_hw_stats, mprc)},
 494         {"mgmt_pkts_dropped", offsetof(struct ixgbe_hw_stats, mngpdc)},
 495         {"rx_crc_errors", offsetof(struct ixgbe_hw_stats, crcerrs)},
 496         {"fdir_match", offsetof(struct ixgbe_hw_stats, fdirmatch)},
 497         {"fdir_miss", offsetof(struct ixgbe_hw_stats, fdirmiss)},
 498         {"tx_flow_control_xon", offsetof(struct ixgbe_hw_stats, lxontxc)},
 499         {"rx_flow_control_xon", offsetof(struct ixgbe_hw_stats, lxonrxc)},
 500         {"tx_flow_control_xoff", offsetof(struct ixgbe_hw_stats, lxofftxc)},
 501         {"rx_flow_control_xoff", offsetof(struct ixgbe_hw_stats, lxoffrxc)},
 502 };
 503
 504 #define IXGBE_NB_XSTATS (sizeof(rte_ixgbe_stats_strings) /      \
 505                 sizeof(rte_ixgbe_stats_strings[0]))
 506
 507 /**
 508  * Atomically reads the link status information from global
 509  * structure rte_eth_dev.
 510  *
 511  * @param dev
 512  *   - Pointer to the structure rte_eth_dev to read from.
 513  *   - Pointer to the buffer to be saved with the link status.
 514  *
 515  * @return
 516  *   - On success, zero.
 517  *   - On failure, negative value.
 518  */
 519 static inline int
 520 rte_ixgbe_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 521                                 struct rte_eth_link *link)
 522 {
 523         struct rte_eth_link *dst = link;
 524         struct rte_eth_link *src = &(dev->data->dev_link);
 525
 526         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 527                                         *(uint64_t *)src) == 0)
 528                 return -1;
 529
 530         return 0;
 531 }
 532
 533 /**
 534  * Atomically writes the link status information into global
 535  * structure rte_eth_dev.
 536  *
 537  * @param dev
 538  *   - Pointer to the structure rte_eth_dev to read from.
 539  *   - Pointer to the buffer to be saved with the link status.
 540  *
 541  * @return
 542  *   - On success, zero.
 543  *   - On failure, negative value.
 544  */
 545 static inline int
 546 rte_ixgbe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 547                                 struct rte_eth_link *link)
 548 {
 549         struct rte_eth_link *dst = &(dev->data->dev_link);
 550         struct rte_eth_link *src = link;
 551
 552         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 553                                         *(uint64_t *)src) == 0)
 554                 return -1;
 555
 556         return 0;
 557 }
 558
 559 /*
 560  * This function is the same as ixgbe_is_sfp() in base/ixgbe.h.
 561  */
 562 static inline int
 563 ixgbe_is_sfp(struct ixgbe_hw *hw)
 564 {
 565         switch (hw->phy.type) {
 566         case ixgbe_phy_sfp_avago:
 567         case ixgbe_phy_sfp_ftl:
 568         case ixgbe_phy_sfp_intel:
 569         case ixgbe_phy_sfp_unknown:
 570         case ixgbe_phy_sfp_passive_tyco:
 571         case ixgbe_phy_sfp_passive_unknown:
 572                 return 1;
 573         default:
 574                 return 0;
 575         }
 576 }
 577
 578 static inline int32_t
 579 ixgbe_pf_reset_hw(struct ixgbe_hw *hw)
 580 {
 581         uint32_t ctrl_ext;
 582         int32_t status;
 583
 584         status = ixgbe_reset_hw(hw);
 585
 586         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 587         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 588         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
 589         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 590         IXGBE_WRITE_FLUSH(hw);
 591
 592         return status;
 593 }
 594
 595 static inline void
 596 ixgbe_enable_intr(struct rte_eth_dev *dev)
 597 {
 598         struct ixgbe_interrupt *intr =
 599                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 600         struct ixgbe_hw *hw =
 601                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 602
 603         IXGBE_WRITE_REG(hw, IXGBE_EIMS, intr->mask);
 604         IXGBE_WRITE_FLUSH(hw);
 605 }
 606
 607 /*
 608  * This function is based on ixgbe_disable_intr() in base/ixgbe.h.
 609  */
 610 static void
 611 ixgbe_disable_intr(struct ixgbe_hw *hw)
 612 {
 613         PMD_INIT_FUNC_TRACE();
 614
 615         if (hw->mac.type == ixgbe_mac_82598EB) {
 616                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ~0);
 617         } else {
 618                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xFFFF0000);
 619                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), ~0);
 620                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), ~0);
 621         }
 622         IXGBE_WRITE_FLUSH(hw);
 623 }
 624
 625 /*
 626  * This function resets queue statistics mapping registers.
 627  * From Niantic datasheet, Initialization of Statistics section:
 628  * "...if software requires the queue counters, the RQSMR and TQSM registers
 629  * must be re-programmed following a device reset.
 630  */
 631 static void
 632 ixgbe_reset_qstat_mappings(struct ixgbe_hw *hw)
 633 {
 634         uint32_t i;
 635
 636         for(i = 0; i != IXGBE_NB_STAT_MAPPING_REGS; i++) {
 637                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), 0);
 638                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), 0);
 639         }
 640 }
 641
 642
 643 static int
 644 ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 645                                   uint16_t queue_id,
 646                                   uint8_t stat_idx,
 647                                   uint8_t is_rx)
 648 {
 649 #define QSM_REG_NB_BITS_PER_QMAP_FIELD 8
 650 #define NB_QMAP_FIELDS_PER_QSM_REG 4
 651 #define QMAP_FIELD_RESERVED_BITS_MASK 0x0f
 652
 653         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 654         struct ixgbe_stat_mapping_registers *stat_mappings =
 655                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(eth_dev->data->dev_private);
 656         uint32_t qsmr_mask = 0;
 657         uint32_t clearing_mask = QMAP_FIELD_RESERVED_BITS_MASK;
 658         uint32_t q_map;
 659         uint8_t n, offset;
 660
 661         if ((hw->mac.type != ixgbe_mac_82599EB) &&
 662                 (hw->mac.type != ixgbe_mac_X540) &&
 663                 (hw->mac.type != ixgbe_mac_X550) &&
 664                 (hw->mac.type != ixgbe_mac_X550EM_x))
 665                 return -ENOSYS;
 666
 667         PMD_INIT_LOG(INFO, "Setting port %d, %s queue_id %d to stat index %d",
 668                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX",
 669                      queue_id, stat_idx);
 670
 671         n = (uint8_t)(queue_id / NB_QMAP_FIELDS_PER_QSM_REG);
 672         if (n >= IXGBE_NB_STAT_MAPPING_REGS) {
 673                 PMD_INIT_LOG(ERR, "Nb of stat mapping registers exceeded");
 674                 return -EIO;
 675         }
 676         offset = (uint8_t)(queue_id % NB_QMAP_FIELDS_PER_QSM_REG);
 677
 678         /* Now clear any previous stat_idx set */
 679         clearing_mask <<= (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 680         if (!is_rx)
 681                 stat_mappings->tqsm[n] &= ~clearing_mask;
 682         else
 683                 stat_mappings->rqsmr[n] &= ~clearing_mask;
 684
 685         q_map = (uint32_t)stat_idx;
 686         q_map &= QMAP_FIELD_RESERVED_BITS_MASK;
 687         qsmr_mask = q_map << (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 688         if (!is_rx)
 689                 stat_mappings->tqsm[n] |= qsmr_mask;
 690         else
 691                 stat_mappings->rqsmr[n] |= qsmr_mask;
 692
 693         PMD_INIT_LOG(INFO, "Set port %d, %s queue_id %d to stat index %d",
 694                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX",
 695                      queue_id, stat_idx);
 696         PMD_INIT_LOG(INFO, "%s[%d] = 0x%08x", is_rx ? "RQSMR" : "TQSM", n,
 697                      is_rx ? stat_mappings->rqsmr[n] : stat_mappings->tqsm[n]);
 698
 699         /* Now write the mapping in the appropriate register */
 700         if (is_rx) {
 701                 PMD_INIT_LOG(INFO, "Write 0x%x to RX IXGBE stat mapping reg:%d",
 702                              stat_mappings->rqsmr[n], n);
 703                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(n), stat_mappings->rqsmr[n]);
 704         }
 705         else {
 706                 PMD_INIT_LOG(INFO, "Write 0x%x to TX IXGBE stat mapping reg:%d",
 707                              stat_mappings->tqsm[n], n);
 708                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(n), stat_mappings->tqsm[n]);
 709         }
 710         return 0;
 711 }
 712
 713 static void
 714 ixgbe_restore_statistics_mapping(struct rte_eth_dev * dev)
 715 {
 716         struct ixgbe_stat_mapping_registers *stat_mappings =
 717                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(dev->data->dev_private);
 718         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 719         int i;
 720
 721         /* write whatever was in stat mapping table to the NIC */
 722         for (i = 0; i < IXGBE_NB_STAT_MAPPING_REGS; i++) {
 723                 /* rx */
 724                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), stat_mappings->rqsmr[i]);
 725
 726                 /* tx */
 727                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), stat_mappings->tqsm[i]);
 728         }
 729 }
 730
 731 static void
 732 ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config)
 733 {
 734         uint8_t i;
 735         struct ixgbe_dcb_tc_config *tc;
 736         uint8_t dcb_max_tc = IXGBE_DCB_MAX_TRAFFIC_CLASS;
 737
 738         dcb_config->num_tcs.pg_tcs = dcb_max_tc;
 739         dcb_config->num_tcs.pfc_tcs = dcb_max_tc;
 740         for (i = 0; i < dcb_max_tc; i++) {
 741                 tc = &dcb_config->tc_config[i];
 742                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_id = i;
 743                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
 744                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 745                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_id = i;
 746                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
 747                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 748                 tc->pfc = ixgbe_dcb_pfc_disabled;
 749         }
 750
 751         /* Initialize default user to priority mapping, UPx->TC0 */
 752         tc = &dcb_config->tc_config[0];
 753         tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
 754         tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
 755         for (i = 0; i< IXGBE_DCB_MAX_BW_GROUP; i++) {
 756                 dcb_config->bw_percentage[IXGBE_DCB_TX_CONFIG][i] = 100;
 757                 dcb_config->bw_percentage[IXGBE_DCB_RX_CONFIG][i] = 100;
 758         }
 759         dcb_config->rx_pba_cfg = ixgbe_dcb_pba_equal;
 760         dcb_config->pfc_mode_enable = false;
 761         dcb_config->vt_mode = true;
 762         dcb_config->round_robin_enable = false;
 763         /* support all DCB capabilities in 82599 */
 764         dcb_config->support.capabilities = 0xFF;
 765
 766         /*we only support 4 Tcs for X540, X550 */
 767         if (hw->mac.type == ixgbe_mac_X540 ||
 768                 hw->mac.type == ixgbe_mac_X550 ||
 769                 hw->mac.type == ixgbe_mac_X550EM_x) {
 770                 dcb_config->num_tcs.pg_tcs = 4;
 771                 dcb_config->num_tcs.pfc_tcs = 4;
 772         }
 773 }
 774
 775 /*
 776  * Ensure that all locks are released before first NVM or PHY access
 777  */
 778 static void
 779 ixgbe_swfw_lock_reset(struct ixgbe_hw *hw)
 780 {
 781         uint16_t mask;
 782
 783         /*
 784          * Phy lock should not fail in this early stage. If this is the case,
 785          * it is due to an improper exit of the application.
 786          * So force the release of the faulty lock. Release of common lock
 787          * is done automatically by swfw_sync function.
 788          */
 789         mask = IXGBE_GSSR_PHY0_SM << hw->bus.func;
 790         if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) {
 791                 PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released", hw->bus.func);
 792         }
 793         ixgbe_release_swfw_semaphore(hw, mask);
 794
 795         /*
 796          * These ones are more tricky since they are common to all ports; but
 797          * swfw_sync retries last long enough (1s) to be almost sure that if
 798          * lock can not be taken it is due to an improper lock of the
 799          * semaphore.
 800          */
 801         mask = IXGBE_GSSR_EEP_SM | IXGBE_GSSR_MAC_CSR_SM | IXGBE_GSSR_SW_MNG_SM;
 802         if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) {
 803                 PMD_DRV_LOG(DEBUG, "SWFW common locks released");
 804         }
 805         ixgbe_release_swfw_semaphore(hw, mask);
 806 }
 807
 808 /*
 809  * This function is based on code in ixgbe_attach() in base/ixgbe.c.
 810  * It returns 0 on success.
 811  */
 812 static int
 813 eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev)
 814 {
 815         struct rte_pci_device *pci_dev;
 816         struct ixgbe_hw *hw =
 817                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 818         struct ixgbe_vfta * shadow_vfta =
 819                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 820         struct ixgbe_hwstrip *hwstrip =
 821                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 822         struct ixgbe_dcb_config *dcb_config =
 823                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private);
 824         struct ixgbe_filter_info *filter_info =
 825                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private);
 826         uint32_t ctrl_ext;
 827         uint16_t csum;
 828         int diag, i;
 829
 830         PMD_INIT_FUNC_TRACE();
 831
 832         eth_dev->dev_ops = &ixgbe_eth_dev_ops;
 833         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
 834         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
 835
 836         /*
 837          * For secondary processes, we don't initialise any further as primary
 838          * has already done this work. Only check we don't need a different
 839          * RX and TX function.
 840          */
 841         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 842                 struct ixgbe_tx_queue *txq;
 843                 /* TX queue function in primary, set by last queue initialized
 844                  * Tx queue may not initialized by primary process */
 845                 if (eth_dev->data->tx_queues) {
 846                         txq = eth_dev->data->tx_queues[eth_dev->data->nb_tx_queues-1];
 847                         ixgbe_set_tx_function(eth_dev, txq);
 848                 } else {
 849                         /* Use default TX function if we get here */
 850                         PMD_INIT_LOG(INFO, "No TX queues configured yet. "
 851                                            "Using default TX function.");
 852                 }
 853
 854                 ixgbe_set_rx_function(eth_dev);
 855
 856                 return 0;
 857         }
 858         pci_dev = eth_dev->pci_dev;
 859
 860         /* Vendor and Device ID need to be set before init of shared code */
 861         hw->device_id = pci_dev->id.device_id;
 862         hw->vendor_id = pci_dev->id.vendor_id;
 863         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 864         hw->allow_unsupported_sfp = 1;
 865
 866         /* Initialize the shared code (base driver) */
 867 #ifdef RTE_NIC_BYPASS
 868         diag = ixgbe_bypass_init_shared_code(hw);
 869 #else
 870         diag = ixgbe_init_shared_code(hw);
 871 #endif /* RTE_NIC_BYPASS */
 872
 873         if (diag != IXGBE_SUCCESS) {
 874                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
 875                 return -EIO;
 876         }
 877
 878         /* pick up the PCI bus settings for reporting later */
 879         ixgbe_get_bus_info(hw);
 880
 881         /* Unlock any pending hardware semaphore */
 882         ixgbe_swfw_lock_reset(hw);
 883
 884         /* Initialize DCB configuration*/
 885         memset(dcb_config, 0, sizeof(struct ixgbe_dcb_config));
 886         ixgbe_dcb_init(hw,dcb_config);
 887         /* Get Hardware Flow Control setting */
 888         hw->fc.requested_mode = ixgbe_fc_full;
 889         hw->fc.current_mode = ixgbe_fc_full;
 890         hw->fc.pause_time = IXGBE_FC_PAUSE;
 891         for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
 892                 hw->fc.low_water[i] = IXGBE_FC_LO;
 893                 hw->fc.high_water[i] = IXGBE_FC_HI;
 894         }
 895         hw->fc.send_xon = 1;
 896
 897         /* Make sure we have a good EEPROM before we read from it */
 898         diag = ixgbe_validate_eeprom_checksum(hw, &csum);
 899         if (diag != IXGBE_SUCCESS) {
 900                 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag);
 901                 return -EIO;
 902         }
 903
 904 #ifdef RTE_NIC_BYPASS
 905         diag = ixgbe_bypass_init_hw(hw);
 906 #else
 907         diag = ixgbe_init_hw(hw);
 908 #endif /* RTE_NIC_BYPASS */
 909
 910         /*
 911          * Devices with copper phys will fail to initialise if ixgbe_init_hw()
 912          * is called too soon after the kernel driver unbinding/binding occurs.
 913          * The failure occurs in ixgbe_identify_phy_generic() for all devices,
 914          * but for non-copper devies, ixgbe_identify_sfp_module_generic() is
 915          * also called. See ixgbe_identify_phy_82599(). The reason for the
 916          * failure is not known, and only occuts when virtualisation features
 917          * are disabled in the bios. A delay of 100ms  was found to be enough by
 918          * trial-and-error, and is doubled to be safe.
 919          */
 920         if (diag && (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper)) {
 921                 rte_delay_ms(200);
 922                 diag = ixgbe_init_hw(hw);
 923         }
 924
 925         if (diag == IXGBE_ERR_EEPROM_VERSION) {
 926                 PMD_INIT_LOG(ERR, "This device is a pre-production adapter/"
 927                     "LOM.  Please be aware there may be issues associated "
 928                     "with your hardware.");
 929                 PMD_INIT_LOG(ERR, "If you are experiencing problems "
 930                     "please contact your Intel or hardware representative "
 931                     "who provided you with this hardware.");
 932         } else if (diag == IXGBE_ERR_SFP_NOT_SUPPORTED)
 933                 PMD_INIT_LOG(ERR, "Unsupported SFP+ Module");
 934         if (diag) {
 935                 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag);
 936                 return -EIO;
 937         }
 938
 939         /* Reset the hw statistics */
 940         ixgbe_dev_stats_reset(eth_dev);
 941
 942         /* disable interrupt */
 943         ixgbe_disable_intr(hw);
 944
 945         /* reset mappings for queue statistics hw counters*/
 946         ixgbe_reset_qstat_mappings(hw);
 947
 948         /* Allocate memory for storing MAC addresses */
 949         eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 950                         hw->mac.num_rar_entries, 0);
 951         if (eth_dev->data->mac_addrs == NULL) {
 952                 PMD_INIT_LOG(ERR,
 953                         "Failed to allocate %u bytes needed to store "
 954                         "MAC addresses",
 955                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 956                 return -ENOMEM;
 957         }
 958         /* Copy the permanent MAC address */
 959         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 960                         &eth_dev->data->mac_addrs[0]);
 961
 962         /* Allocate memory for storing hash filter MAC addresses */
 963         eth_dev->data->hash_mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 964                         IXGBE_VMDQ_NUM_UC_MAC, 0);
 965         if (eth_dev->data->hash_mac_addrs == NULL) {
 966                 PMD_INIT_LOG(ERR,
 967                         "Failed to allocate %d bytes needed to store MAC addresses",
 968                         ETHER_ADDR_LEN * IXGBE_VMDQ_NUM_UC_MAC);
 969                 return -ENOMEM;
 970         }
 971
 972         /* initialize the vfta */
 973         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 974
 975         /* initialize the hw strip bitmap*/
 976         memset(hwstrip, 0, sizeof(*hwstrip));
 977
 978         /* initialize PF if max_vfs not zero */
 979         ixgbe_pf_host_init(eth_dev);
 980
 981         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 982         /* let hardware know driver is loaded */
 983         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 984         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 985         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
 986         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 987         IXGBE_WRITE_FLUSH(hw);
 988
 989         if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
 990                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d, SFP+: %d",
 991                              (int) hw->mac.type, (int) hw->phy.type,
 992                              (int) hw->phy.sfp_type);
 993         else
 994                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d",
 995                              (int) hw->mac.type, (int) hw->phy.type);
 996
 997         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
 998                         eth_dev->data->port_id, pci_dev->id.vendor_id,
 999                         pci_dev->id.device_id);
1000
1001         rte_intr_callback_register(&(pci_dev->intr_handle),
1002                 ixgbe_dev_interrupt_handler, (void *)eth_dev);
1003
1004         /* enable uio intr after callback register */
1005         rte_intr_enable(&(pci_dev->intr_handle));
1006
1007         /* enable support intr */
1008         ixgbe_enable_intr(eth_dev);
1009
1010         /* initialize 5tuple filter list */
1011         TAILQ_INIT(&filter_info->fivetuple_list);
1012         memset(filter_info->fivetuple_mask, 0,
1013                 sizeof(uint32_t) * IXGBE_5TUPLE_ARRAY_SIZE);
1014
1015         return 0;
1016 }
1017
1018
1019 /*
1020  * Negotiate mailbox API version with the PF.
1021  * After reset API version is always set to the basic one (ixgbe_mbox_api_10).
1022  * Then we try to negotiate starting with the most recent one.
1023  * If all negotiation attempts fail, then we will proceed with
1024  * the default one (ixgbe_mbox_api_10).
1025  */
1026 static void
1027 ixgbevf_negotiate_api(struct ixgbe_hw *hw)
1028 {
1029         int32_t i;
1030
1031         /* start with highest supported, proceed down */
1032         static const enum ixgbe_pfvf_api_rev sup_ver[] = {
1033                 ixgbe_mbox_api_11,
1034                 ixgbe_mbox_api_10,
1035         };
1036
1037         for (i = 0;
1038                         i != RTE_DIM(sup_ver) &&
1039                         ixgbevf_negotiate_api_version(hw, sup_ver[i]) != 0;
1040                         i++)
1041                 ;
1042 }
1043
1044 static void
1045 generate_random_mac_addr(struct ether_addr *mac_addr)
1046 {
1047         uint64_t random;
1048
1049         /* Set Organizationally Unique Identifier (OUI) prefix. */
1050         mac_addr->addr_bytes[0] = 0x00;
1051         mac_addr->addr_bytes[1] = 0x09;
1052         mac_addr->addr_bytes[2] = 0xC0;
1053         /* Force indication of locally assigned MAC address. */
1054         mac_addr->addr_bytes[0] |= ETHER_LOCAL_ADMIN_ADDR;
1055         /* Generate the last 3 bytes of the MAC address with a random number. */
1056         random = rte_rand();
1057         memcpy(&mac_addr->addr_bytes[3], &random, 3);
1058 }
1059
1060 /*
1061  * Virtual Function device init
1062  */
1063 static int
1064 eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev)
1065 {
1066         int diag;
1067         uint32_t tc, tcs;
1068         struct rte_pci_device *pci_dev;
1069         struct ixgbe_hw *hw =
1070                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1071         struct ixgbe_vfta * shadow_vfta =
1072                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
1073         struct ixgbe_hwstrip *hwstrip =
1074                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
1075         struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr;
1076
1077         PMD_INIT_FUNC_TRACE();
1078
1079         eth_dev->dev_ops = &ixgbevf_eth_dev_ops;
1080         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
1081         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
1082
1083         /* for secondary processes, we don't initialise any further as primary
1084          * has already done this work. Only check we don't need a different
1085          * RX function */
1086         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
1087                 if (eth_dev->data->scattered_rx)
1088                         eth_dev->rx_pkt_burst = ixgbe_recv_pkts_lro_single_alloc;
1089                 return 0;
1090         }
1091
1092         pci_dev = eth_dev->pci_dev;
1093
1094         hw->device_id = pci_dev->id.device_id;
1095         hw->vendor_id = pci_dev->id.vendor_id;
1096         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
1097
1098         /* initialize the vfta */
1099         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
1100
1101         /* initialize the hw strip bitmap*/
1102         memset(hwstrip, 0, sizeof(*hwstrip));
1103
1104         /* Initialize the shared code (base driver) */
1105         diag = ixgbe_init_shared_code(hw);
1106         if (diag != IXGBE_SUCCESS) {
1107                 PMD_INIT_LOG(ERR, "Shared code init failed for ixgbevf: %d", diag);
1108                 return -EIO;
1109         }
1110
1111         /* init_mailbox_params */
1112         hw->mbx.ops.init_params(hw);
1113
1114         /* Reset the hw statistics */
1115         ixgbevf_dev_stats_reset(eth_dev);
1116
1117         /* Disable the interrupts for VF */
1118         ixgbevf_intr_disable(hw);
1119
1120         hw->mac.num_rar_entries = 128; /* The MAX of the underlying PF */
1121         diag = hw->mac.ops.reset_hw(hw);
1122
1123         /*
1124          * The VF reset operation returns the IXGBE_ERR_INVALID_MAC_ADDR when
1125          * the underlying PF driver has not assigned a MAC address to the VF.
1126          * In this case, assign a random MAC address.
1127          */
1128         if ((diag != IXGBE_SUCCESS) && (diag != IXGBE_ERR_INVALID_MAC_ADDR)) {
1129                 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
1130                 return (diag);
1131         }
1132
1133         /* negotiate mailbox API version to use with the PF. */
1134         ixgbevf_negotiate_api(hw);
1135
1136         /* Get Rx/Tx queue count via mailbox, which is ready after reset_hw */
1137         ixgbevf_get_queues(hw, &tcs, &tc);
1138
1139         /* Allocate memory for storing MAC addresses */
1140         eth_dev->data->mac_addrs = rte_zmalloc("ixgbevf", ETHER_ADDR_LEN *
1141                         hw->mac.num_rar_entries, 0);
1142         if (eth_dev->data->mac_addrs == NULL) {
1143                 PMD_INIT_LOG(ERR,
1144                         "Failed to allocate %u bytes needed to store "
1145                         "MAC addresses",
1146                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
1147                 return -ENOMEM;
1148         }
1149
1150         /* Generate a random MAC address, if none was assigned by PF. */
1151         if (is_zero_ether_addr(perm_addr)) {
1152                 generate_random_mac_addr(perm_addr);
1153                 diag = ixgbe_set_rar_vf(hw, 1, perm_addr->addr_bytes, 0, 1);
1154                 if (diag) {
1155                         rte_free(eth_dev->data->mac_addrs);
1156                         eth_dev->data->mac_addrs = NULL;
1157                         return diag;
1158                 }
1159                 PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF");
1160                 PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address "
1161                              "%02x:%02x:%02x:%02x:%02x:%02x",
1162                              perm_addr->addr_bytes[0],
1163                              perm_addr->addr_bytes[1],
1164                              perm_addr->addr_bytes[2],
1165                              perm_addr->addr_bytes[3],
1166                              perm_addr->addr_bytes[4],
1167                              perm_addr->addr_bytes[5]);
1168         }
1169
1170         /* Copy the permanent MAC address */
1171         ether_addr_copy(perm_addr, &eth_dev->data->mac_addrs[0]);
1172
1173         /* reset the hardware with the new settings */
1174         diag = hw->mac.ops.start_hw(hw);
1175         switch (diag) {
1176                 case  0:
1177                         break;
1178
1179                 default:
1180                         PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
1181                         return (-EIO);
1182         }
1183
1184         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x mac.type=%s",
1185                      eth_dev->data->port_id, pci_dev->id.vendor_id,
1186                      pci_dev->id.device_id, "ixgbe_mac_82599_vf");
1187
1188         return 0;
1189 }
1190
1191 static struct eth_driver rte_ixgbe_pmd = {
1192         .pci_drv = {
1193                 .name = "rte_ixgbe_pmd",
1194                 .id_table = pci_id_ixgbe_map,
1195                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
1196         },
1197         .eth_dev_init = eth_ixgbe_dev_init,
1198         .dev_private_size = sizeof(struct ixgbe_adapter),
1199 };
1200
1201 /*
1202  * virtual function driver struct
1203  */
1204 static struct eth_driver rte_ixgbevf_pmd = {
1205         .pci_drv = {
1206                 .name = "rte_ixgbevf_pmd",
1207                 .id_table = pci_id_ixgbevf_map,
1208                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
1209         },
1210         .eth_dev_init = eth_ixgbevf_dev_init,
1211         .dev_private_size = sizeof(struct ixgbe_adapter),
1212 };
1213
1214 /*
1215  * Driver initialization routine.
1216  * Invoked once at EAL init time.
1217  * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
1218  */
1219 static int
1220 rte_ixgbe_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
1221 {
1222         PMD_INIT_FUNC_TRACE();
1223
1224         rte_eth_driver_register(&rte_ixgbe_pmd);
1225         return 0;
1226 }
1227
1228 /*
1229  * VF Driver initialization routine.
1230  * Invoked one at EAL init time.
1231  * Register itself as the [Virtual Poll Mode] Driver of PCI niantic devices.
1232  */
1233 static int
1234 rte_ixgbevf_pmd_init(const char *name __rte_unused, const char *param __rte_unused)
1235 {
1236         PMD_INIT_FUNC_TRACE();
1237
1238         rte_eth_driver_register(&rte_ixgbevf_pmd);
1239         return (0);
1240 }
1241
1242 static int
1243 ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1244 {
1245         struct ixgbe_hw *hw =
1246                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1247         struct ixgbe_vfta * shadow_vfta =
1248                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1249         uint32_t vfta;
1250         uint32_t vid_idx;
1251         uint32_t vid_bit;
1252
1253         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
1254         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
1255         vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(vid_idx));
1256         if (on)
1257                 vfta |= vid_bit;
1258         else
1259                 vfta &= ~vid_bit;
1260         IXGBE_WRITE_REG(hw, IXGBE_VFTA(vid_idx), vfta);
1261
1262         /* update local VFTA copy */
1263         shadow_vfta->vfta[vid_idx] = vfta;
1264
1265         return 0;
1266 }
1267
1268 static void
1269 ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
1270 {
1271         if (on)
1272                 ixgbe_vlan_hw_strip_enable(dev, queue);
1273         else
1274                 ixgbe_vlan_hw_strip_disable(dev, queue);
1275 }
1276
1277 static void
1278 ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
1279 {
1280         struct ixgbe_hw *hw =
1281                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1282
1283         /* Only the high 16-bits is valid */
1284         IXGBE_WRITE_REG(hw, IXGBE_EXVET, tpid << 16);
1285 }
1286
1287 void
1288 ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev)
1289 {
1290         struct ixgbe_hw *hw =
1291                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1292         uint32_t vlnctrl;
1293
1294         PMD_INIT_FUNC_TRACE();
1295
1296         /* Filter Table Disable */
1297         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1298         vlnctrl &= ~IXGBE_VLNCTRL_VFE;
1299
1300         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
1301 }
1302
1303 void
1304 ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1305 {
1306         struct ixgbe_hw *hw =
1307                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1308         struct ixgbe_vfta * shadow_vfta =
1309                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1310         uint32_t vlnctrl;
1311         uint16_t i;
1312
1313         PMD_INIT_FUNC_TRACE();
1314
1315         /* Filter Table Enable */
1316         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1317         vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
1318         vlnctrl |= IXGBE_VLNCTRL_VFE;
1319
1320         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
1321
1322         /* write whatever is in local vfta copy */
1323         for (i = 0; i < IXGBE_VFTA_SIZE; i++)
1324                 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), shadow_vfta->vfta[i]);
1325 }
1326
1327 static void
1328 ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on)
1329 {
1330         struct ixgbe_hwstrip *hwstrip =
1331                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(dev->data->dev_private);
1332
1333         if(queue >= IXGBE_MAX_RX_QUEUE_NUM)
1334                 return;
1335
1336         if (on)
1337                 IXGBE_SET_HWSTRIP(hwstrip, queue);
1338         else
1339                 IXGBE_CLEAR_HWSTRIP(hwstrip, queue);
1340 }
1341
1342 static void
1343 ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue)
1344 {
1345         struct ixgbe_hw *hw =
1346                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1347         uint32_t ctrl;
1348
1349         PMD_INIT_FUNC_TRACE();
1350
1351         if (hw->mac.type == ixgbe_mac_82598EB) {
1352                 /* No queue level support */
1353                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
1354                 return;
1355         }
1356         else {
1357                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1358                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1359                 ctrl &= ~IXGBE_RXDCTL_VME;
1360                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
1361         }
1362         /* record those setting for HW strip per queue */
1363         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 0);
1364 }
1365
1366 static void
1367 ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue)
1368 {
1369         struct ixgbe_hw *hw =
1370                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1371         uint32_t ctrl;
1372
1373         PMD_INIT_FUNC_TRACE();
1374
1375         if (hw->mac.type == ixgbe_mac_82598EB) {
1376                 /* No queue level supported */
1377                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
1378                 return;
1379         }
1380         else {
1381                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1382                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1383                 ctrl |= IXGBE_RXDCTL_VME;
1384                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
1385         }
1386         /* record those setting for HW strip per queue */
1387         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 1);
1388 }
1389
1390 void
1391 ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev)
1392 {
1393         struct ixgbe_hw *hw =
1394                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1395         uint32_t ctrl;
1396         uint16_t i;
1397
1398         PMD_INIT_FUNC_TRACE();
1399
1400         if (hw->mac.type == ixgbe_mac_82598EB) {
1401                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1402                 ctrl &= ~IXGBE_VLNCTRL_VME;
1403                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1404         }
1405         else {
1406                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1407                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1408                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1409                         ctrl &= ~IXGBE_RXDCTL_VME;
1410                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1411
1412                         /* record those setting for HW strip per queue */
1413                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 0);
1414                 }
1415         }
1416 }
1417
1418 void
1419 ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev)
1420 {
1421         struct ixgbe_hw *hw =
1422                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1423         uint32_t ctrl;
1424         uint16_t i;
1425
1426         PMD_INIT_FUNC_TRACE();
1427
1428         if (hw->mac.type == ixgbe_mac_82598EB) {
1429                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1430                 ctrl |= IXGBE_VLNCTRL_VME;
1431                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1432         }
1433         else {
1434                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1435                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1436                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1437                         ctrl |= IXGBE_RXDCTL_VME;
1438                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1439
1440                         /* record those setting for HW strip per queue */
1441                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 1);
1442                 }
1443         }
1444 }
1445
1446 static void
1447 ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1448 {
1449         struct ixgbe_hw *hw =
1450                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1451         uint32_t ctrl;
1452
1453         PMD_INIT_FUNC_TRACE();
1454
1455         /* DMATXCTRL: Geric Double VLAN Disable */
1456         ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1457         ctrl &= ~IXGBE_DMATXCTL_GDV;
1458         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1459
1460         /* CTRL_EXT: Global Double VLAN Disable */
1461         ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1462         ctrl &= ~IXGBE_EXTENDED_VLAN;
1463         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1464
1465 }
1466
1467 static void
1468 ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1469 {
1470         struct ixgbe_hw *hw =
1471                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1472         uint32_t ctrl;
1473
1474         PMD_INIT_FUNC_TRACE();
1475
1476         /* DMATXCTRL: Geric Double VLAN Enable */
1477         ctrl  = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1478         ctrl |= IXGBE_DMATXCTL_GDV;
1479         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1480
1481         /* CTRL_EXT: Global Double VLAN Enable */
1482         ctrl  = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1483         ctrl |= IXGBE_EXTENDED_VLAN;
1484         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1485
1486         /*
1487          * VET EXT field in the EXVET register = 0x8100 by default
1488          * So no need to change. Same to VT field of DMATXCTL register
1489          */
1490 }
1491
1492 static void
1493 ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1494 {
1495         if(mask & ETH_VLAN_STRIP_MASK){
1496                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1497                         ixgbe_vlan_hw_strip_enable_all(dev);
1498                 else
1499                         ixgbe_vlan_hw_strip_disable_all(dev);
1500         }
1501
1502         if(mask & ETH_VLAN_FILTER_MASK){
1503                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1504                         ixgbe_vlan_hw_filter_enable(dev);
1505                 else
1506                         ixgbe_vlan_hw_filter_disable(dev);
1507         }
1508
1509         if(mask & ETH_VLAN_EXTEND_MASK){
1510                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1511                         ixgbe_vlan_hw_extend_enable(dev);
1512                 else
1513                         ixgbe_vlan_hw_extend_disable(dev);
1514         }
1515 }
1516
1517 static void
1518 ixgbe_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1519 {
1520         struct ixgbe_hw *hw =
1521                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1522         /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
1523         uint32_t vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1524         vlanctrl |= IXGBE_VLNCTRL_VFE ; /* enable vlan filters */
1525         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
1526 }
1527
1528 static int
1529 ixgbe_dev_configure(struct rte_eth_dev *dev)
1530 {
1531         struct ixgbe_interrupt *intr =
1532                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1533         struct ixgbe_adapter *adapter =
1534                 (struct ixgbe_adapter *)dev->data->dev_private;
1535
1536         PMD_INIT_FUNC_TRACE();
1537
1538         /* set flag to update link status after init */
1539         intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1540
1541         /*
1542          * Initialize to TRUE. If any of Rx queues doesn't meet the bulk
1543          * allocation or vector Rx preconditions we will reset it.
1544          */
1545         adapter->rx_bulk_alloc_allowed = true;
1546         adapter->rx_vec_allowed = true;
1547
1548         return 0;
1549 }
1550
1551 /*
1552  * Configure device link speed and setup link.
1553  * It returns 0 on success.
1554  */
1555 static int
1556 ixgbe_dev_start(struct rte_eth_dev *dev)
1557 {
1558         struct ixgbe_hw *hw =
1559                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1560         struct ixgbe_vf_info *vfinfo =
1561                 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private);
1562         int err, link_up = 0, negotiate = 0;
1563         uint32_t speed = 0;
1564         int mask = 0;
1565         int status;
1566         uint16_t vf, idx;
1567
1568         PMD_INIT_FUNC_TRACE();
1569
1570         /* IXGBE devices don't support half duplex */
1571         if ((dev->data->dev_conf.link_duplex != ETH_LINK_AUTONEG_DUPLEX) &&
1572                         (dev->data->dev_conf.link_duplex != ETH_LINK_FULL_DUPLEX)) {
1573                 PMD_INIT_LOG(ERR, "Invalid link_duplex (%hu) for port %hhu",
1574                              dev->data->dev_conf.link_duplex,
1575                              dev->data->port_id);
1576                 return -EINVAL;
1577         }
1578
1579         /* stop adapter */
1580         hw->adapter_stopped = FALSE;
1581         ixgbe_stop_adapter(hw);
1582
1583         /* reinitialize adapter
1584          * this calls reset and start */
1585         status = ixgbe_pf_reset_hw(hw);
1586         if (status != 0)
1587                 return -1;
1588         hw->mac.ops.start_hw(hw);
1589         hw->mac.get_link_status = true;
1590
1591         /* configure PF module if SRIOV enabled */
1592         ixgbe_pf_host_configure(dev);
1593
1594         /* initialize transmission unit */
1595         ixgbe_dev_tx_init(dev);
1596
1597         /* This can fail when allocating mbufs for descriptor rings */
1598         err = ixgbe_dev_rx_init(dev);
1599         if (err) {
1600                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
1601                 goto error;
1602         }
1603
1604         err = ixgbe_dev_rxtx_start(dev);
1605         if (err < 0) {
1606                 PMD_INIT_LOG(ERR, "Unable to start rxtx queues");
1607                 goto error;
1608         }
1609
1610         /* Skip link setup if loopback mode is enabled for 82599. */
1611         if (hw->mac.type == ixgbe_mac_82599EB &&
1612                         dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_82599_TX_RX)
1613                 goto skip_link_setup;
1614
1615         if (ixgbe_is_sfp(hw) && hw->phy.multispeed_fiber) {
1616                 err = hw->mac.ops.setup_sfp(hw);
1617                 if (err)
1618                         goto error;
1619         }
1620
1621         if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
1622                 /* Turn on the copper */
1623                 ixgbe_set_phy_power(hw, true);
1624         } else {
1625                 /* Turn on the laser */
1626                 ixgbe_enable_tx_laser(hw);
1627         }
1628
1629         err = ixgbe_check_link(hw, &speed, &link_up, 0);
1630         if (err)
1631                 goto error;
1632         dev->data->dev_link.link_status = link_up;
1633
1634         err = ixgbe_get_link_capabilities(hw, &speed, &negotiate);
1635         if (err)
1636                 goto error;
1637
1638         switch(dev->data->dev_conf.link_speed) {
1639         case ETH_LINK_SPEED_AUTONEG:
1640                 speed = (hw->mac.type != ixgbe_mac_82598EB) ?
1641                                 IXGBE_LINK_SPEED_82599_AUTONEG :
1642                                 IXGBE_LINK_SPEED_82598_AUTONEG;
1643                 break;
1644         case ETH_LINK_SPEED_100:
1645                 /*
1646                  * Invalid for 82598 but error will be detected by
1647                  * ixgbe_setup_link()
1648                  */
1649                 speed = IXGBE_LINK_SPEED_100_FULL;
1650                 break;
1651         case ETH_LINK_SPEED_1000:
1652                 speed = IXGBE_LINK_SPEED_1GB_FULL;
1653                 break;
1654         case ETH_LINK_SPEED_10000:
1655                 speed = IXGBE_LINK_SPEED_10GB_FULL;
1656                 break;
1657         default:
1658                 PMD_INIT_LOG(ERR, "Invalid link_speed (%hu) for port %hhu",
1659                              dev->data->dev_conf.link_speed,
1660                              dev->data->port_id);
1661                 goto error;
1662         }
1663
1664         err = ixgbe_setup_link(hw, speed, link_up);
1665         if (err)
1666                 goto error;
1667
1668 skip_link_setup:
1669
1670         /* check if lsc interrupt is enabled */
1671         if (dev->data->dev_conf.intr_conf.lsc != 0)
1672                 ixgbe_dev_lsc_interrupt_setup(dev);
1673
1674         /* resume enabled intr since hw reset */
1675         ixgbe_enable_intr(dev);
1676
1677         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
1678                 ETH_VLAN_EXTEND_MASK;
1679         ixgbe_vlan_offload_set(dev, mask);
1680
1681         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
1682                 /* Enable vlan filtering for VMDq */
1683                 ixgbe_vmdq_vlan_hw_filter_enable(dev);
1684         }
1685
1686         /* Configure DCB hw */
1687         ixgbe_configure_dcb(dev);
1688
1689         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) {
1690                 err = ixgbe_fdir_configure(dev);
1691                 if (err)
1692                         goto error;
1693         }
1694
1695         /* Restore vf rate limit */
1696         if (vfinfo != NULL) {
1697                 for (vf = 0; vf < dev->pci_dev->max_vfs; vf++)
1698                         for (idx = 0; idx < IXGBE_MAX_QUEUE_NUM_PER_VF; idx++)
1699                                 if (vfinfo[vf].tx_rate[idx] != 0)
1700                                         ixgbe_set_vf_rate_limit(dev, vf,
1701                                                 vfinfo[vf].tx_rate[idx],
1702                                                 1 << idx);
1703         }
1704
1705         ixgbe_restore_statistics_mapping(dev);
1706
1707         return (0);
1708
1709 error:
1710         PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err);
1711         ixgbe_dev_clear_queues(dev);
1712         return -EIO;
1713 }
1714
1715 /*
1716  * Stop device: disable rx and tx functions to allow for reconfiguring.
1717  */
1718 static void
1719 ixgbe_dev_stop(struct rte_eth_dev *dev)
1720 {
1721         struct rte_eth_link link;
1722         struct ixgbe_hw *hw =
1723                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1724         struct ixgbe_vf_info *vfinfo =
1725                 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private);
1726         struct ixgbe_filter_info *filter_info =
1727                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
1728         struct ixgbe_5tuple_filter *p_5tuple, *p_5tuple_next;
1729         int vf;
1730
1731         PMD_INIT_FUNC_TRACE();
1732
1733         /* disable interrupts */
1734         ixgbe_disable_intr(hw);
1735
1736         /* reset the NIC */
1737         ixgbe_pf_reset_hw(hw);
1738         hw->adapter_stopped = FALSE;
1739
1740         /* stop adapter */
1741         ixgbe_stop_adapter(hw);
1742
1743         for (vf = 0; vfinfo != NULL &&
1744                      vf < dev->pci_dev->max_vfs; vf++)
1745                 vfinfo[vf].clear_to_send = false;
1746
1747         if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
1748                 /* Turn off the copper */
1749                 ixgbe_set_phy_power(hw, false);
1750         } else {
1751                 /* Turn off the laser */
1752                 ixgbe_disable_tx_laser(hw);
1753         }
1754
1755         ixgbe_dev_clear_queues(dev);
1756
1757         /* Clear stored conf */
1758         dev->data->scattered_rx = 0;
1759         dev->data->lro = 0;
1760
1761         /* Clear recorded link status */
1762         memset(&link, 0, sizeof(link));
1763         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1764
1765         /* Remove all ntuple filters of the device */
1766         for (p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list);
1767              p_5tuple != NULL; p_5tuple = p_5tuple_next) {
1768                 p_5tuple_next = TAILQ_NEXT(p_5tuple, entries);
1769                 TAILQ_REMOVE(&filter_info->fivetuple_list,
1770                              p_5tuple, entries);
1771                 rte_free(p_5tuple);
1772         }
1773         memset(filter_info->fivetuple_mask, 0,
1774                 sizeof(uint32_t) * IXGBE_5TUPLE_ARRAY_SIZE);
1775
1776 }
1777
1778 /*
1779  * Set device link up: enable tx.
1780  */
1781 static int
1782 ixgbe_dev_set_link_up(struct rte_eth_dev *dev)
1783 {
1784         struct ixgbe_hw *hw =
1785                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1786         if (hw->mac.type == ixgbe_mac_82599EB) {
1787 #ifdef RTE_NIC_BYPASS
1788                 if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) {
1789                         /* Not suported in bypass mode */
1790                         PMD_INIT_LOG(ERR, "Set link up is not supported "
1791                                      "by device id 0x%x", hw->device_id);
1792                         return -ENOTSUP;
1793                 }
1794 #endif
1795         }
1796
1797         if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
1798                 /* Turn on the copper */
1799                 ixgbe_set_phy_power(hw, true);
1800         } else {
1801                 /* Turn on the laser */
1802                 ixgbe_enable_tx_laser(hw);
1803         }
1804
1805         return 0;
1806 }
1807
1808 /*
1809  * Set device link down: disable tx.
1810  */
1811 static int
1812 ixgbe_dev_set_link_down(struct rte_eth_dev *dev)
1813 {
1814         struct ixgbe_hw *hw =
1815                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1816         if (hw->mac.type == ixgbe_mac_82599EB) {
1817 #ifdef RTE_NIC_BYPASS
1818                 if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) {
1819                         /* Not suported in bypass mode */
1820                         PMD_INIT_LOG(ERR, "Set link down is not supported "
1821                                      "by device id 0x%x", hw->device_id);
1822                         return -ENOTSUP;
1823                 }
1824 #endif
1825         }
1826
1827         if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
1828                 /* Turn off the copper */
1829                 ixgbe_set_phy_power(hw, false);
1830         } else {
1831                 /* Turn off the laser */
1832                 ixgbe_disable_tx_laser(hw);
1833         }
1834
1835         return 0;
1836 }
1837
1838 /*
1839  * Reest and stop device.
1840  */
1841 static void
1842 ixgbe_dev_close(struct rte_eth_dev *dev)
1843 {
1844         struct ixgbe_hw *hw =
1845                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1846
1847         PMD_INIT_FUNC_TRACE();
1848
1849         ixgbe_pf_reset_hw(hw);
1850
1851         ixgbe_dev_stop(dev);
1852         hw->adapter_stopped = 1;
1853
1854         ixgbe_disable_pcie_master(hw);
1855
1856         /* reprogram the RAR[0] in case user changed it. */
1857         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
1858 }
1859
1860 static void
1861 ixgbe_read_stats_registers(struct ixgbe_hw *hw, struct ixgbe_hw_stats
1862                                                    *hw_stats, uint64_t *total_missed_rx,
1863                                                    uint64_t *total_qbrc, uint64_t *total_qprc,
1864                                                    uint64_t *rxnfgpc, uint64_t *txdgpc,
1865                                                    uint64_t *total_qprdc)
1866 {
1867         uint32_t bprc, lxon, lxoff, total;
1868         unsigned i;
1869
1870         hw_stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
1871         hw_stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
1872         hw_stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
1873         hw_stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
1874
1875         for (i = 0; i < 8; i++) {
1876                 uint32_t mp;
1877                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
1878                 /* global total per queue */
1879                 hw_stats->mpc[i] += mp;
1880                 /* Running comprehensive total for stats display */
1881                 *total_missed_rx += hw_stats->mpc[i];
1882                 if (hw->mac.type == ixgbe_mac_82598EB)
1883                         hw_stats->rnbc[i] +=
1884                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
1885                 hw_stats->pxontxc[i] +=
1886                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
1887                 hw_stats->pxonrxc[i] +=
1888                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
1889                 hw_stats->pxofftxc[i] +=
1890                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
1891                 hw_stats->pxoffrxc[i] +=
1892                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
1893                 hw_stats->pxon2offc[i] +=
1894                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
1895         }
1896         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1897                 hw_stats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
1898                 hw_stats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
1899                 hw_stats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
1900                 hw_stats->qbrc[i] +=
1901                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32);
1902                 hw_stats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
1903                 hw_stats->qbtc[i] +=
1904                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32);
1905                 *total_qprdc += hw_stats->qprdc[i] +=
1906                                 IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
1907
1908                 *total_qprc += hw_stats->qprc[i];
1909                 *total_qbrc += hw_stats->qbrc[i];
1910         }
1911         hw_stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
1912         hw_stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
1913         hw_stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
1914
1915         /* Note that gprc counts missed packets */
1916         hw_stats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
1917         *rxnfgpc += IXGBE_READ_REG(hw, IXGBE_RXNFGPC);
1918         *txdgpc += IXGBE_READ_REG(hw, IXGBE_TXDGPC);
1919
1920         if (hw->mac.type != ixgbe_mac_82598EB) {
1921                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
1922                 hw_stats->gorc += ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
1923                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
1924                 hw_stats->gotc += ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
1925                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
1926                 hw_stats->tor += ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
1927                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
1928                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
1929         } else {
1930                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
1931                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
1932                 /* 82598 only has a counter in the high register */
1933                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
1934                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
1935                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
1936         }
1937
1938         /*
1939          * Workaround: mprc hardware is incorrectly counting
1940          * broadcasts, so for now we subtract those.
1941          */
1942         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
1943         hw_stats->bprc += bprc;
1944         hw_stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
1945         if (hw->mac.type == ixgbe_mac_82598EB)
1946                 hw_stats->mprc -= bprc;
1947
1948         hw_stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
1949         hw_stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
1950         hw_stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
1951         hw_stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
1952         hw_stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
1953         hw_stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
1954
1955         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
1956         hw_stats->lxontxc += lxon;
1957         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
1958         hw_stats->lxofftxc += lxoff;
1959         total = lxon + lxoff;
1960
1961         hw_stats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
1962         hw_stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
1963         hw_stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
1964         hw_stats->gptc -= total;
1965         hw_stats->mptc -= total;
1966         hw_stats->ptc64 -= total;
1967         hw_stats->gotc -= total * ETHER_MIN_LEN;
1968
1969         hw_stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
1970         hw_stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
1971         hw_stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
1972         hw_stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
1973         hw_stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
1974         hw_stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
1975         hw_stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
1976         hw_stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
1977         hw_stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
1978         hw_stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
1979         hw_stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
1980         hw_stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
1981         hw_stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
1982         hw_stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
1983         hw_stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
1984         hw_stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);
1985         hw_stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
1986         hw_stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
1987         /* Only read FCOE on 82599 */
1988         if (hw->mac.type != ixgbe_mac_82598EB) {
1989                 hw_stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
1990                 hw_stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
1991                 hw_stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
1992                 hw_stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
1993                 hw_stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
1994         }
1995
1996         /* Flow Director Stats registers */
1997         hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
1998         hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
1999 }
2000
2001 /*
2002  * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c
2003  */
2004 static void
2005 ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
2006 {
2007         struct ixgbe_hw *hw =
2008                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2009         struct ixgbe_hw_stats *hw_stats =
2010                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2011         uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc;
2012         uint64_t rxnfgpc, txdgpc;
2013         unsigned i;
2014
2015         total_missed_rx = 0;
2016         total_qbrc = 0;
2017         total_qprc = 0;
2018         total_qprdc = 0;
2019         rxnfgpc = 0;
2020         txdgpc = 0;
2021
2022         ixgbe_read_stats_registers(hw, hw_stats, &total_missed_rx, &total_qbrc,
2023                         &total_qprc, &rxnfgpc, &txdgpc, &total_qprdc);
2024
2025         if (stats == NULL)
2026                 return;
2027
2028         /* Fill out the rte_eth_stats statistics structure */
2029         stats->ipackets = total_qprc;
2030         stats->ibytes = total_qbrc;
2031         stats->opackets = hw_stats->gptc;
2032         stats->obytes = hw_stats->gotc;
2033
2034         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
2035                 stats->q_ipackets[i] = hw_stats->qprc[i];
2036                 stats->q_opackets[i] = hw_stats->qptc[i];
2037                 stats->q_ibytes[i] = hw_stats->qbrc[i];
2038                 stats->q_obytes[i] = hw_stats->qbtc[i];
2039                 stats->q_errors[i] = hw_stats->qprdc[i];
2040         }
2041
2042         /* Rx Errors */
2043         stats->ierrors  = hw_stats->crcerrs +
2044                           hw_stats->rlec +
2045                           hw_stats->ruc +
2046                           hw_stats->roc +
2047                           total_missed_rx +
2048                           hw_stats->illerrc +
2049                           hw_stats->errbc +
2050                           hw_stats->xec +
2051                           hw_stats->mlfc +
2052                           hw_stats->mrfc +
2053                           hw_stats->rfc +
2054                           hw_stats->rjc +
2055                           hw_stats->fccrc +
2056                           hw_stats->fclast +
2057                           (rxnfgpc - hw_stats->gprc); /* PHY Errors*/
2058
2059         /* Tx Errors */
2060         /*txdgpc: packets that are DMA'ed*/
2061         /*gptc: packets that are sent*/
2062         stats->oerrors  = txdgpc - hw_stats->gptc;
2063 }
2064
2065 static void
2066 ixgbe_dev_stats_reset(struct rte_eth_dev *dev)
2067 {
2068         struct ixgbe_hw_stats *stats =
2069                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2070
2071         /* HW registers are cleared on read */
2072         ixgbe_dev_stats_get(dev, NULL);
2073
2074         /* Reset software totals */
2075         memset(stats, 0, sizeof(*stats));
2076 }
2077
2078 static int
2079 ixgbe_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstats *xstats,
2080                                          unsigned n)
2081 {
2082         struct ixgbe_hw *hw =
2083                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2084         struct ixgbe_hw_stats *hw_stats =
2085                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2086         uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc;
2087         uint64_t rxnfgpc, txdgpc;
2088         unsigned i, count = IXGBE_NB_XSTATS;
2089
2090         if (n < count)
2091                 return count;
2092
2093         total_missed_rx = 0;
2094         total_qbrc = 0;
2095         total_qprc = 0;
2096         total_qprdc = 0;
2097         rxnfgpc = 0;
2098         txdgpc = 0;
2099
2100         ixgbe_read_stats_registers(hw, hw_stats, &total_missed_rx, &total_qbrc,
2101                                                            &total_qprc, &rxnfgpc, &txdgpc, &total_qprdc);
2102
2103         /* If this is a reset xstats is NULL, and we have cleared the
2104          * registers by reading them.
2105          */
2106         if (!xstats)
2107                 return 0;
2108
2109         /* Extended stats */
2110         for (i = 0; i < IXGBE_NB_XSTATS; i++) {
2111                 snprintf(xstats[i].name, sizeof(xstats[i].name),
2112                                 "%s", rte_ixgbe_stats_strings[i].name);
2113                 xstats[i].value = *(uint64_t *)(((char *)hw_stats) +
2114                                                         rte_ixgbe_stats_strings[i].offset);
2115         }
2116
2117         return count;
2118 }
2119
2120 static void
2121 ixgbe_dev_xstats_reset(struct rte_eth_dev *dev)
2122 {
2123         struct ixgbe_hw_stats *stats =
2124                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2125
2126         /* HW registers are cleared on read */
2127         ixgbe_dev_xstats_get(dev, NULL, IXGBE_NB_XSTATS);
2128
2129         /* Reset software totals */
2130         memset(stats, 0, sizeof(*stats));
2131 }
2132
2133 static void
2134 ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
2135 {
2136         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2137         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
2138                           IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2139
2140         /* Good Rx packet, include VF loopback */
2141         UPDATE_VF_STAT(IXGBE_VFGPRC,
2142             hw_stats->last_vfgprc, hw_stats->vfgprc);
2143
2144         /* Good Rx octets, include VF loopback */
2145         UPDATE_VF_STAT_36BIT(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
2146             hw_stats->last_vfgorc, hw_stats->vfgorc);
2147
2148         /* Good Tx packet, include VF loopback */
2149         UPDATE_VF_STAT(IXGBE_VFGPTC,
2150             hw_stats->last_vfgptc, hw_stats->vfgptc);
2151
2152         /* Good Tx octets, include VF loopback */
2153         UPDATE_VF_STAT_36BIT(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
2154             hw_stats->last_vfgotc, hw_stats->vfgotc);
2155
2156         /* Rx Multicst Packet */
2157         UPDATE_VF_STAT(IXGBE_VFMPRC,
2158             hw_stats->last_vfmprc, hw_stats->vfmprc);
2159
2160         if (stats == NULL)
2161                 return;
2162
2163         stats->ipackets = hw_stats->vfgprc;
2164         stats->ibytes = hw_stats->vfgorc;
2165         stats->opackets = hw_stats->vfgptc;
2166         stats->obytes = hw_stats->vfgotc;
2167         stats->imcasts = hw_stats->vfmprc;
2168         /* stats->imcasts should be removed as imcasts is deprecated */
2169 }
2170
2171 static void
2172 ixgbevf_dev_stats_reset(struct rte_eth_dev *dev)
2173 {
2174         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
2175                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2176
2177         /* Sync HW register to the last stats */
2178         ixgbevf_dev_stats_get(dev, NULL);
2179
2180         /* reset HW current stats*/
2181         hw_stats->vfgprc = 0;
2182         hw_stats->vfgorc = 0;
2183         hw_stats->vfgptc = 0;
2184         hw_stats->vfgotc = 0;
2185         hw_stats->vfmprc = 0;
2186
2187 }
2188
2189 static void
2190 ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
2191 {
2192         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2193
2194         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
2195         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
2196         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL register */
2197         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */
2198         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
2199         dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC;
2200         dev_info->max_vfs = dev->pci_dev->max_vfs;
2201         if (hw->mac.type == ixgbe_mac_82598EB)
2202                 dev_info->max_vmdq_pools = ETH_16_POOLS;
2203         else
2204                 dev_info->max_vmdq_pools = ETH_64_POOLS;
2205         dev_info->vmdq_queue_num = dev_info->max_rx_queues;
2206         dev_info->rx_offload_capa =
2207                 DEV_RX_OFFLOAD_VLAN_STRIP |
2208                 DEV_RX_OFFLOAD_IPV4_CKSUM |
2209                 DEV_RX_OFFLOAD_UDP_CKSUM  |
2210                 DEV_RX_OFFLOAD_TCP_CKSUM;
2211
2212         /*
2213          * RSC is only supported by 82599 and x540 PF devices in a non-SR-IOV
2214          * mode.
2215          */
2216         if ((hw->mac.type == ixgbe_mac_82599EB ||
2217              hw->mac.type == ixgbe_mac_X540) &&
2218             !RTE_ETH_DEV_SRIOV(dev).active)
2219                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_LRO;
2220
2221         dev_info->tx_offload_capa =
2222                 DEV_TX_OFFLOAD_VLAN_INSERT |
2223                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
2224                 DEV_TX_OFFLOAD_UDP_CKSUM   |
2225                 DEV_TX_OFFLOAD_TCP_CKSUM   |
2226                 DEV_TX_OFFLOAD_SCTP_CKSUM  |
2227                 DEV_TX_OFFLOAD_TCP_TSO;
2228
2229         dev_info->default_rxconf = (struct rte_eth_rxconf) {
2230                 .rx_thresh = {
2231                         .pthresh = IXGBE_DEFAULT_RX_PTHRESH,
2232                         .hthresh = IXGBE_DEFAULT_RX_HTHRESH,
2233                         .wthresh = IXGBE_DEFAULT_RX_WTHRESH,
2234                 },
2235                 .rx_free_thresh = IXGBE_DEFAULT_RX_FREE_THRESH,
2236                 .rx_drop_en = 0,
2237         };
2238
2239         dev_info->default_txconf = (struct rte_eth_txconf) {
2240                 .tx_thresh = {
2241                         .pthresh = IXGBE_DEFAULT_TX_PTHRESH,
2242                         .hthresh = IXGBE_DEFAULT_TX_HTHRESH,
2243                         .wthresh = IXGBE_DEFAULT_TX_WTHRESH,
2244                 },
2245                 .tx_free_thresh = IXGBE_DEFAULT_TX_FREE_THRESH,
2246                 .tx_rs_thresh = IXGBE_DEFAULT_TX_RSBIT_THRESH,
2247                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
2248                                 ETH_TXQ_FLAGS_NOOFFLOADS,
2249         };
2250         dev_info->hash_key_size = IXGBE_HKEY_MAX_INDEX * sizeof(uint32_t);
2251         dev_info->reta_size = ETH_RSS_RETA_SIZE_128;
2252         dev_info->flow_type_rss_offloads = IXGBE_RSS_OFFLOAD_ALL;
2253 }
2254
2255 static void
2256 ixgbevf_dev_info_get(struct rte_eth_dev *dev,
2257                      struct rte_eth_dev_info *dev_info)
2258 {
2259         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2260
2261         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
2262         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
2263         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL reg */
2264         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS reg */
2265         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
2266         dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC;
2267         dev_info->max_vfs = dev->pci_dev->max_vfs;
2268         if (hw->mac.type == ixgbe_mac_82598EB)
2269                 dev_info->max_vmdq_pools = ETH_16_POOLS;
2270         else
2271                 dev_info->max_vmdq_pools = ETH_64_POOLS;
2272         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
2273                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
2274                                 DEV_RX_OFFLOAD_UDP_CKSUM  |
2275                                 DEV_RX_OFFLOAD_TCP_CKSUM;
2276         dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
2277                                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
2278                                 DEV_TX_OFFLOAD_UDP_CKSUM   |
2279                                 DEV_TX_OFFLOAD_TCP_CKSUM   |
2280                                 DEV_TX_OFFLOAD_SCTP_CKSUM;
2281
2282         dev_info->default_rxconf = (struct rte_eth_rxconf) {
2283                 .rx_thresh = {
2284                         .pthresh = IXGBE_DEFAULT_RX_PTHRESH,
2285                         .hthresh = IXGBE_DEFAULT_RX_HTHRESH,
2286                         .wthresh = IXGBE_DEFAULT_RX_WTHRESH,
2287                 },
2288                 .rx_free_thresh = IXGBE_DEFAULT_RX_FREE_THRESH,
2289                 .rx_drop_en = 0,
2290         };
2291
2292         dev_info->default_txconf = (struct rte_eth_txconf) {
2293                 .tx_thresh = {
2294                         .pthresh = IXGBE_DEFAULT_TX_PTHRESH,
2295                         .hthresh = IXGBE_DEFAULT_TX_HTHRESH,
2296                         .wthresh = IXGBE_DEFAULT_TX_WTHRESH,
2297                 },
2298                 .tx_free_thresh = IXGBE_DEFAULT_TX_FREE_THRESH,
2299                 .tx_rs_thresh = IXGBE_DEFAULT_TX_RSBIT_THRESH,
2300                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
2301                                 ETH_TXQ_FLAGS_NOOFFLOADS,
2302         };
2303 }
2304
2305 /* return 0 means link status changed, -1 means not changed */
2306 static int
2307 ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
2308 {
2309         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2310         struct rte_eth_link link, old;
2311         ixgbe_link_speed link_speed = IXGBE_LINK_SPEED_UNKNOWN;
2312         int link_up;
2313         int diag;
2314
2315         link.link_status = 0;
2316         link.link_speed = 0;
2317         link.link_duplex = 0;
2318         memset(&old, 0, sizeof(old));
2319         rte_ixgbe_dev_atomic_read_link_status(dev, &old);
2320
2321         hw->mac.get_link_status = true;
2322
2323         /* check if it needs to wait to complete, if lsc interrupt is enabled */
2324         if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
2325                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 0);
2326         else
2327                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 1);
2328
2329         if (diag != 0) {
2330                 link.link_speed = ETH_LINK_SPEED_100;
2331                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
2332                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
2333                 if (link.link_status == old.link_status)
2334                         return -1;
2335                 return 0;
2336         }
2337
2338         if (link_up == 0) {
2339                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
2340                 if (link.link_status == old.link_status)
2341                         return -1;
2342                 return 0;
2343         }
2344         link.link_status = 1;
2345         link.link_duplex = ETH_LINK_FULL_DUPLEX;
2346
2347         switch (link_speed) {
2348         default:
2349         case IXGBE_LINK_SPEED_UNKNOWN:
2350                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
2351                 link.link_speed = ETH_LINK_SPEED_100;
2352                 break;
2353
2354         case IXGBE_LINK_SPEED_100_FULL:
2355                 link.link_speed = ETH_LINK_SPEED_100;
2356                 break;
2357
2358         case IXGBE_LINK_SPEED_1GB_FULL:
2359                 link.link_speed = ETH_LINK_SPEED_1000;
2360                 break;
2361
2362         case IXGBE_LINK_SPEED_10GB_FULL:
2363                 link.link_speed = ETH_LINK_SPEED_10000;
2364                 break;
2365         }
2366         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
2367
2368         if (link.link_status == old.link_status)
2369                 return -1;
2370
2371         return 0;
2372 }
2373
2374 static void
2375 ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev)
2376 {
2377         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2378         uint32_t fctrl;
2379
2380         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2381         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
2382         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2383 }
2384
2385 static void
2386 ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev)
2387 {
2388         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2389         uint32_t fctrl;
2390
2391         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2392         fctrl &= (~IXGBE_FCTRL_UPE);
2393         if (dev->data->all_multicast == 1)
2394                 fctrl |= IXGBE_FCTRL_MPE;
2395         else
2396                 fctrl &= (~IXGBE_FCTRL_MPE);
2397         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2398 }
2399
2400 static void
2401 ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev)
2402 {
2403         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2404         uint32_t fctrl;
2405
2406         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2407         fctrl |= IXGBE_FCTRL_MPE;
2408         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2409 }
2410
2411 static void
2412 ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev)
2413 {
2414         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2415         uint32_t fctrl;
2416
2417         if (dev->data->promiscuous == 1)
2418                 return; /* must remain in all_multicast mode */
2419
2420         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2421         fctrl &= (~IXGBE_FCTRL_MPE);
2422         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2423 }
2424
2425 /**
2426  * It clears the interrupt causes and enables the interrupt.
2427  * It will be called once only during nic initialized.
2428  *
2429  * @param dev
2430  *  Pointer to struct rte_eth_dev.
2431  *
2432  * @return
2433  *  - On success, zero.
2434  *  - On failure, a negative value.
2435  */
2436 static int
2437 ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev)
2438 {
2439         struct ixgbe_interrupt *intr =
2440                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2441
2442         ixgbe_dev_link_status_print(dev);
2443         intr->mask |= IXGBE_EICR_LSC;
2444
2445         return 0;
2446 }
2447
2448 /*
2449  * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update.
2450  *
2451  * @param dev
2452  *  Pointer to struct rte_eth_dev.
2453  *
2454  * @return
2455  *  - On success, zero.
2456  *  - On failure, a negative value.
2457  */
2458 static int
2459 ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
2460 {
2461         uint32_t eicr;
2462         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2463         struct ixgbe_interrupt *intr =
2464                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2465
2466         /* clear all cause mask */
2467         ixgbe_disable_intr(hw);
2468
2469         /* read-on-clear nic registers here */
2470         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
2471         PMD_DRV_LOG(INFO, "eicr %x", eicr);
2472
2473         intr->flags = 0;
2474         if (eicr & IXGBE_EICR_LSC) {
2475                 /* set flag for async link update */
2476                 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
2477         }
2478
2479         if (eicr & IXGBE_EICR_MAILBOX)
2480                 intr->flags |= IXGBE_FLAG_MAILBOX;
2481
2482         return 0;
2483 }
2484
2485 /**
2486  * It gets and then prints the link status.
2487  *
2488  * @param dev
2489  *  Pointer to struct rte_eth_dev.
2490  *
2491  * @return
2492  *  - On success, zero.
2493  *  - On failure, a negative value.
2494  */
2495 static void
2496 ixgbe_dev_link_status_print(struct rte_eth_dev *dev)
2497 {
2498         struct rte_eth_link link;
2499
2500         memset(&link, 0, sizeof(link));
2501         rte_ixgbe_dev_atomic_read_link_status(dev, &link);
2502         if (link.link_status) {
2503                 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
2504                                         (int)(dev->data->port_id),
2505                                         (unsigned)link.link_speed,
2506                         link.link_duplex == ETH_LINK_FULL_DUPLEX ?
2507                                         "full-duplex" : "half-duplex");
2508         } else {
2509                 PMD_INIT_LOG(INFO, " Port %d: Link Down",
2510                                 (int)(dev->data->port_id));
2511         }
2512         PMD_INIT_LOG(INFO, "PCI Address: %04d:%02d:%02d:%d",
2513                                 dev->pci_dev->addr.domain,
2514                                 dev->pci_dev->addr.bus,
2515                                 dev->pci_dev->addr.devid,
2516                                 dev->pci_dev->addr.function);
2517 }
2518
2519 /*
2520  * It executes link_update after knowing an interrupt occurred.
2521  *
2522  * @param dev
2523  *  Pointer to struct rte_eth_dev.
2524  *
2525  * @return
2526  *  - On success, zero.
2527  *  - On failure, a negative value.
2528  */
2529 static int
2530 ixgbe_dev_interrupt_action(struct rte_eth_dev *dev)
2531 {
2532         struct ixgbe_interrupt *intr =
2533                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2534         int64_t timeout;
2535         struct rte_eth_link link;
2536         int intr_enable_delay = false;
2537
2538         PMD_DRV_LOG(DEBUG, "intr action type %d", intr->flags);
2539
2540         if (intr->flags & IXGBE_FLAG_MAILBOX) {
2541                 ixgbe_pf_mbx_process(dev);
2542                 intr->flags &= ~IXGBE_FLAG_MAILBOX;
2543         }
2544
2545         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
2546                 /* get the link status before link update, for predicting later */
2547                 memset(&link, 0, sizeof(link));
2548                 rte_ixgbe_dev_atomic_read_link_status(dev, &link);
2549
2550                 ixgbe_dev_link_update(dev, 0);
2551
2552                 /* likely to up */
2553                 if (!link.link_status)
2554                         /* handle it 1 sec later, wait it being stable */
2555                         timeout = IXGBE_LINK_UP_CHECK_TIMEOUT;
2556                 /* likely to down */
2557                 else
2558                         /* handle it 4 sec later, wait it being stable */
2559                         timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT;
2560
2561                 ixgbe_dev_link_status_print(dev);
2562
2563                 intr_enable_delay = true;
2564         }
2565
2566         if (intr_enable_delay) {
2567                 if (rte_eal_alarm_set(timeout * 1000,
2568                                       ixgbe_dev_interrupt_delayed_handler, (void*)dev) < 0)
2569                         PMD_DRV_LOG(ERR, "Error setting alarm");
2570         } else {
2571                 PMD_DRV_LOG(DEBUG, "enable intr immediately");
2572                 ixgbe_enable_intr(dev);
2573                 rte_intr_enable(&(dev->pci_dev->intr_handle));
2574         }
2575
2576
2577         return 0;
2578 }
2579
2580 /**
2581  * Interrupt handler which shall be registered for alarm callback for delayed
2582  * handling specific interrupt to wait for the stable nic state. As the
2583  * NIC interrupt state is not stable for ixgbe after link is just down,
2584  * it needs to wait 4 seconds to get the stable status.
2585  *
2586  * @param handle
2587  *  Pointer to interrupt handle.
2588  * @param param
2589  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2590  *
2591  * @return
2592  *  void
2593  */
2594 static void
2595 ixgbe_dev_interrupt_delayed_handler(void *param)
2596 {
2597         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2598         struct ixgbe_interrupt *intr =
2599                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2600         struct ixgbe_hw *hw =
2601                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2602         uint32_t eicr;
2603
2604         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
2605         if (eicr & IXGBE_EICR_MAILBOX)
2606                 ixgbe_pf_mbx_process(dev);
2607
2608         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
2609                 ixgbe_dev_link_update(dev, 0);
2610                 intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
2611                 ixgbe_dev_link_status_print(dev);
2612                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
2613         }
2614
2615         PMD_DRV_LOG(DEBUG, "enable intr in delayed handler S[%08x]", eicr);
2616         ixgbe_enable_intr(dev);
2617         rte_intr_enable(&(dev->pci_dev->intr_handle));
2618 }
2619
2620 /**
2621  * Interrupt handler triggered by NIC  for handling
2622  * specific interrupt.
2623  *
2624  * @param handle
2625  *  Pointer to interrupt handle.
2626  * @param param
2627  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2628  *
2629  * @return
2630  *  void
2631  */
2632 static void
2633 ixgbe_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
2634                                                         void *param)
2635 {
2636         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2637         ixgbe_dev_interrupt_get_status(dev);
2638         ixgbe_dev_interrupt_action(dev);
2639 }
2640
2641 static int
2642 ixgbe_dev_led_on(struct rte_eth_dev *dev)
2643 {
2644         struct ixgbe_hw *hw;
2645
2646         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2647         return (ixgbe_led_on(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
2648 }
2649
2650 static int
2651 ixgbe_dev_led_off(struct rte_eth_dev *dev)
2652 {
2653         struct ixgbe_hw *hw;
2654
2655         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2656         return (ixgbe_led_off(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
2657 }
2658
2659 static int
2660 ixgbe_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2661 {
2662         struct ixgbe_hw *hw;
2663         uint32_t mflcn_reg;
2664         uint32_t fccfg_reg;
2665         int rx_pause;
2666         int tx_pause;
2667
2668         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2669
2670         fc_conf->pause_time = hw->fc.pause_time;
2671         fc_conf->high_water = hw->fc.high_water[0];
2672         fc_conf->low_water = hw->fc.low_water[0];
2673         fc_conf->send_xon = hw->fc.send_xon;
2674         fc_conf->autoneg = !hw->fc.disable_fc_autoneg;
2675
2676         /*
2677          * Return rx_pause status according to actual setting of
2678          * MFLCN register.
2679          */
2680         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
2681         if (mflcn_reg & (IXGBE_MFLCN_RPFCE | IXGBE_MFLCN_RFCE))
2682                 rx_pause = 1;
2683         else
2684                 rx_pause = 0;
2685
2686         /*
2687          * Return tx_pause status according to actual setting of
2688          * FCCFG register.
2689          */
2690         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
2691         if (fccfg_reg & (IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY))
2692                 tx_pause = 1;
2693         else
2694                 tx_pause = 0;
2695
2696         if (rx_pause && tx_pause)
2697                 fc_conf->mode = RTE_FC_FULL;
2698         else if (rx_pause)
2699                 fc_conf->mode = RTE_FC_RX_PAUSE;
2700         else if (tx_pause)
2701                 fc_conf->mode = RTE_FC_TX_PAUSE;
2702         else
2703                 fc_conf->mode = RTE_FC_NONE;
2704
2705         return 0;
2706 }
2707
2708 static int
2709 ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2710 {
2711         struct ixgbe_hw *hw;
2712         int err;
2713         uint32_t rx_buf_size;
2714         uint32_t max_high_water;
2715         uint32_t mflcn;
2716         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
2717                 ixgbe_fc_none,
2718                 ixgbe_fc_rx_pause,
2719                 ixgbe_fc_tx_pause,
2720                 ixgbe_fc_full
2721         };
2722
2723         PMD_INIT_FUNC_TRACE();
2724
2725         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2726         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0));
2727         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
2728
2729         /*
2730          * At least reserve one Ethernet frame for watermark
2731          * high_water/low_water in kilo bytes for ixgbe
2732          */
2733         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
2734         if ((fc_conf->high_water > max_high_water) ||
2735                 (fc_conf->high_water < fc_conf->low_water)) {
2736                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB");
2737                 PMD_INIT_LOG(ERR, "High_water must <= 0x%x", max_high_water);
2738                 return (-EINVAL);
2739         }
2740
2741         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[fc_conf->mode];
2742         hw->fc.pause_time     = fc_conf->pause_time;
2743         hw->fc.high_water[0]  = fc_conf->high_water;
2744         hw->fc.low_water[0]   = fc_conf->low_water;
2745         hw->fc.send_xon       = fc_conf->send_xon;
2746         hw->fc.disable_fc_autoneg = !fc_conf->autoneg;
2747
2748         err = ixgbe_fc_enable(hw);
2749
2750         /* Not negotiated is not an error case */
2751         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) {
2752
2753                 /* check if we want to forward MAC frames - driver doesn't have native
2754                  * capability to do that, so we'll write the registers ourselves */
2755
2756                 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN);
2757
2758                 /* set or clear MFLCN.PMCF bit depending on configuration */
2759                 if (fc_conf->mac_ctrl_frame_fwd != 0)
2760                         mflcn |= IXGBE_MFLCN_PMCF;
2761                 else
2762                         mflcn &= ~IXGBE_MFLCN_PMCF;
2763
2764                 IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn);
2765                 IXGBE_WRITE_FLUSH(hw);
2766
2767                 return 0;
2768         }
2769
2770         PMD_INIT_LOG(ERR, "ixgbe_fc_enable = 0x%x", err);
2771         return -EIO;
2772 }
2773
2774 /**
2775  *  ixgbe_pfc_enable_generic - Enable flow control
2776  *  @hw: pointer to hardware structure
2777  *  @tc_num: traffic class number
2778  *  Enable flow control according to the current settings.
2779  */
2780 static int
2781 ixgbe_dcb_pfc_enable_generic(struct ixgbe_hw *hw,uint8_t tc_num)
2782 {
2783         int ret_val = 0;
2784         uint32_t mflcn_reg, fccfg_reg;
2785         uint32_t reg;
2786         uint32_t fcrtl, fcrth;
2787         uint8_t i;
2788         uint8_t nb_rx_en;
2789
2790         /* Validate the water mark configuration */
2791         if (!hw->fc.pause_time) {
2792                 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2793                 goto out;
2794         }
2795
2796         /* Low water mark of zero causes XOFF floods */
2797         if (hw->fc.current_mode & ixgbe_fc_tx_pause) {
2798                  /* High/Low water can not be 0 */
2799                 if( (!hw->fc.high_water[tc_num])|| (!hw->fc.low_water[tc_num])) {
2800                         PMD_INIT_LOG(ERR, "Invalid water mark configuration");
2801                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2802                         goto out;
2803                 }
2804
2805                 if(hw->fc.low_water[tc_num] >= hw->fc.high_water[tc_num]) {
2806                         PMD_INIT_LOG(ERR, "Invalid water mark configuration");
2807                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2808                         goto out;
2809                 }
2810         }
2811         /* Negotiate the fc mode to use */
2812         ixgbe_fc_autoneg(hw);
2813
2814         /* Disable any previous flow control settings */
2815         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
2816         mflcn_reg &= ~(IXGBE_MFLCN_RPFCE_SHIFT | IXGBE_MFLCN_RFCE|IXGBE_MFLCN_RPFCE);
2817
2818         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
2819         fccfg_reg &= ~(IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY);
2820
2821         switch (hw->fc.current_mode) {
2822         case ixgbe_fc_none:
2823                 /*
2824                  * If the count of enabled RX Priority Flow control >1,
2825                  * and the TX pause can not be disabled
2826                  */
2827                 nb_rx_en = 0;
2828                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
2829                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
2830                         if (reg & IXGBE_FCRTH_FCEN)
2831                                 nb_rx_en++;
2832                 }
2833                 if (nb_rx_en > 1)
2834                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2835                 break;
2836         case ixgbe_fc_rx_pause:
2837                 /*
2838                  * Rx Flow control is enabled and Tx Flow control is
2839                  * disabled by software override. Since there really
2840                  * isn't a way to advertise that we are capable of RX
2841                  * Pause ONLY, we will advertise that we support both
2842                  * symmetric and asymmetric Rx PAUSE.  Later, we will
2843                  * disable the adapter's ability to send PAUSE frames.
2844                  */
2845                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
2846                 /*
2847                  * If the count of enabled RX Priority Flow control >1,
2848                  * and the TX pause can not be disabled
2849                  */
2850                 nb_rx_en = 0;
2851                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
2852                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
2853                         if (reg & IXGBE_FCRTH_FCEN)
2854                                 nb_rx_en++;
2855                 }
2856                 if (nb_rx_en > 1)
2857                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2858                 break;
2859         case ixgbe_fc_tx_pause:
2860                 /*
2861                  * Tx Flow control is enabled, and Rx Flow control is
2862                  * disabled by software override.
2863                  */
2864                 fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2865                 break;
2866         case ixgbe_fc_full:
2867                 /* Flow control (both Rx and Tx) is enabled by SW override. */
2868                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
2869                 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
2870                 break;
2871         default:
2872                 PMD_DRV_LOG(DEBUG, "Flow control param set incorrectly");
2873                 ret_val = IXGBE_ERR_CONFIG;
2874                 goto out;
2875                 break;
2876         }
2877
2878         /* Set 802.3x based flow control settings. */
2879         mflcn_reg |= IXGBE_MFLCN_DPF;
2880         IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg);
2881         IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg);
2882
2883         /* Set up and enable Rx high/low water mark thresholds, enable XON. */
2884         if ((hw->fc.current_mode & ixgbe_fc_tx_pause) &&
2885                 hw->fc.high_water[tc_num]) {
2886                 fcrtl = (hw->fc.low_water[tc_num] << 10) | IXGBE_FCRTL_XONE;
2887                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), fcrtl);
2888                 fcrth = (hw->fc.high_water[tc_num] << 10) | IXGBE_FCRTH_FCEN;
2889         } else {
2890                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), 0);
2891                 /*
2892                  * In order to prevent Tx hangs when the internal Tx
2893                  * switch is enabled we must set the high water mark
2894                  * to the maximum FCRTH value.  This allows the Tx
2895                  * switch to function even under heavy Rx workloads.
2896                  */
2897                 fcrth = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)) - 32;
2898         }
2899         IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(tc_num), fcrth);
2900
2901         /* Configure pause time (2 TCs per register) */
2902         reg = hw->fc.pause_time * 0x00010001;
2903         for (i = 0; i < (IXGBE_DCB_MAX_TRAFFIC_CLASS / 2); i++)
2904                 IXGBE_WRITE_REG(hw, IXGBE_FCTTV(i), reg);
2905
2906         /* Configure flow control refresh threshold value */
2907         IXGBE_WRITE_REG(hw, IXGBE_FCRTV, hw->fc.pause_time / 2);
2908
2909 out:
2910         return ret_val;
2911 }
2912
2913 static int
2914 ixgbe_dcb_pfc_enable(struct rte_eth_dev *dev,uint8_t tc_num)
2915 {
2916         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2917         int32_t ret_val = IXGBE_NOT_IMPLEMENTED;
2918
2919         if(hw->mac.type != ixgbe_mac_82598EB) {
2920                 ret_val = ixgbe_dcb_pfc_enable_generic(hw,tc_num);
2921         }
2922         return ret_val;
2923 }
2924
2925 static int
2926 ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf)
2927 {
2928         int err;
2929         uint32_t rx_buf_size;
2930         uint32_t max_high_water;
2931         uint8_t tc_num;
2932         uint8_t  map[IXGBE_DCB_MAX_USER_PRIORITY] = { 0 };
2933         struct ixgbe_hw *hw =
2934                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2935         struct ixgbe_dcb_config *dcb_config =
2936                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
2937
2938         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
2939                 ixgbe_fc_none,
2940                 ixgbe_fc_rx_pause,
2941                 ixgbe_fc_tx_pause,
2942                 ixgbe_fc_full
2943         };
2944
2945         PMD_INIT_FUNC_TRACE();
2946
2947         ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
2948         tc_num = map[pfc_conf->priority];
2949         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num));
2950         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
2951         /*
2952          * At least reserve one Ethernet frame for watermark
2953          * high_water/low_water in kilo bytes for ixgbe
2954          */
2955         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
2956         if ((pfc_conf->fc.high_water > max_high_water) ||
2957             (pfc_conf->fc.high_water <= pfc_conf->fc.low_water)) {
2958                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB");
2959                 PMD_INIT_LOG(ERR, "High_water must <= 0x%x", max_high_water);
2960                 return (-EINVAL);
2961         }
2962
2963         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[pfc_conf->fc.mode];
2964         hw->fc.pause_time = pfc_conf->fc.pause_time;
2965         hw->fc.send_xon = pfc_conf->fc.send_xon;
2966         hw->fc.low_water[tc_num] =  pfc_conf->fc.low_water;
2967         hw->fc.high_water[tc_num] = pfc_conf->fc.high_water;
2968
2969         err = ixgbe_dcb_pfc_enable(dev,tc_num);
2970
2971         /* Not negotiated is not an error case */
2972         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED))
2973                 return 0;
2974
2975         PMD_INIT_LOG(ERR, "ixgbe_dcb_pfc_enable = 0x%x", err);
2976         return -EIO;
2977 }
2978
2979 static int
2980 ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
2981                           struct rte_eth_rss_reta_entry64 *reta_conf,
2982                           uint16_t reta_size)
2983 {
2984         uint8_t i, j, mask;
2985         uint32_t reta, r;
2986         uint16_t idx, shift;
2987         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2988
2989         PMD_INIT_FUNC_TRACE();
2990         if (reta_size != ETH_RSS_RETA_SIZE_128) {
2991                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2992                         "(%d) doesn't match the number hardware can supported "
2993                         "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
2994                 return -EINVAL;
2995         }
2996
2997         for (i = 0; i < reta_size; i += IXGBE_4_BIT_WIDTH) {
2998                 idx = i / RTE_RETA_GROUP_SIZE;
2999                 shift = i % RTE_RETA_GROUP_SIZE;
3000                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
3001                                                 IXGBE_4_BIT_MASK);
3002                 if (!mask)
3003                         continue;
3004                 if (mask == IXGBE_4_BIT_MASK)
3005                         r = 0;
3006                 else
3007                         r = IXGBE_READ_REG(hw, IXGBE_RETA(i >> 2));
3008                 for (j = 0, reta = 0; j < IXGBE_4_BIT_WIDTH; j++) {
3009                         if (mask & (0x1 << j))
3010                                 reta |= reta_conf[idx].reta[shift + j] <<
3011                                                         (CHAR_BIT * j);
3012                         else
3013                                 reta |= r & (IXGBE_8_BIT_MASK <<
3014                                                 (CHAR_BIT * j));
3015                 }
3016                 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
3017         }
3018
3019         return 0;
3020 }
3021
3022 static int
3023 ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
3024                          struct rte_eth_rss_reta_entry64 *reta_conf,
3025                          uint16_t reta_size)
3026 {
3027         uint8_t i, j, mask;
3028         uint32_t reta;
3029         uint16_t idx, shift;
3030         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3031
3032         PMD_INIT_FUNC_TRACE();
3033         if (reta_size != ETH_RSS_RETA_SIZE_128) {
3034                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
3035                         "(%d) doesn't match the number hardware can supported "
3036                                 "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
3037                 return -EINVAL;
3038         }
3039
3040         for (i = 0; i < ETH_RSS_RETA_SIZE_128; i += IXGBE_4_BIT_WIDTH) {
3041                 idx = i / RTE_RETA_GROUP_SIZE;
3042                 shift = i % RTE_RETA_GROUP_SIZE;
3043                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
3044                                                 IXGBE_4_BIT_MASK);
3045                 if (!mask)
3046                         continue;
3047
3048                 reta = IXGBE_READ_REG(hw, IXGBE_RETA(i >> 2));
3049                 for (j = 0; j < IXGBE_4_BIT_WIDTH; j++) {
3050                         if (mask & (0x1 << j))
3051                                 reta_conf[idx].reta[shift + j] =
3052                                         ((reta >> (CHAR_BIT * j)) &
3053                                                 IXGBE_8_BIT_MASK);
3054                 }
3055         }
3056
3057         return 0;
3058 }
3059
3060 static void
3061 ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
3062                                 uint32_t index, uint32_t pool)
3063 {
3064         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3065         uint32_t enable_addr = 1;
3066
3067         ixgbe_set_rar(hw, index, mac_addr->addr_bytes, pool, enable_addr);
3068 }
3069
3070 static void
3071 ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
3072 {
3073         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3074
3075         ixgbe_clear_rar(hw, index);
3076 }
3077
3078 static void
3079 ixgbe_set_default_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
3080 {
3081         ixgbe_remove_rar(dev, 0);
3082
3083         ixgbe_add_rar(dev, addr, 0, 0);
3084 }
3085
3086 static int
3087 ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
3088 {
3089         uint32_t hlreg0;
3090         uint32_t maxfrs;
3091         struct ixgbe_hw *hw;
3092         struct rte_eth_dev_info dev_info;
3093         uint32_t frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
3094
3095         ixgbe_dev_info_get(dev, &dev_info);
3096
3097         /* check that mtu is within the allowed range */
3098         if ((mtu < ETHER_MIN_MTU) || (frame_size > dev_info.max_rx_pktlen))
3099                 return -EINVAL;
3100
3101         /* refuse mtu that requires the support of scattered packets when this
3102          * feature has not been enabled before. */
3103         if (!dev->data->scattered_rx &&
3104             (frame_size + 2 * IXGBE_VLAN_TAG_SIZE >
3105              dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM))
3106                 return -EINVAL;
3107
3108         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3109         hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
3110
3111         /* switch to jumbo mode if needed */
3112         if (frame_size > ETHER_MAX_LEN) {
3113                 dev->data->dev_conf.rxmode.jumbo_frame = 1;
3114                 hlreg0 |= IXGBE_HLREG0_JUMBOEN;
3115         } else {
3116                 dev->data->dev_conf.rxmode.jumbo_frame = 0;
3117                 hlreg0 &= ~IXGBE_HLREG0_JUMBOEN;
3118         }
3119         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
3120
3121         /* update max frame size */
3122         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
3123
3124         maxfrs = IXGBE_READ_REG(hw, IXGBE_MAXFRS);
3125         maxfrs &= 0x0000FFFF;
3126         maxfrs |= (dev->data->dev_conf.rxmode.max_rx_pkt_len << 16);
3127         IXGBE_WRITE_REG(hw, IXGBE_MAXFRS, maxfrs);
3128
3129         return 0;
3130 }
3131
3132 /*
3133  * Virtual Function operations
3134  */
3135 static void
3136 ixgbevf_intr_disable(struct ixgbe_hw *hw)
3137 {
3138         PMD_INIT_FUNC_TRACE();
3139
3140         /* Clear interrupt mask to stop from interrupts being generated */
3141         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, IXGBE_VF_IRQ_CLEAR_MASK);
3142
3143         IXGBE_WRITE_FLUSH(hw);
3144 }
3145
3146 static int
3147 ixgbevf_dev_configure(struct rte_eth_dev *dev)
3148 {
3149         struct rte_eth_conf* conf = &dev->data->dev_conf;
3150         struct ixgbe_adapter *adapter =
3151                         (struct ixgbe_adapter *)dev->data->dev_private;
3152
3153         PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d",
3154                      dev->data->port_id);
3155
3156         /*
3157          * VF has no ability to enable/disable HW CRC
3158          * Keep the persistent behavior the same as Host PF
3159          */
3160 #ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC
3161         if (!conf->rxmode.hw_strip_crc) {
3162                 PMD_INIT_LOG(INFO, "VF can't disable HW CRC Strip");
3163                 conf->rxmode.hw_strip_crc = 1;
3164         }
3165 #else
3166         if (conf->rxmode.hw_strip_crc) {
3167                 PMD_INIT_LOG(INFO, "VF can't enable HW CRC Strip");
3168                 conf->rxmode.hw_strip_crc = 0;
3169         }
3170 #endif
3171
3172         /*
3173          * Initialize to TRUE. If any of Rx queues doesn't meet the bulk
3174          * allocation or vector Rx preconditions we will reset it.
3175          */
3176         adapter->rx_bulk_alloc_allowed = true;
3177         adapter->rx_vec_allowed = true;
3178
3179         return 0;
3180 }
3181
3182 static int
3183 ixgbevf_dev_start(struct rte_eth_dev *dev)
3184 {
3185         struct ixgbe_hw *hw =
3186                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3187         int err, mask = 0;
3188
3189         PMD_INIT_FUNC_TRACE();
3190
3191         hw->mac.ops.reset_hw(hw);
3192         hw->mac.get_link_status = true;
3193
3194         /* negotiate mailbox API version to use with the PF. */
3195         ixgbevf_negotiate_api(hw);
3196
3197         ixgbevf_dev_tx_init(dev);
3198
3199         /* This can fail when allocating mbufs for descriptor rings */
3200         err = ixgbevf_dev_rx_init(dev);
3201         if (err) {
3202                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware (%d)", err);
3203                 ixgbe_dev_clear_queues(dev);
3204                 return err;
3205         }
3206
3207         /* Set vfta */
3208         ixgbevf_set_vfta_all(dev,1);
3209
3210         /* Set HW strip */
3211         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
3212                 ETH_VLAN_EXTEND_MASK;
3213         ixgbevf_vlan_offload_set(dev, mask);
3214
3215         ixgbevf_dev_rxtx_start(dev);
3216
3217         return 0;
3218 }
3219
3220 static void
3221 ixgbevf_dev_stop(struct rte_eth_dev *dev)
3222 {
3223         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3224
3225         PMD_INIT_FUNC_TRACE();
3226
3227         hw->adapter_stopped = TRUE;
3228         ixgbe_stop_adapter(hw);
3229
3230         /*
3231           * Clear what we set, but we still keep shadow_vfta to
3232           * restore after device starts
3233           */
3234         ixgbevf_set_vfta_all(dev,0);
3235
3236         /* Clear stored conf */
3237         dev->data->scattered_rx = 0;
3238
3239         ixgbe_dev_clear_queues(dev);
3240 }
3241
3242 static void
3243 ixgbevf_dev_close(struct rte_eth_dev *dev)
3244 {
3245         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3246
3247         PMD_INIT_FUNC_TRACE();
3248
3249         ixgbe_reset_hw(hw);
3250
3251         ixgbevf_dev_stop(dev);
3252
3253         /* reprogram the RAR[0] in case user changed it. */
3254         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
3255 }
3256
3257 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on)
3258 {
3259         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3260         struct ixgbe_vfta * shadow_vfta =
3261                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
3262         int i = 0, j = 0, vfta = 0, mask = 1;
3263
3264         for (i = 0; i < IXGBE_VFTA_SIZE; i++){
3265                 vfta = shadow_vfta->vfta[i];
3266                 if(vfta){
3267                         mask = 1;
3268                         for (j = 0; j < 32; j++){
3269                                 if(vfta & mask)
3270                                         ixgbe_set_vfta(hw, (i<<5)+j, 0, on);
3271                                 mask<<=1;
3272                         }
3273                 }
3274         }
3275
3276 }
3277
3278 static int
3279 ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
3280 {
3281         struct ixgbe_hw *hw =
3282                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3283         struct ixgbe_vfta * shadow_vfta =
3284                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
3285         uint32_t vid_idx = 0;
3286         uint32_t vid_bit = 0;
3287         int ret = 0;
3288
3289         PMD_INIT_FUNC_TRACE();
3290
3291         /* vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf */
3292         ret = ixgbe_set_vfta(hw, vlan_id, 0, !!on);
3293         if(ret){
3294                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
3295                 return ret;
3296         }
3297         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
3298         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
3299
3300         /* Save what we set and retore it after device reset */
3301         if (on)
3302                 shadow_vfta->vfta[vid_idx] |= vid_bit;
3303         else
3304                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
3305
3306         return 0;
3307 }
3308
3309 static void
3310 ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
3311 {
3312         struct ixgbe_hw *hw =
3313                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3314         uint32_t ctrl;
3315
3316         PMD_INIT_FUNC_TRACE();
3317
3318         if(queue >= hw->mac.max_rx_queues)
3319                 return;
3320
3321         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
3322         if(on)
3323                 ctrl |= IXGBE_RXDCTL_VME;
3324         else
3325                 ctrl &= ~IXGBE_RXDCTL_VME;
3326         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
3327
3328         ixgbe_vlan_hw_strip_bitmap_set( dev, queue, on);
3329 }
3330
3331 static void
3332 ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
3333 {
3334         struct ixgbe_hw *hw =
3335                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3336         uint16_t i;
3337         int on = 0;
3338
3339         /* VF function only support hw strip feature, others are not support */
3340         if(mask & ETH_VLAN_STRIP_MASK){
3341                 on = !!(dev->data->dev_conf.rxmode.hw_vlan_strip);
3342
3343                 for(i=0; i < hw->mac.max_rx_queues; i++)
3344                         ixgbevf_vlan_strip_queue_set(dev,i,on);
3345         }
3346 }
3347
3348 static int
3349 ixgbe_vmdq_mode_check(struct ixgbe_hw *hw)
3350 {
3351         uint32_t reg_val;
3352
3353         /* we only need to do this if VMDq is enabled */
3354         reg_val = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
3355         if (!(reg_val & IXGBE_VT_CTL_VT_ENABLE)) {
3356                 PMD_INIT_LOG(ERR, "VMDq must be enabled for this setting");
3357                 return (-1);
3358         }
3359
3360         return 0;
3361 }
3362
3363 static uint32_t
3364 ixgbe_uta_vector(struct ixgbe_hw *hw, struct ether_addr* uc_addr)
3365 {
3366         uint32_t vector = 0;
3367         switch (hw->mac.mc_filter_type) {
3368         case 0:   /* use bits [47:36] of the address */
3369                 vector = ((uc_addr->addr_bytes[4] >> 4) |
3370                         (((uint16_t)uc_addr->addr_bytes[5]) << 4));
3371                 break;
3372         case 1:   /* use bits [46:35] of the address */
3373                 vector = ((uc_addr->addr_bytes[4] >> 3) |
3374                         (((uint16_t)uc_addr->addr_bytes[5]) << 5));
3375                 break;
3376         case 2:   /* use bits [45:34] of the address */
3377                 vector = ((uc_addr->addr_bytes[4] >> 2) |
3378                         (((uint16_t)uc_addr->addr_bytes[5]) << 6));
3379                 break;
3380         case 3:   /* use bits [43:32] of the address */
3381                 vector = ((uc_addr->addr_bytes[4]) |
3382                         (((uint16_t)uc_addr->addr_bytes[5]) << 8));
3383                 break;
3384         default:  /* Invalid mc_filter_type */
3385                 break;
3386         }
3387
3388         /* vector can only be 12-bits or boundary will be exceeded */
3389         vector &= 0xFFF;
3390         return vector;
3391 }
3392
3393 static int
3394 ixgbe_uc_hash_table_set(struct rte_eth_dev *dev,struct ether_addr* mac_addr,
3395                                uint8_t on)
3396 {
3397         uint32_t vector;
3398         uint32_t uta_idx;
3399         uint32_t reg_val;
3400         uint32_t uta_shift;
3401         uint32_t rc;
3402         const uint32_t ixgbe_uta_idx_mask = 0x7F;
3403         const uint32_t ixgbe_uta_bit_shift = 5;
3404         const uint32_t ixgbe_uta_bit_mask = (0x1 << ixgbe_uta_bit_shift) - 1;
3405         const uint32_t bit1 = 0x1;
3406
3407         struct ixgbe_hw *hw =
3408                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3409         struct ixgbe_uta_info *uta_info =
3410                 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private);
3411
3412         /* The UTA table only exists on 82599 hardware and newer */
3413         if (hw->mac.type < ixgbe_mac_82599EB)
3414                 return (-ENOTSUP);
3415
3416         vector = ixgbe_uta_vector(hw,mac_addr);
3417         uta_idx = (vector >> ixgbe_uta_bit_shift) & ixgbe_uta_idx_mask;
3418         uta_shift = vector & ixgbe_uta_bit_mask;
3419
3420         rc = ((uta_info->uta_shadow[uta_idx] >> uta_shift & bit1) != 0);
3421         if(rc == on)
3422                 return 0;
3423
3424         reg_val = IXGBE_READ_REG(hw, IXGBE_UTA(uta_idx));
3425         if (on) {
3426                 uta_info->uta_in_use++;
3427                 reg_val |= (bit1 << uta_shift);
3428                 uta_info->uta_shadow[uta_idx] |= (bit1 << uta_shift);
3429         } else {
3430                 uta_info->uta_in_use--;
3431                 reg_val &= ~(bit1 << uta_shift);
3432                 uta_info->uta_shadow[uta_idx] &= ~(bit1 << uta_shift);
3433         }
3434
3435         IXGBE_WRITE_REG(hw, IXGBE_UTA(uta_idx), reg_val);
3436
3437         if (uta_info->uta_in_use > 0)
3438                 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL,
3439                                 IXGBE_MCSTCTRL_MFE | hw->mac.mc_filter_type);
3440         else
3441                 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL,hw->mac.mc_filter_type);
3442
3443         return 0;
3444 }
3445
3446 static int
3447 ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on)
3448 {
3449         int i;
3450         struct ixgbe_hw *hw =
3451                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3452         struct ixgbe_uta_info *uta_info =
3453                 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private);
3454
3455         /* The UTA table only exists on 82599 hardware and newer */
3456         if (hw->mac.type < ixgbe_mac_82599EB)
3457                 return (-ENOTSUP);
3458
3459         if(on) {
3460                 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
3461                         uta_info->uta_shadow[i] = ~0;
3462                         IXGBE_WRITE_REG(hw, IXGBE_UTA(i), ~0);
3463                 }
3464         } else {
3465                 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
3466                         uta_info->uta_shadow[i] = 0;
3467                         IXGBE_WRITE_REG(hw, IXGBE_UTA(i), 0);
3468                 }
3469         }
3470         return 0;
3471
3472 }
3473
3474 uint32_t
3475 ixgbe_convert_vm_rx_mask_to_val(uint16_t rx_mask, uint32_t orig_val)
3476 {
3477         uint32_t new_val = orig_val;
3478
3479         if (rx_mask & ETH_VMDQ_ACCEPT_UNTAG)
3480                 new_val |= IXGBE_VMOLR_AUPE;
3481         if (rx_mask & ETH_VMDQ_ACCEPT_HASH_MC)
3482                 new_val |= IXGBE_VMOLR_ROMPE;
3483         if (rx_mask & ETH_VMDQ_ACCEPT_HASH_UC)
3484                 new_val |= IXGBE_VMOLR_ROPE;
3485         if (rx_mask & ETH_VMDQ_ACCEPT_BROADCAST)
3486                 new_val |= IXGBE_VMOLR_BAM;
3487         if (rx_mask & ETH_VMDQ_ACCEPT_MULTICAST)
3488                 new_val |= IXGBE_VMOLR_MPE;
3489
3490         return new_val;
3491 }
3492
3493 static int
3494 ixgbe_set_pool_rx_mode(struct rte_eth_dev *dev, uint16_t pool,
3495                                uint16_t rx_mask, uint8_t on)
3496 {
3497         int val = 0;
3498
3499         struct ixgbe_hw *hw =
3500                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3501         uint32_t vmolr = IXGBE_READ_REG(hw, IXGBE_VMOLR(pool));
3502
3503         if (hw->mac.type == ixgbe_mac_82598EB) {
3504                 PMD_INIT_LOG(ERR, "setting VF receive mode set should be done"
3505                              " on 82599 hardware and newer");
3506                 return (-ENOTSUP);
3507         }
3508         if (ixgbe_vmdq_mode_check(hw) < 0)
3509                 return (-ENOTSUP);
3510
3511         val = ixgbe_convert_vm_rx_mask_to_val(rx_mask, val);
3512
3513         if (on)
3514                 vmolr |= val;
3515         else
3516                 vmolr &= ~val;
3517
3518         IXGBE_WRITE_REG(hw, IXGBE_VMOLR(pool), vmolr);
3519
3520         return 0;
3521 }
3522
3523 static int
3524 ixgbe_set_pool_rx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on)
3525 {
3526         uint32_t reg,addr;
3527         uint32_t val;
3528         const uint8_t bit1 = 0x1;
3529
3530         struct ixgbe_hw *hw =
3531                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3532
3533         if (ixgbe_vmdq_mode_check(hw) < 0)
3534                 return (-ENOTSUP);
3535
3536         addr = IXGBE_VFRE(pool >= ETH_64_POOLS/2);
3537         reg = IXGBE_READ_REG(hw, addr);
3538         val = bit1 << pool;
3539
3540         if (on)
3541                 reg |= val;
3542         else
3543                 reg &= ~val;
3544
3545         IXGBE_WRITE_REG(hw, addr,reg);
3546
3547         return 0;
3548 }
3549
3550 static int
3551 ixgbe_set_pool_tx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on)
3552 {
3553         uint32_t reg,addr;
3554         uint32_t val;
3555         const uint8_t bit1 = 0x1;
3556
3557         struct ixgbe_hw *hw =
3558                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3559
3560         if (ixgbe_vmdq_mode_check(hw) < 0)
3561                 return (-ENOTSUP);
3562
3563         addr = IXGBE_VFTE(pool >= ETH_64_POOLS/2);
3564         reg = IXGBE_READ_REG(hw, addr);
3565         val = bit1 << pool;
3566
3567         if (on)
3568                 reg |= val;
3569         else
3570                 reg &= ~val;
3571
3572         IXGBE_WRITE_REG(hw, addr,reg);
3573
3574         return 0;
3575 }
3576
3577 static int
3578 ixgbe_set_pool_vlan_filter(struct rte_eth_dev *dev, uint16_t vlan,
3579                         uint64_t pool_mask, uint8_t vlan_on)
3580 {
3581         int ret = 0;
3582         uint16_t pool_idx;
3583         struct ixgbe_hw *hw =
3584                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3585
3586         if (ixgbe_vmdq_mode_check(hw) < 0)
3587                 return (-ENOTSUP);
3588         for (pool_idx = 0; pool_idx < ETH_64_POOLS; pool_idx++) {
3589                 if (pool_mask & ((uint64_t)(1ULL << pool_idx)))
3590                         ret = hw->mac.ops.set_vfta(hw,vlan,pool_idx,vlan_on);
3591                         if (ret < 0)
3592                                 return ret;
3593         }
3594
3595         return ret;
3596 }
3597
3598 #define IXGBE_MRCTL_VPME  0x01 /* Virtual Pool Mirroring. */
3599 #define IXGBE_MRCTL_UPME  0x02 /* Uplink Port Mirroring. */
3600 #define IXGBE_MRCTL_DPME  0x04 /* Downlink Port Mirroring. */
3601 #define IXGBE_MRCTL_VLME  0x08 /* VLAN Mirroring. */
3602 #define IXGBE_INVALID_MIRROR_TYPE(mirror_type) \
3603         ((mirror_type) & ~(uint8_t)(ETH_MIRROR_VIRTUAL_POOL_UP | \
3604         ETH_MIRROR_UPLINK_PORT | ETH_MIRROR_DOWNLINK_PORT | ETH_MIRROR_VLAN))
3605
3606 static int
3607 ixgbe_mirror_rule_set(struct rte_eth_dev *dev,
3608                         struct rte_eth_mirror_conf *mirror_conf,
3609                         uint8_t rule_id, uint8_t on)
3610 {
3611         uint32_t mr_ctl,vlvf;
3612         uint32_t mp_lsb = 0;
3613         uint32_t mv_msb = 0;
3614         uint32_t mv_lsb = 0;
3615         uint32_t mp_msb = 0;
3616         uint8_t i = 0;
3617         int reg_index = 0;
3618         uint64_t vlan_mask = 0;
3619
3620         const uint8_t pool_mask_offset = 32;
3621         const uint8_t vlan_mask_offset = 32;
3622         const uint8_t dst_pool_offset = 8;
3623         const uint8_t rule_mr_offset  = 4;
3624         const uint8_t mirror_rule_mask= 0x0F;
3625
3626         struct ixgbe_mirror_info *mr_info =
3627                         (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private));
3628         struct ixgbe_hw *hw =
3629                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3630         uint8_t mirror_type = 0;
3631
3632         if (ixgbe_vmdq_mode_check(hw) < 0)
3633                 return -ENOTSUP;
3634
3635         if (rule_id >= IXGBE_MAX_MIRROR_RULES)
3636                 return -EINVAL;
3637
3638         if (IXGBE_INVALID_MIRROR_TYPE(mirror_conf->rule_type)) {
3639                 PMD_DRV_LOG(ERR, "unsupported mirror type 0x%x.",
3640                         mirror_conf->rule_type);
3641                 return -EINVAL;
3642         }
3643
3644         if (mirror_conf->rule_type & ETH_MIRROR_VLAN) {
3645                 mirror_type |= IXGBE_MRCTL_VLME;
3646                 /* Check if vlan id is valid and find conresponding VLAN ID index in VLVF */
3647                 for (i = 0;i < IXGBE_VLVF_ENTRIES; i++) {
3648                         if (mirror_conf->vlan.vlan_mask & (1ULL << i)) {
3649                                 /* search vlan id related pool vlan filter index */
3650                                 reg_index = ixgbe_find_vlvf_slot(hw,
3651                                                 mirror_conf->vlan.vlan_id[i]);
3652                                 if(reg_index < 0)
3653                                         return -EINVAL;
3654                                 vlvf = IXGBE_READ_REG(hw, IXGBE_VLVF(reg_index));
3655                                 if ((vlvf & IXGBE_VLVF_VIEN) &&
3656                                     ((vlvf & IXGBE_VLVF_VLANID_MASK) ==
3657                                       mirror_conf->vlan.vlan_id[i]))
3658                                         vlan_mask |= (1ULL << reg_index);
3659                                 else
3660                                         return -EINVAL;
3661                         }
3662                 }
3663
3664                 if (on) {
3665                         mv_lsb = vlan_mask & 0xFFFFFFFF;
3666                         mv_msb = vlan_mask >> vlan_mask_offset;
3667
3668                         mr_info->mr_conf[rule_id].vlan.vlan_mask =
3669                                                 mirror_conf->vlan.vlan_mask;
3670                         for(i = 0 ;i < ETH_VMDQ_MAX_VLAN_FILTERS; i++) {
3671                                 if(mirror_conf->vlan.vlan_mask & (1ULL << i))
3672                                         mr_info->mr_conf[rule_id].vlan.vlan_id[i] =
3673                                                 mirror_conf->vlan.vlan_id[i];
3674                         }
3675                 } else {
3676                         mv_lsb = 0;
3677                         mv_msb = 0;
3678                         mr_info->mr_conf[rule_id].vlan.vlan_mask = 0;
3679                         for(i = 0 ;i < ETH_VMDQ_MAX_VLAN_FILTERS; i++)
3680                                 mr_info->mr_conf[rule_id].vlan.vlan_id[i] = 0;
3681                 }
3682         }
3683
3684         /*
3685          * if enable pool mirror, write related pool mask register,if disable
3686          * pool mirror, clear PFMRVM register
3687          */
3688         if (mirror_conf->rule_type & ETH_MIRROR_VIRTUAL_POOL_UP) {
3689                 mirror_type |= IXGBE_MRCTL_VPME;
3690                 if (on) {
3691                         mp_lsb = mirror_conf->pool_mask & 0xFFFFFFFF;
3692                         mp_msb = mirror_conf->pool_mask >> pool_mask_offset;
3693                         mr_info->mr_conf[rule_id].pool_mask =
3694                                         mirror_conf->pool_mask;
3695
3696                 } else {
3697                         mp_lsb = 0;
3698                         mp_msb = 0;
3699                         mr_info->mr_conf[rule_id].pool_mask = 0;
3700                 }
3701         }
3702         if (mirror_conf->rule_type & ETH_MIRROR_UPLINK_PORT)
3703                 mirror_type |= IXGBE_MRCTL_UPME;
3704         if (mirror_conf->rule_type & ETH_MIRROR_DOWNLINK_PORT)
3705                 mirror_type |= IXGBE_MRCTL_DPME;
3706
3707         /* read  mirror control register and recalculate it */
3708         mr_ctl = IXGBE_READ_REG(hw, IXGBE_MRCTL(rule_id));
3709
3710         if (on) {
3711                 mr_ctl |= mirror_type;
3712                 mr_ctl &= mirror_rule_mask;
3713                 mr_ctl |= mirror_conf->dst_pool << dst_pool_offset;
3714         } else
3715                 mr_ctl &= ~(mirror_conf->rule_type & mirror_rule_mask);
3716
3717         mr_info->mr_conf[rule_id].rule_type = mirror_conf->rule_type;
3718         mr_info->mr_conf[rule_id].dst_pool = mirror_conf->dst_pool;
3719
3720         /* write mirrror control  register */
3721         IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl);
3722
3723         /* write pool mirrror control  register */
3724         if (mirror_conf->rule_type == ETH_MIRROR_VIRTUAL_POOL_UP) {
3725                 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), mp_lsb);
3726                 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset),
3727                                 mp_msb);
3728         }
3729         /* write VLAN mirrror control  register */
3730         if (mirror_conf->rule_type == ETH_MIRROR_VLAN) {
3731                 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), mv_lsb);
3732                 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset),
3733                                 mv_msb);
3734         }
3735
3736         return 0;
3737 }
3738
3739 static int
3740 ixgbe_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t rule_id)
3741 {
3742         int mr_ctl = 0;
3743         uint32_t lsb_val = 0;
3744         uint32_t msb_val = 0;
3745         const uint8_t rule_mr_offset = 4;
3746
3747         struct ixgbe_hw *hw =
3748                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3749         struct ixgbe_mirror_info *mr_info =
3750                 (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private));
3751
3752         if (ixgbe_vmdq_mode_check(hw) < 0)
3753                 return (-ENOTSUP);
3754
3755         memset(&mr_info->mr_conf[rule_id], 0,
3756                 sizeof(struct rte_eth_mirror_conf));
3757
3758         /* clear PFVMCTL register */
3759         IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl);
3760
3761         /* clear pool mask register */
3762         IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), lsb_val);
3763         IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset), msb_val);
3764
3765         /* clear vlan mask register */
3766         IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), lsb_val);
3767         IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset), msb_val);
3768
3769         return 0;
3770 }
3771
3772 static int ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev,
3773         uint16_t queue_idx, uint16_t tx_rate)
3774 {
3775         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3776         uint32_t rf_dec, rf_int;
3777         uint32_t bcnrc_val;
3778         uint16_t link_speed = dev->data->dev_link.link_speed;
3779
3780         if (queue_idx >= hw->mac.max_tx_queues)
3781                 return -EINVAL;
3782
3783         if (tx_rate != 0) {
3784                 /* Calculate the rate factor values to set */
3785                 rf_int = (uint32_t)link_speed / (uint32_t)tx_rate;
3786                 rf_dec = (uint32_t)link_speed % (uint32_t)tx_rate;
3787                 rf_dec = (rf_dec << IXGBE_RTTBCNRC_RF_INT_SHIFT) / tx_rate;
3788
3789                 bcnrc_val = IXGBE_RTTBCNRC_RS_ENA;
3790                 bcnrc_val |= ((rf_int << IXGBE_RTTBCNRC_RF_INT_SHIFT) &
3791                                 IXGBE_RTTBCNRC_RF_INT_MASK_M);
3792                 bcnrc_val |= (rf_dec & IXGBE_RTTBCNRC_RF_DEC_MASK);
3793         } else {
3794                 bcnrc_val = 0;
3795         }
3796
3797         /*
3798          * Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
3799          * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported, otherwise
3800          * set as 0x4.
3801          */
3802         if ((dev->data->dev_conf.rxmode.jumbo_frame == 1) &&
3803                 (dev->data->dev_conf.rxmode.max_rx_pkt_len >=
3804                                 IXGBE_MAX_JUMBO_FRAME_SIZE))
3805                 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRM,
3806                         IXGBE_MMW_SIZE_JUMBO_FRAME);
3807         else
3808                 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRM,
3809                         IXGBE_MMW_SIZE_DEFAULT);
3810
3811         /* Set RTTBCNRC of queue X */
3812         IXGBE_WRITE_REG(hw, IXGBE_RTTDQSEL, queue_idx);
3813         IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRC, bcnrc_val);
3814         IXGBE_WRITE_FLUSH(hw);
3815
3816         return 0;
3817 }
3818
3819 static int ixgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf,
3820         uint16_t tx_rate, uint64_t q_msk)
3821 {
3822         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3823         struct ixgbe_vf_info *vfinfo =
3824                 *(IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private));
3825         uint8_t  nb_q_per_pool = RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool;
3826         uint32_t queue_stride =
3827                 IXGBE_MAX_RX_QUEUE_NUM / RTE_ETH_DEV_SRIOV(dev).active;
3828         uint32_t queue_idx = vf * queue_stride, idx = 0, vf_idx;
3829         uint32_t queue_end = queue_idx + nb_q_per_pool - 1;
3830         uint16_t total_rate = 0;
3831
3832         if (queue_end >= hw->mac.max_tx_queues)
3833                 return -EINVAL;
3834
3835         if (vfinfo != NULL) {
3836                 for (vf_idx = 0; vf_idx < dev->pci_dev->max_vfs; vf_idx++) {
3837                         if (vf_idx == vf)
3838                                 continue;
3839                         for (idx = 0; idx < RTE_DIM(vfinfo[vf_idx].tx_rate);
3840                                 idx++)
3841                                 total_rate += vfinfo[vf_idx].tx_rate[idx];
3842                 }
3843         } else
3844                 return -EINVAL;
3845
3846         /* Store tx_rate for this vf. */
3847         for (idx = 0; idx < nb_q_per_pool; idx++) {
3848                 if (((uint64_t)0x1 << idx) & q_msk) {
3849                         if (vfinfo[vf].tx_rate[idx] != tx_rate)
3850                                 vfinfo[vf].tx_rate[idx] = tx_rate;
3851                         total_rate += tx_rate;
3852                 }
3853         }
3854
3855         if (total_rate > dev->data->dev_link.link_speed) {
3856                 /*
3857                  * Reset stored TX rate of the VF if it causes exceed
3858                  * link speed.
3859                  */
3860                 memset(vfinfo[vf].tx_rate, 0, sizeof(vfinfo[vf].tx_rate));
3861                 return -EINVAL;
3862         }
3863
3864         /* Set RTTBCNRC of each queue/pool for vf X  */
3865         for (; queue_idx <= queue_end; queue_idx++) {
3866                 if (0x1 & q_msk)
3867                         ixgbe_set_queue_rate_limit(dev, queue_idx, tx_rate);
3868                 q_msk = q_msk >> 1;
3869         }
3870
3871         return 0;
3872 }
3873
3874 static void
3875 ixgbevf_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
3876                      __attribute__((unused)) uint32_t index,
3877                      __attribute__((unused)) uint32_t pool)
3878 {
3879         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3880         int diag;
3881
3882         /*
3883          * On a 82599 VF, adding again the same MAC addr is not an idempotent
3884          * operation. Trap this case to avoid exhausting the [very limited]
3885          * set of PF resources used to store VF MAC addresses.
3886          */
3887         if (memcmp(hw->mac.perm_addr, mac_addr, sizeof(struct ether_addr)) == 0)
3888                 return;
3889         diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes);
3890         if (diag == 0)
3891                 return;
3892         PMD_DRV_LOG(ERR, "Unable to add MAC address - diag=%d", diag);
3893 }
3894
3895 static void
3896 ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index)
3897 {
3898         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3899         struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr;
3900         struct ether_addr *mac_addr;
3901         uint32_t i;
3902         int diag;
3903
3904         /*
3905          * The IXGBE_VF_SET_MACVLAN command of the ixgbe-pf driver does
3906          * not support the deletion of a given MAC address.
3907          * Instead, it imposes to delete all MAC addresses, then to add again
3908          * all MAC addresses with the exception of the one to be deleted.
3909          */
3910         (void) ixgbevf_set_uc_addr_vf(hw, 0, NULL);
3911
3912         /*
3913          * Add again all MAC addresses, with the exception of the deleted one
3914          * and of the permanent MAC address.
3915          */
3916         for (i = 0, mac_addr = dev->data->mac_addrs;
3917              i < hw->mac.num_rar_entries; i++, mac_addr++) {
3918                 /* Skip the deleted MAC address */
3919                 if (i == index)
3920                         continue;
3921                 /* Skip NULL MAC addresses */
3922                 if (is_zero_ether_addr(mac_addr))
3923                         continue;
3924                 /* Skip the permanent MAC address */
3925                 if (memcmp(perm_addr, mac_addr, sizeof(struct ether_addr)) == 0)
3926                         continue;
3927                 diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes);
3928                 if (diag != 0)
3929                         PMD_DRV_LOG(ERR,
3930                                     "Adding again MAC address "
3931                                     "%02x:%02x:%02x:%02x:%02x:%02x failed "
3932                                     "diag=%d",
3933                                     mac_addr->addr_bytes[0],
3934                                     mac_addr->addr_bytes[1],
3935                                     mac_addr->addr_bytes[2],
3936                                     mac_addr->addr_bytes[3],
3937                                     mac_addr->addr_bytes[4],
3938                                     mac_addr->addr_bytes[5],
3939                                     diag);
3940         }
3941 }
3942
3943 static void
3944 ixgbevf_set_default_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
3945 {
3946         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3947
3948         hw->mac.ops.set_rar(hw, 0, (void *)addr, 0, 0);
3949 }
3950
3951 #define MAC_TYPE_FILTER_SUP(type)    do {\
3952         if ((type) != ixgbe_mac_82599EB && (type) != ixgbe_mac_X540 &&\
3953                 (type) != ixgbe_mac_X550)\
3954                 return -ENOTSUP;\
3955 } while (0)
3956
3957 static int
3958 ixgbe_syn_filter_set(struct rte_eth_dev *dev,
3959                         struct rte_eth_syn_filter *filter,
3960                         bool add)
3961 {
3962         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3963         uint32_t synqf;
3964
3965         if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM)
3966                 return -EINVAL;
3967
3968         synqf = IXGBE_READ_REG(hw, IXGBE_SYNQF);
3969
3970         if (add) {
3971                 if (synqf & IXGBE_SYN_FILTER_ENABLE)
3972                         return -EINVAL;
3973                 synqf = (uint32_t)(((filter->queue << IXGBE_SYN_FILTER_QUEUE_SHIFT) &
3974                         IXGBE_SYN_FILTER_QUEUE) | IXGBE_SYN_FILTER_ENABLE);
3975
3976                 if (filter->hig_pri)
3977                         synqf |= IXGBE_SYN_FILTER_SYNQFP;
3978                 else
3979                         synqf &= ~IXGBE_SYN_FILTER_SYNQFP;
3980         } else {
3981                 if (!(synqf & IXGBE_SYN_FILTER_ENABLE))
3982                         return -ENOENT;
3983                 synqf &= ~(IXGBE_SYN_FILTER_QUEUE | IXGBE_SYN_FILTER_ENABLE);
3984         }
3985         IXGBE_WRITE_REG(hw, IXGBE_SYNQF, synqf);
3986         IXGBE_WRITE_FLUSH(hw);
3987         return 0;
3988 }
3989
3990 static int
3991 ixgbe_syn_filter_get(struct rte_eth_dev *dev,
3992                         struct rte_eth_syn_filter *filter)
3993 {
3994         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3995         uint32_t synqf = IXGBE_READ_REG(hw, IXGBE_SYNQF);
3996
3997         if (synqf & IXGBE_SYN_FILTER_ENABLE) {
3998                 filter->hig_pri = (synqf & IXGBE_SYN_FILTER_SYNQFP) ? 1 : 0;
3999                 filter->queue = (uint16_t)((synqf & IXGBE_SYN_FILTER_QUEUE) >> 1);
4000                 return 0;
4001         }
4002         return -ENOENT;
4003 }
4004
4005 static int
4006 ixgbe_syn_filter_handle(struct rte_eth_dev *dev,
4007                         enum rte_filter_op filter_op,
4008                         void *arg)
4009 {
4010         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4011         int ret;
4012
4013         MAC_TYPE_FILTER_SUP(hw->mac.type);
4014
4015         if (filter_op == RTE_ETH_FILTER_NOP)
4016                 return 0;
4017
4018         if (arg == NULL) {
4019                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
4020                             filter_op);
4021                 return -EINVAL;
4022         }
4023
4024         switch (filter_op) {
4025         case RTE_ETH_FILTER_ADD:
4026                 ret = ixgbe_syn_filter_set(dev,
4027                                 (struct rte_eth_syn_filter *)arg,
4028                                 TRUE);
4029                 break;
4030         case RTE_ETH_FILTER_DELETE:
4031                 ret = ixgbe_syn_filter_set(dev,
4032                                 (struct rte_eth_syn_filter *)arg,
4033                                 FALSE);
4034                 break;
4035         case RTE_ETH_FILTER_GET:
4036                 ret = ixgbe_syn_filter_get(dev,
4037                                 (struct rte_eth_syn_filter *)arg);
4038                 break;
4039         default:
4040                 PMD_DRV_LOG(ERR, "unsupported operation %u\n", filter_op);
4041                 ret = -EINVAL;
4042                 break;
4043         }
4044
4045         return ret;
4046 }
4047
4048
4049 static inline enum ixgbe_5tuple_protocol
4050 convert_protocol_type(uint8_t protocol_value)
4051 {
4052         if (protocol_value == IPPROTO_TCP)
4053                 return IXGBE_FILTER_PROTOCOL_TCP;
4054         else if (protocol_value == IPPROTO_UDP)
4055                 return IXGBE_FILTER_PROTOCOL_UDP;
4056         else if (protocol_value == IPPROTO_SCTP)
4057                 return IXGBE_FILTER_PROTOCOL_SCTP;
4058         else
4059                 return IXGBE_FILTER_PROTOCOL_NONE;
4060 }
4061
4062 /*
4063  * add a 5tuple filter
4064  *
4065  * @param
4066  * dev: Pointer to struct rte_eth_dev.
4067  * index: the index the filter allocates.
4068  * filter: ponter to the filter that will be added.
4069  * rx_queue: the queue id the filter assigned to.
4070  *
4071  * @return
4072  *    - On success, zero.
4073  *    - On failure, a negative value.
4074  */
4075 static int
4076 ixgbe_add_5tuple_filter(struct rte_eth_dev *dev,
4077                         struct ixgbe_5tuple_filter *filter)
4078 {
4079         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4080         struct ixgbe_filter_info *filter_info =
4081                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4082         int i, idx, shift;
4083         uint32_t ftqf, sdpqf;
4084         uint32_t l34timir = 0;
4085         uint8_t mask = 0xff;
4086
4087         /*
4088          * look for an unused 5tuple filter index,
4089          * and insert the filter to list.
4090          */
4091         for (i = 0; i < IXGBE_MAX_FTQF_FILTERS; i++) {
4092                 idx = i / (sizeof(uint32_t) * NBBY);
4093                 shift = i % (sizeof(uint32_t) * NBBY);
4094                 if (!(filter_info->fivetuple_mask[idx] & (1 << shift))) {
4095                         filter_info->fivetuple_mask[idx] |= 1 << shift;
4096                         filter->index = i;
4097                         TAILQ_INSERT_TAIL(&filter_info->fivetuple_list,
4098                                           filter,
4099                                           entries);
4100                         break;
4101                 }
4102         }
4103         if (i >= IXGBE_MAX_FTQF_FILTERS) {
4104                 PMD_DRV_LOG(ERR, "5tuple filters are full.");
4105                 return -ENOSYS;
4106         }
4107
4108         sdpqf = (uint32_t)(filter->filter_info.dst_port <<
4109                                 IXGBE_SDPQF_DSTPORT_SHIFT);
4110         sdpqf = sdpqf | (filter->filter_info.src_port & IXGBE_SDPQF_SRCPORT);
4111
4112         ftqf = (uint32_t)(filter->filter_info.proto &
4113                 IXGBE_FTQF_PROTOCOL_MASK);
4114         ftqf |= (uint32_t)((filter->filter_info.priority &
4115                 IXGBE_FTQF_PRIORITY_MASK) << IXGBE_FTQF_PRIORITY_SHIFT);
4116         if (filter->filter_info.src_ip_mask == 0) /* 0 means compare. */
4117                 mask &= IXGBE_FTQF_SOURCE_ADDR_MASK;
4118         if (filter->filter_info.dst_ip_mask == 0)
4119                 mask &= IXGBE_FTQF_DEST_ADDR_MASK;
4120         if (filter->filter_info.src_port_mask == 0)
4121                 mask &= IXGBE_FTQF_SOURCE_PORT_MASK;
4122         if (filter->filter_info.dst_port_mask == 0)
4123                 mask &= IXGBE_FTQF_DEST_PORT_MASK;
4124         if (filter->filter_info.proto_mask == 0)
4125                 mask &= IXGBE_FTQF_PROTOCOL_COMP_MASK;
4126         ftqf |= mask << IXGBE_FTQF_5TUPLE_MASK_SHIFT;
4127         ftqf |= IXGBE_FTQF_POOL_MASK_EN;
4128         ftqf |= IXGBE_FTQF_QUEUE_ENABLE;
4129
4130         IXGBE_WRITE_REG(hw, IXGBE_DAQF(i), filter->filter_info.dst_ip);
4131         IXGBE_WRITE_REG(hw, IXGBE_SAQF(i), filter->filter_info.src_ip);
4132         IXGBE_WRITE_REG(hw, IXGBE_SDPQF(i), sdpqf);
4133         IXGBE_WRITE_REG(hw, IXGBE_FTQF(i), ftqf);
4134
4135         l34timir |= IXGBE_L34T_IMIR_RESERVE;
4136         l34timir |= (uint32_t)(filter->queue <<
4137                                 IXGBE_L34T_IMIR_QUEUE_SHIFT);
4138         IXGBE_WRITE_REG(hw, IXGBE_L34T_IMIR(i), l34timir);
4139         return 0;
4140 }
4141
4142 /*
4143  * remove a 5tuple filter
4144  *
4145  * @param
4146  * dev: Pointer to struct rte_eth_dev.
4147  * filter: the pointer of the filter will be removed.
4148  */
4149 static void
4150 ixgbe_remove_5tuple_filter(struct rte_eth_dev *dev,
4151                         struct ixgbe_5tuple_filter *filter)
4152 {
4153         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4154         struct ixgbe_filter_info *filter_info =
4155                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4156         uint16_t index = filter->index;
4157
4158         filter_info->fivetuple_mask[index / (sizeof(uint32_t) * NBBY)] &=
4159                                 ~(1 << (index % (sizeof(uint32_t) * NBBY)));
4160         TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries);
4161         rte_free(filter);
4162
4163         IXGBE_WRITE_REG(hw, IXGBE_DAQF(index), 0);
4164         IXGBE_WRITE_REG(hw, IXGBE_SAQF(index), 0);
4165         IXGBE_WRITE_REG(hw, IXGBE_SDPQF(index), 0);
4166         IXGBE_WRITE_REG(hw, IXGBE_FTQF(index), 0);
4167         IXGBE_WRITE_REG(hw, IXGBE_L34T_IMIR(index), 0);
4168 }
4169
4170 static int
4171 ixgbevf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
4172 {
4173         struct ixgbe_hw *hw;
4174         uint32_t max_frame = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
4175
4176         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4177
4178         if ((mtu < ETHER_MIN_MTU) || (max_frame > ETHER_MAX_JUMBO_FRAME_LEN))
4179                 return -EINVAL;
4180
4181         /* refuse mtu that requires the support of scattered packets when this
4182          * feature has not been enabled before. */
4183         if (!dev->data->scattered_rx &&
4184             (max_frame + 2 * IXGBE_VLAN_TAG_SIZE >
4185              dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM))
4186                 return -EINVAL;
4187
4188         /*
4189          * When supported by the underlying PF driver, use the IXGBE_VF_SET_MTU
4190          * request of the version 2.0 of the mailbox API.
4191          * For now, use the IXGBE_VF_SET_LPE request of the version 1.0
4192          * of the mailbox API.
4193          * This call to IXGBE_SET_LPE action won't work with ixgbe pf drivers
4194          * prior to 3.11.33 which contains the following change:
4195          * "ixgbe: Enable jumbo frames support w/ SR-IOV"
4196          */
4197         ixgbevf_rlpml_set_vf(hw, max_frame);
4198
4199         /* update max frame size */
4200         dev->data->dev_conf.rxmode.max_rx_pkt_len = max_frame;
4201         return 0;
4202 }
4203
4204 #define MAC_TYPE_FILTER_SUP_EXT(type)    do {\
4205         if ((type) != ixgbe_mac_82599EB && (type) != ixgbe_mac_X540)\
4206                 return -ENOTSUP;\
4207 } while (0)
4208
4209 static inline struct ixgbe_5tuple_filter *
4210 ixgbe_5tuple_filter_lookup(struct ixgbe_5tuple_filter_list *filter_list,
4211                         struct ixgbe_5tuple_filter_info *key)
4212 {
4213         struct ixgbe_5tuple_filter *it;
4214
4215         TAILQ_FOREACH(it, filter_list, entries) {
4216                 if (memcmp(key, &it->filter_info,
4217                         sizeof(struct ixgbe_5tuple_filter_info)) == 0) {
4218                         return it;
4219                 }
4220         }
4221         return NULL;
4222 }
4223
4224 /* translate elements in struct rte_eth_ntuple_filter to struct ixgbe_5tuple_filter_info*/
4225 static inline int
4226 ntuple_filter_to_5tuple(struct rte_eth_ntuple_filter *filter,
4227                         struct ixgbe_5tuple_filter_info *filter_info)
4228 {
4229         if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM ||
4230                 filter->priority > IXGBE_5TUPLE_MAX_PRI ||
4231                 filter->priority < IXGBE_5TUPLE_MIN_PRI)
4232                 return -EINVAL;
4233
4234         switch (filter->dst_ip_mask) {
4235         case UINT32_MAX:
4236                 filter_info->dst_ip_mask = 0;
4237                 filter_info->dst_ip = filter->dst_ip;
4238                 break;
4239         case 0:
4240                 filter_info->dst_ip_mask = 1;
4241                 break;
4242         default:
4243                 PMD_DRV_LOG(ERR, "invalid dst_ip mask.");
4244                 return -EINVAL;
4245         }
4246
4247         switch (filter->src_ip_mask) {
4248         case UINT32_MAX:
4249                 filter_info->src_ip_mask = 0;
4250                 filter_info->src_ip = filter->src_ip;
4251                 break;
4252         case 0:
4253                 filter_info->src_ip_mask = 1;
4254                 break;
4255         default:
4256                 PMD_DRV_LOG(ERR, "invalid src_ip mask.");
4257                 return -EINVAL;
4258         }
4259
4260         switch (filter->dst_port_mask) {
4261         case UINT16_MAX:
4262                 filter_info->dst_port_mask = 0;
4263                 filter_info->dst_port = filter->dst_port;
4264                 break;
4265         case 0:
4266                 filter_info->dst_port_mask = 1;
4267                 break;
4268         default:
4269                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
4270                 return -EINVAL;
4271         }
4272
4273         switch (filter->src_port_mask) {
4274         case UINT16_MAX:
4275                 filter_info->src_port_mask = 0;
4276                 filter_info->src_port = filter->src_port;
4277                 break;
4278         case 0:
4279                 filter_info->src_port_mask = 1;
4280                 break;
4281         default:
4282                 PMD_DRV_LOG(ERR, "invalid src_port mask.");
4283                 return -EINVAL;
4284         }
4285
4286         switch (filter->proto_mask) {
4287         case UINT8_MAX:
4288                 filter_info->proto_mask = 0;
4289                 filter_info->proto =
4290                         convert_protocol_type(filter->proto);
4291                 break;
4292         case 0:
4293                 filter_info->proto_mask = 1;
4294                 break;
4295         default:
4296                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
4297                 return -EINVAL;
4298         }
4299
4300         filter_info->priority = (uint8_t)filter->priority;
4301         return 0;
4302 }
4303
4304 /*
4305  * add or delete a ntuple filter
4306  *
4307  * @param
4308  * dev: Pointer to struct rte_eth_dev.
4309  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
4310  * add: if true, add filter, if false, remove filter
4311  *
4312  * @return
4313  *    - On success, zero.
4314  *    - On failure, a negative value.
4315  */
4316 static int
4317 ixgbe_add_del_ntuple_filter(struct rte_eth_dev *dev,
4318                         struct rte_eth_ntuple_filter *ntuple_filter,
4319                         bool add)
4320 {
4321         struct ixgbe_filter_info *filter_info =
4322                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4323         struct ixgbe_5tuple_filter_info filter_5tuple;
4324         struct ixgbe_5tuple_filter *filter;
4325         int ret;
4326
4327         if (ntuple_filter->flags != RTE_5TUPLE_FLAGS) {
4328                 PMD_DRV_LOG(ERR, "only 5tuple is supported.");
4329                 return -EINVAL;
4330         }
4331
4332         memset(&filter_5tuple, 0, sizeof(struct ixgbe_5tuple_filter_info));
4333         ret = ntuple_filter_to_5tuple(ntuple_filter, &filter_5tuple);
4334         if (ret < 0)
4335                 return ret;
4336
4337         filter = ixgbe_5tuple_filter_lookup(&filter_info->fivetuple_list,
4338                                          &filter_5tuple);
4339         if (filter != NULL && add) {
4340                 PMD_DRV_LOG(ERR, "filter exists.");
4341                 return -EEXIST;
4342         }
4343         if (filter == NULL && !add) {
4344                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
4345                 return -ENOENT;
4346         }
4347
4348         if (add) {
4349                 filter = rte_zmalloc("ixgbe_5tuple_filter",
4350                                 sizeof(struct ixgbe_5tuple_filter), 0);
4351                 if (filter == NULL)
4352                         return -ENOMEM;
4353                 (void)rte_memcpy(&filter->filter_info,
4354                                  &filter_5tuple,
4355                                  sizeof(struct ixgbe_5tuple_filter_info));
4356                 filter->queue = ntuple_filter->queue;
4357                 ret = ixgbe_add_5tuple_filter(dev, filter);
4358                 if (ret < 0) {
4359                         rte_free(filter);
4360                         return ret;
4361                 }
4362         } else
4363                 ixgbe_remove_5tuple_filter(dev, filter);
4364
4365         return 0;
4366 }
4367
4368 /*
4369  * get a ntuple filter
4370  *
4371  * @param
4372  * dev: Pointer to struct rte_eth_dev.
4373  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
4374  *
4375  * @return
4376  *    - On success, zero.
4377  *    - On failure, a negative value.
4378  */
4379 static int
4380 ixgbe_get_ntuple_filter(struct rte_eth_dev *dev,
4381                         struct rte_eth_ntuple_filter *ntuple_filter)
4382 {
4383         struct ixgbe_filter_info *filter_info =
4384                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4385         struct ixgbe_5tuple_filter_info filter_5tuple;
4386         struct ixgbe_5tuple_filter *filter;
4387         int ret;
4388
4389         if (ntuple_filter->flags != RTE_5TUPLE_FLAGS) {
4390                 PMD_DRV_LOG(ERR, "only 5tuple is supported.");
4391                 return -EINVAL;
4392         }
4393
4394         memset(&filter_5tuple, 0, sizeof(struct ixgbe_5tuple_filter_info));
4395         ret = ntuple_filter_to_5tuple(ntuple_filter, &filter_5tuple);
4396         if (ret < 0)
4397                 return ret;
4398
4399         filter = ixgbe_5tuple_filter_lookup(&filter_info->fivetuple_list,
4400                                          &filter_5tuple);
4401         if (filter == NULL) {
4402                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
4403                 return -ENOENT;
4404         }
4405         ntuple_filter->queue = filter->queue;
4406         return 0;
4407 }
4408
4409 /*
4410  * ixgbe_ntuple_filter_handle - Handle operations for ntuple filter.
4411  * @dev: pointer to rte_eth_dev structure
4412  * @filter_op:operation will be taken.
4413  * @arg: a pointer to specific structure corresponding to the filter_op
4414  *
4415  * @return
4416  *    - On success, zero.
4417  *    - On failure, a negative value.
4418  */
4419 static int
4420 ixgbe_ntuple_filter_handle(struct rte_eth_dev *dev,
4421                                 enum rte_filter_op filter_op,
4422                                 void *arg)
4423 {
4424         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4425         int ret;
4426
4427         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
4428
4429         if (filter_op == RTE_ETH_FILTER_NOP)
4430                 return 0;
4431
4432         if (arg == NULL) {
4433                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
4434                             filter_op);
4435                 return -EINVAL;
4436         }
4437
4438         switch (filter_op) {
4439         case RTE_ETH_FILTER_ADD:
4440                 ret = ixgbe_add_del_ntuple_filter(dev,
4441                         (struct rte_eth_ntuple_filter *)arg,
4442                         TRUE);
4443                 break;
4444         case RTE_ETH_FILTER_DELETE:
4445                 ret = ixgbe_add_del_ntuple_filter(dev,
4446                         (struct rte_eth_ntuple_filter *)arg,
4447                         FALSE);
4448                 break;
4449         case RTE_ETH_FILTER_GET:
4450                 ret = ixgbe_get_ntuple_filter(dev,
4451                         (struct rte_eth_ntuple_filter *)arg);
4452                 break;
4453         default:
4454                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
4455                 ret = -EINVAL;
4456                 break;
4457         }
4458         return ret;
4459 }
4460
4461 static inline int
4462 ixgbe_ethertype_filter_lookup(struct ixgbe_filter_info *filter_info,
4463                         uint16_t ethertype)
4464 {
4465         int i;
4466
4467         for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) {
4468                 if (filter_info->ethertype_filters[i] == ethertype &&
4469                     (filter_info->ethertype_mask & (1 << i)))
4470                         return i;
4471         }
4472         return -1;
4473 }
4474
4475 static inline int
4476 ixgbe_ethertype_filter_insert(struct ixgbe_filter_info *filter_info,
4477                         uint16_t ethertype)
4478 {
4479         int i;
4480
4481         for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) {
4482                 if (!(filter_info->ethertype_mask & (1 << i))) {
4483                         filter_info->ethertype_mask |= 1 << i;
4484                         filter_info->ethertype_filters[i] = ethertype;
4485                         return i;
4486                 }
4487         }
4488         return -1;
4489 }
4490
4491 static inline int
4492 ixgbe_ethertype_filter_remove(struct ixgbe_filter_info *filter_info,
4493                         uint8_t idx)
4494 {
4495         if (idx >= IXGBE_MAX_ETQF_FILTERS)
4496                 return -1;
4497         filter_info->ethertype_mask &= ~(1 << idx);
4498         filter_info->ethertype_filters[idx] = 0;
4499         return idx;
4500 }
4501
4502 static int
4503 ixgbe_add_del_ethertype_filter(struct rte_eth_dev *dev,
4504                         struct rte_eth_ethertype_filter *filter,
4505                         bool add)
4506 {
4507         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4508         struct ixgbe_filter_info *filter_info =
4509                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4510         uint32_t etqf = 0;
4511         uint32_t etqs = 0;
4512         int ret;
4513
4514         if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM)
4515                 return -EINVAL;
4516
4517         if (filter->ether_type == ETHER_TYPE_IPv4 ||
4518                 filter->ether_type == ETHER_TYPE_IPv6) {
4519                 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in"
4520                         " ethertype filter.", filter->ether_type);
4521                 return -EINVAL;
4522         }
4523
4524         if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) {
4525                 PMD_DRV_LOG(ERR, "mac compare is unsupported.");
4526                 return -EINVAL;
4527         }
4528         if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) {
4529                 PMD_DRV_LOG(ERR, "drop option is unsupported.");
4530                 return -EINVAL;
4531         }
4532
4533         ret = ixgbe_ethertype_filter_lookup(filter_info, filter->ether_type);
4534         if (ret >= 0 && add) {
4535                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.",
4536                             filter->ether_type);
4537                 return -EEXIST;
4538         }
4539         if (ret < 0 && !add) {
4540                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
4541                             filter->ether_type);
4542                 return -ENOENT;
4543         }
4544
4545         if (add) {
4546                 ret = ixgbe_ethertype_filter_insert(filter_info,
4547                         filter->ether_type);
4548                 if (ret < 0) {
4549                         PMD_DRV_LOG(ERR, "ethertype filters are full.");
4550                         return -ENOSYS;
4551                 }
4552                 etqf = IXGBE_ETQF_FILTER_EN;
4553                 etqf |= (uint32_t)filter->ether_type;
4554                 etqs |= (uint32_t)((filter->queue <<
4555                                     IXGBE_ETQS_RX_QUEUE_SHIFT) &
4556                                     IXGBE_ETQS_RX_QUEUE);
4557                 etqs |= IXGBE_ETQS_QUEUE_EN;
4558         } else {
4559                 ret = ixgbe_ethertype_filter_remove(filter_info, (uint8_t)ret);
4560                 if (ret < 0)
4561                         return -ENOSYS;
4562         }
4563         IXGBE_WRITE_REG(hw, IXGBE_ETQF(ret), etqf);
4564         IXGBE_WRITE_REG(hw, IXGBE_ETQS(ret), etqs);
4565         IXGBE_WRITE_FLUSH(hw);
4566
4567         return 0;
4568 }
4569
4570 static int
4571 ixgbe_get_ethertype_filter(struct rte_eth_dev *dev,
4572                         struct rte_eth_ethertype_filter *filter)
4573 {
4574         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4575         struct ixgbe_filter_info *filter_info =
4576                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4577         uint32_t etqf, etqs;
4578         int ret;
4579
4580         ret = ixgbe_ethertype_filter_lookup(filter_info, filter->ether_type);
4581         if (ret < 0) {
4582                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
4583                             filter->ether_type);
4584                 return -ENOENT;
4585         }
4586
4587         etqf = IXGBE_READ_REG(hw, IXGBE_ETQF(ret));
4588         if (etqf & IXGBE_ETQF_FILTER_EN) {
4589                 etqs = IXGBE_READ_REG(hw, IXGBE_ETQS(ret));
4590                 filter->ether_type = etqf & IXGBE_ETQF_ETHERTYPE;
4591                 filter->flags = 0;
4592                 filter->queue = (etqs & IXGBE_ETQS_RX_QUEUE) >>
4593                                IXGBE_ETQS_RX_QUEUE_SHIFT;
4594                 return 0;
4595         }
4596         return -ENOENT;
4597 }
4598
4599 /*
4600  * ixgbe_ethertype_filter_handle - Handle operations for ethertype filter.
4601  * @dev: pointer to rte_eth_dev structure
4602  * @filter_op:operation will be taken.
4603  * @arg: a pointer to specific structure corresponding to the filter_op
4604  */
4605 static int
4606 ixgbe_ethertype_filter_handle(struct rte_eth_dev *dev,
4607                                 enum rte_filter_op filter_op,
4608                                 void *arg)
4609 {
4610         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4611         int ret;
4612
4613         MAC_TYPE_FILTER_SUP(hw->mac.type);
4614
4615         if (filter_op == RTE_ETH_FILTER_NOP)
4616                 return 0;
4617
4618         if (arg == NULL) {
4619                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
4620                             filter_op);
4621                 return -EINVAL;
4622         }
4623
4624         switch (filter_op) {
4625         case RTE_ETH_FILTER_ADD:
4626                 ret = ixgbe_add_del_ethertype_filter(dev,
4627                         (struct rte_eth_ethertype_filter *)arg,
4628                         TRUE);
4629                 break;
4630         case RTE_ETH_FILTER_DELETE:
4631                 ret = ixgbe_add_del_ethertype_filter(dev,
4632                         (struct rte_eth_ethertype_filter *)arg,
4633                         FALSE);
4634                 break;
4635         case RTE_ETH_FILTER_GET:
4636                 ret = ixgbe_get_ethertype_filter(dev,
4637                         (struct rte_eth_ethertype_filter *)arg);
4638                 break;
4639         default:
4640                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
4641                 ret = -EINVAL;
4642                 break;
4643         }
4644         return ret;
4645 }
4646
4647 static int
4648 ixgbe_dev_filter_ctrl(struct rte_eth_dev *dev,
4649                      enum rte_filter_type filter_type,
4650                      enum rte_filter_op filter_op,
4651                      void *arg)
4652 {
4653         int ret = -EINVAL;
4654
4655         switch (filter_type) {
4656         case RTE_ETH_FILTER_NTUPLE:
4657                 ret = ixgbe_ntuple_filter_handle(dev, filter_op, arg);
4658                 break;
4659         case RTE_ETH_FILTER_ETHERTYPE:
4660                 ret = ixgbe_ethertype_filter_handle(dev, filter_op, arg);
4661                 break;
4662         case RTE_ETH_FILTER_SYN:
4663                 ret = ixgbe_syn_filter_handle(dev, filter_op, arg);
4664                 break;
4665         case RTE_ETH_FILTER_FDIR:
4666                 ret = ixgbe_fdir_ctrl_func(dev, filter_op, arg);
4667                 break;
4668         default:
4669                 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported",
4670                                                         filter_type);
4671                 break;
4672         }
4673
4674         return ret;
4675 }
4676
4677 static u8 *
4678 ixgbe_dev_addr_list_itr(__attribute__((unused)) struct ixgbe_hw *hw,
4679                         u8 **mc_addr_ptr, u32 *vmdq)
4680 {
4681         u8 *mc_addr;
4682
4683         *vmdq = 0;
4684         mc_addr = *mc_addr_ptr;
4685         *mc_addr_ptr = (mc_addr + sizeof(struct ether_addr));
4686         return mc_addr;
4687 }
4688
4689 static int
4690 ixgbe_dev_set_mc_addr_list(struct rte_eth_dev *dev,
4691                           struct ether_addr *mc_addr_set,
4692                           uint32_t nb_mc_addr)
4693 {
4694         struct ixgbe_hw *hw;
4695         u8 *mc_addr_list;
4696
4697         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4698         mc_addr_list = (u8 *)mc_addr_set;
4699         return ixgbe_update_mc_addr_list(hw, mc_addr_list, nb_mc_addr,
4700                                          ixgbe_dev_addr_list_itr, TRUE);
4701 }
4702
4703 static int
4704 ixgbe_timesync_enable(struct rte_eth_dev *dev)
4705 {
4706         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4707         uint32_t tsync_ctl;
4708         uint32_t tsauxc;
4709
4710         /* Enable system time for platforms where it isn't on by default. */
4711         tsauxc = IXGBE_READ_REG(hw, IXGBE_TSAUXC);
4712         tsauxc &= ~IXGBE_TSAUXC_DISABLE_SYSTIME;
4713         IXGBE_WRITE_REG(hw, IXGBE_TSAUXC, tsauxc);
4714
4715         /* Start incrementing the register used to timestamp PTP packets. */
4716         IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, IXGBE_TIMINCA_INIT);
4717
4718         /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
4719         IXGBE_WRITE_REG(hw, IXGBE_ETQF(IXGBE_ETQF_FILTER_1588),
4720                         (ETHER_TYPE_1588 |
4721                          IXGBE_ETQF_FILTER_EN |
4722                          IXGBE_ETQF_1588));
4723
4724         /* Enable timestamping of received PTP packets. */
4725         tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL);
4726         tsync_ctl |= IXGBE_TSYNCRXCTL_ENABLED;
4727         IXGBE_WRITE_REG(hw, IXGBE_TSYNCRXCTL, tsync_ctl);
4728
4729         /* Enable timestamping of transmitted PTP packets. */
4730         tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL);
4731         tsync_ctl |= IXGBE_TSYNCTXCTL_ENABLED;
4732         IXGBE_WRITE_REG(hw, IXGBE_TSYNCTXCTL, tsync_ctl);
4733
4734         return 0;
4735 }
4736
4737 static int
4738 ixgbe_timesync_disable(struct rte_eth_dev *dev)
4739 {
4740         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4741         uint32_t tsync_ctl;
4742
4743         /* Disable timestamping of transmitted PTP packets. */
4744         tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL);
4745         tsync_ctl &= ~IXGBE_TSYNCTXCTL_ENABLED;
4746         IXGBE_WRITE_REG(hw, IXGBE_TSYNCTXCTL, tsync_ctl);
4747
4748         /* Disable timestamping of received PTP packets. */
4749         tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL);
4750         tsync_ctl &= ~IXGBE_TSYNCRXCTL_ENABLED;
4751         IXGBE_WRITE_REG(hw, IXGBE_TSYNCRXCTL, tsync_ctl);
4752
4753         /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
4754         IXGBE_WRITE_REG(hw, IXGBE_ETQF(IXGBE_ETQF_FILTER_1588), 0);
4755
4756         /* Stop incrementating the System Time registers. */
4757         IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, 0);
4758
4759         return 0;
4760 }
4761
4762 static int
4763 ixgbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
4764                                  struct timespec *timestamp,
4765                                  uint32_t flags __rte_unused)
4766 {
4767         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4768         uint32_t tsync_rxctl;
4769         uint32_t rx_stmpl;
4770         uint32_t rx_stmph;
4771
4772         tsync_rxctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL);
4773         if ((tsync_rxctl & IXGBE_TSYNCRXCTL_VALID) == 0)
4774                 return -EINVAL;
4775
4776         rx_stmpl = IXGBE_READ_REG(hw, IXGBE_RXSTMPL);
4777         rx_stmph = IXGBE_READ_REG(hw, IXGBE_RXSTMPH);
4778
4779         timestamp->tv_sec = (uint64_t)(((uint64_t)rx_stmph << 32) | rx_stmpl);
4780         timestamp->tv_nsec = 0;
4781
4782         return  0;
4783 }
4784
4785 static int
4786 ixgbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
4787                                  struct timespec *timestamp)
4788 {
4789         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4790         uint32_t tsync_txctl;
4791         uint32_t tx_stmpl;
4792         uint32_t tx_stmph;
4793
4794         tsync_txctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL);
4795         if ((tsync_txctl & IXGBE_TSYNCTXCTL_VALID) == 0)
4796                 return -EINVAL;
4797
4798         tx_stmpl = IXGBE_READ_REG(hw, IXGBE_TXSTMPL);
4799         tx_stmph = IXGBE_READ_REG(hw, IXGBE_TXSTMPH);
4800
4801         timestamp->tv_sec = (uint64_t)(((uint64_t)tx_stmph << 32) | tx_stmpl);
4802         timestamp->tv_nsec = 0;
4803
4804         return  0;
4805 }
4806
4807 static int
4808 ixgbe_get_reg_length(struct rte_eth_dev *dev)
4809 {
4810         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4811         int count = 0;
4812         int g_ind = 0;
4813         const struct reg_info *reg_group;
4814         const struct reg_info **reg_set = (hw->mac.type == ixgbe_mac_82598EB) ?
4815                                     ixgbe_regs_mac_82598EB : ixgbe_regs_others;
4816
4817         while ((reg_group = reg_set[g_ind++]))
4818                 count += ixgbe_regs_group_count(reg_group);
4819
4820         return count;
4821 }
4822
4823 static int
4824 ixgbevf_get_reg_length(struct rte_eth_dev *dev __rte_unused)
4825 {
4826         int count = 0;
4827         int g_ind = 0;
4828         const struct reg_info *reg_group;
4829
4830         while ((reg_group = ixgbevf_regs[g_ind++]))
4831                 count += ixgbe_regs_group_count(reg_group);
4832
4833         return count;
4834 }
4835
4836 static int
4837 ixgbe_get_regs(struct rte_eth_dev *dev,
4838               struct rte_dev_reg_info *regs)
4839 {
4840         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4841         uint32_t *data = regs->data;
4842         int g_ind = 0;
4843         int count = 0;
4844         const struct reg_info *reg_group;
4845         const struct reg_info **reg_set = (hw->mac.type == ixgbe_mac_82598EB) ?
4846                                     ixgbe_regs_mac_82598EB : ixgbe_regs_others;
4847
4848         /* Support only full register dump */
4849         if ((regs->length == 0) ||
4850             (regs->length == (uint32_t)ixgbe_get_reg_length(dev))) {
4851                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
4852                         hw->device_id;
4853                 while ((reg_group = reg_set[g_ind++]))
4854                         count += ixgbe_read_regs_group(dev, &data[count],
4855                                 reg_group);
4856                 return 0;
4857         }
4858
4859         return -ENOTSUP;
4860 }
4861
4862 static int
4863 ixgbevf_get_regs(struct rte_eth_dev *dev,
4864                 struct rte_dev_reg_info *regs)
4865 {
4866         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4867         uint32_t *data = regs->data;
4868         int g_ind = 0;
4869         int count = 0;
4870         const struct reg_info *reg_group;
4871
4872         /* Support only full register dump */
4873         if ((regs->length == 0) ||
4874             (regs->length == (uint32_t)ixgbevf_get_reg_length(dev))) {
4875                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
4876                         hw->device_id;
4877                 while ((reg_group = ixgbevf_regs[g_ind++]))
4878                         count += ixgbe_read_regs_group(dev, &data[count],
4879                                                       reg_group);
4880                 return 0;
4881         }
4882
4883         return -ENOTSUP;
4884 }
4885
4886 static int
4887 ixgbe_get_eeprom_length(struct rte_eth_dev *dev)
4888 {
4889         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4890
4891         /* Return unit is byte count */
4892         return hw->eeprom.word_size * 2;
4893 }
4894
4895 static int
4896 ixgbe_get_eeprom(struct rte_eth_dev *dev,
4897                 struct rte_dev_eeprom_info *in_eeprom)
4898 {
4899         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4900         struct ixgbe_eeprom_info *eeprom = &hw->eeprom;
4901         uint16_t *data = in_eeprom->data;
4902         int first, length;
4903
4904         first = in_eeprom->offset >> 1;
4905         length = in_eeprom->length >> 1;
4906         if ((first >= hw->eeprom.word_size) ||
4907             ((first + length) >= hw->eeprom.word_size))
4908                 return -EINVAL;
4909
4910         in_eeprom->magic = hw->vendor_id | (hw->device_id << 16);
4911
4912         return eeprom->ops.read_buffer(hw, first, length, data);
4913 }
4914
4915 static int
4916 ixgbe_set_eeprom(struct rte_eth_dev *dev,
4917                 struct rte_dev_eeprom_info *in_eeprom)
4918 {
4919         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4920         struct ixgbe_eeprom_info *eeprom = &hw->eeprom;
4921         uint16_t *data = in_eeprom->data;
4922         int first, length;
4923
4924         first = in_eeprom->offset >> 1;
4925         length = in_eeprom->length >> 1;
4926         if ((first >= hw->eeprom.word_size) ||
4927             ((first + length) >= hw->eeprom.word_size))
4928                 return -EINVAL;
4929
4930         in_eeprom->magic = hw->vendor_id | (hw->device_id << 16);
4931
4932         return eeprom->ops.write_buffer(hw,  first, length, data);
4933 }
4934
4935 static struct rte_driver rte_ixgbe_driver = {
4936         .type = PMD_PDEV,
4937         .init = rte_ixgbe_pmd_init,
4938 };
4939
4940 static struct rte_driver rte_ixgbevf_driver = {
4941         .type = PMD_PDEV,
4942         .init = rte_ixgbevf_pmd_init,
4943 };
4944
4945 PMD_REGISTER_DRIVER(rte_ixgbe_driver);
4946 PMD_REGISTER_DRIVER(rte_ixgbevf_driver);