drivers/net/ixgbe/ixgbe_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <inttypes.h>
  42 #include <netinet/in.h>
  43 #include <rte_byteorder.h>
  44 #include <rte_common.h>
  45 #include <rte_cycles.h>
  46
  47 #include <rte_interrupts.h>
  48 #include <rte_log.h>
  49 #include <rte_debug.h>
  50 #include <rte_pci.h>
  51 #include <rte_atomic.h>
  52 #include <rte_branch_prediction.h>
  53 #include <rte_memory.h>
  54 #include <rte_memzone.h>
  55 #include <rte_eal.h>
  56 #include <rte_alarm.h>
  57 #include <rte_ether.h>
  58 #include <rte_ethdev.h>
  59 #include <rte_atomic.h>
  60 #include <rte_malloc.h>
  61 #include <rte_random.h>
  62 #include <rte_dev.h>
  63
  64 #include "ixgbe_logs.h"
  65 #include "base/ixgbe_api.h"
  66 #include "base/ixgbe_vf.h"
  67 #include "base/ixgbe_common.h"
  68 #include "ixgbe_ethdev.h"
  69 #include "ixgbe_bypass.h"
  70 #include "ixgbe_rxtx.h"
  71 #include "base/ixgbe_type.h"
  72 #include "base/ixgbe_phy.h"
  73 #include "ixgbe_regs.h"
  74
  75 /*
  76  * High threshold controlling when to start sending XOFF frames. Must be at
  77  * least 8 bytes less than receive packet buffer size. This value is in units
  78  * of 1024 bytes.
  79  */
  80 #define IXGBE_FC_HI    0x80
  81
  82 /*
  83  * Low threshold controlling when to start sending XON frames. This value is
  84  * in units of 1024 bytes.
  85  */
  86 #define IXGBE_FC_LO    0x40
  87
  88 /* Timer value included in XOFF frames. */
  89 #define IXGBE_FC_PAUSE 0x680
  90
  91 #define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */
  92 #define IXGBE_LINK_UP_CHECK_TIMEOUT   1000 /* ms */
  93 #define IXGBE_VMDQ_NUM_UC_MAC         4096 /* Maximum nb. of UC MAC addr. */
  94
  95 #define IXGBE_MMW_SIZE_DEFAULT        0x4
  96 #define IXGBE_MMW_SIZE_JUMBO_FRAME    0x14
  97 #define IXGBE_MAX_RING_DESC           4096 /* replicate define from rxtx */
  98
  99 /*
 100  *  Default values for RX/TX configuration
 101  */
 102 #define IXGBE_DEFAULT_RX_FREE_THRESH  32
 103 #define IXGBE_DEFAULT_RX_PTHRESH      8
 104 #define IXGBE_DEFAULT_RX_HTHRESH      8
 105 #define IXGBE_DEFAULT_RX_WTHRESH      0
 106
 107 #define IXGBE_DEFAULT_TX_FREE_THRESH  32
 108 #define IXGBE_DEFAULT_TX_PTHRESH      32
 109 #define IXGBE_DEFAULT_TX_HTHRESH      0
 110 #define IXGBE_DEFAULT_TX_WTHRESH      0
 111 #define IXGBE_DEFAULT_TX_RSBIT_THRESH 32
 112
 113 /* Bit shift and mask */
 114 #define IXGBE_4_BIT_WIDTH  (CHAR_BIT / 2)
 115 #define IXGBE_4_BIT_MASK   RTE_LEN2MASK(IXGBE_4_BIT_WIDTH, uint8_t)
 116 #define IXGBE_8_BIT_WIDTH  CHAR_BIT
 117 #define IXGBE_8_BIT_MASK   UINT8_MAX
 118
 119 #define IXGBEVF_PMD_NAME "rte_ixgbevf_pmd" /* PMD name */
 120
 121 #define IXGBE_QUEUE_STAT_COUNTERS (sizeof(hw_stats->qprc) / sizeof(hw_stats->qprc[0]))
 122
 123 #define IXGBE_HKEY_MAX_INDEX 10
 124
 125 /* Additional timesync values. */
 126 #define IXGBE_TIMINCA_16NS_SHIFT 24
 127 #define IXGBE_TIMINCA_INCVALUE   16000000
 128 #define IXGBE_TIMINCA_INIT       ((0x02 << IXGBE_TIMINCA_16NS_SHIFT) \
 129                                   | IXGBE_TIMINCA_INCVALUE)
 130
 131 static int eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev);
 132 static int  ixgbe_dev_configure(struct rte_eth_dev *dev);
 133 static int  ixgbe_dev_start(struct rte_eth_dev *dev);
 134 static void ixgbe_dev_stop(struct rte_eth_dev *dev);
 135 static int  ixgbe_dev_set_link_up(struct rte_eth_dev *dev);
 136 static int  ixgbe_dev_set_link_down(struct rte_eth_dev *dev);
 137 static void ixgbe_dev_close(struct rte_eth_dev *dev);
 138 static void ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev);
 139 static void ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev);
 140 static void ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev);
 141 static void ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev);
 142 static int ixgbe_dev_link_update(struct rte_eth_dev *dev,
 143                                 int wait_to_complete);
 144 static void ixgbe_dev_stats_get(struct rte_eth_dev *dev,
 145                                 struct rte_eth_stats *stats);
 146 static int ixgbe_dev_xstats_get(struct rte_eth_dev *dev,
 147                                 struct rte_eth_xstats *xstats, unsigned n);
 148 static void ixgbe_dev_stats_reset(struct rte_eth_dev *dev);
 149 static void ixgbe_dev_xstats_reset(struct rte_eth_dev *dev);
 150 static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 151                                              uint16_t queue_id,
 152                                              uint8_t stat_idx,
 153                                              uint8_t is_rx);
 154 static void ixgbe_dev_info_get(struct rte_eth_dev *dev,
 155                                struct rte_eth_dev_info *dev_info);
 156 static void ixgbevf_dev_info_get(struct rte_eth_dev *dev,
 157                                  struct rte_eth_dev_info *dev_info);
 158 static int ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 159
 160 static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev,
 161                 uint16_t vlan_id, int on);
 162 static void ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 163 static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev,
 164                 uint16_t queue, bool on);
 165 static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue,
 166                 int on);
 167 static void ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 168 static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue);
 169 static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue);
 170 static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 171 static void ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 172
 173 static int ixgbe_dev_led_on(struct rte_eth_dev *dev);
 174 static int ixgbe_dev_led_off(struct rte_eth_dev *dev);
 175 static int ixgbe_flow_ctrl_get(struct rte_eth_dev *dev,
 176                                struct rte_eth_fc_conf *fc_conf);
 177 static int ixgbe_flow_ctrl_set(struct rte_eth_dev *dev,
 178                                struct rte_eth_fc_conf *fc_conf);
 179 static int ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 180                 struct rte_eth_pfc_conf *pfc_conf);
 181 static int ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
 182                         struct rte_eth_rss_reta_entry64 *reta_conf,
 183                         uint16_t reta_size);
 184 static int ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
 185                         struct rte_eth_rss_reta_entry64 *reta_conf,
 186                         uint16_t reta_size);
 187 static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev);
 188 static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev);
 189 static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev);
 190 static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev);
 191 static void ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle,
 192                 void *param);
 193 static void ixgbe_dev_interrupt_delayed_handler(void *param);
 194 static void ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 195                 uint32_t index, uint32_t pool);
 196 static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index);
 197 static void ixgbe_set_default_mac_addr(struct rte_eth_dev *dev,
 198                                            struct ether_addr *mac_addr);
 199 static void ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config);
 200
 201 /* For Virtual Function support */
 202 static int eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev);
 203 static int  ixgbevf_dev_configure(struct rte_eth_dev *dev);
 204 static int  ixgbevf_dev_start(struct rte_eth_dev *dev);
 205 static void ixgbevf_dev_stop(struct rte_eth_dev *dev);
 206 static void ixgbevf_dev_close(struct rte_eth_dev *dev);
 207 static void ixgbevf_intr_disable(struct ixgbe_hw *hw);
 208 static void ixgbevf_dev_stats_get(struct rte_eth_dev *dev,
 209                 struct rte_eth_stats *stats);
 210 static void ixgbevf_dev_stats_reset(struct rte_eth_dev *dev);
 211 static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev,
 212                 uint16_t vlan_id, int on);
 213 static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev,
 214                 uint16_t queue, int on);
 215 static void ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 216 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 217
 218 /* For Eth VMDQ APIs support */
 219 static int ixgbe_uc_hash_table_set(struct rte_eth_dev *dev, struct
 220                 ether_addr* mac_addr,uint8_t on);
 221 static int ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev,uint8_t on);
 222 static int  ixgbe_set_pool_rx_mode(struct rte_eth_dev *dev,  uint16_t pool,
 223                 uint16_t rx_mask, uint8_t on);
 224 static int ixgbe_set_pool_rx(struct rte_eth_dev *dev,uint16_t pool,uint8_t on);
 225 static int ixgbe_set_pool_tx(struct rte_eth_dev *dev,uint16_t pool,uint8_t on);
 226 static int ixgbe_set_pool_vlan_filter(struct rte_eth_dev *dev, uint16_t vlan,
 227                 uint64_t pool_mask,uint8_t vlan_on);
 228 static int ixgbe_mirror_rule_set(struct rte_eth_dev *dev,
 229                 struct rte_eth_mirror_conf *mirror_conf,
 230                 uint8_t rule_id, uint8_t on);
 231 static int ixgbe_mirror_rule_reset(struct rte_eth_dev *dev,
 232                 uint8_t rule_id);
 233
 234 static int ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev,
 235                 uint16_t queue_idx, uint16_t tx_rate);
 236 static int ixgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf,
 237                 uint16_t tx_rate, uint64_t q_msk);
 238
 239 static void ixgbevf_add_mac_addr(struct rte_eth_dev *dev,
 240                                  struct ether_addr *mac_addr,
 241                                  uint32_t index, uint32_t pool);
 242 static void ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index);
 243 static void ixgbevf_set_default_mac_addr(struct rte_eth_dev *dev,
 244                                              struct ether_addr *mac_addr);
 245 static int ixgbe_syn_filter_set(struct rte_eth_dev *dev,
 246                         struct rte_eth_syn_filter *filter,
 247                         bool add);
 248 static int ixgbe_syn_filter_get(struct rte_eth_dev *dev,
 249                         struct rte_eth_syn_filter *filter);
 250 static int ixgbe_syn_filter_handle(struct rte_eth_dev *dev,
 251                         enum rte_filter_op filter_op,
 252                         void *arg);
 253 static int ixgbe_add_5tuple_filter(struct rte_eth_dev *dev,
 254                         struct ixgbe_5tuple_filter *filter);
 255 static void ixgbe_remove_5tuple_filter(struct rte_eth_dev *dev,
 256                         struct ixgbe_5tuple_filter *filter);
 257 static int ixgbe_add_del_ntuple_filter(struct rte_eth_dev *dev,
 258                         struct rte_eth_ntuple_filter *filter,
 259                         bool add);
 260 static int ixgbe_ntuple_filter_handle(struct rte_eth_dev *dev,
 261                                 enum rte_filter_op filter_op,
 262                                 void *arg);
 263 static int ixgbe_get_ntuple_filter(struct rte_eth_dev *dev,
 264                         struct rte_eth_ntuple_filter *filter);
 265 static int ixgbe_add_del_ethertype_filter(struct rte_eth_dev *dev,
 266                         struct rte_eth_ethertype_filter *filter,
 267                         bool add);
 268 static int ixgbe_ethertype_filter_handle(struct rte_eth_dev *dev,
 269                                 enum rte_filter_op filter_op,
 270                                 void *arg);
 271 static int ixgbe_get_ethertype_filter(struct rte_eth_dev *dev,
 272                         struct rte_eth_ethertype_filter *filter);
 273 static int ixgbe_dev_filter_ctrl(struct rte_eth_dev *dev,
 274                      enum rte_filter_type filter_type,
 275                      enum rte_filter_op filter_op,
 276                      void *arg);
 277 static int ixgbevf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu);
 278
 279 static int ixgbe_dev_set_mc_addr_list(struct rte_eth_dev *dev,
 280                                       struct ether_addr *mc_addr_set,
 281                                       uint32_t nb_mc_addr);
 282
 283 static int ixgbe_get_reg_length(struct rte_eth_dev *dev);
 284 static int ixgbe_get_regs(struct rte_eth_dev *dev,
 285                             struct rte_dev_reg_info *regs);
 286 static int ixgbe_get_eeprom_length(struct rte_eth_dev *dev);
 287 static int ixgbe_get_eeprom(struct rte_eth_dev *dev,
 288                                 struct rte_dev_eeprom_info *eeprom);
 289 static int ixgbe_set_eeprom(struct rte_eth_dev *dev,
 290                                 struct rte_dev_eeprom_info *eeprom);
 291
 292 static int ixgbevf_get_reg_length(struct rte_eth_dev *dev);
 293 static int ixgbevf_get_regs(struct rte_eth_dev *dev,
 294                                 struct rte_dev_reg_info *regs);
 295
 296 static int ixgbe_timesync_enable(struct rte_eth_dev *dev);
 297 static int ixgbe_timesync_disable(struct rte_eth_dev *dev);
 298 static int ixgbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
 299                                             struct timespec *timestamp,
 300                                             uint32_t flags);
 301 static int ixgbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
 302                                             struct timespec *timestamp);
 303
 304 /*
 305  * Define VF Stats MACRO for Non "cleared on read" register
 306  */
 307 #define UPDATE_VF_STAT(reg, last, cur)                          \
 308 {                                                               \
 309         u32 latest = IXGBE_READ_REG(hw, reg);                   \
 310         cur += latest - last;                                   \
 311         last = latest;                                          \
 312 }
 313
 314 #define UPDATE_VF_STAT_36BIT(lsb, msb, last, cur)                \
 315 {                                                                \
 316         u64 new_lsb = IXGBE_READ_REG(hw, lsb);                   \
 317         u64 new_msb = IXGBE_READ_REG(hw, msb);                   \
 318         u64 latest = ((new_msb << 32) | new_lsb);                \
 319         cur += (0x1000000000LL + latest - last) & 0xFFFFFFFFFLL; \
 320         last = latest;                                           \
 321 }
 322
 323 #define IXGBE_SET_HWSTRIP(h, q) do{\
 324                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 325                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 326                 (h)->bitmap[idx] |= 1 << bit;\
 327         }while(0)
 328
 329 #define IXGBE_CLEAR_HWSTRIP(h, q) do{\
 330                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 331                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 332                 (h)->bitmap[idx] &= ~(1 << bit);\
 333         }while(0)
 334
 335 #define IXGBE_GET_HWSTRIP(h, q, r) do{\
 336                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 337                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 338                 (r) = (h)->bitmap[idx] >> bit & 1;\
 339         }while(0)
 340
 341 /*
 342  * The set of PCI devices this driver supports
 343  */
 344 static const struct rte_pci_id pci_id_ixgbe_map[] = {
 345
 346 #define RTE_PCI_DEV_ID_DECL_IXGBE(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 347 #include "rte_pci_dev_ids.h"
 348
 349 { .vendor_id = 0, /* sentinel */ },
 350 };
 351
 352
 353 /*
 354  * The set of PCI devices this driver supports (for 82599 VF)
 355  */
 356 static const struct rte_pci_id pci_id_ixgbevf_map[] = {
 357
 358 #define RTE_PCI_DEV_ID_DECL_IXGBEVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 359 #include "rte_pci_dev_ids.h"
 360 { .vendor_id = 0, /* sentinel */ },
 361
 362 };
 363
 364 static const struct eth_dev_ops ixgbe_eth_dev_ops = {
 365         .dev_configure        = ixgbe_dev_configure,
 366         .dev_start            = ixgbe_dev_start,
 367         .dev_stop             = ixgbe_dev_stop,
 368         .dev_set_link_up    = ixgbe_dev_set_link_up,
 369         .dev_set_link_down  = ixgbe_dev_set_link_down,
 370         .dev_close            = ixgbe_dev_close,
 371         .promiscuous_enable   = ixgbe_dev_promiscuous_enable,
 372         .promiscuous_disable  = ixgbe_dev_promiscuous_disable,
 373         .allmulticast_enable  = ixgbe_dev_allmulticast_enable,
 374         .allmulticast_disable = ixgbe_dev_allmulticast_disable,
 375         .link_update          = ixgbe_dev_link_update,
 376         .stats_get            = ixgbe_dev_stats_get,
 377         .xstats_get           = ixgbe_dev_xstats_get,
 378         .stats_reset          = ixgbe_dev_stats_reset,
 379         .xstats_reset         = ixgbe_dev_xstats_reset,
 380         .queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set,
 381         .dev_infos_get        = ixgbe_dev_info_get,
 382         .mtu_set              = ixgbe_dev_mtu_set,
 383         .vlan_filter_set      = ixgbe_vlan_filter_set,
 384         .vlan_tpid_set        = ixgbe_vlan_tpid_set,
 385         .vlan_offload_set     = ixgbe_vlan_offload_set,
 386         .vlan_strip_queue_set = ixgbe_vlan_strip_queue_set,
 387         .rx_queue_start       = ixgbe_dev_rx_queue_start,
 388         .rx_queue_stop        = ixgbe_dev_rx_queue_stop,
 389         .tx_queue_start       = ixgbe_dev_tx_queue_start,
 390         .tx_queue_stop        = ixgbe_dev_tx_queue_stop,
 391         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 392         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 393         .rx_queue_count       = ixgbe_dev_rx_queue_count,
 394         .rx_descriptor_done   = ixgbe_dev_rx_descriptor_done,
 395         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 396         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 397         .dev_led_on           = ixgbe_dev_led_on,
 398         .dev_led_off          = ixgbe_dev_led_off,
 399         .flow_ctrl_get        = ixgbe_flow_ctrl_get,
 400         .flow_ctrl_set        = ixgbe_flow_ctrl_set,
 401         .priority_flow_ctrl_set = ixgbe_priority_flow_ctrl_set,
 402         .mac_addr_add         = ixgbe_add_rar,
 403         .mac_addr_remove      = ixgbe_remove_rar,
 404         .mac_addr_set         = ixgbe_set_default_mac_addr,
 405         .uc_hash_table_set    = ixgbe_uc_hash_table_set,
 406         .uc_all_hash_table_set  = ixgbe_uc_all_hash_table_set,
 407         .mirror_rule_set      = ixgbe_mirror_rule_set,
 408         .mirror_rule_reset    = ixgbe_mirror_rule_reset,
 409         .set_vf_rx_mode       = ixgbe_set_pool_rx_mode,
 410         .set_vf_rx            = ixgbe_set_pool_rx,
 411         .set_vf_tx            = ixgbe_set_pool_tx,
 412         .set_vf_vlan_filter   = ixgbe_set_pool_vlan_filter,
 413         .set_queue_rate_limit = ixgbe_set_queue_rate_limit,
 414         .set_vf_rate_limit    = ixgbe_set_vf_rate_limit,
 415         .reta_update          = ixgbe_dev_rss_reta_update,
 416         .reta_query           = ixgbe_dev_rss_reta_query,
 417 #ifdef RTE_NIC_BYPASS
 418         .bypass_init          = ixgbe_bypass_init,
 419         .bypass_state_set     = ixgbe_bypass_state_store,
 420         .bypass_state_show    = ixgbe_bypass_state_show,
 421         .bypass_event_set     = ixgbe_bypass_event_store,
 422         .bypass_event_show    = ixgbe_bypass_event_show,
 423         .bypass_wd_timeout_set  = ixgbe_bypass_wd_timeout_store,
 424         .bypass_wd_timeout_show = ixgbe_bypass_wd_timeout_show,
 425         .bypass_ver_show      = ixgbe_bypass_ver_show,
 426         .bypass_wd_reset      = ixgbe_bypass_wd_reset,
 427 #endif /* RTE_NIC_BYPASS */
 428         .rss_hash_update      = ixgbe_dev_rss_hash_update,
 429         .rss_hash_conf_get    = ixgbe_dev_rss_hash_conf_get,
 430         .filter_ctrl          = ixgbe_dev_filter_ctrl,
 431         .set_mc_addr_list     = ixgbe_dev_set_mc_addr_list,
 432         .timesync_enable      = ixgbe_timesync_enable,
 433         .timesync_disable     = ixgbe_timesync_disable,
 434         .timesync_read_rx_timestamp = ixgbe_timesync_read_rx_timestamp,
 435         .timesync_read_tx_timestamp = ixgbe_timesync_read_tx_timestamp,
 436         .get_reg_length       = ixgbe_get_reg_length,
 437         .get_reg              = ixgbe_get_regs,
 438         .get_eeprom_length    = ixgbe_get_eeprom_length,
 439         .get_eeprom           = ixgbe_get_eeprom,
 440         .set_eeprom           = ixgbe_set_eeprom,
 441 };
 442
 443 /*
 444  * dev_ops for virtual function, bare necessities for basic vf
 445  * operation have been implemented
 446  */
 447 static const struct eth_dev_ops ixgbevf_eth_dev_ops = {
 448         .dev_configure        = ixgbevf_dev_configure,
 449         .dev_start            = ixgbevf_dev_start,
 450         .dev_stop             = ixgbevf_dev_stop,
 451         .link_update          = ixgbe_dev_link_update,
 452         .stats_get            = ixgbevf_dev_stats_get,
 453         .stats_reset          = ixgbevf_dev_stats_reset,
 454         .dev_close            = ixgbevf_dev_close,
 455         .dev_infos_get        = ixgbevf_dev_info_get,
 456         .mtu_set              = ixgbevf_dev_set_mtu,
 457         .vlan_filter_set      = ixgbevf_vlan_filter_set,
 458         .vlan_strip_queue_set = ixgbevf_vlan_strip_queue_set,
 459         .vlan_offload_set     = ixgbevf_vlan_offload_set,
 460         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 461         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 462         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 463         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 464         .mac_addr_add         = ixgbevf_add_mac_addr,
 465         .mac_addr_remove      = ixgbevf_remove_mac_addr,
 466         .set_mc_addr_list     = ixgbe_dev_set_mc_addr_list,
 467         .mac_addr_set         = ixgbevf_set_default_mac_addr,
 468         .get_reg_length       = ixgbevf_get_reg_length,
 469         .get_reg              = ixgbevf_get_regs,
 470 };
 471
 472 /* store statistics names and its offset in stats structure */
 473 struct rte_ixgbe_xstats_name_off {
 474         char name[RTE_ETH_XSTATS_NAME_SIZE];
 475         unsigned offset;
 476 };
 477
 478 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_stats_strings[] = {
 479         {"rx_illegal_byte_err", offsetof(struct ixgbe_hw_stats, errbc)},
 480         {"rx_len_err", offsetof(struct ixgbe_hw_stats, rlec)},
 481         {"rx_undersize_count", offsetof(struct ixgbe_hw_stats, ruc)},
 482         {"rx_oversize_count", offsetof(struct ixgbe_hw_stats, roc)},
 483         {"rx_fragment_count", offsetof(struct ixgbe_hw_stats, rfc)},
 484         {"rx_jabber_count", offsetof(struct ixgbe_hw_stats, rjc)},
 485         {"l3_l4_xsum_error", offsetof(struct ixgbe_hw_stats, xec)},
 486         {"mac_local_fault", offsetof(struct ixgbe_hw_stats, mlfc)},
 487         {"mac_remote_fault", offsetof(struct ixgbe_hw_stats, mrfc)},
 488         {"mac_short_pkt_discard", offsetof(struct ixgbe_hw_stats, mspdc)},
 489         {"fccrc_error", offsetof(struct ixgbe_hw_stats, fccrc)},
 490         {"fcoe_drop", offsetof(struct ixgbe_hw_stats, fcoerpdc)},
 491         {"fc_last_error", offsetof(struct ixgbe_hw_stats, fclast)},
 492         {"rx_broadcast_packets", offsetof(struct ixgbe_hw_stats, bprc)},
 493         {"rx_phy_multicast_packets", offsetof(struct ixgbe_hw_stats, mprc)},
 494         {"mgmt_pkts_dropped", offsetof(struct ixgbe_hw_stats, mngpdc)},
 495         {"rx_crc_errors", offsetof(struct ixgbe_hw_stats, crcerrs)},
 496         {"fdir_match", offsetof(struct ixgbe_hw_stats, fdirmatch)},
 497         {"fdir_miss", offsetof(struct ixgbe_hw_stats, fdirmiss)},
 498         {"tx_flow_control_xon", offsetof(struct ixgbe_hw_stats, lxontxc)},
 499         {"rx_flow_control_xon", offsetof(struct ixgbe_hw_stats, lxonrxc)},
 500         {"tx_flow_control_xoff", offsetof(struct ixgbe_hw_stats, lxofftxc)},
 501         {"rx_flow_control_xoff", offsetof(struct ixgbe_hw_stats, lxoffrxc)},
 502 };
 503
 504 #define IXGBE_NB_XSTATS (sizeof(rte_ixgbe_stats_strings) /      \
 505                 sizeof(rte_ixgbe_stats_strings[0]))
 506
 507 /**
 508  * Atomically reads the link status information from global
 509  * structure rte_eth_dev.
 510  *
 511  * @param dev
 512  *   - Pointer to the structure rte_eth_dev to read from.
 513  *   - Pointer to the buffer to be saved with the link status.
 514  *
 515  * @return
 516  *   - On success, zero.
 517  *   - On failure, negative value.
 518  */
 519 static inline int
 520 rte_ixgbe_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 521                                 struct rte_eth_link *link)
 522 {
 523         struct rte_eth_link *dst = link;
 524         struct rte_eth_link *src = &(dev->data->dev_link);
 525
 526         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 527                                         *(uint64_t *)src) == 0)
 528                 return -1;
 529
 530         return 0;
 531 }
 532
 533 /**
 534  * Atomically writes the link status information into global
 535  * structure rte_eth_dev.
 536  *
 537  * @param dev
 538  *   - Pointer to the structure rte_eth_dev to read from.
 539  *   - Pointer to the buffer to be saved with the link status.
 540  *
 541  * @return
 542  *   - On success, zero.
 543  *   - On failure, negative value.
 544  */
 545 static inline int
 546 rte_ixgbe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 547                                 struct rte_eth_link *link)
 548 {
 549         struct rte_eth_link *dst = &(dev->data->dev_link);
 550         struct rte_eth_link *src = link;
 551
 552         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 553                                         *(uint64_t *)src) == 0)
 554                 return -1;
 555
 556         return 0;
 557 }
 558
 559 /*
 560  * This function is the same as ixgbe_is_sfp() in base/ixgbe.h.
 561  */
 562 static inline int
 563 ixgbe_is_sfp(struct ixgbe_hw *hw)
 564 {
 565         switch (hw->phy.type) {
 566         case ixgbe_phy_sfp_avago:
 567         case ixgbe_phy_sfp_ftl:
 568         case ixgbe_phy_sfp_intel:
 569         case ixgbe_phy_sfp_unknown:
 570         case ixgbe_phy_sfp_passive_tyco:
 571         case ixgbe_phy_sfp_passive_unknown:
 572                 return 1;
 573         default:
 574                 return 0;
 575         }
 576 }
 577
 578 static inline int32_t
 579 ixgbe_pf_reset_hw(struct ixgbe_hw *hw)
 580 {
 581         uint32_t ctrl_ext;
 582         int32_t status;
 583
 584         status = ixgbe_reset_hw(hw);
 585
 586         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 587         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 588         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
 589         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 590         IXGBE_WRITE_FLUSH(hw);
 591
 592         return status;
 593 }
 594
 595 static inline void
 596 ixgbe_enable_intr(struct rte_eth_dev *dev)
 597 {
 598         struct ixgbe_interrupt *intr =
 599                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 600         struct ixgbe_hw *hw =
 601                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 602
 603         IXGBE_WRITE_REG(hw, IXGBE_EIMS, intr->mask);
 604         IXGBE_WRITE_FLUSH(hw);
 605 }
 606
 607 /*
 608  * This function is based on ixgbe_disable_intr() in base/ixgbe.h.
 609  */
 610 static void
 611 ixgbe_disable_intr(struct ixgbe_hw *hw)
 612 {
 613         PMD_INIT_FUNC_TRACE();
 614
 615         if (hw->mac.type == ixgbe_mac_82598EB) {
 616                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ~0);
 617         } else {
 618                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xFFFF0000);
 619                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), ~0);
 620                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), ~0);
 621         }
 622         IXGBE_WRITE_FLUSH(hw);
 623 }
 624
 625 /*
 626  * This function resets queue statistics mapping registers.
 627  * From Niantic datasheet, Initialization of Statistics section:
 628  * "...if software requires the queue counters, the RQSMR and TQSM registers
 629  * must be re-programmed following a device reset.
 630  */
 631 static void
 632 ixgbe_reset_qstat_mappings(struct ixgbe_hw *hw)
 633 {
 634         uint32_t i;
 635
 636         for(i = 0; i != IXGBE_NB_STAT_MAPPING_REGS; i++) {
 637                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), 0);
 638                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), 0);
 639         }
 640 }
 641
 642
 643 static int
 644 ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 645                                   uint16_t queue_id,
 646                                   uint8_t stat_idx,
 647                                   uint8_t is_rx)
 648 {
 649 #define QSM_REG_NB_BITS_PER_QMAP_FIELD 8
 650 #define NB_QMAP_FIELDS_PER_QSM_REG 4
 651 #define QMAP_FIELD_RESERVED_BITS_MASK 0x0f
 652
 653         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 654         struct ixgbe_stat_mapping_registers *stat_mappings =
 655                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(eth_dev->data->dev_private);
 656         uint32_t qsmr_mask = 0;
 657         uint32_t clearing_mask = QMAP_FIELD_RESERVED_BITS_MASK;
 658         uint32_t q_map;
 659         uint8_t n, offset;
 660
 661         if ((hw->mac.type != ixgbe_mac_82599EB) &&
 662                 (hw->mac.type != ixgbe_mac_X540) &&
 663                 (hw->mac.type != ixgbe_mac_X550) &&
 664                 (hw->mac.type != ixgbe_mac_X550EM_x))
 665                 return -ENOSYS;
 666
 667         PMD_INIT_LOG(INFO, "Setting port %d, %s queue_id %d to stat index %d",
 668                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX",
 669                      queue_id, stat_idx);
 670
 671         n = (uint8_t)(queue_id / NB_QMAP_FIELDS_PER_QSM_REG);
 672         if (n >= IXGBE_NB_STAT_MAPPING_REGS) {
 673                 PMD_INIT_LOG(ERR, "Nb of stat mapping registers exceeded");
 674                 return -EIO;
 675         }
 676         offset = (uint8_t)(queue_id % NB_QMAP_FIELDS_PER_QSM_REG);
 677
 678         /* Now clear any previous stat_idx set */
 679         clearing_mask <<= (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 680         if (!is_rx)
 681                 stat_mappings->tqsm[n] &= ~clearing_mask;
 682         else
 683                 stat_mappings->rqsmr[n] &= ~clearing_mask;
 684
 685         q_map = (uint32_t)stat_idx;
 686         q_map &= QMAP_FIELD_RESERVED_BITS_MASK;
 687         qsmr_mask = q_map << (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 688         if (!is_rx)
 689                 stat_mappings->tqsm[n] |= qsmr_mask;
 690         else
 691                 stat_mappings->rqsmr[n] |= qsmr_mask;
 692
 693         PMD_INIT_LOG(INFO, "Set port %d, %s queue_id %d to stat index %d",
 694                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX",
 695                      queue_id, stat_idx);
 696         PMD_INIT_LOG(INFO, "%s[%d] = 0x%08x", is_rx ? "RQSMR" : "TQSM", n,
 697                      is_rx ? stat_mappings->rqsmr[n] : stat_mappings->tqsm[n]);
 698
 699         /* Now write the mapping in the appropriate register */
 700         if (is_rx) {
 701                 PMD_INIT_LOG(INFO, "Write 0x%x to RX IXGBE stat mapping reg:%d",
 702                              stat_mappings->rqsmr[n], n);
 703                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(n), stat_mappings->rqsmr[n]);
 704         }
 705         else {
 706                 PMD_INIT_LOG(INFO, "Write 0x%x to TX IXGBE stat mapping reg:%d",
 707                              stat_mappings->tqsm[n], n);
 708                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(n), stat_mappings->tqsm[n]);
 709         }
 710         return 0;
 711 }
 712
 713 static void
 714 ixgbe_restore_statistics_mapping(struct rte_eth_dev * dev)
 715 {
 716         struct ixgbe_stat_mapping_registers *stat_mappings =
 717                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(dev->data->dev_private);
 718         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 719         int i;
 720
 721         /* write whatever was in stat mapping table to the NIC */
 722         for (i = 0; i < IXGBE_NB_STAT_MAPPING_REGS; i++) {
 723                 /* rx */
 724                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), stat_mappings->rqsmr[i]);
 725
 726                 /* tx */
 727                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), stat_mappings->tqsm[i]);
 728         }
 729 }
 730
 731 static void
 732 ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config)
 733 {
 734         uint8_t i;
 735         struct ixgbe_dcb_tc_config *tc;
 736         uint8_t dcb_max_tc = IXGBE_DCB_MAX_TRAFFIC_CLASS;
 737
 738         dcb_config->num_tcs.pg_tcs = dcb_max_tc;
 739         dcb_config->num_tcs.pfc_tcs = dcb_max_tc;
 740         for (i = 0; i < dcb_max_tc; i++) {
 741                 tc = &dcb_config->tc_config[i];
 742                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_id = i;
 743                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
 744                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 745                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_id = i;
 746                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
 747                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 748                 tc->pfc = ixgbe_dcb_pfc_disabled;
 749         }
 750
 751         /* Initialize default user to priority mapping, UPx->TC0 */
 752         tc = &dcb_config->tc_config[0];
 753         tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
 754         tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
 755         for (i = 0; i< IXGBE_DCB_MAX_BW_GROUP; i++) {
 756                 dcb_config->bw_percentage[IXGBE_DCB_TX_CONFIG][i] = 100;
 757                 dcb_config->bw_percentage[IXGBE_DCB_RX_CONFIG][i] = 100;
 758         }
 759         dcb_config->rx_pba_cfg = ixgbe_dcb_pba_equal;
 760         dcb_config->pfc_mode_enable = false;
 761         dcb_config->vt_mode = true;
 762         dcb_config->round_robin_enable = false;
 763         /* support all DCB capabilities in 82599 */
 764         dcb_config->support.capabilities = 0xFF;
 765
 766         /*we only support 4 Tcs for X540, X550 */
 767         if (hw->mac.type == ixgbe_mac_X540 ||
 768                 hw->mac.type == ixgbe_mac_X550 ||
 769                 hw->mac.type == ixgbe_mac_X550EM_x) {
 770                 dcb_config->num_tcs.pg_tcs = 4;
 771                 dcb_config->num_tcs.pfc_tcs = 4;
 772         }
 773 }
 774
 775 /*
 776  * Ensure that all locks are released before first NVM or PHY access
 777  */
 778 static void
 779 ixgbe_swfw_lock_reset(struct ixgbe_hw *hw)
 780 {
 781         uint16_t mask;
 782
 783         /*
 784          * Phy lock should not fail in this early stage. If this is the case,
 785          * it is due to an improper exit of the application.
 786          * So force the release of the faulty lock. Release of common lock
 787          * is done automatically by swfw_sync function.
 788          */
 789         mask = IXGBE_GSSR_PHY0_SM << hw->bus.func;
 790         if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) {
 791                 PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released", hw->bus.func);
 792         }
 793         ixgbe_release_swfw_semaphore(hw, mask);
 794
 795         /*
 796          * These ones are more tricky since they are common to all ports; but
 797          * swfw_sync retries last long enough (1s) to be almost sure that if
 798          * lock can not be taken it is due to an improper lock of the
 799          * semaphore.
 800          */
 801         mask = IXGBE_GSSR_EEP_SM | IXGBE_GSSR_MAC_CSR_SM | IXGBE_GSSR_SW_MNG_SM;
 802         if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) {
 803                 PMD_DRV_LOG(DEBUG, "SWFW common locks released");
 804         }
 805         ixgbe_release_swfw_semaphore(hw, mask);
 806 }
 807
 808 /*
 809  * This function is based on code in ixgbe_attach() in base/ixgbe.c.
 810  * It returns 0 on success.
 811  */
 812 static int
 813 eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev)
 814 {
 815         struct rte_pci_device *pci_dev;
 816         struct ixgbe_hw *hw =
 817                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 818         struct ixgbe_vfta * shadow_vfta =
 819                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 820         struct ixgbe_hwstrip *hwstrip =
 821                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 822         struct ixgbe_dcb_config *dcb_config =
 823                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private);
 824         struct ixgbe_filter_info *filter_info =
 825                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private);
 826         uint32_t ctrl_ext;
 827         uint16_t csum;
 828         int diag, i;
 829
 830         PMD_INIT_FUNC_TRACE();
 831
 832         eth_dev->dev_ops = &ixgbe_eth_dev_ops;
 833         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
 834         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
 835
 836         /*
 837          * For secondary processes, we don't initialise any further as primary
 838          * has already done this work. Only check we don't need a different
 839          * RX and TX function.
 840          */
 841         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 842                 struct ixgbe_tx_queue *txq;
 843                 /* TX queue function in primary, set by last queue initialized
 844                  * Tx queue may not initialized by primary process */
 845                 if (eth_dev->data->tx_queues) {
 846                         txq = eth_dev->data->tx_queues[eth_dev->data->nb_tx_queues-1];
 847                         ixgbe_set_tx_function(eth_dev, txq);
 848                 } else {
 849                         /* Use default TX function if we get here */
 850                         PMD_INIT_LOG(INFO, "No TX queues configured yet. "
 851                                            "Using default TX function.");
 852                 }
 853
 854                 ixgbe_set_rx_function(eth_dev);
 855
 856                 return 0;
 857         }
 858         pci_dev = eth_dev->pci_dev;
 859
 860         /* Vendor and Device ID need to be set before init of shared code */
 861         hw->device_id = pci_dev->id.device_id;
 862         hw->vendor_id = pci_dev->id.vendor_id;
 863         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 864         hw->allow_unsupported_sfp = 1;
 865
 866         /* Initialize the shared code (base driver) */
 867 #ifdef RTE_NIC_BYPASS
 868         diag = ixgbe_bypass_init_shared_code(hw);
 869 #else
 870         diag = ixgbe_init_shared_code(hw);
 871 #endif /* RTE_NIC_BYPASS */
 872
 873         if (diag != IXGBE_SUCCESS) {
 874                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
 875                 return -EIO;
 876         }
 877
 878         /* pick up the PCI bus settings for reporting later */
 879         ixgbe_get_bus_info(hw);
 880
 881         /* Unlock any pending hardware semaphore */
 882         ixgbe_swfw_lock_reset(hw);
 883
 884         /* Initialize DCB configuration*/
 885         memset(dcb_config, 0, sizeof(struct ixgbe_dcb_config));
 886         ixgbe_dcb_init(hw,dcb_config);
 887         /* Get Hardware Flow Control setting */
 888         hw->fc.requested_mode = ixgbe_fc_full;
 889         hw->fc.current_mode = ixgbe_fc_full;
 890         hw->fc.pause_time = IXGBE_FC_PAUSE;
 891         for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
 892                 hw->fc.low_water[i] = IXGBE_FC_LO;
 893                 hw->fc.high_water[i] = IXGBE_FC_HI;
 894         }
 895         hw->fc.send_xon = 1;
 896
 897         /* Make sure we have a good EEPROM before we read from it */
 898         diag = ixgbe_validate_eeprom_checksum(hw, &csum);
 899         if (diag != IXGBE_SUCCESS) {
 900                 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag);
 901                 return -EIO;
 902         }
 903
 904 #ifdef RTE_NIC_BYPASS
 905         diag = ixgbe_bypass_init_hw(hw);
 906 #else
 907         diag = ixgbe_init_hw(hw);
 908 #endif /* RTE_NIC_BYPASS */
 909
 910         /*
 911          * Devices with copper phys will fail to initialise if ixgbe_init_hw()
 912          * is called too soon after the kernel driver unbinding/binding occurs.
 913          * The failure occurs in ixgbe_identify_phy_generic() for all devices,
 914          * but for non-copper devies, ixgbe_identify_sfp_module_generic() is
 915          * also called. See ixgbe_identify_phy_82599(). The reason for the
 916          * failure is not known, and only occuts when virtualisation features
 917          * are disabled in the bios. A delay of 100ms  was found to be enough by
 918          * trial-and-error, and is doubled to be safe.
 919          */
 920         if (diag && (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper)) {
 921                 rte_delay_ms(200);
 922                 diag = ixgbe_init_hw(hw);
 923         }
 924
 925         if (diag == IXGBE_ERR_EEPROM_VERSION) {
 926                 PMD_INIT_LOG(ERR, "This device is a pre-production adapter/"
 927                     "LOM.  Please be aware there may be issues associated "
 928                     "with your hardware.");
 929                 PMD_INIT_LOG(ERR, "If you are experiencing problems "
 930                     "please contact your Intel or hardware representative "
 931                     "who provided you with this hardware.");
 932         } else if (diag == IXGBE_ERR_SFP_NOT_SUPPORTED)
 933                 PMD_INIT_LOG(ERR, "Unsupported SFP+ Module");
 934         if (diag) {
 935                 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag);
 936                 return -EIO;
 937         }
 938
 939         /* Reset the hw statistics */
 940         ixgbe_dev_stats_reset(eth_dev);
 941
 942         /* disable interrupt */
 943         ixgbe_disable_intr(hw);
 944
 945         /* reset mappings for queue statistics hw counters*/
 946         ixgbe_reset_qstat_mappings(hw);
 947
 948         /* Allocate memory for storing MAC addresses */
 949         eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 950                         hw->mac.num_rar_entries, 0);
 951         if (eth_dev->data->mac_addrs == NULL) {
 952                 PMD_INIT_LOG(ERR,
 953                         "Failed to allocate %u bytes needed to store "
 954                         "MAC addresses",
 955                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 956                 return -ENOMEM;
 957         }
 958         /* Copy the permanent MAC address */
 959         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 960                         &eth_dev->data->mac_addrs[0]);
 961
 962         /* Allocate memory for storing hash filter MAC addresses */
 963         eth_dev->data->hash_mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 964                         IXGBE_VMDQ_NUM_UC_MAC, 0);
 965         if (eth_dev->data->hash_mac_addrs == NULL) {
 966                 PMD_INIT_LOG(ERR,
 967                         "Failed to allocate %d bytes needed to store MAC addresses",
 968                         ETHER_ADDR_LEN * IXGBE_VMDQ_NUM_UC_MAC);
 969                 return -ENOMEM;
 970         }
 971
 972         /* initialize the vfta */
 973         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 974
 975         /* initialize the hw strip bitmap*/
 976         memset(hwstrip, 0, sizeof(*hwstrip));
 977
 978         /* initialize PF if max_vfs not zero */
 979         ixgbe_pf_host_init(eth_dev);
 980
 981         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 982         /* let hardware know driver is loaded */
 983         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 984         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 985         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
 986         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 987         IXGBE_WRITE_FLUSH(hw);
 988
 989         if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
 990                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d, SFP+: %d",
 991                              (int) hw->mac.type, (int) hw->phy.type,
 992                              (int) hw->phy.sfp_type);
 993         else
 994                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d",
 995                              (int) hw->mac.type, (int) hw->phy.type);
 996
 997         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
 998                         eth_dev->data->port_id, pci_dev->id.vendor_id,
 999                         pci_dev->id.device_id);
1000
1001         rte_intr_callback_register(&(pci_dev->intr_handle),
1002                 ixgbe_dev_interrupt_handler, (void *)eth_dev);
1003
1004         /* enable uio intr after callback register */
1005         rte_intr_enable(&(pci_dev->intr_handle));
1006
1007         /* enable support intr */
1008         ixgbe_enable_intr(eth_dev);
1009
1010         /* initialize 5tuple filter list */
1011         TAILQ_INIT(&filter_info->fivetuple_list);
1012         memset(filter_info->fivetuple_mask, 0,
1013                 sizeof(uint32_t) * IXGBE_5TUPLE_ARRAY_SIZE);
1014
1015         return 0;
1016 }
1017
1018
1019 /*
1020  * Negotiate mailbox API version with the PF.
1021  * After reset API version is always set to the basic one (ixgbe_mbox_api_10).
1022  * Then we try to negotiate starting with the most recent one.
1023  * If all negotiation attempts fail, then we will proceed with
1024  * the default one (ixgbe_mbox_api_10).
1025  */
1026 static void
1027 ixgbevf_negotiate_api(struct ixgbe_hw *hw)
1028 {
1029         int32_t i;
1030
1031         /* start with highest supported, proceed down */
1032         static const enum ixgbe_pfvf_api_rev sup_ver[] = {
1033                 ixgbe_mbox_api_11,
1034                 ixgbe_mbox_api_10,
1035         };
1036
1037         for (i = 0;
1038                         i != RTE_DIM(sup_ver) &&
1039                         ixgbevf_negotiate_api_version(hw, sup_ver[i]) != 0;
1040                         i++)
1041                 ;
1042 }
1043
1044 static void
1045 generate_random_mac_addr(struct ether_addr *mac_addr)
1046 {
1047         uint64_t random;
1048
1049         /* Set Organizationally Unique Identifier (OUI) prefix. */
1050         mac_addr->addr_bytes[0] = 0x00;
1051         mac_addr->addr_bytes[1] = 0x09;
1052         mac_addr->addr_bytes[2] = 0xC0;
1053         /* Force indication of locally assigned MAC address. */
1054         mac_addr->addr_bytes[0] |= ETHER_LOCAL_ADMIN_ADDR;
1055         /* Generate the last 3 bytes of the MAC address with a random number. */
1056         random = rte_rand();
1057         memcpy(&mac_addr->addr_bytes[3], &random, 3);
1058 }
1059
1060 /*
1061  * Virtual Function device init
1062  */
1063 static int
1064 eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev)
1065 {
1066         int diag;
1067         uint32_t tc, tcs;
1068         struct rte_pci_device *pci_dev;
1069         struct ixgbe_hw *hw =
1070                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1071         struct ixgbe_vfta * shadow_vfta =
1072                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
1073         struct ixgbe_hwstrip *hwstrip =
1074                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
1075         struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr;
1076
1077         PMD_INIT_FUNC_TRACE();
1078
1079         eth_dev->dev_ops = &ixgbevf_eth_dev_ops;
1080         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
1081         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
1082
1083         /* for secondary processes, we don't initialise any further as primary
1084          * has already done this work. Only check we don't need a different
1085          * RX function */
1086         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
1087                 if (eth_dev->data->scattered_rx)
1088                         eth_dev->rx_pkt_burst = ixgbe_recv_pkts_lro_single_alloc;
1089                 return 0;
1090         }
1091
1092         pci_dev = eth_dev->pci_dev;
1093
1094         hw->device_id = pci_dev->id.device_id;
1095         hw->vendor_id = pci_dev->id.vendor_id;
1096         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
1097
1098         /* initialize the vfta */
1099         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
1100
1101         /* initialize the hw strip bitmap*/
1102         memset(hwstrip, 0, sizeof(*hwstrip));
1103
1104         /* Initialize the shared code (base driver) */
1105         diag = ixgbe_init_shared_code(hw);
1106         if (diag != IXGBE_SUCCESS) {
1107                 PMD_INIT_LOG(ERR, "Shared code init failed for ixgbevf: %d", diag);
1108                 return -EIO;
1109         }
1110
1111         /* init_mailbox_params */
1112         hw->mbx.ops.init_params(hw);
1113
1114         /* Reset the hw statistics */
1115         ixgbevf_dev_stats_reset(eth_dev);
1116
1117         /* Disable the interrupts for VF */
1118         ixgbevf_intr_disable(hw);
1119
1120         hw->mac.num_rar_entries = 128; /* The MAX of the underlying PF */
1121         diag = hw->mac.ops.reset_hw(hw);
1122
1123         /*
1124          * The VF reset operation returns the IXGBE_ERR_INVALID_MAC_ADDR when
1125          * the underlying PF driver has not assigned a MAC address to the VF.
1126          * In this case, assign a random MAC address.
1127          */
1128         if ((diag != IXGBE_SUCCESS) && (diag != IXGBE_ERR_INVALID_MAC_ADDR)) {
1129                 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
1130                 return (diag);
1131         }
1132
1133         /* negotiate mailbox API version to use with the PF. */
1134         ixgbevf_negotiate_api(hw);
1135
1136         /* Get Rx/Tx queue count via mailbox, which is ready after reset_hw */
1137         ixgbevf_get_queues(hw, &tcs, &tc);
1138
1139         /* Allocate memory for storing MAC addresses */
1140         eth_dev->data->mac_addrs = rte_zmalloc("ixgbevf", ETHER_ADDR_LEN *
1141                         hw->mac.num_rar_entries, 0);
1142         if (eth_dev->data->mac_addrs == NULL) {
1143                 PMD_INIT_LOG(ERR,
1144                         "Failed to allocate %u bytes needed to store "
1145                         "MAC addresses",
1146                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
1147                 return -ENOMEM;
1148         }
1149
1150         /* Generate a random MAC address, if none was assigned by PF. */
1151         if (is_zero_ether_addr(perm_addr)) {
1152                 generate_random_mac_addr(perm_addr);
1153                 diag = ixgbe_set_rar_vf(hw, 1, perm_addr->addr_bytes, 0, 1);
1154                 if (diag) {
1155                         rte_free(eth_dev->data->mac_addrs);
1156                         eth_dev->data->mac_addrs = NULL;
1157                         return diag;
1158                 }
1159                 PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF");
1160                 PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address "
1161                              "%02x:%02x:%02x:%02x:%02x:%02x",
1162                              perm_addr->addr_bytes[0],
1163                              perm_addr->addr_bytes[1],
1164                              perm_addr->addr_bytes[2],
1165                              perm_addr->addr_bytes[3],
1166                              perm_addr->addr_bytes[4],
1167                              perm_addr->addr_bytes[5]);
1168         }
1169
1170         /* Copy the permanent MAC address */
1171         ether_addr_copy(perm_addr, &eth_dev->data->mac_addrs[0]);
1172
1173         /* reset the hardware with the new settings */
1174         diag = hw->mac.ops.start_hw(hw);
1175         switch (diag) {
1176                 case  0:
1177                         break;
1178
1179                 default:
1180                         PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
1181                         return (-EIO);
1182         }
1183
1184         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x mac.type=%s",
1185                      eth_dev->data->port_id, pci_dev->id.vendor_id,
1186                      pci_dev->id.device_id, "ixgbe_mac_82599_vf");
1187
1188         return 0;
1189 }
1190
1191 static struct eth_driver rte_ixgbe_pmd = {
1192         .pci_drv = {
1193                 .name = "rte_ixgbe_pmd",
1194                 .id_table = pci_id_ixgbe_map,
1195                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
1196         },
1197         .eth_dev_init = eth_ixgbe_dev_init,
1198         .dev_private_size = sizeof(struct ixgbe_adapter),
1199 };
1200
1201 /*
1202  * virtual function driver struct
1203  */
1204 static struct eth_driver rte_ixgbevf_pmd = {
1205         .pci_drv = {
1206                 .name = "rte_ixgbevf_pmd",
1207                 .id_table = pci_id_ixgbevf_map,
1208                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
1209         },
1210         .eth_dev_init = eth_ixgbevf_dev_init,
1211         .dev_private_size = sizeof(struct ixgbe_adapter),
1212 };
1213
1214 /*
1215  * Driver initialization routine.
1216  * Invoked once at EAL init time.
1217  * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
1218  */
1219 static int
1220 rte_ixgbe_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
1221 {
1222         PMD_INIT_FUNC_TRACE();
1223
1224         rte_eth_driver_register(&rte_ixgbe_pmd);
1225         return 0;
1226 }
1227
1228 /*
1229  * VF Driver initialization routine.
1230  * Invoked one at EAL init time.
1231  * Register itself as the [Virtual Poll Mode] Driver of PCI niantic devices.
1232  */
1233 static int
1234 rte_ixgbevf_pmd_init(const char *name __rte_unused, const char *param __rte_unused)
1235 {
1236         PMD_INIT_FUNC_TRACE();
1237
1238         rte_eth_driver_register(&rte_ixgbevf_pmd);
1239         return (0);
1240 }
1241
1242 static int
1243 ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1244 {
1245         struct ixgbe_hw *hw =
1246                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1247         struct ixgbe_vfta * shadow_vfta =
1248                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1249         uint32_t vfta;
1250         uint32_t vid_idx;
1251         uint32_t vid_bit;
1252
1253         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
1254         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
1255         vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(vid_idx));
1256         if (on)
1257                 vfta |= vid_bit;
1258         else
1259                 vfta &= ~vid_bit;
1260         IXGBE_WRITE_REG(hw, IXGBE_VFTA(vid_idx), vfta);
1261
1262         /* update local VFTA copy */
1263         shadow_vfta->vfta[vid_idx] = vfta;
1264
1265         return 0;
1266 }
1267
1268 static void
1269 ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
1270 {
1271         if (on)
1272                 ixgbe_vlan_hw_strip_enable(dev, queue);
1273         else
1274                 ixgbe_vlan_hw_strip_disable(dev, queue);
1275 }
1276
1277 static void
1278 ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
1279 {
1280         struct ixgbe_hw *hw =
1281                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1282
1283         /* Only the high 16-bits is valid */
1284         IXGBE_WRITE_REG(hw, IXGBE_EXVET, tpid << 16);
1285 }
1286
1287 void
1288 ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev)
1289 {
1290         struct ixgbe_hw *hw =
1291                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1292         uint32_t vlnctrl;
1293
1294         PMD_INIT_FUNC_TRACE();
1295
1296         /* Filter Table Disable */
1297         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1298         vlnctrl &= ~IXGBE_VLNCTRL_VFE;
1299
1300         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
1301 }
1302
1303 void
1304 ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1305 {
1306         struct ixgbe_hw *hw =
1307                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1308         struct ixgbe_vfta * shadow_vfta =
1309                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1310         uint32_t vlnctrl;
1311         uint16_t i;
1312
1313         PMD_INIT_FUNC_TRACE();
1314
1315         /* Filter Table Enable */
1316         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1317         vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
1318         vlnctrl |= IXGBE_VLNCTRL_VFE;
1319
1320         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
1321
1322         /* write whatever is in local vfta copy */
1323         for (i = 0; i < IXGBE_VFTA_SIZE; i++)
1324                 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), shadow_vfta->vfta[i]);
1325 }
1326
1327 static void
1328 ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on)
1329 {
1330         struct ixgbe_hwstrip *hwstrip =
1331                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(dev->data->dev_private);
1332
1333         if(queue >= IXGBE_MAX_RX_QUEUE_NUM)
1334                 return;
1335
1336         if (on)
1337                 IXGBE_SET_HWSTRIP(hwstrip, queue);
1338         else
1339                 IXGBE_CLEAR_HWSTRIP(hwstrip, queue);
1340 }
1341
1342 static void
1343 ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue)
1344 {
1345         struct ixgbe_hw *hw =
1346                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1347         uint32_t ctrl;
1348
1349         PMD_INIT_FUNC_TRACE();
1350
1351         if (hw->mac.type == ixgbe_mac_82598EB) {
1352                 /* No queue level support */
1353                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
1354                 return;
1355         }
1356         else {
1357                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1358                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1359                 ctrl &= ~IXGBE_RXDCTL_VME;
1360                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
1361         }
1362         /* record those setting for HW strip per queue */
1363         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 0);
1364 }
1365
1366 static void
1367 ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue)
1368 {
1369         struct ixgbe_hw *hw =
1370                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1371         uint32_t ctrl;
1372
1373         PMD_INIT_FUNC_TRACE();
1374
1375         if (hw->mac.type == ixgbe_mac_82598EB) {
1376                 /* No queue level supported */
1377                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
1378                 return;
1379         }
1380         else {
1381                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1382                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1383                 ctrl |= IXGBE_RXDCTL_VME;
1384                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
1385         }
1386         /* record those setting for HW strip per queue */
1387         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 1);
1388 }
1389
1390 void
1391 ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev)
1392 {
1393         struct ixgbe_hw *hw =
1394                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1395         uint32_t ctrl;
1396         uint16_t i;
1397
1398         PMD_INIT_FUNC_TRACE();
1399
1400         if (hw->mac.type == ixgbe_mac_82598EB) {
1401                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1402                 ctrl &= ~IXGBE_VLNCTRL_VME;
1403                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1404         }
1405         else {
1406                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1407                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1408                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1409                         ctrl &= ~IXGBE_RXDCTL_VME;
1410                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1411
1412                         /* record those setting for HW strip per queue */
1413                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 0);
1414                 }
1415         }
1416 }
1417
1418 void
1419 ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev)
1420 {
1421         struct ixgbe_hw *hw =
1422                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1423         uint32_t ctrl;
1424         uint16_t i;
1425
1426         PMD_INIT_FUNC_TRACE();
1427
1428         if (hw->mac.type == ixgbe_mac_82598EB) {
1429                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1430                 ctrl |= IXGBE_VLNCTRL_VME;
1431                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1432         }
1433         else {
1434                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1435                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1436                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1437                         ctrl |= IXGBE_RXDCTL_VME;
1438                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1439
1440                         /* record those setting for HW strip per queue */
1441                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 1);
1442                 }
1443         }
1444 }
1445
1446 static void
1447 ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1448 {
1449         struct ixgbe_hw *hw =
1450                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1451         uint32_t ctrl;
1452
1453         PMD_INIT_FUNC_TRACE();
1454
1455         /* DMATXCTRL: Geric Double VLAN Disable */
1456         ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1457         ctrl &= ~IXGBE_DMATXCTL_GDV;
1458         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1459
1460         /* CTRL_EXT: Global Double VLAN Disable */
1461         ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1462         ctrl &= ~IXGBE_EXTENDED_VLAN;
1463         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1464
1465 }
1466
1467 static void
1468 ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1469 {
1470         struct ixgbe_hw *hw =
1471                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1472         uint32_t ctrl;
1473
1474         PMD_INIT_FUNC_TRACE();
1475
1476         /* DMATXCTRL: Geric Double VLAN Enable */
1477         ctrl  = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1478         ctrl |= IXGBE_DMATXCTL_GDV;
1479         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1480
1481         /* CTRL_EXT: Global Double VLAN Enable */
1482         ctrl  = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1483         ctrl |= IXGBE_EXTENDED_VLAN;
1484         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1485
1486         /*
1487          * VET EXT field in the EXVET register = 0x8100 by default
1488          * So no need to change. Same to VT field of DMATXCTL register
1489          */
1490 }
1491
1492 static void
1493 ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1494 {
1495         if(mask & ETH_VLAN_STRIP_MASK){
1496                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1497                         ixgbe_vlan_hw_strip_enable_all(dev);
1498                 else
1499                         ixgbe_vlan_hw_strip_disable_all(dev);
1500         }
1501
1502         if(mask & ETH_VLAN_FILTER_MASK){
1503                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1504                         ixgbe_vlan_hw_filter_enable(dev);
1505                 else
1506                         ixgbe_vlan_hw_filter_disable(dev);
1507         }
1508
1509         if(mask & ETH_VLAN_EXTEND_MASK){
1510                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1511                         ixgbe_vlan_hw_extend_enable(dev);
1512                 else
1513                         ixgbe_vlan_hw_extend_disable(dev);
1514         }
1515 }
1516
1517 static void
1518 ixgbe_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1519 {
1520         struct ixgbe_hw *hw =
1521                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1522         /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
1523         uint32_t vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1524         vlanctrl |= IXGBE_VLNCTRL_VFE ; /* enable vlan filters */
1525         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
1526 }
1527
1528 static int
1529 ixgbe_dev_configure(struct rte_eth_dev *dev)
1530 {
1531         struct ixgbe_interrupt *intr =
1532                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1533         struct ixgbe_adapter *adapter =
1534                 (struct ixgbe_adapter *)dev->data->dev_private;
1535
1536         PMD_INIT_FUNC_TRACE();
1537
1538         /* set flag to update link status after init */
1539         intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1540
1541         /*
1542          * Initialize to TRUE. If any of Rx queues doesn't meet the bulk
1543          * allocation or vector Rx preconditions we will reset it.
1544          */
1545         adapter->rx_bulk_alloc_allowed = true;
1546         adapter->rx_vec_allowed = true;
1547
1548         return 0;
1549 }
1550
1551 /*
1552  * Configure device link speed and setup link.
1553  * It returns 0 on success.
1554  */
1555 static int
1556 ixgbe_dev_start(struct rte_eth_dev *dev)
1557 {
1558         struct ixgbe_hw *hw =
1559                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1560         struct ixgbe_vf_info *vfinfo =
1561                 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private);
1562         int err, link_up = 0, negotiate = 0;
1563         uint32_t speed = 0;
1564         int mask = 0;
1565         int status;
1566         uint16_t vf, idx;
1567
1568         PMD_INIT_FUNC_TRACE();
1569
1570         /* IXGBE devices don't support half duplex */
1571         if ((dev->data->dev_conf.link_duplex != ETH_LINK_AUTONEG_DUPLEX) &&
1572                         (dev->data->dev_conf.link_duplex != ETH_LINK_FULL_DUPLEX)) {
1573                 PMD_INIT_LOG(ERR, "Invalid link_duplex (%hu) for port %hhu",
1574                              dev->data->dev_conf.link_duplex,
1575                              dev->data->port_id);
1576                 return -EINVAL;
1577         }
1578
1579         /* stop adapter */
1580         hw->adapter_stopped = FALSE;
1581         ixgbe_stop_adapter(hw);
1582
1583         /* reinitialize adapter
1584          * this calls reset and start */
1585         status = ixgbe_pf_reset_hw(hw);
1586         if (status != 0)
1587                 return -1;
1588         hw->mac.ops.start_hw(hw);
1589         hw->mac.get_link_status = true;
1590
1591         /* configure PF module if SRIOV enabled */
1592         ixgbe_pf_host_configure(dev);
1593
1594         /* initialize transmission unit */
1595         ixgbe_dev_tx_init(dev);
1596
1597         /* This can fail when allocating mbufs for descriptor rings */
1598         err = ixgbe_dev_rx_init(dev);
1599         if (err) {
1600                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
1601                 goto error;
1602         }
1603
1604         err = ixgbe_dev_rxtx_start(dev);
1605         if (err < 0) {
1606                 PMD_INIT_LOG(ERR, "Unable to start rxtx queues");
1607                 goto error;
1608         }
1609
1610         /* Skip link setup if loopback mode is enabled for 82599. */
1611         if (hw->mac.type == ixgbe_mac_82599EB &&
1612                         dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_82599_TX_RX)
1613                 goto skip_link_setup;
1614
1615         if (ixgbe_is_sfp(hw) && hw->phy.multispeed_fiber) {
1616                 err = hw->mac.ops.setup_sfp(hw);
1617                 if (err)
1618                         goto error;
1619         }
1620
1621         if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
1622                 /* Turn on the copper */
1623                 ixgbe_set_phy_power(hw, true);
1624         } else {
1625                 /* Turn on the laser */
1626                 ixgbe_enable_tx_laser(hw);
1627         }
1628
1629         err = ixgbe_check_link(hw, &speed, &link_up, 0);
1630         if (err)
1631                 goto error;
1632         dev->data->dev_link.link_status = link_up;
1633
1634         err = ixgbe_get_link_capabilities(hw, &speed, &negotiate);
1635         if (err)
1636                 goto error;
1637
1638         switch(dev->data->dev_conf.link_speed) {
1639         case ETH_LINK_SPEED_AUTONEG:
1640                 speed = (hw->mac.type != ixgbe_mac_82598EB) ?
1641                                 IXGBE_LINK_SPEED_82599_AUTONEG :
1642                                 IXGBE_LINK_SPEED_82598_AUTONEG;
1643                 break;
1644         case ETH_LINK_SPEED_100:
1645                 /*
1646                  * Invalid for 82598 but error will be detected by
1647                  * ixgbe_setup_link()
1648                  */
1649                 speed = IXGBE_LINK_SPEED_100_FULL;
1650                 break;
1651         case ETH_LINK_SPEED_1000:
1652                 speed = IXGBE_LINK_SPEED_1GB_FULL;
1653                 break;
1654         case ETH_LINK_SPEED_10000:
1655                 speed = IXGBE_LINK_SPEED_10GB_FULL;
1656                 break;
1657         default:
1658                 PMD_INIT_LOG(ERR, "Invalid link_speed (%hu) for port %hhu",
1659                              dev->data->dev_conf.link_speed,
1660                              dev->data->port_id);
1661                 goto error;
1662         }
1663
1664         err = ixgbe_setup_link(hw, speed, link_up);
1665         if (err)
1666                 goto error;
1667
1668 skip_link_setup:
1669
1670         /* check if lsc interrupt is enabled */
1671         if (dev->data->dev_conf.intr_conf.lsc != 0)
1672                 ixgbe_dev_lsc_interrupt_setup(dev);
1673
1674         /* resume enabled intr since hw reset */
1675         ixgbe_enable_intr(dev);
1676
1677         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
1678                 ETH_VLAN_EXTEND_MASK;
1679         ixgbe_vlan_offload_set(dev, mask);
1680
1681         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
1682                 /* Enable vlan filtering for VMDq */
1683                 ixgbe_vmdq_vlan_hw_filter_enable(dev);
1684         }
1685
1686         /* Configure DCB hw */
1687         ixgbe_configure_dcb(dev);
1688
1689         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) {
1690                 err = ixgbe_fdir_configure(dev);
1691                 if (err)
1692                         goto error;
1693         }
1694
1695         /* Restore vf rate limit */
1696         if (vfinfo != NULL) {
1697                 for (vf = 0; vf < dev->pci_dev->max_vfs; vf++)
1698                         for (idx = 0; idx < IXGBE_MAX_QUEUE_NUM_PER_VF; idx++)
1699                                 if (vfinfo[vf].tx_rate[idx] != 0)
1700                                         ixgbe_set_vf_rate_limit(dev, vf,
1701                                                 vfinfo[vf].tx_rate[idx],
1702                                                 1 << idx);
1703         }
1704
1705         ixgbe_restore_statistics_mapping(dev);
1706
1707         return (0);
1708
1709 error:
1710         PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err);
1711         ixgbe_dev_clear_queues(dev);
1712         return -EIO;
1713 }
1714
1715 /*
1716  * Stop device: disable rx and tx functions to allow for reconfiguring.
1717  */
1718 static void
1719 ixgbe_dev_stop(struct rte_eth_dev *dev)
1720 {
1721         struct rte_eth_link link;
1722         struct ixgbe_hw *hw =
1723                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1724         struct ixgbe_vf_info *vfinfo =
1725                 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private);
1726         struct ixgbe_filter_info *filter_info =
1727                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
1728         struct ixgbe_5tuple_filter *p_5tuple, *p_5tuple_next;
1729         int vf;
1730
1731         PMD_INIT_FUNC_TRACE();
1732
1733         /* disable interrupts */
1734         ixgbe_disable_intr(hw);
1735
1736         /* reset the NIC */
1737         ixgbe_pf_reset_hw(hw);
1738         hw->adapter_stopped = FALSE;
1739
1740         /* stop adapter */
1741         ixgbe_stop_adapter(hw);
1742
1743         for (vf = 0; vfinfo != NULL &&
1744                      vf < dev->pci_dev->max_vfs; vf++)
1745                 vfinfo[vf].clear_to_send = false;
1746
1747         if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
1748                 /* Turn off the copper */
1749                 ixgbe_set_phy_power(hw, false);
1750         } else {
1751                 /* Turn off the laser */
1752                 ixgbe_disable_tx_laser(hw);
1753         }
1754
1755         ixgbe_dev_clear_queues(dev);
1756
1757         /* Clear stored conf */
1758         dev->data->scattered_rx = 0;
1759         dev->data->lro = 0;
1760
1761         /* Clear recorded link status */
1762         memset(&link, 0, sizeof(link));
1763         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1764
1765         /* Remove all ntuple filters of the device */
1766         for (p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list);
1767              p_5tuple != NULL; p_5tuple = p_5tuple_next) {
1768                 p_5tuple_next = TAILQ_NEXT(p_5tuple, entries);
1769                 TAILQ_REMOVE(&filter_info->fivetuple_list,
1770                              p_5tuple, entries);
1771                 rte_free(p_5tuple);
1772         }
1773         memset(filter_info->fivetuple_mask, 0,
1774                 sizeof(uint32_t) * IXGBE_5TUPLE_ARRAY_SIZE);
1775
1776 }
1777
1778 /*
1779  * Set device link up: enable tx.
1780  */
1781 static int
1782 ixgbe_dev_set_link_up(struct rte_eth_dev *dev)
1783 {
1784         struct ixgbe_hw *hw =
1785                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1786         if (hw->mac.type == ixgbe_mac_82599EB) {
1787 #ifdef RTE_NIC_BYPASS
1788                 if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) {
1789                         /* Not suported in bypass mode */
1790                         PMD_INIT_LOG(ERR, "Set link up is not supported "
1791                                      "by device id 0x%x", hw->device_id);
1792                         return -ENOTSUP;
1793                 }
1794 #endif
1795         }
1796
1797         if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
1798                 /* Turn on the copper */
1799                 ixgbe_set_phy_power(hw, true);
1800         } else {
1801                 /* Turn on the laser */
1802                 ixgbe_enable_tx_laser(hw);
1803         }
1804
1805         return 0;
1806 }
1807
1808 /*
1809  * Set device link down: disable tx.
1810  */
1811 static int
1812 ixgbe_dev_set_link_down(struct rte_eth_dev *dev)
1813 {
1814         struct ixgbe_hw *hw =
1815                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1816         if (hw->mac.type == ixgbe_mac_82599EB) {
1817 #ifdef RTE_NIC_BYPASS
1818                 if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) {
1819                         /* Not suported in bypass mode */
1820                         PMD_INIT_LOG(ERR, "Set link down is not supported "
1821                                      "by device id 0x%x", hw->device_id);
1822                         return -ENOTSUP;
1823                 }
1824 #endif
1825         }
1826
1827         if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
1828                 /* Turn off the copper */
1829                 ixgbe_set_phy_power(hw, false);
1830         } else {
1831                 /* Turn off the laser */
1832                 ixgbe_disable_tx_laser(hw);
1833         }
1834
1835         return 0;
1836 }
1837
1838 /*
1839  * Reest and stop device.
1840  */
1841 static void
1842 ixgbe_dev_close(struct rte_eth_dev *dev)
1843 {
1844         struct ixgbe_hw *hw =
1845                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1846
1847         PMD_INIT_FUNC_TRACE();
1848
1849         ixgbe_pf_reset_hw(hw);
1850
1851         ixgbe_dev_stop(dev);
1852         hw->adapter_stopped = 1;
1853
1854         ixgbe_disable_pcie_master(hw);
1855
1856         /* reprogram the RAR[0] in case user changed it. */
1857         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
1858 }
1859
1860 static void
1861 ixgbe_read_stats_registers(struct ixgbe_hw *hw, struct ixgbe_hw_stats
1862                                                    *hw_stats, uint64_t *total_missed_rx,
1863                                                    uint64_t *total_qbrc, uint64_t *total_qprc,
1864                                                    uint64_t *rxnfgpc, uint64_t *txdgpc,
1865                                                    uint64_t *total_qprdc)
1866 {
1867         uint32_t bprc, lxon, lxoff, total;
1868         unsigned i;
1869
1870         hw_stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
1871         hw_stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
1872         hw_stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
1873         hw_stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
1874
1875         for (i = 0; i < 8; i++) {
1876                 uint32_t mp;
1877                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
1878                 /* global total per queue */
1879                 hw_stats->mpc[i] += mp;
1880                 /* Running comprehensive total for stats display */
1881                 *total_missed_rx += hw_stats->mpc[i];
1882                 if (hw->mac.type == ixgbe_mac_82598EB)
1883                         hw_stats->rnbc[i] +=
1884                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
1885                 hw_stats->pxontxc[i] +=
1886                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
1887                 hw_stats->pxonrxc[i] +=
1888                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
1889                 hw_stats->pxofftxc[i] +=
1890                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
1891                 hw_stats->pxoffrxc[i] +=
1892                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
1893                 hw_stats->pxon2offc[i] +=
1894                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
1895         }
1896         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1897                 hw_stats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
1898                 hw_stats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
1899                 hw_stats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
1900                 hw_stats->qbrc[i] +=
1901                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32);
1902                 hw_stats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
1903                 hw_stats->qbtc[i] +=
1904                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32);
1905                 *total_qprdc += hw_stats->qprdc[i] +=
1906                                 IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
1907
1908                 *total_qprc += hw_stats->qprc[i];
1909                 *total_qbrc += hw_stats->qbrc[i];
1910         }
1911         hw_stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
1912         hw_stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
1913         hw_stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
1914
1915         /* Note that gprc counts missed packets */
1916         hw_stats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
1917         *rxnfgpc += IXGBE_READ_REG(hw, IXGBE_RXNFGPC);
1918         *txdgpc += IXGBE_READ_REG(hw, IXGBE_TXDGPC);
1919
1920         if (hw->mac.type != ixgbe_mac_82598EB) {
1921                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
1922                 hw_stats->gorc += ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
1923                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
1924                 hw_stats->gotc += ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
1925                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
1926                 hw_stats->tor += ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
1927                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
1928                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
1929         } else {
1930                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
1931                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
1932                 /* 82598 only has a counter in the high register */
1933                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
1934                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
1935                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
1936         }
1937
1938         /*
1939          * Workaround: mprc hardware is incorrectly counting
1940          * broadcasts, so for now we subtract those.
1941          */
1942         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
1943         hw_stats->bprc += bprc;
1944         hw_stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
1945         if (hw->mac.type == ixgbe_mac_82598EB)
1946                 hw_stats->mprc -= bprc;
1947
1948         hw_stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
1949         hw_stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
1950         hw_stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
1951         hw_stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
1952         hw_stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
1953         hw_stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
1954
1955         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
1956         hw_stats->lxontxc += lxon;
1957         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
1958         hw_stats->lxofftxc += lxoff;
1959         total = lxon + lxoff;
1960
1961         hw_stats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
1962         hw_stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
1963         hw_stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
1964         hw_stats->gptc -= total;
1965         hw_stats->mptc -= total;
1966         hw_stats->ptc64 -= total;
1967         hw_stats->gotc -= total * ETHER_MIN_LEN;
1968
1969         hw_stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
1970         hw_stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
1971         hw_stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
1972         hw_stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
1973         hw_stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
1974         hw_stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
1975         hw_stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
1976         hw_stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
1977         hw_stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
1978         hw_stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
1979         hw_stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
1980         hw_stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
1981         hw_stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
1982         hw_stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
1983         hw_stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
1984         hw_stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);
1985         hw_stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
1986         hw_stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
1987         /* Only read FCOE on 82599 */
1988         if (hw->mac.type != ixgbe_mac_82598EB) {
1989                 hw_stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
1990                 hw_stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
1991                 hw_stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
1992                 hw_stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
1993                 hw_stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
1994         }
1995
1996         /* Flow Director Stats registers */
1997         hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
1998         hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
1999 }
2000
2001 /*
2002  * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c
2003  */
2004 static void
2005 ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
2006 {
2007         struct ixgbe_hw *hw =
2008                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2009         struct ixgbe_hw_stats *hw_stats =
2010                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2011         uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc;
2012         uint64_t rxnfgpc, txdgpc;
2013         unsigned i;
2014
2015         total_missed_rx = 0;
2016         total_qbrc = 0;
2017         total_qprc = 0;
2018         total_qprdc = 0;
2019         rxnfgpc = 0;
2020         txdgpc = 0;
2021
2022         ixgbe_read_stats_registers(hw, hw_stats, &total_missed_rx, &total_qbrc,
2023                         &total_qprc, &rxnfgpc, &txdgpc, &total_qprdc);
2024
2025         if (stats == NULL)
2026                 return;
2027
2028         /* Fill out the rte_eth_stats statistics structure */
2029         stats->ipackets = total_qprc;
2030         stats->ibytes = total_qbrc;
2031         stats->opackets = hw_stats->gptc;
2032         stats->obytes = hw_stats->gotc;
2033
2034         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
2035                 stats->q_ipackets[i] = hw_stats->qprc[i];
2036                 stats->q_opackets[i] = hw_stats->qptc[i];
2037                 stats->q_ibytes[i] = hw_stats->qbrc[i];
2038                 stats->q_obytes[i] = hw_stats->qbtc[i];
2039                 stats->q_errors[i] = hw_stats->qprdc[i];
2040         }
2041
2042         /* Rx Errors */
2043         stats->ierrors  = hw_stats->crcerrs +
2044                           hw_stats->rlec +
2045                           hw_stats->ruc +
2046                           hw_stats->roc +
2047                           total_missed_rx +
2048                           hw_stats->illerrc + hw_stats->errbc;
2049
2050         /* Tx Errors */
2051         /*txdgpc: packets that are DMA'ed*/
2052         /*gptc: packets that are sent*/
2053         stats->oerrors  = txdgpc - hw_stats->gptc;
2054 }
2055
2056 static void
2057 ixgbe_dev_stats_reset(struct rte_eth_dev *dev)
2058 {
2059         struct ixgbe_hw_stats *stats =
2060                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2061
2062         /* HW registers are cleared on read */
2063         ixgbe_dev_stats_get(dev, NULL);
2064
2065         /* Reset software totals */
2066         memset(stats, 0, sizeof(*stats));
2067 }
2068
2069 static int
2070 ixgbe_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstats *xstats,
2071                                          unsigned n)
2072 {
2073         struct ixgbe_hw *hw =
2074                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2075         struct ixgbe_hw_stats *hw_stats =
2076                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2077         uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc;
2078         uint64_t rxnfgpc, txdgpc;
2079         unsigned i, count = IXGBE_NB_XSTATS;
2080
2081         if (n < count)
2082                 return count;
2083
2084         total_missed_rx = 0;
2085         total_qbrc = 0;
2086         total_qprc = 0;
2087         total_qprdc = 0;
2088         rxnfgpc = 0;
2089         txdgpc = 0;
2090
2091         ixgbe_read_stats_registers(hw, hw_stats, &total_missed_rx, &total_qbrc,
2092                                                            &total_qprc, &rxnfgpc, &txdgpc, &total_qprdc);
2093
2094         /* If this is a reset xstats is NULL, and we have cleared the
2095          * registers by reading them.
2096          */
2097         if (!xstats)
2098                 return 0;
2099
2100         /* Extended stats */
2101         for (i = 0; i < IXGBE_NB_XSTATS; i++) {
2102                 snprintf(xstats[i].name, sizeof(xstats[i].name),
2103                                 "%s", rte_ixgbe_stats_strings[i].name);
2104                 xstats[i].value = *(uint64_t *)(((char *)hw_stats) +
2105                                                         rte_ixgbe_stats_strings[i].offset);
2106         }
2107
2108         return count;
2109 }
2110
2111 static void
2112 ixgbe_dev_xstats_reset(struct rte_eth_dev *dev)
2113 {
2114         struct ixgbe_hw_stats *stats =
2115                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2116
2117         /* HW registers are cleared on read */
2118         ixgbe_dev_xstats_get(dev, NULL, IXGBE_NB_XSTATS);
2119
2120         /* Reset software totals */
2121         memset(stats, 0, sizeof(*stats));
2122 }
2123
2124 static void
2125 ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
2126 {
2127         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2128         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
2129                           IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2130
2131         /* Good Rx packet, include VF loopback */
2132         UPDATE_VF_STAT(IXGBE_VFGPRC,
2133             hw_stats->last_vfgprc, hw_stats->vfgprc);
2134
2135         /* Good Rx octets, include VF loopback */
2136         UPDATE_VF_STAT_36BIT(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
2137             hw_stats->last_vfgorc, hw_stats->vfgorc);
2138
2139         /* Good Tx packet, include VF loopback */
2140         UPDATE_VF_STAT(IXGBE_VFGPTC,
2141             hw_stats->last_vfgptc, hw_stats->vfgptc);
2142
2143         /* Good Tx octets, include VF loopback */
2144         UPDATE_VF_STAT_36BIT(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
2145             hw_stats->last_vfgotc, hw_stats->vfgotc);
2146
2147         /* Rx Multicst Packet */
2148         UPDATE_VF_STAT(IXGBE_VFMPRC,
2149             hw_stats->last_vfmprc, hw_stats->vfmprc);
2150
2151         if (stats == NULL)
2152                 return;
2153
2154         stats->ipackets = hw_stats->vfgprc;
2155         stats->ibytes = hw_stats->vfgorc;
2156         stats->opackets = hw_stats->vfgptc;
2157         stats->obytes = hw_stats->vfgotc;
2158         stats->imcasts = hw_stats->vfmprc;
2159         /* stats->imcasts should be removed as imcasts is deprecated */
2160 }
2161
2162 static void
2163 ixgbevf_dev_stats_reset(struct rte_eth_dev *dev)
2164 {
2165         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
2166                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2167
2168         /* Sync HW register to the last stats */
2169         ixgbevf_dev_stats_get(dev, NULL);
2170
2171         /* reset HW current stats*/
2172         hw_stats->vfgprc = 0;
2173         hw_stats->vfgorc = 0;
2174         hw_stats->vfgptc = 0;
2175         hw_stats->vfgotc = 0;
2176         hw_stats->vfmprc = 0;
2177
2178 }
2179
2180 static void
2181 ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
2182 {
2183         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2184
2185         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
2186         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
2187         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL register */
2188         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */
2189         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
2190         dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC;
2191         dev_info->max_vfs = dev->pci_dev->max_vfs;
2192         if (hw->mac.type == ixgbe_mac_82598EB)
2193                 dev_info->max_vmdq_pools = ETH_16_POOLS;
2194         else
2195                 dev_info->max_vmdq_pools = ETH_64_POOLS;
2196         dev_info->vmdq_queue_num = dev_info->max_rx_queues;
2197         dev_info->rx_offload_capa =
2198                 DEV_RX_OFFLOAD_VLAN_STRIP |
2199                 DEV_RX_OFFLOAD_IPV4_CKSUM |
2200                 DEV_RX_OFFLOAD_UDP_CKSUM  |
2201                 DEV_RX_OFFLOAD_TCP_CKSUM;
2202
2203         /*
2204          * RSC is only supported by 82599 and x540 PF devices in a non-SR-IOV
2205          * mode.
2206          */
2207         if ((hw->mac.type == ixgbe_mac_82599EB ||
2208              hw->mac.type == ixgbe_mac_X540) &&
2209             !RTE_ETH_DEV_SRIOV(dev).active)
2210                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_LRO;
2211
2212         dev_info->tx_offload_capa =
2213                 DEV_TX_OFFLOAD_VLAN_INSERT |
2214                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
2215                 DEV_TX_OFFLOAD_UDP_CKSUM   |
2216                 DEV_TX_OFFLOAD_TCP_CKSUM   |
2217                 DEV_TX_OFFLOAD_SCTP_CKSUM  |
2218                 DEV_TX_OFFLOAD_TCP_TSO;
2219
2220         dev_info->default_rxconf = (struct rte_eth_rxconf) {
2221                 .rx_thresh = {
2222                         .pthresh = IXGBE_DEFAULT_RX_PTHRESH,
2223                         .hthresh = IXGBE_DEFAULT_RX_HTHRESH,
2224                         .wthresh = IXGBE_DEFAULT_RX_WTHRESH,
2225                 },
2226                 .rx_free_thresh = IXGBE_DEFAULT_RX_FREE_THRESH,
2227                 .rx_drop_en = 0,
2228         };
2229
2230         dev_info->default_txconf = (struct rte_eth_txconf) {
2231                 .tx_thresh = {
2232                         .pthresh = IXGBE_DEFAULT_TX_PTHRESH,
2233                         .hthresh = IXGBE_DEFAULT_TX_HTHRESH,
2234                         .wthresh = IXGBE_DEFAULT_TX_WTHRESH,
2235                 },
2236                 .tx_free_thresh = IXGBE_DEFAULT_TX_FREE_THRESH,
2237                 .tx_rs_thresh = IXGBE_DEFAULT_TX_RSBIT_THRESH,
2238                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
2239                                 ETH_TXQ_FLAGS_NOOFFLOADS,
2240         };
2241         dev_info->hash_key_size = IXGBE_HKEY_MAX_INDEX * sizeof(uint32_t);
2242         dev_info->reta_size = ETH_RSS_RETA_SIZE_128;
2243         dev_info->flow_type_rss_offloads = IXGBE_RSS_OFFLOAD_ALL;
2244 }
2245
2246 static void
2247 ixgbevf_dev_info_get(struct rte_eth_dev *dev,
2248                      struct rte_eth_dev_info *dev_info)
2249 {
2250         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2251
2252         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
2253         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
2254         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL reg */
2255         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS reg */
2256         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
2257         dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC;
2258         dev_info->max_vfs = dev->pci_dev->max_vfs;
2259         if (hw->mac.type == ixgbe_mac_82598EB)
2260                 dev_info->max_vmdq_pools = ETH_16_POOLS;
2261         else
2262                 dev_info->max_vmdq_pools = ETH_64_POOLS;
2263         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
2264                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
2265                                 DEV_RX_OFFLOAD_UDP_CKSUM  |
2266                                 DEV_RX_OFFLOAD_TCP_CKSUM;
2267         dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
2268                                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
2269                                 DEV_TX_OFFLOAD_UDP_CKSUM   |
2270                                 DEV_TX_OFFLOAD_TCP_CKSUM   |
2271                                 DEV_TX_OFFLOAD_SCTP_CKSUM;
2272
2273         dev_info->default_rxconf = (struct rte_eth_rxconf) {
2274                 .rx_thresh = {
2275                         .pthresh = IXGBE_DEFAULT_RX_PTHRESH,
2276                         .hthresh = IXGBE_DEFAULT_RX_HTHRESH,
2277                         .wthresh = IXGBE_DEFAULT_RX_WTHRESH,
2278                 },
2279                 .rx_free_thresh = IXGBE_DEFAULT_RX_FREE_THRESH,
2280                 .rx_drop_en = 0,
2281         };
2282
2283         dev_info->default_txconf = (struct rte_eth_txconf) {
2284                 .tx_thresh = {
2285                         .pthresh = IXGBE_DEFAULT_TX_PTHRESH,
2286                         .hthresh = IXGBE_DEFAULT_TX_HTHRESH,
2287                         .wthresh = IXGBE_DEFAULT_TX_WTHRESH,
2288                 },
2289                 .tx_free_thresh = IXGBE_DEFAULT_TX_FREE_THRESH,
2290                 .tx_rs_thresh = IXGBE_DEFAULT_TX_RSBIT_THRESH,
2291                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
2292                                 ETH_TXQ_FLAGS_NOOFFLOADS,
2293         };
2294 }
2295
2296 /* return 0 means link status changed, -1 means not changed */
2297 static int
2298 ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
2299 {
2300         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2301         struct rte_eth_link link, old;
2302         ixgbe_link_speed link_speed = IXGBE_LINK_SPEED_UNKNOWN;
2303         int link_up;
2304         int diag;
2305
2306         link.link_status = 0;
2307         link.link_speed = 0;
2308         link.link_duplex = 0;
2309         memset(&old, 0, sizeof(old));
2310         rte_ixgbe_dev_atomic_read_link_status(dev, &old);
2311
2312         hw->mac.get_link_status = true;
2313
2314         /* check if it needs to wait to complete, if lsc interrupt is enabled */
2315         if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
2316                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 0);
2317         else
2318                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 1);
2319
2320         if (diag != 0) {
2321                 link.link_speed = ETH_LINK_SPEED_100;
2322                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
2323                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
2324                 if (link.link_status == old.link_status)
2325                         return -1;
2326                 return 0;
2327         }
2328
2329         if (link_up == 0) {
2330                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
2331                 if (link.link_status == old.link_status)
2332                         return -1;
2333                 return 0;
2334         }
2335         link.link_status = 1;
2336         link.link_duplex = ETH_LINK_FULL_DUPLEX;
2337
2338         switch (link_speed) {
2339         default:
2340         case IXGBE_LINK_SPEED_UNKNOWN:
2341                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
2342                 link.link_speed = ETH_LINK_SPEED_100;
2343                 break;
2344
2345         case IXGBE_LINK_SPEED_100_FULL:
2346                 link.link_speed = ETH_LINK_SPEED_100;
2347                 break;
2348
2349         case IXGBE_LINK_SPEED_1GB_FULL:
2350                 link.link_speed = ETH_LINK_SPEED_1000;
2351                 break;
2352
2353         case IXGBE_LINK_SPEED_10GB_FULL:
2354                 link.link_speed = ETH_LINK_SPEED_10000;
2355                 break;
2356         }
2357         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
2358
2359         if (link.link_status == old.link_status)
2360                 return -1;
2361
2362         return 0;
2363 }
2364
2365 static void
2366 ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev)
2367 {
2368         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2369         uint32_t fctrl;
2370
2371         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2372         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
2373         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2374 }
2375
2376 static void
2377 ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev)
2378 {
2379         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2380         uint32_t fctrl;
2381
2382         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2383         fctrl &= (~IXGBE_FCTRL_UPE);
2384         if (dev->data->all_multicast == 1)
2385                 fctrl |= IXGBE_FCTRL_MPE;
2386         else
2387                 fctrl &= (~IXGBE_FCTRL_MPE);
2388         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2389 }
2390
2391 static void
2392 ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev)
2393 {
2394         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2395         uint32_t fctrl;
2396
2397         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2398         fctrl |= IXGBE_FCTRL_MPE;
2399         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2400 }
2401
2402 static void
2403 ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev)
2404 {
2405         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2406         uint32_t fctrl;
2407
2408         if (dev->data->promiscuous == 1)
2409                 return; /* must remain in all_multicast mode */
2410
2411         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2412         fctrl &= (~IXGBE_FCTRL_MPE);
2413         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2414 }
2415
2416 /**
2417  * It clears the interrupt causes and enables the interrupt.
2418  * It will be called once only during nic initialized.
2419  *
2420  * @param dev
2421  *  Pointer to struct rte_eth_dev.
2422  *
2423  * @return
2424  *  - On success, zero.
2425  *  - On failure, a negative value.
2426  */
2427 static int
2428 ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev)
2429 {
2430         struct ixgbe_interrupt *intr =
2431                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2432
2433         ixgbe_dev_link_status_print(dev);
2434         intr->mask |= IXGBE_EICR_LSC;
2435
2436         return 0;
2437 }
2438
2439 /*
2440  * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update.
2441  *
2442  * @param dev
2443  *  Pointer to struct rte_eth_dev.
2444  *
2445  * @return
2446  *  - On success, zero.
2447  *  - On failure, a negative value.
2448  */
2449 static int
2450 ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
2451 {
2452         uint32_t eicr;
2453         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2454         struct ixgbe_interrupt *intr =
2455                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2456
2457         /* clear all cause mask */
2458         ixgbe_disable_intr(hw);
2459
2460         /* read-on-clear nic registers here */
2461         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
2462         PMD_DRV_LOG(INFO, "eicr %x", eicr);
2463
2464         intr->flags = 0;
2465         if (eicr & IXGBE_EICR_LSC) {
2466                 /* set flag for async link update */
2467                 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
2468         }
2469
2470         if (eicr & IXGBE_EICR_MAILBOX)
2471                 intr->flags |= IXGBE_FLAG_MAILBOX;
2472
2473         return 0;
2474 }
2475
2476 /**
2477  * It gets and then prints the link status.
2478  *
2479  * @param dev
2480  *  Pointer to struct rte_eth_dev.
2481  *
2482  * @return
2483  *  - On success, zero.
2484  *  - On failure, a negative value.
2485  */
2486 static void
2487 ixgbe_dev_link_status_print(struct rte_eth_dev *dev)
2488 {
2489         struct rte_eth_link link;
2490
2491         memset(&link, 0, sizeof(link));
2492         rte_ixgbe_dev_atomic_read_link_status(dev, &link);
2493         if (link.link_status) {
2494                 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
2495                                         (int)(dev->data->port_id),
2496                                         (unsigned)link.link_speed,
2497                         link.link_duplex == ETH_LINK_FULL_DUPLEX ?
2498                                         "full-duplex" : "half-duplex");
2499         } else {
2500                 PMD_INIT_LOG(INFO, " Port %d: Link Down",
2501                                 (int)(dev->data->port_id));
2502         }
2503         PMD_INIT_LOG(INFO, "PCI Address: %04d:%02d:%02d:%d",
2504                                 dev->pci_dev->addr.domain,
2505                                 dev->pci_dev->addr.bus,
2506                                 dev->pci_dev->addr.devid,
2507                                 dev->pci_dev->addr.function);
2508 }
2509
2510 /*
2511  * It executes link_update after knowing an interrupt occurred.
2512  *
2513  * @param dev
2514  *  Pointer to struct rte_eth_dev.
2515  *
2516  * @return
2517  *  - On success, zero.
2518  *  - On failure, a negative value.
2519  */
2520 static int
2521 ixgbe_dev_interrupt_action(struct rte_eth_dev *dev)
2522 {
2523         struct ixgbe_interrupt *intr =
2524                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2525         int64_t timeout;
2526         struct rte_eth_link link;
2527         int intr_enable_delay = false;
2528
2529         PMD_DRV_LOG(DEBUG, "intr action type %d", intr->flags);
2530
2531         if (intr->flags & IXGBE_FLAG_MAILBOX) {
2532                 ixgbe_pf_mbx_process(dev);
2533                 intr->flags &= ~IXGBE_FLAG_MAILBOX;
2534         }
2535
2536         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
2537                 /* get the link status before link update, for predicting later */
2538                 memset(&link, 0, sizeof(link));
2539                 rte_ixgbe_dev_atomic_read_link_status(dev, &link);
2540
2541                 ixgbe_dev_link_update(dev, 0);
2542
2543                 /* likely to up */
2544                 if (!link.link_status)
2545                         /* handle it 1 sec later, wait it being stable */
2546                         timeout = IXGBE_LINK_UP_CHECK_TIMEOUT;
2547                 /* likely to down */
2548                 else
2549                         /* handle it 4 sec later, wait it being stable */
2550                         timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT;
2551
2552                 ixgbe_dev_link_status_print(dev);
2553
2554                 intr_enable_delay = true;
2555         }
2556
2557         if (intr_enable_delay) {
2558                 if (rte_eal_alarm_set(timeout * 1000,
2559                                       ixgbe_dev_interrupt_delayed_handler, (void*)dev) < 0)
2560                         PMD_DRV_LOG(ERR, "Error setting alarm");
2561         } else {
2562                 PMD_DRV_LOG(DEBUG, "enable intr immediately");
2563                 ixgbe_enable_intr(dev);
2564                 rte_intr_enable(&(dev->pci_dev->intr_handle));
2565         }
2566
2567
2568         return 0;
2569 }
2570
2571 /**
2572  * Interrupt handler which shall be registered for alarm callback for delayed
2573  * handling specific interrupt to wait for the stable nic state. As the
2574  * NIC interrupt state is not stable for ixgbe after link is just down,
2575  * it needs to wait 4 seconds to get the stable status.
2576  *
2577  * @param handle
2578  *  Pointer to interrupt handle.
2579  * @param param
2580  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2581  *
2582  * @return
2583  *  void
2584  */
2585 static void
2586 ixgbe_dev_interrupt_delayed_handler(void *param)
2587 {
2588         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2589         struct ixgbe_interrupt *intr =
2590                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2591         struct ixgbe_hw *hw =
2592                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2593         uint32_t eicr;
2594
2595         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
2596         if (eicr & IXGBE_EICR_MAILBOX)
2597                 ixgbe_pf_mbx_process(dev);
2598
2599         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
2600                 ixgbe_dev_link_update(dev, 0);
2601                 intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
2602                 ixgbe_dev_link_status_print(dev);
2603                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
2604         }
2605
2606         PMD_DRV_LOG(DEBUG, "enable intr in delayed handler S[%08x]", eicr);
2607         ixgbe_enable_intr(dev);
2608         rte_intr_enable(&(dev->pci_dev->intr_handle));
2609 }
2610
2611 /**
2612  * Interrupt handler triggered by NIC  for handling
2613  * specific interrupt.
2614  *
2615  * @param handle
2616  *  Pointer to interrupt handle.
2617  * @param param
2618  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2619  *
2620  * @return
2621  *  void
2622  */
2623 static void
2624 ixgbe_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
2625                                                         void *param)
2626 {
2627         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2628         ixgbe_dev_interrupt_get_status(dev);
2629         ixgbe_dev_interrupt_action(dev);
2630 }
2631
2632 static int
2633 ixgbe_dev_led_on(struct rte_eth_dev *dev)
2634 {
2635         struct ixgbe_hw *hw;
2636
2637         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2638         return (ixgbe_led_on(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
2639 }
2640
2641 static int
2642 ixgbe_dev_led_off(struct rte_eth_dev *dev)
2643 {
2644         struct ixgbe_hw *hw;
2645
2646         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2647         return (ixgbe_led_off(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
2648 }
2649
2650 static int
2651 ixgbe_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2652 {
2653         struct ixgbe_hw *hw;
2654         uint32_t mflcn_reg;
2655         uint32_t fccfg_reg;
2656         int rx_pause;
2657         int tx_pause;
2658
2659         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2660
2661         fc_conf->pause_time = hw->fc.pause_time;
2662         fc_conf->high_water = hw->fc.high_water[0];
2663         fc_conf->low_water = hw->fc.low_water[0];
2664         fc_conf->send_xon = hw->fc.send_xon;
2665         fc_conf->autoneg = !hw->fc.disable_fc_autoneg;
2666
2667         /*
2668          * Return rx_pause status according to actual setting of
2669          * MFLCN register.
2670          */
2671         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
2672         if (mflcn_reg & (IXGBE_MFLCN_RPFCE | IXGBE_MFLCN_RFCE))
2673                 rx_pause = 1;
2674         else
2675                 rx_pause = 0;
2676
2677         /*
2678          * Return tx_pause status according to actual setting of
2679          * FCCFG register.
2680          */
2681         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
2682         if (fccfg_reg & (IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY))
2683                 tx_pause = 1;
2684         else
2685                 tx_pause = 0;
2686
2687         if (rx_pause && tx_pause)
2688                 fc_conf->mode = RTE_FC_FULL;
2689         else if (rx_pause)
2690                 fc_conf->mode = RTE_FC_RX_PAUSE;
2691         else if (tx_pause)
2692                 fc_conf->mode = RTE_FC_TX_PAUSE;
2693         else
2694                 fc_conf->mode = RTE_FC_NONE;
2695
2696         return 0;
2697 }
2698
2699 static int
2700 ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2701 {
2702         struct ixgbe_hw *hw;
2703         int err;
2704         uint32_t rx_buf_size;
2705         uint32_t max_high_water;
2706         uint32_t mflcn;
2707         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
2708                 ixgbe_fc_none,
2709                 ixgbe_fc_rx_pause,
2710                 ixgbe_fc_tx_pause,
2711                 ixgbe_fc_full
2712         };
2713
2714         PMD_INIT_FUNC_TRACE();
2715
2716         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2717         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0));
2718         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
2719
2720         /*
2721          * At least reserve one Ethernet frame for watermark
2722          * high_water/low_water in kilo bytes for ixgbe
2723          */
2724         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
2725         if ((fc_conf->high_water > max_high_water) ||
2726                 (fc_conf->high_water < fc_conf->low_water)) {
2727                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB");
2728                 PMD_INIT_LOG(ERR, "High_water must <= 0x%x", max_high_water);
2729                 return (-EINVAL);
2730         }
2731
2732         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[fc_conf->mode];
2733         hw->fc.pause_time     = fc_conf->pause_time;
2734         hw->fc.high_water[0]  = fc_conf->high_water;
2735         hw->fc.low_water[0]   = fc_conf->low_water;
2736         hw->fc.send_xon       = fc_conf->send_xon;
2737         hw->fc.disable_fc_autoneg = !fc_conf->autoneg;
2738
2739         err = ixgbe_fc_enable(hw);
2740
2741         /* Not negotiated is not an error case */
2742         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) {
2743
2744                 /* check if we want to forward MAC frames - driver doesn't have native
2745                  * capability to do that, so we'll write the registers ourselves */
2746
2747                 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN);
2748
2749                 /* set or clear MFLCN.PMCF bit depending on configuration */
2750                 if (fc_conf->mac_ctrl_frame_fwd != 0)
2751                         mflcn |= IXGBE_MFLCN_PMCF;
2752                 else
2753                         mflcn &= ~IXGBE_MFLCN_PMCF;
2754
2755                 IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn);
2756                 IXGBE_WRITE_FLUSH(hw);
2757
2758                 return 0;
2759         }
2760
2761         PMD_INIT_LOG(ERR, "ixgbe_fc_enable = 0x%x", err);
2762         return -EIO;
2763 }
2764
2765 /**
2766  *  ixgbe_pfc_enable_generic - Enable flow control
2767  *  @hw: pointer to hardware structure
2768  *  @tc_num: traffic class number
2769  *  Enable flow control according to the current settings.
2770  */
2771 static int
2772 ixgbe_dcb_pfc_enable_generic(struct ixgbe_hw *hw,uint8_t tc_num)
2773 {
2774         int ret_val = 0;
2775         uint32_t mflcn_reg, fccfg_reg;
2776         uint32_t reg;
2777         uint32_t fcrtl, fcrth;
2778         uint8_t i;
2779         uint8_t nb_rx_en;
2780
2781         /* Validate the water mark configuration */
2782         if (!hw->fc.pause_time) {
2783                 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2784                 goto out;
2785         }
2786
2787         /* Low water mark of zero causes XOFF floods */
2788         if (hw->fc.current_mode & ixgbe_fc_tx_pause) {
2789                  /* High/Low water can not be 0 */
2790                 if( (!hw->fc.high_water[tc_num])|| (!hw->fc.low_water[tc_num])) {
2791                         PMD_INIT_LOG(ERR, "Invalid water mark configuration");
2792                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2793                         goto out;
2794                 }
2795
2796                 if(hw->fc.low_water[tc_num] >= hw->fc.high_water[tc_num]) {
2797                         PMD_INIT_LOG(ERR, "Invalid water mark configuration");
2798                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2799                         goto out;
2800                 }
2801         }
2802         /* Negotiate the fc mode to use */
2803         ixgbe_fc_autoneg(hw);
2804
2805         /* Disable any previous flow control settings */
2806         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
2807         mflcn_reg &= ~(IXGBE_MFLCN_RPFCE_SHIFT | IXGBE_MFLCN_RFCE|IXGBE_MFLCN_RPFCE);
2808
2809         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
2810         fccfg_reg &= ~(IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY);
2811
2812         switch (hw->fc.current_mode) {
2813         case ixgbe_fc_none:
2814                 /*
2815                  * If the count of enabled RX Priority Flow control >1,
2816                  * and the TX pause can not be disabled
2817                  */
2818                 nb_rx_en = 0;
2819                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
2820                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
2821                         if (reg & IXGBE_FCRTH_FCEN)
2822                                 nb_rx_en++;
2823                 }
2824                 if (nb_rx_en > 1)
2825                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2826                 break;
2827         case ixgbe_fc_rx_pause:
2828                 /*
2829                  * Rx Flow control is enabled and Tx Flow control is
2830                  * disabled by software override. Since there really
2831                  * isn't a way to advertise that we are capable of RX
2832                  * Pause ONLY, we will advertise that we support both
2833                  * symmetric and asymmetric Rx PAUSE.  Later, we will
2834                  * disable the adapter's ability to send PAUSE frames.
2835                  */
2836                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
2837                 /*
2838                  * If the count of enabled RX Priority Flow control >1,
2839                  * and the TX pause can not be disabled
2840                  */
2841                 nb_rx_en = 0;
2842                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
2843                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
2844                         if (reg & IXGBE_FCRTH_FCEN)
2845                                 nb_rx_en++;
2846                 }
2847                 if (nb_rx_en > 1)
2848                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2849                 break;
2850         case ixgbe_fc_tx_pause:
2851                 /*
2852                  * Tx Flow control is enabled, and Rx Flow control is
2853                  * disabled by software override.
2854                  */
2855                 fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2856                 break;
2857         case ixgbe_fc_full:
2858                 /* Flow control (both Rx and Tx) is enabled by SW override. */
2859                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
2860                 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
2861                 break;
2862         default:
2863                 PMD_DRV_LOG(DEBUG, "Flow control param set incorrectly");
2864                 ret_val = IXGBE_ERR_CONFIG;
2865                 goto out;
2866                 break;
2867         }
2868
2869         /* Set 802.3x based flow control settings. */
2870         mflcn_reg |= IXGBE_MFLCN_DPF;
2871         IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg);
2872         IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg);
2873
2874         /* Set up and enable Rx high/low water mark thresholds, enable XON. */
2875         if ((hw->fc.current_mode & ixgbe_fc_tx_pause) &&
2876                 hw->fc.high_water[tc_num]) {
2877                 fcrtl = (hw->fc.low_water[tc_num] << 10) | IXGBE_FCRTL_XONE;
2878                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), fcrtl);
2879                 fcrth = (hw->fc.high_water[tc_num] << 10) | IXGBE_FCRTH_FCEN;
2880         } else {
2881                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), 0);
2882                 /*
2883                  * In order to prevent Tx hangs when the internal Tx
2884                  * switch is enabled we must set the high water mark
2885                  * to the maximum FCRTH value.  This allows the Tx
2886                  * switch to function even under heavy Rx workloads.
2887                  */
2888                 fcrth = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)) - 32;
2889         }
2890         IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(tc_num), fcrth);
2891
2892         /* Configure pause time (2 TCs per register) */
2893         reg = hw->fc.pause_time * 0x00010001;
2894         for (i = 0; i < (IXGBE_DCB_MAX_TRAFFIC_CLASS / 2); i++)
2895                 IXGBE_WRITE_REG(hw, IXGBE_FCTTV(i), reg);
2896
2897         /* Configure flow control refresh threshold value */
2898         IXGBE_WRITE_REG(hw, IXGBE_FCRTV, hw->fc.pause_time / 2);
2899
2900 out:
2901         return ret_val;
2902 }
2903
2904 static int
2905 ixgbe_dcb_pfc_enable(struct rte_eth_dev *dev,uint8_t tc_num)
2906 {
2907         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2908         int32_t ret_val = IXGBE_NOT_IMPLEMENTED;
2909
2910         if(hw->mac.type != ixgbe_mac_82598EB) {
2911                 ret_val = ixgbe_dcb_pfc_enable_generic(hw,tc_num);
2912         }
2913         return ret_val;
2914 }
2915
2916 static int
2917 ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf)
2918 {
2919         int err;
2920         uint32_t rx_buf_size;
2921         uint32_t max_high_water;
2922         uint8_t tc_num;
2923         uint8_t  map[IXGBE_DCB_MAX_USER_PRIORITY] = { 0 };
2924         struct ixgbe_hw *hw =
2925                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2926         struct ixgbe_dcb_config *dcb_config =
2927                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
2928
2929         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
2930                 ixgbe_fc_none,
2931                 ixgbe_fc_rx_pause,
2932                 ixgbe_fc_tx_pause,
2933                 ixgbe_fc_full
2934         };
2935
2936         PMD_INIT_FUNC_TRACE();
2937
2938         ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
2939         tc_num = map[pfc_conf->priority];
2940         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num));
2941         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
2942         /*
2943          * At least reserve one Ethernet frame for watermark
2944          * high_water/low_water in kilo bytes for ixgbe
2945          */
2946         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
2947         if ((pfc_conf->fc.high_water > max_high_water) ||
2948             (pfc_conf->fc.high_water <= pfc_conf->fc.low_water)) {
2949                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB");
2950                 PMD_INIT_LOG(ERR, "High_water must <= 0x%x", max_high_water);
2951                 return (-EINVAL);
2952         }
2953
2954         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[pfc_conf->fc.mode];
2955         hw->fc.pause_time = pfc_conf->fc.pause_time;
2956         hw->fc.send_xon = pfc_conf->fc.send_xon;
2957         hw->fc.low_water[tc_num] =  pfc_conf->fc.low_water;
2958         hw->fc.high_water[tc_num] = pfc_conf->fc.high_water;
2959
2960         err = ixgbe_dcb_pfc_enable(dev,tc_num);
2961
2962         /* Not negotiated is not an error case */
2963         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED))
2964                 return 0;
2965
2966         PMD_INIT_LOG(ERR, "ixgbe_dcb_pfc_enable = 0x%x", err);
2967         return -EIO;
2968 }
2969
2970 static int
2971 ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
2972                           struct rte_eth_rss_reta_entry64 *reta_conf,
2973                           uint16_t reta_size)
2974 {
2975         uint8_t i, j, mask;
2976         uint32_t reta, r;
2977         uint16_t idx, shift;
2978         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2979
2980         PMD_INIT_FUNC_TRACE();
2981         if (reta_size != ETH_RSS_RETA_SIZE_128) {
2982                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2983                         "(%d) doesn't match the number hardware can supported "
2984                         "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
2985                 return -EINVAL;
2986         }
2987
2988         for (i = 0; i < reta_size; i += IXGBE_4_BIT_WIDTH) {
2989                 idx = i / RTE_RETA_GROUP_SIZE;
2990                 shift = i % RTE_RETA_GROUP_SIZE;
2991                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2992                                                 IXGBE_4_BIT_MASK);
2993                 if (!mask)
2994                         continue;
2995                 if (mask == IXGBE_4_BIT_MASK)
2996                         r = 0;
2997                 else
2998                         r = IXGBE_READ_REG(hw, IXGBE_RETA(i >> 2));
2999                 for (j = 0, reta = 0; j < IXGBE_4_BIT_WIDTH; j++) {
3000                         if (mask & (0x1 << j))
3001                                 reta |= reta_conf[idx].reta[shift + j] <<
3002                                                         (CHAR_BIT * j);
3003                         else
3004                                 reta |= r & (IXGBE_8_BIT_MASK <<
3005                                                 (CHAR_BIT * j));
3006                 }
3007                 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
3008         }
3009
3010         return 0;
3011 }
3012
3013 static int
3014 ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
3015                          struct rte_eth_rss_reta_entry64 *reta_conf,
3016                          uint16_t reta_size)
3017 {
3018         uint8_t i, j, mask;
3019         uint32_t reta;
3020         uint16_t idx, shift;
3021         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3022
3023         PMD_INIT_FUNC_TRACE();
3024         if (reta_size != ETH_RSS_RETA_SIZE_128) {
3025                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
3026                         "(%d) doesn't match the number hardware can supported "
3027                                 "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
3028                 return -EINVAL;
3029         }
3030
3031         for (i = 0; i < ETH_RSS_RETA_SIZE_128; i += IXGBE_4_BIT_WIDTH) {
3032                 idx = i / RTE_RETA_GROUP_SIZE;
3033                 shift = i % RTE_RETA_GROUP_SIZE;
3034                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
3035                                                 IXGBE_4_BIT_MASK);
3036                 if (!mask)
3037                         continue;
3038
3039                 reta = IXGBE_READ_REG(hw, IXGBE_RETA(i >> 2));
3040                 for (j = 0; j < IXGBE_4_BIT_WIDTH; j++) {
3041                         if (mask & (0x1 << j))
3042                                 reta_conf[idx].reta[shift + j] =
3043                                         ((reta >> (CHAR_BIT * j)) &
3044                                                 IXGBE_8_BIT_MASK);
3045                 }
3046         }
3047
3048         return 0;
3049 }
3050
3051 static void
3052 ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
3053                                 uint32_t index, uint32_t pool)
3054 {
3055         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3056         uint32_t enable_addr = 1;
3057
3058         ixgbe_set_rar(hw, index, mac_addr->addr_bytes, pool, enable_addr);
3059 }
3060
3061 static void
3062 ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
3063 {
3064         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3065
3066         ixgbe_clear_rar(hw, index);
3067 }
3068
3069 static void
3070 ixgbe_set_default_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
3071 {
3072         ixgbe_remove_rar(dev, 0);
3073
3074         ixgbe_add_rar(dev, addr, 0, 0);
3075 }
3076
3077 static int
3078 ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
3079 {
3080         uint32_t hlreg0;
3081         uint32_t maxfrs;
3082         struct ixgbe_hw *hw;
3083         struct rte_eth_dev_info dev_info;
3084         uint32_t frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
3085
3086         ixgbe_dev_info_get(dev, &dev_info);
3087
3088         /* check that mtu is within the allowed range */
3089         if ((mtu < ETHER_MIN_MTU) || (frame_size > dev_info.max_rx_pktlen))
3090                 return -EINVAL;
3091
3092         /* refuse mtu that requires the support of scattered packets when this
3093          * feature has not been enabled before. */
3094         if (!dev->data->scattered_rx &&
3095             (frame_size + 2 * IXGBE_VLAN_TAG_SIZE >
3096              dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM))
3097                 return -EINVAL;
3098
3099         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3100         hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
3101
3102         /* switch to jumbo mode if needed */
3103         if (frame_size > ETHER_MAX_LEN) {
3104                 dev->data->dev_conf.rxmode.jumbo_frame = 1;
3105                 hlreg0 |= IXGBE_HLREG0_JUMBOEN;
3106         } else {
3107                 dev->data->dev_conf.rxmode.jumbo_frame = 0;
3108                 hlreg0 &= ~IXGBE_HLREG0_JUMBOEN;
3109         }
3110         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
3111
3112         /* update max frame size */
3113         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
3114
3115         maxfrs = IXGBE_READ_REG(hw, IXGBE_MAXFRS);
3116         maxfrs &= 0x0000FFFF;
3117         maxfrs |= (dev->data->dev_conf.rxmode.max_rx_pkt_len << 16);
3118         IXGBE_WRITE_REG(hw, IXGBE_MAXFRS, maxfrs);
3119
3120         return 0;
3121 }
3122
3123 /*
3124  * Virtual Function operations
3125  */
3126 static void
3127 ixgbevf_intr_disable(struct ixgbe_hw *hw)
3128 {
3129         PMD_INIT_FUNC_TRACE();
3130
3131         /* Clear interrupt mask to stop from interrupts being generated */
3132         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, IXGBE_VF_IRQ_CLEAR_MASK);
3133
3134         IXGBE_WRITE_FLUSH(hw);
3135 }
3136
3137 static int
3138 ixgbevf_dev_configure(struct rte_eth_dev *dev)
3139 {
3140         struct rte_eth_conf* conf = &dev->data->dev_conf;
3141         struct ixgbe_adapter *adapter =
3142                         (struct ixgbe_adapter *)dev->data->dev_private;
3143
3144         PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d",
3145                      dev->data->port_id);
3146
3147         /*
3148          * VF has no ability to enable/disable HW CRC
3149          * Keep the persistent behavior the same as Host PF
3150          */
3151 #ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC
3152         if (!conf->rxmode.hw_strip_crc) {
3153                 PMD_INIT_LOG(INFO, "VF can't disable HW CRC Strip");
3154                 conf->rxmode.hw_strip_crc = 1;
3155         }
3156 #else
3157         if (conf->rxmode.hw_strip_crc) {
3158                 PMD_INIT_LOG(INFO, "VF can't enable HW CRC Strip");
3159                 conf->rxmode.hw_strip_crc = 0;
3160         }
3161 #endif
3162
3163         /*
3164          * Initialize to TRUE. If any of Rx queues doesn't meet the bulk
3165          * allocation or vector Rx preconditions we will reset it.
3166          */
3167         adapter->rx_bulk_alloc_allowed = true;
3168         adapter->rx_vec_allowed = true;
3169
3170         return 0;
3171 }
3172
3173 static int
3174 ixgbevf_dev_start(struct rte_eth_dev *dev)
3175 {
3176         struct ixgbe_hw *hw =
3177                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3178         int err, mask = 0;
3179
3180         PMD_INIT_FUNC_TRACE();
3181
3182         hw->mac.ops.reset_hw(hw);
3183         hw->mac.get_link_status = true;
3184
3185         /* negotiate mailbox API version to use with the PF. */
3186         ixgbevf_negotiate_api(hw);
3187
3188         ixgbevf_dev_tx_init(dev);
3189
3190         /* This can fail when allocating mbufs for descriptor rings */
3191         err = ixgbevf_dev_rx_init(dev);
3192         if (err) {
3193                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware (%d)", err);
3194                 ixgbe_dev_clear_queues(dev);
3195                 return err;
3196         }
3197
3198         /* Set vfta */
3199         ixgbevf_set_vfta_all(dev,1);
3200
3201         /* Set HW strip */
3202         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
3203                 ETH_VLAN_EXTEND_MASK;
3204         ixgbevf_vlan_offload_set(dev, mask);
3205
3206         ixgbevf_dev_rxtx_start(dev);
3207
3208         return 0;
3209 }
3210
3211 static void
3212 ixgbevf_dev_stop(struct rte_eth_dev *dev)
3213 {
3214         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3215
3216         PMD_INIT_FUNC_TRACE();
3217
3218         hw->adapter_stopped = TRUE;
3219         ixgbe_stop_adapter(hw);
3220
3221         /*
3222           * Clear what we set, but we still keep shadow_vfta to
3223           * restore after device starts
3224           */
3225         ixgbevf_set_vfta_all(dev,0);
3226
3227         /* Clear stored conf */
3228         dev->data->scattered_rx = 0;
3229
3230         ixgbe_dev_clear_queues(dev);
3231 }
3232
3233 static void
3234 ixgbevf_dev_close(struct rte_eth_dev *dev)
3235 {
3236         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3237
3238         PMD_INIT_FUNC_TRACE();
3239
3240         ixgbe_reset_hw(hw);
3241
3242         ixgbevf_dev_stop(dev);
3243
3244         /* reprogram the RAR[0] in case user changed it. */
3245         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
3246 }
3247
3248 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on)
3249 {
3250         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3251         struct ixgbe_vfta * shadow_vfta =
3252                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
3253         int i = 0, j = 0, vfta = 0, mask = 1;
3254
3255         for (i = 0; i < IXGBE_VFTA_SIZE; i++){
3256                 vfta = shadow_vfta->vfta[i];
3257                 if(vfta){
3258                         mask = 1;
3259                         for (j = 0; j < 32; j++){
3260                                 if(vfta & mask)
3261                                         ixgbe_set_vfta(hw, (i<<5)+j, 0, on);
3262                                 mask<<=1;
3263                         }
3264                 }
3265         }
3266
3267 }
3268
3269 static int
3270 ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
3271 {
3272         struct ixgbe_hw *hw =
3273                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3274         struct ixgbe_vfta * shadow_vfta =
3275                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
3276         uint32_t vid_idx = 0;
3277         uint32_t vid_bit = 0;
3278         int ret = 0;
3279
3280         PMD_INIT_FUNC_TRACE();
3281
3282         /* vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf */
3283         ret = ixgbe_set_vfta(hw, vlan_id, 0, !!on);
3284         if(ret){
3285                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
3286                 return ret;
3287         }
3288         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
3289         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
3290
3291         /* Save what we set and retore it after device reset */
3292         if (on)
3293                 shadow_vfta->vfta[vid_idx] |= vid_bit;
3294         else
3295                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
3296
3297         return 0;
3298 }
3299
3300 static void
3301 ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
3302 {
3303         struct ixgbe_hw *hw =
3304                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3305         uint32_t ctrl;
3306
3307         PMD_INIT_FUNC_TRACE();
3308
3309         if(queue >= hw->mac.max_rx_queues)
3310                 return;
3311
3312         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
3313         if(on)
3314                 ctrl |= IXGBE_RXDCTL_VME;
3315         else
3316                 ctrl &= ~IXGBE_RXDCTL_VME;
3317         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
3318
3319         ixgbe_vlan_hw_strip_bitmap_set( dev, queue, on);
3320 }
3321
3322 static void
3323 ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
3324 {
3325         struct ixgbe_hw *hw =
3326                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3327         uint16_t i;
3328         int on = 0;
3329
3330         /* VF function only support hw strip feature, others are not support */
3331         if(mask & ETH_VLAN_STRIP_MASK){
3332                 on = !!(dev->data->dev_conf.rxmode.hw_vlan_strip);
3333
3334                 for(i=0; i < hw->mac.max_rx_queues; i++)
3335                         ixgbevf_vlan_strip_queue_set(dev,i,on);
3336         }
3337 }
3338
3339 static int
3340 ixgbe_vmdq_mode_check(struct ixgbe_hw *hw)
3341 {
3342         uint32_t reg_val;
3343
3344         /* we only need to do this if VMDq is enabled */
3345         reg_val = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
3346         if (!(reg_val & IXGBE_VT_CTL_VT_ENABLE)) {
3347                 PMD_INIT_LOG(ERR, "VMDq must be enabled for this setting");
3348                 return (-1);
3349         }
3350
3351         return 0;
3352 }
3353
3354 static uint32_t
3355 ixgbe_uta_vector(struct ixgbe_hw *hw, struct ether_addr* uc_addr)
3356 {
3357         uint32_t vector = 0;
3358         switch (hw->mac.mc_filter_type) {
3359         case 0:   /* use bits [47:36] of the address */
3360                 vector = ((uc_addr->addr_bytes[4] >> 4) |
3361                         (((uint16_t)uc_addr->addr_bytes[5]) << 4));
3362                 break;
3363         case 1:   /* use bits [46:35] of the address */
3364                 vector = ((uc_addr->addr_bytes[4] >> 3) |
3365                         (((uint16_t)uc_addr->addr_bytes[5]) << 5));
3366                 break;
3367         case 2:   /* use bits [45:34] of the address */
3368                 vector = ((uc_addr->addr_bytes[4] >> 2) |
3369                         (((uint16_t)uc_addr->addr_bytes[5]) << 6));
3370                 break;
3371         case 3:   /* use bits [43:32] of the address */
3372                 vector = ((uc_addr->addr_bytes[4]) |
3373                         (((uint16_t)uc_addr->addr_bytes[5]) << 8));
3374                 break;
3375         default:  /* Invalid mc_filter_type */
3376                 break;
3377         }
3378
3379         /* vector can only be 12-bits or boundary will be exceeded */
3380         vector &= 0xFFF;
3381         return vector;
3382 }
3383
3384 static int
3385 ixgbe_uc_hash_table_set(struct rte_eth_dev *dev,struct ether_addr* mac_addr,
3386                                uint8_t on)
3387 {
3388         uint32_t vector;
3389         uint32_t uta_idx;
3390         uint32_t reg_val;
3391         uint32_t uta_shift;
3392         uint32_t rc;
3393         const uint32_t ixgbe_uta_idx_mask = 0x7F;
3394         const uint32_t ixgbe_uta_bit_shift = 5;
3395         const uint32_t ixgbe_uta_bit_mask = (0x1 << ixgbe_uta_bit_shift) - 1;
3396         const uint32_t bit1 = 0x1;
3397
3398         struct ixgbe_hw *hw =
3399                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3400         struct ixgbe_uta_info *uta_info =
3401                 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private);
3402
3403         /* The UTA table only exists on 82599 hardware and newer */
3404         if (hw->mac.type < ixgbe_mac_82599EB)
3405                 return (-ENOTSUP);
3406
3407         vector = ixgbe_uta_vector(hw,mac_addr);
3408         uta_idx = (vector >> ixgbe_uta_bit_shift) & ixgbe_uta_idx_mask;
3409         uta_shift = vector & ixgbe_uta_bit_mask;
3410
3411         rc = ((uta_info->uta_shadow[uta_idx] >> uta_shift & bit1) != 0);
3412         if(rc == on)
3413                 return 0;
3414
3415         reg_val = IXGBE_READ_REG(hw, IXGBE_UTA(uta_idx));
3416         if (on) {
3417                 uta_info->uta_in_use++;
3418                 reg_val |= (bit1 << uta_shift);
3419                 uta_info->uta_shadow[uta_idx] |= (bit1 << uta_shift);
3420         } else {
3421                 uta_info->uta_in_use--;
3422                 reg_val &= ~(bit1 << uta_shift);
3423                 uta_info->uta_shadow[uta_idx] &= ~(bit1 << uta_shift);
3424         }
3425
3426         IXGBE_WRITE_REG(hw, IXGBE_UTA(uta_idx), reg_val);
3427
3428         if (uta_info->uta_in_use > 0)
3429                 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL,
3430                                 IXGBE_MCSTCTRL_MFE | hw->mac.mc_filter_type);
3431         else
3432                 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL,hw->mac.mc_filter_type);
3433
3434         return 0;
3435 }
3436
3437 static int
3438 ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on)
3439 {
3440         int i;
3441         struct ixgbe_hw *hw =
3442                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3443         struct ixgbe_uta_info *uta_info =
3444                 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private);
3445
3446         /* The UTA table only exists on 82599 hardware and newer */
3447         if (hw->mac.type < ixgbe_mac_82599EB)
3448                 return (-ENOTSUP);
3449
3450         if(on) {
3451                 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
3452                         uta_info->uta_shadow[i] = ~0;
3453                         IXGBE_WRITE_REG(hw, IXGBE_UTA(i), ~0);
3454                 }
3455         } else {
3456                 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
3457                         uta_info->uta_shadow[i] = 0;
3458                         IXGBE_WRITE_REG(hw, IXGBE_UTA(i), 0);
3459                 }
3460         }
3461         return 0;
3462
3463 }
3464
3465 uint32_t
3466 ixgbe_convert_vm_rx_mask_to_val(uint16_t rx_mask, uint32_t orig_val)
3467 {
3468         uint32_t new_val = orig_val;
3469
3470         if (rx_mask & ETH_VMDQ_ACCEPT_UNTAG)
3471                 new_val |= IXGBE_VMOLR_AUPE;
3472         if (rx_mask & ETH_VMDQ_ACCEPT_HASH_MC)
3473                 new_val |= IXGBE_VMOLR_ROMPE;
3474         if (rx_mask & ETH_VMDQ_ACCEPT_HASH_UC)
3475                 new_val |= IXGBE_VMOLR_ROPE;
3476         if (rx_mask & ETH_VMDQ_ACCEPT_BROADCAST)
3477                 new_val |= IXGBE_VMOLR_BAM;
3478         if (rx_mask & ETH_VMDQ_ACCEPT_MULTICAST)
3479                 new_val |= IXGBE_VMOLR_MPE;
3480
3481         return new_val;
3482 }
3483
3484 static int
3485 ixgbe_set_pool_rx_mode(struct rte_eth_dev *dev, uint16_t pool,
3486                                uint16_t rx_mask, uint8_t on)
3487 {
3488         int val = 0;
3489
3490         struct ixgbe_hw *hw =
3491                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3492         uint32_t vmolr = IXGBE_READ_REG(hw, IXGBE_VMOLR(pool));
3493
3494         if (hw->mac.type == ixgbe_mac_82598EB) {
3495                 PMD_INIT_LOG(ERR, "setting VF receive mode set should be done"
3496                              " on 82599 hardware and newer");
3497                 return (-ENOTSUP);
3498         }
3499         if (ixgbe_vmdq_mode_check(hw) < 0)
3500                 return (-ENOTSUP);
3501
3502         val = ixgbe_convert_vm_rx_mask_to_val(rx_mask, val);
3503
3504         if (on)
3505                 vmolr |= val;
3506         else
3507                 vmolr &= ~val;
3508
3509         IXGBE_WRITE_REG(hw, IXGBE_VMOLR(pool), vmolr);
3510
3511         return 0;
3512 }
3513
3514 static int
3515 ixgbe_set_pool_rx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on)
3516 {
3517         uint32_t reg,addr;
3518         uint32_t val;
3519         const uint8_t bit1 = 0x1;
3520
3521         struct ixgbe_hw *hw =
3522                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3523
3524         if (ixgbe_vmdq_mode_check(hw) < 0)
3525                 return (-ENOTSUP);
3526
3527         addr = IXGBE_VFRE(pool >= ETH_64_POOLS/2);
3528         reg = IXGBE_READ_REG(hw, addr);
3529         val = bit1 << pool;
3530
3531         if (on)
3532                 reg |= val;
3533         else
3534                 reg &= ~val;
3535
3536         IXGBE_WRITE_REG(hw, addr,reg);
3537
3538         return 0;
3539 }
3540
3541 static int
3542 ixgbe_set_pool_tx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on)
3543 {
3544         uint32_t reg,addr;
3545         uint32_t val;
3546         const uint8_t bit1 = 0x1;
3547
3548         struct ixgbe_hw *hw =
3549                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3550
3551         if (ixgbe_vmdq_mode_check(hw) < 0)
3552                 return (-ENOTSUP);
3553
3554         addr = IXGBE_VFTE(pool >= ETH_64_POOLS/2);
3555         reg = IXGBE_READ_REG(hw, addr);
3556         val = bit1 << pool;
3557
3558         if (on)
3559                 reg |= val;
3560         else
3561                 reg &= ~val;
3562
3563         IXGBE_WRITE_REG(hw, addr,reg);
3564
3565         return 0;
3566 }
3567
3568 static int
3569 ixgbe_set_pool_vlan_filter(struct rte_eth_dev *dev, uint16_t vlan,
3570                         uint64_t pool_mask, uint8_t vlan_on)
3571 {
3572         int ret = 0;
3573         uint16_t pool_idx;
3574         struct ixgbe_hw *hw =
3575                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3576
3577         if (ixgbe_vmdq_mode_check(hw) < 0)
3578                 return (-ENOTSUP);
3579         for (pool_idx = 0; pool_idx < ETH_64_POOLS; pool_idx++) {
3580                 if (pool_mask & ((uint64_t)(1ULL << pool_idx)))
3581                         ret = hw->mac.ops.set_vfta(hw,vlan,pool_idx,vlan_on);
3582                         if (ret < 0)
3583                                 return ret;
3584         }
3585
3586         return ret;
3587 }
3588
3589 #define IXGBE_MRCTL_VPME  0x01 /* Virtual Pool Mirroring. */
3590 #define IXGBE_MRCTL_UPME  0x02 /* Uplink Port Mirroring. */
3591 #define IXGBE_MRCTL_DPME  0x04 /* Downlink Port Mirroring. */
3592 #define IXGBE_MRCTL_VLME  0x08 /* VLAN Mirroring. */
3593 #define IXGBE_INVALID_MIRROR_TYPE(mirror_type) \
3594         ((mirror_type) & ~(uint8_t)(ETH_MIRROR_VIRTUAL_POOL_UP | \
3595         ETH_MIRROR_UPLINK_PORT | ETH_MIRROR_DOWNLINK_PORT | ETH_MIRROR_VLAN))
3596
3597 static int
3598 ixgbe_mirror_rule_set(struct rte_eth_dev *dev,
3599                         struct rte_eth_mirror_conf *mirror_conf,
3600                         uint8_t rule_id, uint8_t on)
3601 {
3602         uint32_t mr_ctl,vlvf;
3603         uint32_t mp_lsb = 0;
3604         uint32_t mv_msb = 0;
3605         uint32_t mv_lsb = 0;
3606         uint32_t mp_msb = 0;
3607         uint8_t i = 0;
3608         int reg_index = 0;
3609         uint64_t vlan_mask = 0;
3610
3611         const uint8_t pool_mask_offset = 32;
3612         const uint8_t vlan_mask_offset = 32;
3613         const uint8_t dst_pool_offset = 8;
3614         const uint8_t rule_mr_offset  = 4;
3615         const uint8_t mirror_rule_mask= 0x0F;
3616
3617         struct ixgbe_mirror_info *mr_info =
3618                         (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private));
3619         struct ixgbe_hw *hw =
3620                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3621         uint8_t mirror_type = 0;
3622
3623         if (ixgbe_vmdq_mode_check(hw) < 0)
3624                 return -ENOTSUP;
3625
3626         if (rule_id >= IXGBE_MAX_MIRROR_RULES)
3627                 return -EINVAL;
3628
3629         if (IXGBE_INVALID_MIRROR_TYPE(mirror_conf->rule_type)) {
3630                 PMD_DRV_LOG(ERR, "unsupported mirror type 0x%x.",
3631                         mirror_conf->rule_type);
3632                 return -EINVAL;
3633         }
3634
3635         if (mirror_conf->rule_type & ETH_MIRROR_VLAN) {
3636                 mirror_type |= IXGBE_MRCTL_VLME;
3637                 /* Check if vlan id is valid and find conresponding VLAN ID index in VLVF */
3638                 for (i = 0;i < IXGBE_VLVF_ENTRIES; i++) {
3639                         if (mirror_conf->vlan.vlan_mask & (1ULL << i)) {
3640                                 /* search vlan id related pool vlan filter index */
3641                                 reg_index = ixgbe_find_vlvf_slot(hw,
3642                                                 mirror_conf->vlan.vlan_id[i]);
3643                                 if(reg_index < 0)
3644                                         return -EINVAL;
3645                                 vlvf = IXGBE_READ_REG(hw, IXGBE_VLVF(reg_index));
3646                                 if ((vlvf & IXGBE_VLVF_VIEN) &&
3647                                     ((vlvf & IXGBE_VLVF_VLANID_MASK) ==
3648                                       mirror_conf->vlan.vlan_id[i]))
3649                                         vlan_mask |= (1ULL << reg_index);
3650                                 else
3651                                         return -EINVAL;
3652                         }
3653                 }
3654
3655                 if (on) {
3656                         mv_lsb = vlan_mask & 0xFFFFFFFF;
3657                         mv_msb = vlan_mask >> vlan_mask_offset;
3658
3659                         mr_info->mr_conf[rule_id].vlan.vlan_mask =
3660                                                 mirror_conf->vlan.vlan_mask;
3661                         for(i = 0 ;i < ETH_VMDQ_MAX_VLAN_FILTERS; i++) {
3662                                 if(mirror_conf->vlan.vlan_mask & (1ULL << i))
3663                                         mr_info->mr_conf[rule_id].vlan.vlan_id[i] =
3664                                                 mirror_conf->vlan.vlan_id[i];
3665                         }
3666                 } else {
3667                         mv_lsb = 0;
3668                         mv_msb = 0;
3669                         mr_info->mr_conf[rule_id].vlan.vlan_mask = 0;
3670                         for(i = 0 ;i < ETH_VMDQ_MAX_VLAN_FILTERS; i++)
3671                                 mr_info->mr_conf[rule_id].vlan.vlan_id[i] = 0;
3672                 }
3673         }
3674
3675         /*
3676          * if enable pool mirror, write related pool mask register,if disable
3677          * pool mirror, clear PFMRVM register
3678          */
3679         if (mirror_conf->rule_type & ETH_MIRROR_VIRTUAL_POOL_UP) {
3680                 mirror_type |= IXGBE_MRCTL_VPME;
3681                 if (on) {
3682                         mp_lsb = mirror_conf->pool_mask & 0xFFFFFFFF;
3683                         mp_msb = mirror_conf->pool_mask >> pool_mask_offset;
3684                         mr_info->mr_conf[rule_id].pool_mask =
3685                                         mirror_conf->pool_mask;
3686
3687                 } else {
3688                         mp_lsb = 0;
3689                         mp_msb = 0;
3690                         mr_info->mr_conf[rule_id].pool_mask = 0;
3691                 }
3692         }
3693         if (mirror_conf->rule_type & ETH_MIRROR_UPLINK_PORT)
3694                 mirror_type |= IXGBE_MRCTL_UPME;
3695         if (mirror_conf->rule_type & ETH_MIRROR_DOWNLINK_PORT)
3696                 mirror_type |= IXGBE_MRCTL_DPME;
3697
3698         /* read  mirror control register and recalculate it */
3699         mr_ctl = IXGBE_READ_REG(hw, IXGBE_MRCTL(rule_id));
3700
3701         if (on) {
3702                 mr_ctl |= mirror_type;
3703                 mr_ctl &= mirror_rule_mask;
3704                 mr_ctl |= mirror_conf->dst_pool << dst_pool_offset;
3705         } else
3706                 mr_ctl &= ~(mirror_conf->rule_type & mirror_rule_mask);
3707
3708         mr_info->mr_conf[rule_id].rule_type = mirror_conf->rule_type;
3709         mr_info->mr_conf[rule_id].dst_pool = mirror_conf->dst_pool;
3710
3711         /* write mirrror control  register */
3712         IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl);
3713
3714         /* write pool mirrror control  register */
3715         if (mirror_conf->rule_type == ETH_MIRROR_VIRTUAL_POOL_UP) {
3716                 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), mp_lsb);
3717                 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset),
3718                                 mp_msb);
3719         }
3720         /* write VLAN mirrror control  register */
3721         if (mirror_conf->rule_type == ETH_MIRROR_VLAN) {
3722                 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), mv_lsb);
3723                 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset),
3724                                 mv_msb);
3725         }
3726
3727         return 0;
3728 }
3729
3730 static int
3731 ixgbe_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t rule_id)
3732 {
3733         int mr_ctl = 0;
3734         uint32_t lsb_val = 0;
3735         uint32_t msb_val = 0;
3736         const uint8_t rule_mr_offset = 4;
3737
3738         struct ixgbe_hw *hw =
3739                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3740         struct ixgbe_mirror_info *mr_info =
3741                 (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private));
3742
3743         if (ixgbe_vmdq_mode_check(hw) < 0)
3744                 return (-ENOTSUP);
3745
3746         memset(&mr_info->mr_conf[rule_id], 0,
3747                 sizeof(struct rte_eth_mirror_conf));
3748
3749         /* clear PFVMCTL register */
3750         IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl);
3751
3752         /* clear pool mask register */
3753         IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), lsb_val);
3754         IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset), msb_val);
3755
3756         /* clear vlan mask register */
3757         IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), lsb_val);
3758         IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset), msb_val);
3759
3760         return 0;
3761 }
3762
3763 static int ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev,
3764         uint16_t queue_idx, uint16_t tx_rate)
3765 {
3766         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3767         uint32_t rf_dec, rf_int;
3768         uint32_t bcnrc_val;
3769         uint16_t link_speed = dev->data->dev_link.link_speed;
3770
3771         if (queue_idx >= hw->mac.max_tx_queues)
3772                 return -EINVAL;
3773
3774         if (tx_rate != 0) {
3775                 /* Calculate the rate factor values to set */
3776                 rf_int = (uint32_t)link_speed / (uint32_t)tx_rate;
3777                 rf_dec = (uint32_t)link_speed % (uint32_t)tx_rate;
3778                 rf_dec = (rf_dec << IXGBE_RTTBCNRC_RF_INT_SHIFT) / tx_rate;
3779
3780                 bcnrc_val = IXGBE_RTTBCNRC_RS_ENA;
3781                 bcnrc_val |= ((rf_int << IXGBE_RTTBCNRC_RF_INT_SHIFT) &
3782                                 IXGBE_RTTBCNRC_RF_INT_MASK_M);
3783                 bcnrc_val |= (rf_dec & IXGBE_RTTBCNRC_RF_DEC_MASK);
3784         } else {
3785                 bcnrc_val = 0;
3786         }
3787
3788         /*
3789          * Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
3790          * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported, otherwise
3791          * set as 0x4.
3792          */
3793         if ((dev->data->dev_conf.rxmode.jumbo_frame == 1) &&
3794                 (dev->data->dev_conf.rxmode.max_rx_pkt_len >=
3795                                 IXGBE_MAX_JUMBO_FRAME_SIZE))
3796                 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRM,
3797                         IXGBE_MMW_SIZE_JUMBO_FRAME);
3798         else
3799                 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRM,
3800                         IXGBE_MMW_SIZE_DEFAULT);
3801
3802         /* Set RTTBCNRC of queue X */
3803         IXGBE_WRITE_REG(hw, IXGBE_RTTDQSEL, queue_idx);
3804         IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRC, bcnrc_val);
3805         IXGBE_WRITE_FLUSH(hw);
3806
3807         return 0;
3808 }
3809
3810 static int ixgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf,
3811         uint16_t tx_rate, uint64_t q_msk)
3812 {
3813         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3814         struct ixgbe_vf_info *vfinfo =
3815                 *(IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private));
3816         uint8_t  nb_q_per_pool = RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool;
3817         uint32_t queue_stride =
3818                 IXGBE_MAX_RX_QUEUE_NUM / RTE_ETH_DEV_SRIOV(dev).active;
3819         uint32_t queue_idx = vf * queue_stride, idx = 0, vf_idx;
3820         uint32_t queue_end = queue_idx + nb_q_per_pool - 1;
3821         uint16_t total_rate = 0;
3822
3823         if (queue_end >= hw->mac.max_tx_queues)
3824                 return -EINVAL;
3825
3826         if (vfinfo != NULL) {
3827                 for (vf_idx = 0; vf_idx < dev->pci_dev->max_vfs; vf_idx++) {
3828                         if (vf_idx == vf)
3829                                 continue;
3830                         for (idx = 0; idx < RTE_DIM(vfinfo[vf_idx].tx_rate);
3831                                 idx++)
3832                                 total_rate += vfinfo[vf_idx].tx_rate[idx];
3833                 }
3834         } else
3835                 return -EINVAL;
3836
3837         /* Store tx_rate for this vf. */
3838         for (idx = 0; idx < nb_q_per_pool; idx++) {
3839                 if (((uint64_t)0x1 << idx) & q_msk) {
3840                         if (vfinfo[vf].tx_rate[idx] != tx_rate)
3841                                 vfinfo[vf].tx_rate[idx] = tx_rate;
3842                         total_rate += tx_rate;
3843                 }
3844         }
3845
3846         if (total_rate > dev->data->dev_link.link_speed) {
3847                 /*
3848                  * Reset stored TX rate of the VF if it causes exceed
3849                  * link speed.
3850                  */
3851                 memset(vfinfo[vf].tx_rate, 0, sizeof(vfinfo[vf].tx_rate));
3852                 return -EINVAL;
3853         }
3854
3855         /* Set RTTBCNRC of each queue/pool for vf X  */
3856         for (; queue_idx <= queue_end; queue_idx++) {
3857                 if (0x1 & q_msk)
3858                         ixgbe_set_queue_rate_limit(dev, queue_idx, tx_rate);
3859                 q_msk = q_msk >> 1;
3860         }
3861
3862         return 0;
3863 }
3864
3865 static void
3866 ixgbevf_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
3867                      __attribute__((unused)) uint32_t index,
3868                      __attribute__((unused)) uint32_t pool)
3869 {
3870         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3871         int diag;
3872
3873         /*
3874          * On a 82599 VF, adding again the same MAC addr is not an idempotent
3875          * operation. Trap this case to avoid exhausting the [very limited]
3876          * set of PF resources used to store VF MAC addresses.
3877          */
3878         if (memcmp(hw->mac.perm_addr, mac_addr, sizeof(struct ether_addr)) == 0)
3879                 return;
3880         diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes);
3881         if (diag == 0)
3882                 return;
3883         PMD_DRV_LOG(ERR, "Unable to add MAC address - diag=%d", diag);
3884 }
3885
3886 static void
3887 ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index)
3888 {
3889         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3890         struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr;
3891         struct ether_addr *mac_addr;
3892         uint32_t i;
3893         int diag;
3894
3895         /*
3896          * The IXGBE_VF_SET_MACVLAN command of the ixgbe-pf driver does
3897          * not support the deletion of a given MAC address.
3898          * Instead, it imposes to delete all MAC addresses, then to add again
3899          * all MAC addresses with the exception of the one to be deleted.
3900          */
3901         (void) ixgbevf_set_uc_addr_vf(hw, 0, NULL);
3902
3903         /*
3904          * Add again all MAC addresses, with the exception of the deleted one
3905          * and of the permanent MAC address.
3906          */
3907         for (i = 0, mac_addr = dev->data->mac_addrs;
3908              i < hw->mac.num_rar_entries; i++, mac_addr++) {
3909                 /* Skip the deleted MAC address */
3910                 if (i == index)
3911                         continue;
3912                 /* Skip NULL MAC addresses */
3913                 if (is_zero_ether_addr(mac_addr))
3914                         continue;
3915                 /* Skip the permanent MAC address */
3916                 if (memcmp(perm_addr, mac_addr, sizeof(struct ether_addr)) == 0)
3917                         continue;
3918                 diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes);
3919                 if (diag != 0)
3920                         PMD_DRV_LOG(ERR,
3921                                     "Adding again MAC address "
3922                                     "%02x:%02x:%02x:%02x:%02x:%02x failed "
3923                                     "diag=%d",
3924                                     mac_addr->addr_bytes[0],
3925                                     mac_addr->addr_bytes[1],
3926                                     mac_addr->addr_bytes[2],
3927                                     mac_addr->addr_bytes[3],
3928                                     mac_addr->addr_bytes[4],
3929                                     mac_addr->addr_bytes[5],
3930                                     diag);
3931         }
3932 }
3933
3934 static void
3935 ixgbevf_set_default_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
3936 {
3937         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3938
3939         hw->mac.ops.set_rar(hw, 0, (void *)addr, 0, 0);
3940 }
3941
3942 #define MAC_TYPE_FILTER_SUP(type)    do {\
3943         if ((type) != ixgbe_mac_82599EB && (type) != ixgbe_mac_X540 &&\
3944                 (type) != ixgbe_mac_X550)\
3945                 return -ENOTSUP;\
3946 } while (0)
3947
3948 static int
3949 ixgbe_syn_filter_set(struct rte_eth_dev *dev,
3950                         struct rte_eth_syn_filter *filter,
3951                         bool add)
3952 {
3953         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3954         uint32_t synqf;
3955
3956         if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM)
3957                 return -EINVAL;
3958
3959         synqf = IXGBE_READ_REG(hw, IXGBE_SYNQF);
3960
3961         if (add) {
3962                 if (synqf & IXGBE_SYN_FILTER_ENABLE)
3963                         return -EINVAL;
3964                 synqf = (uint32_t)(((filter->queue << IXGBE_SYN_FILTER_QUEUE_SHIFT) &
3965                         IXGBE_SYN_FILTER_QUEUE) | IXGBE_SYN_FILTER_ENABLE);
3966
3967                 if (filter->hig_pri)
3968                         synqf |= IXGBE_SYN_FILTER_SYNQFP;
3969                 else
3970                         synqf &= ~IXGBE_SYN_FILTER_SYNQFP;
3971         } else {
3972                 if (!(synqf & IXGBE_SYN_FILTER_ENABLE))
3973                         return -ENOENT;
3974                 synqf &= ~(IXGBE_SYN_FILTER_QUEUE | IXGBE_SYN_FILTER_ENABLE);
3975         }
3976         IXGBE_WRITE_REG(hw, IXGBE_SYNQF, synqf);
3977         IXGBE_WRITE_FLUSH(hw);
3978         return 0;
3979 }
3980
3981 static int
3982 ixgbe_syn_filter_get(struct rte_eth_dev *dev,
3983                         struct rte_eth_syn_filter *filter)
3984 {
3985         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3986         uint32_t synqf = IXGBE_READ_REG(hw, IXGBE_SYNQF);
3987
3988         if (synqf & IXGBE_SYN_FILTER_ENABLE) {
3989                 filter->hig_pri = (synqf & IXGBE_SYN_FILTER_SYNQFP) ? 1 : 0;
3990                 filter->queue = (uint16_t)((synqf & IXGBE_SYN_FILTER_QUEUE) >> 1);
3991                 return 0;
3992         }
3993         return -ENOENT;
3994 }
3995
3996 static int
3997 ixgbe_syn_filter_handle(struct rte_eth_dev *dev,
3998                         enum rte_filter_op filter_op,
3999                         void *arg)
4000 {
4001         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4002         int ret;
4003
4004         MAC_TYPE_FILTER_SUP(hw->mac.type);
4005
4006         if (filter_op == RTE_ETH_FILTER_NOP)
4007                 return 0;
4008
4009         if (arg == NULL) {
4010                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
4011                             filter_op);
4012                 return -EINVAL;
4013         }
4014
4015         switch (filter_op) {
4016         case RTE_ETH_FILTER_ADD:
4017                 ret = ixgbe_syn_filter_set(dev,
4018                                 (struct rte_eth_syn_filter *)arg,
4019                                 TRUE);
4020                 break;
4021         case RTE_ETH_FILTER_DELETE:
4022                 ret = ixgbe_syn_filter_set(dev,
4023                                 (struct rte_eth_syn_filter *)arg,
4024                                 FALSE);
4025                 break;
4026         case RTE_ETH_FILTER_GET:
4027                 ret = ixgbe_syn_filter_get(dev,
4028                                 (struct rte_eth_syn_filter *)arg);
4029                 break;
4030         default:
4031                 PMD_DRV_LOG(ERR, "unsupported operation %u\n", filter_op);
4032                 ret = -EINVAL;
4033                 break;
4034         }
4035
4036         return ret;
4037 }
4038
4039
4040 static inline enum ixgbe_5tuple_protocol
4041 convert_protocol_type(uint8_t protocol_value)
4042 {
4043         if (protocol_value == IPPROTO_TCP)
4044                 return IXGBE_FILTER_PROTOCOL_TCP;
4045         else if (protocol_value == IPPROTO_UDP)
4046                 return IXGBE_FILTER_PROTOCOL_UDP;
4047         else if (protocol_value == IPPROTO_SCTP)
4048                 return IXGBE_FILTER_PROTOCOL_SCTP;
4049         else
4050                 return IXGBE_FILTER_PROTOCOL_NONE;
4051 }
4052
4053 /*
4054  * add a 5tuple filter
4055  *
4056  * @param
4057  * dev: Pointer to struct rte_eth_dev.
4058  * index: the index the filter allocates.
4059  * filter: ponter to the filter that will be added.
4060  * rx_queue: the queue id the filter assigned to.
4061  *
4062  * @return
4063  *    - On success, zero.
4064  *    - On failure, a negative value.
4065  */
4066 static int
4067 ixgbe_add_5tuple_filter(struct rte_eth_dev *dev,
4068                         struct ixgbe_5tuple_filter *filter)
4069 {
4070         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4071         struct ixgbe_filter_info *filter_info =
4072                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4073         int i, idx, shift;
4074         uint32_t ftqf, sdpqf;
4075         uint32_t l34timir = 0;
4076         uint8_t mask = 0xff;
4077
4078         /*
4079          * look for an unused 5tuple filter index,
4080          * and insert the filter to list.
4081          */
4082         for (i = 0; i < IXGBE_MAX_FTQF_FILTERS; i++) {
4083                 idx = i / (sizeof(uint32_t) * NBBY);
4084                 shift = i % (sizeof(uint32_t) * NBBY);
4085                 if (!(filter_info->fivetuple_mask[idx] & (1 << shift))) {
4086                         filter_info->fivetuple_mask[idx] |= 1 << shift;
4087                         filter->index = i;
4088                         TAILQ_INSERT_TAIL(&filter_info->fivetuple_list,
4089                                           filter,
4090                                           entries);
4091                         break;
4092                 }
4093         }
4094         if (i >= IXGBE_MAX_FTQF_FILTERS) {
4095                 PMD_DRV_LOG(ERR, "5tuple filters are full.");
4096                 return -ENOSYS;
4097         }
4098
4099         sdpqf = (uint32_t)(filter->filter_info.dst_port <<
4100                                 IXGBE_SDPQF_DSTPORT_SHIFT);
4101         sdpqf = sdpqf | (filter->filter_info.src_port & IXGBE_SDPQF_SRCPORT);
4102
4103         ftqf = (uint32_t)(filter->filter_info.proto &
4104                 IXGBE_FTQF_PROTOCOL_MASK);
4105         ftqf |= (uint32_t)((filter->filter_info.priority &
4106                 IXGBE_FTQF_PRIORITY_MASK) << IXGBE_FTQF_PRIORITY_SHIFT);
4107         if (filter->filter_info.src_ip_mask == 0) /* 0 means compare. */
4108                 mask &= IXGBE_FTQF_SOURCE_ADDR_MASK;
4109         if (filter->filter_info.dst_ip_mask == 0)
4110                 mask &= IXGBE_FTQF_DEST_ADDR_MASK;
4111         if (filter->filter_info.src_port_mask == 0)
4112                 mask &= IXGBE_FTQF_SOURCE_PORT_MASK;
4113         if (filter->filter_info.dst_port_mask == 0)
4114                 mask &= IXGBE_FTQF_DEST_PORT_MASK;
4115         if (filter->filter_info.proto_mask == 0)
4116                 mask &= IXGBE_FTQF_PROTOCOL_COMP_MASK;
4117         ftqf |= mask << IXGBE_FTQF_5TUPLE_MASK_SHIFT;
4118         ftqf |= IXGBE_FTQF_POOL_MASK_EN;
4119         ftqf |= IXGBE_FTQF_QUEUE_ENABLE;
4120
4121         IXGBE_WRITE_REG(hw, IXGBE_DAQF(i), filter->filter_info.dst_ip);
4122         IXGBE_WRITE_REG(hw, IXGBE_SAQF(i), filter->filter_info.src_ip);
4123         IXGBE_WRITE_REG(hw, IXGBE_SDPQF(i), sdpqf);
4124         IXGBE_WRITE_REG(hw, IXGBE_FTQF(i), ftqf);
4125
4126         l34timir |= IXGBE_L34T_IMIR_RESERVE;
4127         l34timir |= (uint32_t)(filter->queue <<
4128                                 IXGBE_L34T_IMIR_QUEUE_SHIFT);
4129         IXGBE_WRITE_REG(hw, IXGBE_L34T_IMIR(i), l34timir);
4130         return 0;
4131 }
4132
4133 /*
4134  * remove a 5tuple filter
4135  *
4136  * @param
4137  * dev: Pointer to struct rte_eth_dev.
4138  * filter: the pointer of the filter will be removed.
4139  */
4140 static void
4141 ixgbe_remove_5tuple_filter(struct rte_eth_dev *dev,
4142                         struct ixgbe_5tuple_filter *filter)
4143 {
4144         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4145         struct ixgbe_filter_info *filter_info =
4146                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4147         uint16_t index = filter->index;
4148
4149         filter_info->fivetuple_mask[index / (sizeof(uint32_t) * NBBY)] &=
4150                                 ~(1 << (index % (sizeof(uint32_t) * NBBY)));
4151         TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries);
4152         rte_free(filter);
4153
4154         IXGBE_WRITE_REG(hw, IXGBE_DAQF(index), 0);
4155         IXGBE_WRITE_REG(hw, IXGBE_SAQF(index), 0);
4156         IXGBE_WRITE_REG(hw, IXGBE_SDPQF(index), 0);
4157         IXGBE_WRITE_REG(hw, IXGBE_FTQF(index), 0);
4158         IXGBE_WRITE_REG(hw, IXGBE_L34T_IMIR(index), 0);
4159 }
4160
4161 static int
4162 ixgbevf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
4163 {
4164         struct ixgbe_hw *hw;
4165         uint32_t max_frame = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
4166
4167         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4168
4169         if ((mtu < ETHER_MIN_MTU) || (max_frame > ETHER_MAX_JUMBO_FRAME_LEN))
4170                 return -EINVAL;
4171
4172         /* refuse mtu that requires the support of scattered packets when this
4173          * feature has not been enabled before. */
4174         if (!dev->data->scattered_rx &&
4175             (max_frame + 2 * IXGBE_VLAN_TAG_SIZE >
4176              dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM))
4177                 return -EINVAL;
4178
4179         /*
4180          * When supported by the underlying PF driver, use the IXGBE_VF_SET_MTU
4181          * request of the version 2.0 of the mailbox API.
4182          * For now, use the IXGBE_VF_SET_LPE request of the version 1.0
4183          * of the mailbox API.
4184          * This call to IXGBE_SET_LPE action won't work with ixgbe pf drivers
4185          * prior to 3.11.33 which contains the following change:
4186          * "ixgbe: Enable jumbo frames support w/ SR-IOV"
4187          */
4188         ixgbevf_rlpml_set_vf(hw, max_frame);
4189
4190         /* update max frame size */
4191         dev->data->dev_conf.rxmode.max_rx_pkt_len = max_frame;
4192         return 0;
4193 }
4194
4195 #define MAC_TYPE_FILTER_SUP_EXT(type)    do {\
4196         if ((type) != ixgbe_mac_82599EB && (type) != ixgbe_mac_X540)\
4197                 return -ENOTSUP;\
4198 } while (0)
4199
4200 static inline struct ixgbe_5tuple_filter *
4201 ixgbe_5tuple_filter_lookup(struct ixgbe_5tuple_filter_list *filter_list,
4202                         struct ixgbe_5tuple_filter_info *key)
4203 {
4204         struct ixgbe_5tuple_filter *it;
4205
4206         TAILQ_FOREACH(it, filter_list, entries) {
4207                 if (memcmp(key, &it->filter_info,
4208                         sizeof(struct ixgbe_5tuple_filter_info)) == 0) {
4209                         return it;
4210                 }
4211         }
4212         return NULL;
4213 }
4214
4215 /* translate elements in struct rte_eth_ntuple_filter to struct ixgbe_5tuple_filter_info*/
4216 static inline int
4217 ntuple_filter_to_5tuple(struct rte_eth_ntuple_filter *filter,
4218                         struct ixgbe_5tuple_filter_info *filter_info)
4219 {
4220         if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM ||
4221                 filter->priority > IXGBE_5TUPLE_MAX_PRI ||
4222                 filter->priority < IXGBE_5TUPLE_MIN_PRI)
4223                 return -EINVAL;
4224
4225         switch (filter->dst_ip_mask) {
4226         case UINT32_MAX:
4227                 filter_info->dst_ip_mask = 0;
4228                 filter_info->dst_ip = filter->dst_ip;
4229                 break;
4230         case 0:
4231                 filter_info->dst_ip_mask = 1;
4232                 break;
4233         default:
4234                 PMD_DRV_LOG(ERR, "invalid dst_ip mask.");
4235                 return -EINVAL;
4236         }
4237
4238         switch (filter->src_ip_mask) {
4239         case UINT32_MAX:
4240                 filter_info->src_ip_mask = 0;
4241                 filter_info->src_ip = filter->src_ip;
4242                 break;
4243         case 0:
4244                 filter_info->src_ip_mask = 1;
4245                 break;
4246         default:
4247                 PMD_DRV_LOG(ERR, "invalid src_ip mask.");
4248                 return -EINVAL;
4249         }
4250
4251         switch (filter->dst_port_mask) {
4252         case UINT16_MAX:
4253                 filter_info->dst_port_mask = 0;
4254                 filter_info->dst_port = filter->dst_port;
4255                 break;
4256         case 0:
4257                 filter_info->dst_port_mask = 1;
4258                 break;
4259         default:
4260                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
4261                 return -EINVAL;
4262         }
4263
4264         switch (filter->src_port_mask) {
4265         case UINT16_MAX:
4266                 filter_info->src_port_mask = 0;
4267                 filter_info->src_port = filter->src_port;
4268                 break;
4269         case 0:
4270                 filter_info->src_port_mask = 1;
4271                 break;
4272         default:
4273                 PMD_DRV_LOG(ERR, "invalid src_port mask.");
4274                 return -EINVAL;
4275         }
4276
4277         switch (filter->proto_mask) {
4278         case UINT8_MAX:
4279                 filter_info->proto_mask = 0;
4280                 filter_info->proto =
4281                         convert_protocol_type(filter->proto);
4282                 break;
4283         case 0:
4284                 filter_info->proto_mask = 1;
4285                 break;
4286         default:
4287                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
4288                 return -EINVAL;
4289         }
4290
4291         filter_info->priority = (uint8_t)filter->priority;
4292         return 0;
4293 }
4294
4295 /*
4296  * add or delete a ntuple filter
4297  *
4298  * @param
4299  * dev: Pointer to struct rte_eth_dev.
4300  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
4301  * add: if true, add filter, if false, remove filter
4302  *
4303  * @return
4304  *    - On success, zero.
4305  *    - On failure, a negative value.
4306  */
4307 static int
4308 ixgbe_add_del_ntuple_filter(struct rte_eth_dev *dev,
4309                         struct rte_eth_ntuple_filter *ntuple_filter,
4310                         bool add)
4311 {
4312         struct ixgbe_filter_info *filter_info =
4313                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4314         struct ixgbe_5tuple_filter_info filter_5tuple;
4315         struct ixgbe_5tuple_filter *filter;
4316         int ret;
4317
4318         if (ntuple_filter->flags != RTE_5TUPLE_FLAGS) {
4319                 PMD_DRV_LOG(ERR, "only 5tuple is supported.");
4320                 return -EINVAL;
4321         }
4322
4323         memset(&filter_5tuple, 0, sizeof(struct ixgbe_5tuple_filter_info));
4324         ret = ntuple_filter_to_5tuple(ntuple_filter, &filter_5tuple);
4325         if (ret < 0)
4326                 return ret;
4327
4328         filter = ixgbe_5tuple_filter_lookup(&filter_info->fivetuple_list,
4329                                          &filter_5tuple);
4330         if (filter != NULL && add) {
4331                 PMD_DRV_LOG(ERR, "filter exists.");
4332                 return -EEXIST;
4333         }
4334         if (filter == NULL && !add) {
4335                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
4336                 return -ENOENT;
4337         }
4338
4339         if (add) {
4340                 filter = rte_zmalloc("ixgbe_5tuple_filter",
4341                                 sizeof(struct ixgbe_5tuple_filter), 0);
4342                 if (filter == NULL)
4343                         return -ENOMEM;
4344                 (void)rte_memcpy(&filter->filter_info,
4345                                  &filter_5tuple,
4346                                  sizeof(struct ixgbe_5tuple_filter_info));
4347                 filter->queue = ntuple_filter->queue;
4348                 ret = ixgbe_add_5tuple_filter(dev, filter);
4349                 if (ret < 0) {
4350                         rte_free(filter);
4351                         return ret;
4352                 }
4353         } else
4354                 ixgbe_remove_5tuple_filter(dev, filter);
4355
4356         return 0;
4357 }
4358
4359 /*
4360  * get a ntuple filter
4361  *
4362  * @param
4363  * dev: Pointer to struct rte_eth_dev.
4364  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
4365  *
4366  * @return
4367  *    - On success, zero.
4368  *    - On failure, a negative value.
4369  */
4370 static int
4371 ixgbe_get_ntuple_filter(struct rte_eth_dev *dev,
4372                         struct rte_eth_ntuple_filter *ntuple_filter)
4373 {
4374         struct ixgbe_filter_info *filter_info =
4375                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4376         struct ixgbe_5tuple_filter_info filter_5tuple;
4377         struct ixgbe_5tuple_filter *filter;
4378         int ret;
4379
4380         if (ntuple_filter->flags != RTE_5TUPLE_FLAGS) {
4381                 PMD_DRV_LOG(ERR, "only 5tuple is supported.");
4382                 return -EINVAL;
4383         }
4384
4385         memset(&filter_5tuple, 0, sizeof(struct ixgbe_5tuple_filter_info));
4386         ret = ntuple_filter_to_5tuple(ntuple_filter, &filter_5tuple);
4387         if (ret < 0)
4388                 return ret;
4389
4390         filter = ixgbe_5tuple_filter_lookup(&filter_info->fivetuple_list,
4391                                          &filter_5tuple);
4392         if (filter == NULL) {
4393                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
4394                 return -ENOENT;
4395         }
4396         ntuple_filter->queue = filter->queue;
4397         return 0;
4398 }
4399
4400 /*
4401  * ixgbe_ntuple_filter_handle - Handle operations for ntuple filter.
4402  * @dev: pointer to rte_eth_dev structure
4403  * @filter_op:operation will be taken.
4404  * @arg: a pointer to specific structure corresponding to the filter_op
4405  *
4406  * @return
4407  *    - On success, zero.
4408  *    - On failure, a negative value.
4409  */
4410 static int
4411 ixgbe_ntuple_filter_handle(struct rte_eth_dev *dev,
4412                                 enum rte_filter_op filter_op,
4413                                 void *arg)
4414 {
4415         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4416         int ret;
4417
4418         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
4419
4420         if (filter_op == RTE_ETH_FILTER_NOP)
4421                 return 0;
4422
4423         if (arg == NULL) {
4424                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
4425                             filter_op);
4426                 return -EINVAL;
4427         }
4428
4429         switch (filter_op) {
4430         case RTE_ETH_FILTER_ADD:
4431                 ret = ixgbe_add_del_ntuple_filter(dev,
4432                         (struct rte_eth_ntuple_filter *)arg,
4433                         TRUE);
4434                 break;
4435         case RTE_ETH_FILTER_DELETE:
4436                 ret = ixgbe_add_del_ntuple_filter(dev,
4437                         (struct rte_eth_ntuple_filter *)arg,
4438                         FALSE);
4439                 break;
4440         case RTE_ETH_FILTER_GET:
4441                 ret = ixgbe_get_ntuple_filter(dev,
4442                         (struct rte_eth_ntuple_filter *)arg);
4443                 break;
4444         default:
4445                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
4446                 ret = -EINVAL;
4447                 break;
4448         }
4449         return ret;
4450 }
4451
4452 static inline int
4453 ixgbe_ethertype_filter_lookup(struct ixgbe_filter_info *filter_info,
4454                         uint16_t ethertype)
4455 {
4456         int i;
4457
4458         for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) {
4459                 if (filter_info->ethertype_filters[i] == ethertype &&
4460                     (filter_info->ethertype_mask & (1 << i)))
4461                         return i;
4462         }
4463         return -1;
4464 }
4465
4466 static inline int
4467 ixgbe_ethertype_filter_insert(struct ixgbe_filter_info *filter_info,
4468                         uint16_t ethertype)
4469 {
4470         int i;
4471
4472         for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) {
4473                 if (!(filter_info->ethertype_mask & (1 << i))) {
4474                         filter_info->ethertype_mask |= 1 << i;
4475                         filter_info->ethertype_filters[i] = ethertype;
4476                         return i;
4477                 }
4478         }
4479         return -1;
4480 }
4481
4482 static inline int
4483 ixgbe_ethertype_filter_remove(struct ixgbe_filter_info *filter_info,
4484                         uint8_t idx)
4485 {
4486         if (idx >= IXGBE_MAX_ETQF_FILTERS)
4487                 return -1;
4488         filter_info->ethertype_mask &= ~(1 << idx);
4489         filter_info->ethertype_filters[idx] = 0;
4490         return idx;
4491 }
4492
4493 static int
4494 ixgbe_add_del_ethertype_filter(struct rte_eth_dev *dev,
4495                         struct rte_eth_ethertype_filter *filter,
4496                         bool add)
4497 {
4498         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4499         struct ixgbe_filter_info *filter_info =
4500                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4501         uint32_t etqf = 0;
4502         uint32_t etqs = 0;
4503         int ret;
4504
4505         if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM)
4506                 return -EINVAL;
4507
4508         if (filter->ether_type == ETHER_TYPE_IPv4 ||
4509                 filter->ether_type == ETHER_TYPE_IPv6) {
4510                 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in"
4511                         " ethertype filter.", filter->ether_type);
4512                 return -EINVAL;
4513         }
4514
4515         if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) {
4516                 PMD_DRV_LOG(ERR, "mac compare is unsupported.");
4517                 return -EINVAL;
4518         }
4519         if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) {
4520                 PMD_DRV_LOG(ERR, "drop option is unsupported.");
4521                 return -EINVAL;
4522         }
4523
4524         ret = ixgbe_ethertype_filter_lookup(filter_info, filter->ether_type);
4525         if (ret >= 0 && add) {
4526                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.",
4527                             filter->ether_type);
4528                 return -EEXIST;
4529         }
4530         if (ret < 0 && !add) {
4531                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
4532                             filter->ether_type);
4533                 return -ENOENT;
4534         }
4535
4536         if (add) {
4537                 ret = ixgbe_ethertype_filter_insert(filter_info,
4538                         filter->ether_type);
4539                 if (ret < 0) {
4540                         PMD_DRV_LOG(ERR, "ethertype filters are full.");
4541                         return -ENOSYS;
4542                 }
4543                 etqf = IXGBE_ETQF_FILTER_EN;
4544                 etqf |= (uint32_t)filter->ether_type;
4545                 etqs |= (uint32_t)((filter->queue <<
4546                                     IXGBE_ETQS_RX_QUEUE_SHIFT) &
4547                                     IXGBE_ETQS_RX_QUEUE);
4548                 etqs |= IXGBE_ETQS_QUEUE_EN;
4549         } else {
4550                 ret = ixgbe_ethertype_filter_remove(filter_info, (uint8_t)ret);
4551                 if (ret < 0)
4552                         return -ENOSYS;
4553         }
4554         IXGBE_WRITE_REG(hw, IXGBE_ETQF(ret), etqf);
4555         IXGBE_WRITE_REG(hw, IXGBE_ETQS(ret), etqs);
4556         IXGBE_WRITE_FLUSH(hw);
4557
4558         return 0;
4559 }
4560
4561 static int
4562 ixgbe_get_ethertype_filter(struct rte_eth_dev *dev,
4563                         struct rte_eth_ethertype_filter *filter)
4564 {
4565         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4566         struct ixgbe_filter_info *filter_info =
4567                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4568         uint32_t etqf, etqs;
4569         int ret;
4570
4571         ret = ixgbe_ethertype_filter_lookup(filter_info, filter->ether_type);
4572         if (ret < 0) {
4573                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
4574                             filter->ether_type);
4575                 return -ENOENT;
4576         }
4577
4578         etqf = IXGBE_READ_REG(hw, IXGBE_ETQF(ret));
4579         if (etqf & IXGBE_ETQF_FILTER_EN) {
4580                 etqs = IXGBE_READ_REG(hw, IXGBE_ETQS(ret));
4581                 filter->ether_type = etqf & IXGBE_ETQF_ETHERTYPE;
4582                 filter->flags = 0;
4583                 filter->queue = (etqs & IXGBE_ETQS_RX_QUEUE) >>
4584                                IXGBE_ETQS_RX_QUEUE_SHIFT;
4585                 return 0;
4586         }
4587         return -ENOENT;
4588 }
4589
4590 /*
4591  * ixgbe_ethertype_filter_handle - Handle operations for ethertype filter.
4592  * @dev: pointer to rte_eth_dev structure
4593  * @filter_op:operation will be taken.
4594  * @arg: a pointer to specific structure corresponding to the filter_op
4595  */
4596 static int
4597 ixgbe_ethertype_filter_handle(struct rte_eth_dev *dev,
4598                                 enum rte_filter_op filter_op,
4599                                 void *arg)
4600 {
4601         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4602         int ret;
4603
4604         MAC_TYPE_FILTER_SUP(hw->mac.type);
4605
4606         if (filter_op == RTE_ETH_FILTER_NOP)
4607                 return 0;
4608
4609         if (arg == NULL) {
4610                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
4611                             filter_op);
4612                 return -EINVAL;
4613         }
4614
4615         switch (filter_op) {
4616         case RTE_ETH_FILTER_ADD:
4617                 ret = ixgbe_add_del_ethertype_filter(dev,
4618                         (struct rte_eth_ethertype_filter *)arg,
4619                         TRUE);
4620                 break;
4621         case RTE_ETH_FILTER_DELETE:
4622                 ret = ixgbe_add_del_ethertype_filter(dev,
4623                         (struct rte_eth_ethertype_filter *)arg,
4624                         FALSE);
4625                 break;
4626         case RTE_ETH_FILTER_GET:
4627                 ret = ixgbe_get_ethertype_filter(dev,
4628                         (struct rte_eth_ethertype_filter *)arg);
4629                 break;
4630         default:
4631                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
4632                 ret = -EINVAL;
4633                 break;
4634         }
4635         return ret;
4636 }
4637
4638 static int
4639 ixgbe_dev_filter_ctrl(struct rte_eth_dev *dev,
4640                      enum rte_filter_type filter_type,
4641                      enum rte_filter_op filter_op,
4642                      void *arg)
4643 {
4644         int ret = -EINVAL;
4645
4646         switch (filter_type) {
4647         case RTE_ETH_FILTER_NTUPLE:
4648                 ret = ixgbe_ntuple_filter_handle(dev, filter_op, arg);
4649                 break;
4650         case RTE_ETH_FILTER_ETHERTYPE:
4651                 ret = ixgbe_ethertype_filter_handle(dev, filter_op, arg);
4652                 break;
4653         case RTE_ETH_FILTER_SYN:
4654                 ret = ixgbe_syn_filter_handle(dev, filter_op, arg);
4655                 break;
4656         case RTE_ETH_FILTER_FDIR:
4657                 ret = ixgbe_fdir_ctrl_func(dev, filter_op, arg);
4658                 break;
4659         default:
4660                 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported",
4661                                                         filter_type);
4662                 break;
4663         }
4664
4665         return ret;
4666 }
4667
4668 static u8 *
4669 ixgbe_dev_addr_list_itr(__attribute__((unused)) struct ixgbe_hw *hw,
4670                         u8 **mc_addr_ptr, u32 *vmdq)
4671 {
4672         u8 *mc_addr;
4673
4674         *vmdq = 0;
4675         mc_addr = *mc_addr_ptr;
4676         *mc_addr_ptr = (mc_addr + sizeof(struct ether_addr));
4677         return mc_addr;
4678 }
4679
4680 static int
4681 ixgbe_dev_set_mc_addr_list(struct rte_eth_dev *dev,
4682                           struct ether_addr *mc_addr_set,
4683                           uint32_t nb_mc_addr)
4684 {
4685         struct ixgbe_hw *hw;
4686         u8 *mc_addr_list;
4687
4688         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4689         mc_addr_list = (u8 *)mc_addr_set;
4690         return ixgbe_update_mc_addr_list(hw, mc_addr_list, nb_mc_addr,
4691                                          ixgbe_dev_addr_list_itr, TRUE);
4692 }
4693
4694 static int
4695 ixgbe_timesync_enable(struct rte_eth_dev *dev)
4696 {
4697         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4698         uint32_t tsync_ctl;
4699         uint32_t tsauxc;
4700
4701         /* Enable system time for platforms where it isn't on by default. */
4702         tsauxc = IXGBE_READ_REG(hw, IXGBE_TSAUXC);
4703         tsauxc &= ~IXGBE_TSAUXC_DISABLE_SYSTIME;
4704         IXGBE_WRITE_REG(hw, IXGBE_TSAUXC, tsauxc);
4705
4706         /* Start incrementing the register used to timestamp PTP packets. */
4707         IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, IXGBE_TIMINCA_INIT);
4708
4709         /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
4710         IXGBE_WRITE_REG(hw, IXGBE_ETQF(IXGBE_ETQF_FILTER_1588),
4711                         (ETHER_TYPE_1588 |
4712                          IXGBE_ETQF_FILTER_EN |
4713                          IXGBE_ETQF_1588));
4714
4715         /* Enable timestamping of received PTP packets. */
4716         tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL);
4717         tsync_ctl |= IXGBE_TSYNCRXCTL_ENABLED;
4718         IXGBE_WRITE_REG(hw, IXGBE_TSYNCRXCTL, tsync_ctl);
4719
4720         /* Enable timestamping of transmitted PTP packets. */
4721         tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL);
4722         tsync_ctl |= IXGBE_TSYNCTXCTL_ENABLED;
4723         IXGBE_WRITE_REG(hw, IXGBE_TSYNCTXCTL, tsync_ctl);
4724
4725         return 0;
4726 }
4727
4728 static int
4729 ixgbe_timesync_disable(struct rte_eth_dev *dev)
4730 {
4731         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4732         uint32_t tsync_ctl;
4733
4734         /* Disable timestamping of transmitted PTP packets. */
4735         tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL);
4736         tsync_ctl &= ~IXGBE_TSYNCTXCTL_ENABLED;
4737         IXGBE_WRITE_REG(hw, IXGBE_TSYNCTXCTL, tsync_ctl);
4738
4739         /* Disable timestamping of received PTP packets. */
4740         tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL);
4741         tsync_ctl &= ~IXGBE_TSYNCRXCTL_ENABLED;
4742         IXGBE_WRITE_REG(hw, IXGBE_TSYNCRXCTL, tsync_ctl);
4743
4744         /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
4745         IXGBE_WRITE_REG(hw, IXGBE_ETQF(IXGBE_ETQF_FILTER_1588), 0);
4746
4747         /* Stop incrementating the System Time registers. */
4748         IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, 0);
4749
4750         return 0;
4751 }
4752
4753 static int
4754 ixgbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
4755                                  struct timespec *timestamp,
4756                                  uint32_t flags __rte_unused)
4757 {
4758         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4759         uint32_t tsync_rxctl;
4760         uint32_t rx_stmpl;
4761         uint32_t rx_stmph;
4762
4763         tsync_rxctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL);
4764         if ((tsync_rxctl & IXGBE_TSYNCRXCTL_VALID) == 0)
4765                 return -EINVAL;
4766
4767         rx_stmpl = IXGBE_READ_REG(hw, IXGBE_RXSTMPL);
4768         rx_stmph = IXGBE_READ_REG(hw, IXGBE_RXSTMPH);
4769
4770         timestamp->tv_sec = (uint64_t)(((uint64_t)rx_stmph << 32) | rx_stmpl);
4771         timestamp->tv_nsec = 0;
4772
4773         return  0;
4774 }
4775
4776 static int
4777 ixgbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
4778                                  struct timespec *timestamp)
4779 {
4780         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4781         uint32_t tsync_txctl;
4782         uint32_t tx_stmpl;
4783         uint32_t tx_stmph;
4784
4785         tsync_txctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL);
4786         if ((tsync_txctl & IXGBE_TSYNCTXCTL_VALID) == 0)
4787                 return -EINVAL;
4788
4789         tx_stmpl = IXGBE_READ_REG(hw, IXGBE_TXSTMPL);
4790         tx_stmph = IXGBE_READ_REG(hw, IXGBE_TXSTMPH);
4791
4792         timestamp->tv_sec = (uint64_t)(((uint64_t)tx_stmph << 32) | tx_stmpl);
4793         timestamp->tv_nsec = 0;
4794
4795         return  0;
4796 }
4797
4798 static int
4799 ixgbe_get_reg_length(struct rte_eth_dev *dev)
4800 {
4801         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4802         int count = 0;
4803         int g_ind = 0;
4804         const struct reg_info *reg_group;
4805         const struct reg_info **reg_set = (hw->mac.type == ixgbe_mac_82598EB) ?
4806                                     ixgbe_regs_mac_82598EB : ixgbe_regs_others;
4807
4808         while ((reg_group = reg_set[g_ind++]))
4809                 count += ixgbe_regs_group_count(reg_group);
4810
4811         return count;
4812 }
4813
4814 static int
4815 ixgbevf_get_reg_length(struct rte_eth_dev *dev __rte_unused)
4816 {
4817         int count = 0;
4818         int g_ind = 0;
4819         const struct reg_info *reg_group;
4820
4821         while ((reg_group = ixgbevf_regs[g_ind++]))
4822                 count += ixgbe_regs_group_count(reg_group);
4823
4824         return count;
4825 }
4826
4827 static int
4828 ixgbe_get_regs(struct rte_eth_dev *dev,
4829               struct rte_dev_reg_info *regs)
4830 {
4831         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4832         uint32_t *data = regs->data;
4833         int g_ind = 0;
4834         int count = 0;
4835         const struct reg_info *reg_group;
4836         const struct reg_info **reg_set = (hw->mac.type == ixgbe_mac_82598EB) ?
4837                                     ixgbe_regs_mac_82598EB : ixgbe_regs_others;
4838
4839         /* Support only full register dump */
4840         if ((regs->length == 0) ||
4841             (regs->length == (uint32_t)ixgbe_get_reg_length(dev))) {
4842                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
4843                         hw->device_id;
4844                 while ((reg_group = reg_set[g_ind++]))
4845                         count += ixgbe_read_regs_group(dev, &data[count],
4846                                 reg_group);
4847                 return 0;
4848         }
4849
4850         return -ENOTSUP;
4851 }
4852
4853 static int
4854 ixgbevf_get_regs(struct rte_eth_dev *dev,
4855                 struct rte_dev_reg_info *regs)
4856 {
4857         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4858         uint32_t *data = regs->data;
4859         int g_ind = 0;
4860         int count = 0;
4861         const struct reg_info *reg_group;
4862
4863         /* Support only full register dump */
4864         if ((regs->length == 0) ||
4865             (regs->length == (uint32_t)ixgbevf_get_reg_length(dev))) {
4866                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
4867                         hw->device_id;
4868                 while ((reg_group = ixgbevf_regs[g_ind++]))
4869                         count += ixgbe_read_regs_group(dev, &data[count],
4870                                                       reg_group);
4871                 return 0;
4872         }
4873
4874         return -ENOTSUP;
4875 }
4876
4877 static int
4878 ixgbe_get_eeprom_length(struct rte_eth_dev *dev)
4879 {
4880         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4881
4882         /* Return unit is byte count */
4883         return hw->eeprom.word_size * 2;
4884 }
4885
4886 static int
4887 ixgbe_get_eeprom(struct rte_eth_dev *dev,
4888                 struct rte_dev_eeprom_info *in_eeprom)
4889 {
4890         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4891         struct ixgbe_eeprom_info *eeprom = &hw->eeprom;
4892         uint16_t *data = in_eeprom->data;
4893         int first, length;
4894
4895         first = in_eeprom->offset >> 1;
4896         length = in_eeprom->length >> 1;
4897         if ((first >= hw->eeprom.word_size) ||
4898             ((first + length) >= hw->eeprom.word_size))
4899                 return -EINVAL;
4900
4901         in_eeprom->magic = hw->vendor_id | (hw->device_id << 16);
4902
4903         return eeprom->ops.read_buffer(hw, first, length, data);
4904 }
4905
4906 static int
4907 ixgbe_set_eeprom(struct rte_eth_dev *dev,
4908                 struct rte_dev_eeprom_info *in_eeprom)
4909 {
4910         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4911         struct ixgbe_eeprom_info *eeprom = &hw->eeprom;
4912         uint16_t *data = in_eeprom->data;
4913         int first, length;
4914
4915         first = in_eeprom->offset >> 1;
4916         length = in_eeprom->length >> 1;
4917         if ((first >= hw->eeprom.word_size) ||
4918             ((first + length) >= hw->eeprom.word_size))
4919                 return -EINVAL;
4920
4921         in_eeprom->magic = hw->vendor_id | (hw->device_id << 16);
4922
4923         return eeprom->ops.write_buffer(hw,  first, length, data);
4924 }
4925
4926 static struct rte_driver rte_ixgbe_driver = {
4927         .type = PMD_PDEV,
4928         .init = rte_ixgbe_pmd_init,
4929 };
4930
4931 static struct rte_driver rte_ixgbevf_driver = {
4932         .type = PMD_PDEV,
4933         .init = rte_ixgbevf_pmd_init,
4934 };
4935
4936 PMD_REGISTER_DRIVER(rte_ixgbe_driver);
4937 PMD_REGISTER_DRIVER(rte_ixgbevf_driver);