drivers/net/ixgbe/ixgbe_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <inttypes.h>
  42 #include <netinet/in.h>
  43 #include <rte_byteorder.h>
  44 #include <rte_common.h>
  45 #include <rte_cycles.h>
  46
  47 #include <rte_interrupts.h>
  48 #include <rte_log.h>
  49 #include <rte_debug.h>
  50 #include <rte_pci.h>
  51 #include <rte_atomic.h>
  52 #include <rte_branch_prediction.h>
  53 #include <rte_memory.h>
  54 #include <rte_memzone.h>
  55 #include <rte_eal.h>
  56 #include <rte_alarm.h>
  57 #include <rte_ether.h>
  58 #include <rte_ethdev.h>
  59 #include <rte_atomic.h>
  60 #include <rte_malloc.h>
  61 #include <rte_random.h>
  62 #include <rte_dev.h>
  63
  64 #include "ixgbe_logs.h"
  65 #include "base/ixgbe_api.h"
  66 #include "base/ixgbe_vf.h"
  67 #include "base/ixgbe_common.h"
  68 #include "ixgbe_ethdev.h"
  69 #include "ixgbe_bypass.h"
  70 #include "ixgbe_rxtx.h"
  71 #include "base/ixgbe_type.h"
  72 #include "base/ixgbe_phy.h"
  73 #include "ixgbe_regs.h"
  74
  75 /*
  76  * High threshold controlling when to start sending XOFF frames. Must be at
  77  * least 8 bytes less than receive packet buffer size. This value is in units
  78  * of 1024 bytes.
  79  */
  80 #define IXGBE_FC_HI    0x80
  81
  82 /*
  83  * Low threshold controlling when to start sending XON frames. This value is
  84  * in units of 1024 bytes.
  85  */
  86 #define IXGBE_FC_LO    0x40
  87
  88 /* Timer value included in XOFF frames. */
  89 #define IXGBE_FC_PAUSE 0x680
  90
  91 #define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */
  92 #define IXGBE_LINK_UP_CHECK_TIMEOUT   1000 /* ms */
  93 #define IXGBE_VMDQ_NUM_UC_MAC         4096 /* Maximum nb. of UC MAC addr. */
  94
  95 #define IXGBE_MMW_SIZE_DEFAULT        0x4
  96 #define IXGBE_MMW_SIZE_JUMBO_FRAME    0x14
  97 #define IXGBE_MAX_RING_DESC           4096 /* replicate define from rxtx */
  98
  99 /*
 100  *  Default values for RX/TX configuration
 101  */
 102 #define IXGBE_DEFAULT_RX_FREE_THRESH  32
 103 #define IXGBE_DEFAULT_RX_PTHRESH      8
 104 #define IXGBE_DEFAULT_RX_HTHRESH      8
 105 #define IXGBE_DEFAULT_RX_WTHRESH      0
 106
 107 #define IXGBE_DEFAULT_TX_FREE_THRESH  32
 108 #define IXGBE_DEFAULT_TX_PTHRESH      32
 109 #define IXGBE_DEFAULT_TX_HTHRESH      0
 110 #define IXGBE_DEFAULT_TX_WTHRESH      0
 111 #define IXGBE_DEFAULT_TX_RSBIT_THRESH 32
 112
 113 /* Bit shift and mask */
 114 #define IXGBE_4_BIT_WIDTH  (CHAR_BIT / 2)
 115 #define IXGBE_4_BIT_MASK   RTE_LEN2MASK(IXGBE_4_BIT_WIDTH, uint8_t)
 116 #define IXGBE_8_BIT_WIDTH  CHAR_BIT
 117 #define IXGBE_8_BIT_MASK   UINT8_MAX
 118
 119 #define IXGBEVF_PMD_NAME "rte_ixgbevf_pmd" /* PMD name */
 120
 121 #define IXGBE_QUEUE_STAT_COUNTERS (sizeof(hw_stats->qprc) / sizeof(hw_stats->qprc[0]))
 122
 123 #define IXGBE_HKEY_MAX_INDEX 10
 124
 125 /* Additional timesync values. */
 126 #define IXGBE_TIMINCA_16NS_SHIFT 24
 127 #define IXGBE_TIMINCA_INCVALUE   16000000
 128 #define IXGBE_TIMINCA_INIT       ((0x02 << IXGBE_TIMINCA_16NS_SHIFT) \
 129                                   | IXGBE_TIMINCA_INCVALUE)
 130
 131 static int eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev);
 132 static int  ixgbe_dev_configure(struct rte_eth_dev *dev);
 133 static int  ixgbe_dev_start(struct rte_eth_dev *dev);
 134 static void ixgbe_dev_stop(struct rte_eth_dev *dev);
 135 static int  ixgbe_dev_set_link_up(struct rte_eth_dev *dev);
 136 static int  ixgbe_dev_set_link_down(struct rte_eth_dev *dev);
 137 static void ixgbe_dev_close(struct rte_eth_dev *dev);
 138 static void ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev);
 139 static void ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev);
 140 static void ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev);
 141 static void ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev);
 142 static int ixgbe_dev_link_update(struct rte_eth_dev *dev,
 143                                 int wait_to_complete);
 144 static void ixgbe_dev_stats_get(struct rte_eth_dev *dev,
 145                                 struct rte_eth_stats *stats);
 146 static int ixgbe_dev_xstats_get(struct rte_eth_dev *dev,
 147                                 struct rte_eth_xstats *xstats, unsigned n);
 148 static void ixgbe_dev_stats_reset(struct rte_eth_dev *dev);
 149 static void ixgbe_dev_xstats_reset(struct rte_eth_dev *dev);
 150 static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 151                                              uint16_t queue_id,
 152                                              uint8_t stat_idx,
 153                                              uint8_t is_rx);
 154 static void ixgbe_dev_info_get(struct rte_eth_dev *dev,
 155                                struct rte_eth_dev_info *dev_info);
 156 static void ixgbevf_dev_info_get(struct rte_eth_dev *dev,
 157                                  struct rte_eth_dev_info *dev_info);
 158 static int ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 159
 160 static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev,
 161                 uint16_t vlan_id, int on);
 162 static void ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 163 static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev,
 164                 uint16_t queue, bool on);
 165 static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue,
 166                 int on);
 167 static void ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 168 static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue);
 169 static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue);
 170 static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 171 static void ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 172
 173 static int ixgbe_dev_led_on(struct rte_eth_dev *dev);
 174 static int ixgbe_dev_led_off(struct rte_eth_dev *dev);
 175 static int ixgbe_flow_ctrl_get(struct rte_eth_dev *dev,
 176                                struct rte_eth_fc_conf *fc_conf);
 177 static int ixgbe_flow_ctrl_set(struct rte_eth_dev *dev,
 178                                struct rte_eth_fc_conf *fc_conf);
 179 static int ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 180                 struct rte_eth_pfc_conf *pfc_conf);
 181 static int ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
 182                         struct rte_eth_rss_reta_entry64 *reta_conf,
 183                         uint16_t reta_size);
 184 static int ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
 185                         struct rte_eth_rss_reta_entry64 *reta_conf,
 186                         uint16_t reta_size);
 187 static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev);
 188 static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev);
 189 static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev);
 190 static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev);
 191 static void ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle,
 192                 void *param);
 193 static void ixgbe_dev_interrupt_delayed_handler(void *param);
 194 static void ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 195                 uint32_t index, uint32_t pool);
 196 static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index);
 197 static void ixgbe_set_default_mac_addr(struct rte_eth_dev *dev,
 198                                            struct ether_addr *mac_addr);
 199 static void ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config);
 200
 201 /* For Virtual Function support */
 202 static int eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev);
 203 static int  ixgbevf_dev_configure(struct rte_eth_dev *dev);
 204 static int  ixgbevf_dev_start(struct rte_eth_dev *dev);
 205 static void ixgbevf_dev_stop(struct rte_eth_dev *dev);
 206 static void ixgbevf_dev_close(struct rte_eth_dev *dev);
 207 static void ixgbevf_intr_disable(struct ixgbe_hw *hw);
 208 static void ixgbevf_dev_stats_get(struct rte_eth_dev *dev,
 209                 struct rte_eth_stats *stats);
 210 static void ixgbevf_dev_stats_reset(struct rte_eth_dev *dev);
 211 static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev,
 212                 uint16_t vlan_id, int on);
 213 static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev,
 214                 uint16_t queue, int on);
 215 static void ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 216 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 217
 218 /* For Eth VMDQ APIs support */
 219 static int ixgbe_uc_hash_table_set(struct rte_eth_dev *dev, struct
 220                 ether_addr* mac_addr,uint8_t on);
 221 static int ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev,uint8_t on);
 222 static int  ixgbe_set_pool_rx_mode(struct rte_eth_dev *dev,  uint16_t pool,
 223                 uint16_t rx_mask, uint8_t on);
 224 static int ixgbe_set_pool_rx(struct rte_eth_dev *dev,uint16_t pool,uint8_t on);
 225 static int ixgbe_set_pool_tx(struct rte_eth_dev *dev,uint16_t pool,uint8_t on);
 226 static int ixgbe_set_pool_vlan_filter(struct rte_eth_dev *dev, uint16_t vlan,
 227                 uint64_t pool_mask,uint8_t vlan_on);
 228 static int ixgbe_mirror_rule_set(struct rte_eth_dev *dev,
 229                 struct rte_eth_mirror_conf *mirror_conf,
 230                 uint8_t rule_id, uint8_t on);
 231 static int ixgbe_mirror_rule_reset(struct rte_eth_dev *dev,
 232                 uint8_t rule_id);
 233
 234 static int ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev,
 235                 uint16_t queue_idx, uint16_t tx_rate);
 236 static int ixgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf,
 237                 uint16_t tx_rate, uint64_t q_msk);
 238
 239 static void ixgbevf_add_mac_addr(struct rte_eth_dev *dev,
 240                                  struct ether_addr *mac_addr,
 241                                  uint32_t index, uint32_t pool);
 242 static void ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index);
 243 static void ixgbevf_set_default_mac_addr(struct rte_eth_dev *dev,
 244                                              struct ether_addr *mac_addr);
 245 static int ixgbe_syn_filter_set(struct rte_eth_dev *dev,
 246                         struct rte_eth_syn_filter *filter,
 247                         bool add);
 248 static int ixgbe_syn_filter_get(struct rte_eth_dev *dev,
 249                         struct rte_eth_syn_filter *filter);
 250 static int ixgbe_syn_filter_handle(struct rte_eth_dev *dev,
 251                         enum rte_filter_op filter_op,
 252                         void *arg);
 253 static int ixgbe_add_5tuple_filter(struct rte_eth_dev *dev,
 254                         struct ixgbe_5tuple_filter *filter);
 255 static void ixgbe_remove_5tuple_filter(struct rte_eth_dev *dev,
 256                         struct ixgbe_5tuple_filter *filter);
 257 static int ixgbe_add_del_ntuple_filter(struct rte_eth_dev *dev,
 258                         struct rte_eth_ntuple_filter *filter,
 259                         bool add);
 260 static int ixgbe_ntuple_filter_handle(struct rte_eth_dev *dev,
 261                                 enum rte_filter_op filter_op,
 262                                 void *arg);
 263 static int ixgbe_get_ntuple_filter(struct rte_eth_dev *dev,
 264                         struct rte_eth_ntuple_filter *filter);
 265 static int ixgbe_add_del_ethertype_filter(struct rte_eth_dev *dev,
 266                         struct rte_eth_ethertype_filter *filter,
 267                         bool add);
 268 static int ixgbe_ethertype_filter_handle(struct rte_eth_dev *dev,
 269                                 enum rte_filter_op filter_op,
 270                                 void *arg);
 271 static int ixgbe_get_ethertype_filter(struct rte_eth_dev *dev,
 272                         struct rte_eth_ethertype_filter *filter);
 273 static int ixgbe_dev_filter_ctrl(struct rte_eth_dev *dev,
 274                      enum rte_filter_type filter_type,
 275                      enum rte_filter_op filter_op,
 276                      void *arg);
 277 static int ixgbevf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu);
 278
 279 static int ixgbe_dev_set_mc_addr_list(struct rte_eth_dev *dev,
 280                                       struct ether_addr *mc_addr_set,
 281                                       uint32_t nb_mc_addr);
 282
 283 static int ixgbe_get_reg_length(struct rte_eth_dev *dev);
 284 static int ixgbe_get_regs(struct rte_eth_dev *dev,
 285                             struct rte_dev_reg_info *regs);
 286 static int ixgbe_get_eeprom_length(struct rte_eth_dev *dev);
 287 static int ixgbe_get_eeprom(struct rte_eth_dev *dev,
 288                                 struct rte_dev_eeprom_info *eeprom);
 289 static int ixgbe_set_eeprom(struct rte_eth_dev *dev,
 290                                 struct rte_dev_eeprom_info *eeprom);
 291
 292 static int ixgbevf_get_reg_length(struct rte_eth_dev *dev);
 293 static int ixgbevf_get_regs(struct rte_eth_dev *dev,
 294                                 struct rte_dev_reg_info *regs);
 295
 296 static int ixgbe_timesync_enable(struct rte_eth_dev *dev);
 297 static int ixgbe_timesync_disable(struct rte_eth_dev *dev);
 298 static int ixgbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
 299                                             struct timespec *timestamp,
 300                                             uint32_t flags);
 301 static int ixgbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
 302                                             struct timespec *timestamp);
 303
 304 /*
 305  * Define VF Stats MACRO for Non "cleared on read" register
 306  */
 307 #define UPDATE_VF_STAT(reg, last, cur)                          \
 308 {                                                               \
 309         u32 latest = IXGBE_READ_REG(hw, reg);                   \
 310         cur += latest - last;                                   \
 311         last = latest;                                          \
 312 }
 313
 314 #define UPDATE_VF_STAT_36BIT(lsb, msb, last, cur)                \
 315 {                                                                \
 316         u64 new_lsb = IXGBE_READ_REG(hw, lsb);                   \
 317         u64 new_msb = IXGBE_READ_REG(hw, msb);                   \
 318         u64 latest = ((new_msb << 32) | new_lsb);                \
 319         cur += (0x1000000000LL + latest - last) & 0xFFFFFFFFFLL; \
 320         last = latest;                                           \
 321 }
 322
 323 #define IXGBE_SET_HWSTRIP(h, q) do{\
 324                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 325                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 326                 (h)->bitmap[idx] |= 1 << bit;\
 327         }while(0)
 328
 329 #define IXGBE_CLEAR_HWSTRIP(h, q) do{\
 330                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 331                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 332                 (h)->bitmap[idx] &= ~(1 << bit);\
 333         }while(0)
 334
 335 #define IXGBE_GET_HWSTRIP(h, q, r) do{\
 336                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 337                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 338                 (r) = (h)->bitmap[idx] >> bit & 1;\
 339         }while(0)
 340
 341 /*
 342  * The set of PCI devices this driver supports
 343  */
 344 static const struct rte_pci_id pci_id_ixgbe_map[] = {
 345
 346 #define RTE_PCI_DEV_ID_DECL_IXGBE(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 347 #include "rte_pci_dev_ids.h"
 348
 349 { .vendor_id = 0, /* sentinel */ },
 350 };
 351
 352
 353 /*
 354  * The set of PCI devices this driver supports (for 82599 VF)
 355  */
 356 static const struct rte_pci_id pci_id_ixgbevf_map[] = {
 357
 358 #define RTE_PCI_DEV_ID_DECL_IXGBEVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 359 #include "rte_pci_dev_ids.h"
 360 { .vendor_id = 0, /* sentinel */ },
 361
 362 };
 363
 364 static const struct eth_dev_ops ixgbe_eth_dev_ops = {
 365         .dev_configure        = ixgbe_dev_configure,
 366         .dev_start            = ixgbe_dev_start,
 367         .dev_stop             = ixgbe_dev_stop,
 368         .dev_set_link_up    = ixgbe_dev_set_link_up,
 369         .dev_set_link_down  = ixgbe_dev_set_link_down,
 370         .dev_close            = ixgbe_dev_close,
 371         .promiscuous_enable   = ixgbe_dev_promiscuous_enable,
 372         .promiscuous_disable  = ixgbe_dev_promiscuous_disable,
 373         .allmulticast_enable  = ixgbe_dev_allmulticast_enable,
 374         .allmulticast_disable = ixgbe_dev_allmulticast_disable,
 375         .link_update          = ixgbe_dev_link_update,
 376         .stats_get            = ixgbe_dev_stats_get,
 377         .xstats_get           = ixgbe_dev_xstats_get,
 378         .stats_reset          = ixgbe_dev_stats_reset,
 379         .xstats_reset         = ixgbe_dev_xstats_reset,
 380         .queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set,
 381         .dev_infos_get        = ixgbe_dev_info_get,
 382         .mtu_set              = ixgbe_dev_mtu_set,
 383         .vlan_filter_set      = ixgbe_vlan_filter_set,
 384         .vlan_tpid_set        = ixgbe_vlan_tpid_set,
 385         .vlan_offload_set     = ixgbe_vlan_offload_set,
 386         .vlan_strip_queue_set = ixgbe_vlan_strip_queue_set,
 387         .rx_queue_start       = ixgbe_dev_rx_queue_start,
 388         .rx_queue_stop        = ixgbe_dev_rx_queue_stop,
 389         .tx_queue_start       = ixgbe_dev_tx_queue_start,
 390         .tx_queue_stop        = ixgbe_dev_tx_queue_stop,
 391         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 392         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 393         .rx_queue_count       = ixgbe_dev_rx_queue_count,
 394         .rx_descriptor_done   = ixgbe_dev_rx_descriptor_done,
 395         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 396         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 397         .dev_led_on           = ixgbe_dev_led_on,
 398         .dev_led_off          = ixgbe_dev_led_off,
 399         .flow_ctrl_get        = ixgbe_flow_ctrl_get,
 400         .flow_ctrl_set        = ixgbe_flow_ctrl_set,
 401         .priority_flow_ctrl_set = ixgbe_priority_flow_ctrl_set,
 402         .mac_addr_add         = ixgbe_add_rar,
 403         .mac_addr_remove      = ixgbe_remove_rar,
 404         .mac_addr_set         = ixgbe_set_default_mac_addr,
 405         .uc_hash_table_set    = ixgbe_uc_hash_table_set,
 406         .uc_all_hash_table_set  = ixgbe_uc_all_hash_table_set,
 407         .mirror_rule_set      = ixgbe_mirror_rule_set,
 408         .mirror_rule_reset    = ixgbe_mirror_rule_reset,
 409         .set_vf_rx_mode       = ixgbe_set_pool_rx_mode,
 410         .set_vf_rx            = ixgbe_set_pool_rx,
 411         .set_vf_tx            = ixgbe_set_pool_tx,
 412         .set_vf_vlan_filter   = ixgbe_set_pool_vlan_filter,
 413         .set_queue_rate_limit = ixgbe_set_queue_rate_limit,
 414         .set_vf_rate_limit    = ixgbe_set_vf_rate_limit,
 415         .reta_update          = ixgbe_dev_rss_reta_update,
 416         .reta_query           = ixgbe_dev_rss_reta_query,
 417 #ifdef RTE_NIC_BYPASS
 418         .bypass_init          = ixgbe_bypass_init,
 419         .bypass_state_set     = ixgbe_bypass_state_store,
 420         .bypass_state_show    = ixgbe_bypass_state_show,
 421         .bypass_event_set     = ixgbe_bypass_event_store,
 422         .bypass_event_show    = ixgbe_bypass_event_show,
 423         .bypass_wd_timeout_set  = ixgbe_bypass_wd_timeout_store,
 424         .bypass_wd_timeout_show = ixgbe_bypass_wd_timeout_show,
 425         .bypass_ver_show      = ixgbe_bypass_ver_show,
 426         .bypass_wd_reset      = ixgbe_bypass_wd_reset,
 427 #endif /* RTE_NIC_BYPASS */
 428         .rss_hash_update      = ixgbe_dev_rss_hash_update,
 429         .rss_hash_conf_get    = ixgbe_dev_rss_hash_conf_get,
 430         .filter_ctrl          = ixgbe_dev_filter_ctrl,
 431         .set_mc_addr_list     = ixgbe_dev_set_mc_addr_list,
 432         .timesync_enable      = ixgbe_timesync_enable,
 433         .timesync_disable     = ixgbe_timesync_disable,
 434         .timesync_read_rx_timestamp = ixgbe_timesync_read_rx_timestamp,
 435         .timesync_read_tx_timestamp = ixgbe_timesync_read_tx_timestamp,
 436         .get_reg_length       = ixgbe_get_reg_length,
 437         .get_reg              = ixgbe_get_regs,
 438         .get_eeprom_length    = ixgbe_get_eeprom_length,
 439         .get_eeprom           = ixgbe_get_eeprom,
 440         .set_eeprom           = ixgbe_set_eeprom,
 441 };
 442
 443 /*
 444  * dev_ops for virtual function, bare necessities for basic vf
 445  * operation have been implemented
 446  */
 447 static const struct eth_dev_ops ixgbevf_eth_dev_ops = {
 448         .dev_configure        = ixgbevf_dev_configure,
 449         .dev_start            = ixgbevf_dev_start,
 450         .dev_stop             = ixgbevf_dev_stop,
 451         .link_update          = ixgbe_dev_link_update,
 452         .stats_get            = ixgbevf_dev_stats_get,
 453         .stats_reset          = ixgbevf_dev_stats_reset,
 454         .dev_close            = ixgbevf_dev_close,
 455         .dev_infos_get        = ixgbevf_dev_info_get,
 456         .mtu_set              = ixgbevf_dev_set_mtu,
 457         .vlan_filter_set      = ixgbevf_vlan_filter_set,
 458         .vlan_strip_queue_set = ixgbevf_vlan_strip_queue_set,
 459         .vlan_offload_set     = ixgbevf_vlan_offload_set,
 460         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 461         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 462         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 463         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 464         .mac_addr_add         = ixgbevf_add_mac_addr,
 465         .mac_addr_remove      = ixgbevf_remove_mac_addr,
 466         .set_mc_addr_list     = ixgbe_dev_set_mc_addr_list,
 467         .mac_addr_set         = ixgbevf_set_default_mac_addr,
 468         .get_reg_length       = ixgbevf_get_reg_length,
 469         .get_reg              = ixgbevf_get_regs,
 470 };
 471
 472 /* store statistics names and its offset in stats structure */
 473 struct rte_ixgbe_xstats_name_off {
 474         char name[RTE_ETH_XSTATS_NAME_SIZE];
 475         unsigned offset;
 476 };
 477
 478 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_stats_strings[] = {
 479         {"rx_illegal_byte_err", offsetof(struct ixgbe_hw_stats, errbc)},
 480         {"rx_len_err", offsetof(struct ixgbe_hw_stats, rlec)},
 481         {"rx_undersize_count", offsetof(struct ixgbe_hw_stats, ruc)},
 482         {"rx_oversize_count", offsetof(struct ixgbe_hw_stats, roc)},
 483         {"rx_fragment_count", offsetof(struct ixgbe_hw_stats, rfc)},
 484         {"rx_jabber_count", offsetof(struct ixgbe_hw_stats, rjc)},
 485         {"l3_l4_xsum_error", offsetof(struct ixgbe_hw_stats, xec)},
 486         {"mac_local_fault", offsetof(struct ixgbe_hw_stats, mlfc)},
 487         {"mac_remote_fault", offsetof(struct ixgbe_hw_stats, mrfc)},
 488         {"mac_short_pkt_discard", offsetof(struct ixgbe_hw_stats, mspdc)},
 489         {"fccrc_error", offsetof(struct ixgbe_hw_stats, fccrc)},
 490         {"fcoe_drop", offsetof(struct ixgbe_hw_stats, fcoerpdc)},
 491         {"fc_last_error", offsetof(struct ixgbe_hw_stats, fclast)},
 492         {"rx_broadcast_packets", offsetof(struct ixgbe_hw_stats, bprc)},
 493         {"mgmt_pkts_dropped", offsetof(struct ixgbe_hw_stats, mngpdc)},
 494 };
 495
 496 #define IXGBE_NB_XSTATS (sizeof(rte_ixgbe_stats_strings) /      \
 497                 sizeof(rte_ixgbe_stats_strings[0]))
 498
 499 /**
 500  * Atomically reads the link status information from global
 501  * structure rte_eth_dev.
 502  *
 503  * @param dev
 504  *   - Pointer to the structure rte_eth_dev to read from.
 505  *   - Pointer to the buffer to be saved with the link status.
 506  *
 507  * @return
 508  *   - On success, zero.
 509  *   - On failure, negative value.
 510  */
 511 static inline int
 512 rte_ixgbe_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 513                                 struct rte_eth_link *link)
 514 {
 515         struct rte_eth_link *dst = link;
 516         struct rte_eth_link *src = &(dev->data->dev_link);
 517
 518         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 519                                         *(uint64_t *)src) == 0)
 520                 return -1;
 521
 522         return 0;
 523 }
 524
 525 /**
 526  * Atomically writes the link status information into global
 527  * structure rte_eth_dev.
 528  *
 529  * @param dev
 530  *   - Pointer to the structure rte_eth_dev to read from.
 531  *   - Pointer to the buffer to be saved with the link status.
 532  *
 533  * @return
 534  *   - On success, zero.
 535  *   - On failure, negative value.
 536  */
 537 static inline int
 538 rte_ixgbe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 539                                 struct rte_eth_link *link)
 540 {
 541         struct rte_eth_link *dst = &(dev->data->dev_link);
 542         struct rte_eth_link *src = link;
 543
 544         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 545                                         *(uint64_t *)src) == 0)
 546                 return -1;
 547
 548         return 0;
 549 }
 550
 551 /*
 552  * This function is the same as ixgbe_is_sfp() in base/ixgbe.h.
 553  */
 554 static inline int
 555 ixgbe_is_sfp(struct ixgbe_hw *hw)
 556 {
 557         switch (hw->phy.type) {
 558         case ixgbe_phy_sfp_avago:
 559         case ixgbe_phy_sfp_ftl:
 560         case ixgbe_phy_sfp_intel:
 561         case ixgbe_phy_sfp_unknown:
 562         case ixgbe_phy_sfp_passive_tyco:
 563         case ixgbe_phy_sfp_passive_unknown:
 564                 return 1;
 565         default:
 566                 return 0;
 567         }
 568 }
 569
 570 static inline int32_t
 571 ixgbe_pf_reset_hw(struct ixgbe_hw *hw)
 572 {
 573         uint32_t ctrl_ext;
 574         int32_t status;
 575
 576         status = ixgbe_reset_hw(hw);
 577
 578         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 579         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 580         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
 581         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 582         IXGBE_WRITE_FLUSH(hw);
 583
 584         return status;
 585 }
 586
 587 static inline void
 588 ixgbe_enable_intr(struct rte_eth_dev *dev)
 589 {
 590         struct ixgbe_interrupt *intr =
 591                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 592         struct ixgbe_hw *hw =
 593                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 594
 595         IXGBE_WRITE_REG(hw, IXGBE_EIMS, intr->mask);
 596         IXGBE_WRITE_FLUSH(hw);
 597 }
 598
 599 /*
 600  * This function is based on ixgbe_disable_intr() in base/ixgbe.h.
 601  */
 602 static void
 603 ixgbe_disable_intr(struct ixgbe_hw *hw)
 604 {
 605         PMD_INIT_FUNC_TRACE();
 606
 607         if (hw->mac.type == ixgbe_mac_82598EB) {
 608                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ~0);
 609         } else {
 610                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xFFFF0000);
 611                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), ~0);
 612                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), ~0);
 613         }
 614         IXGBE_WRITE_FLUSH(hw);
 615 }
 616
 617 /*
 618  * This function resets queue statistics mapping registers.
 619  * From Niantic datasheet, Initialization of Statistics section:
 620  * "...if software requires the queue counters, the RQSMR and TQSM registers
 621  * must be re-programmed following a device reset.
 622  */
 623 static void
 624 ixgbe_reset_qstat_mappings(struct ixgbe_hw *hw)
 625 {
 626         uint32_t i;
 627
 628         for(i = 0; i != IXGBE_NB_STAT_MAPPING_REGS; i++) {
 629                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), 0);
 630                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), 0);
 631         }
 632 }
 633
 634
 635 static int
 636 ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 637                                   uint16_t queue_id,
 638                                   uint8_t stat_idx,
 639                                   uint8_t is_rx)
 640 {
 641 #define QSM_REG_NB_BITS_PER_QMAP_FIELD 8
 642 #define NB_QMAP_FIELDS_PER_QSM_REG 4
 643 #define QMAP_FIELD_RESERVED_BITS_MASK 0x0f
 644
 645         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 646         struct ixgbe_stat_mapping_registers *stat_mappings =
 647                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(eth_dev->data->dev_private);
 648         uint32_t qsmr_mask = 0;
 649         uint32_t clearing_mask = QMAP_FIELD_RESERVED_BITS_MASK;
 650         uint32_t q_map;
 651         uint8_t n, offset;
 652
 653         if ((hw->mac.type != ixgbe_mac_82599EB) &&
 654                 (hw->mac.type != ixgbe_mac_X540) &&
 655                 (hw->mac.type != ixgbe_mac_X550) &&
 656                 (hw->mac.type != ixgbe_mac_X550EM_x))
 657                 return -ENOSYS;
 658
 659         PMD_INIT_LOG(INFO, "Setting port %d, %s queue_id %d to stat index %d",
 660                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX",
 661                      queue_id, stat_idx);
 662
 663         n = (uint8_t)(queue_id / NB_QMAP_FIELDS_PER_QSM_REG);
 664         if (n >= IXGBE_NB_STAT_MAPPING_REGS) {
 665                 PMD_INIT_LOG(ERR, "Nb of stat mapping registers exceeded");
 666                 return -EIO;
 667         }
 668         offset = (uint8_t)(queue_id % NB_QMAP_FIELDS_PER_QSM_REG);
 669
 670         /* Now clear any previous stat_idx set */
 671         clearing_mask <<= (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 672         if (!is_rx)
 673                 stat_mappings->tqsm[n] &= ~clearing_mask;
 674         else
 675                 stat_mappings->rqsmr[n] &= ~clearing_mask;
 676
 677         q_map = (uint32_t)stat_idx;
 678         q_map &= QMAP_FIELD_RESERVED_BITS_MASK;
 679         qsmr_mask = q_map << (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 680         if (!is_rx)
 681                 stat_mappings->tqsm[n] |= qsmr_mask;
 682         else
 683                 stat_mappings->rqsmr[n] |= qsmr_mask;
 684
 685         PMD_INIT_LOG(INFO, "Set port %d, %s queue_id %d to stat index %d",
 686                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX",
 687                      queue_id, stat_idx);
 688         PMD_INIT_LOG(INFO, "%s[%d] = 0x%08x", is_rx ? "RQSMR" : "TQSM", n,
 689                      is_rx ? stat_mappings->rqsmr[n] : stat_mappings->tqsm[n]);
 690
 691         /* Now write the mapping in the appropriate register */
 692         if (is_rx) {
 693                 PMD_INIT_LOG(INFO, "Write 0x%x to RX IXGBE stat mapping reg:%d",
 694                              stat_mappings->rqsmr[n], n);
 695                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(n), stat_mappings->rqsmr[n]);
 696         }
 697         else {
 698                 PMD_INIT_LOG(INFO, "Write 0x%x to TX IXGBE stat mapping reg:%d",
 699                              stat_mappings->tqsm[n], n);
 700                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(n), stat_mappings->tqsm[n]);
 701         }
 702         return 0;
 703 }
 704
 705 static void
 706 ixgbe_restore_statistics_mapping(struct rte_eth_dev * dev)
 707 {
 708         struct ixgbe_stat_mapping_registers *stat_mappings =
 709                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(dev->data->dev_private);
 710         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 711         int i;
 712
 713         /* write whatever was in stat mapping table to the NIC */
 714         for (i = 0; i < IXGBE_NB_STAT_MAPPING_REGS; i++) {
 715                 /* rx */
 716                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), stat_mappings->rqsmr[i]);
 717
 718                 /* tx */
 719                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), stat_mappings->tqsm[i]);
 720         }
 721 }
 722
 723 static void
 724 ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config)
 725 {
 726         uint8_t i;
 727         struct ixgbe_dcb_tc_config *tc;
 728         uint8_t dcb_max_tc = IXGBE_DCB_MAX_TRAFFIC_CLASS;
 729
 730         dcb_config->num_tcs.pg_tcs = dcb_max_tc;
 731         dcb_config->num_tcs.pfc_tcs = dcb_max_tc;
 732         for (i = 0; i < dcb_max_tc; i++) {
 733                 tc = &dcb_config->tc_config[i];
 734                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_id = i;
 735                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
 736                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 737                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_id = i;
 738                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
 739                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 740                 tc->pfc = ixgbe_dcb_pfc_disabled;
 741         }
 742
 743         /* Initialize default user to priority mapping, UPx->TC0 */
 744         tc = &dcb_config->tc_config[0];
 745         tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
 746         tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
 747         for (i = 0; i< IXGBE_DCB_MAX_BW_GROUP; i++) {
 748                 dcb_config->bw_percentage[IXGBE_DCB_TX_CONFIG][i] = 100;
 749                 dcb_config->bw_percentage[IXGBE_DCB_RX_CONFIG][i] = 100;
 750         }
 751         dcb_config->rx_pba_cfg = ixgbe_dcb_pba_equal;
 752         dcb_config->pfc_mode_enable = false;
 753         dcb_config->vt_mode = true;
 754         dcb_config->round_robin_enable = false;
 755         /* support all DCB capabilities in 82599 */
 756         dcb_config->support.capabilities = 0xFF;
 757
 758         /*we only support 4 Tcs for X540, X550 */
 759         if (hw->mac.type == ixgbe_mac_X540 ||
 760                 hw->mac.type == ixgbe_mac_X550 ||
 761                 hw->mac.type == ixgbe_mac_X550EM_x) {
 762                 dcb_config->num_tcs.pg_tcs = 4;
 763                 dcb_config->num_tcs.pfc_tcs = 4;
 764         }
 765 }
 766
 767 /*
 768  * Ensure that all locks are released before first NVM or PHY access
 769  */
 770 static void
 771 ixgbe_swfw_lock_reset(struct ixgbe_hw *hw)
 772 {
 773         uint16_t mask;
 774
 775         /*
 776          * Phy lock should not fail in this early stage. If this is the case,
 777          * it is due to an improper exit of the application.
 778          * So force the release of the faulty lock. Release of common lock
 779          * is done automatically by swfw_sync function.
 780          */
 781         mask = IXGBE_GSSR_PHY0_SM << hw->bus.func;
 782         if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) {
 783                 PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released", hw->bus.func);
 784         }
 785         ixgbe_release_swfw_semaphore(hw, mask);
 786
 787         /*
 788          * These ones are more tricky since they are common to all ports; but
 789          * swfw_sync retries last long enough (1s) to be almost sure that if
 790          * lock can not be taken it is due to an improper lock of the
 791          * semaphore.
 792          */
 793         mask = IXGBE_GSSR_EEP_SM | IXGBE_GSSR_MAC_CSR_SM | IXGBE_GSSR_SW_MNG_SM;
 794         if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) {
 795                 PMD_DRV_LOG(DEBUG, "SWFW common locks released");
 796         }
 797         ixgbe_release_swfw_semaphore(hw, mask);
 798 }
 799
 800 /*
 801  * This function is based on code in ixgbe_attach() in base/ixgbe.c.
 802  * It returns 0 on success.
 803  */
 804 static int
 805 eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev)
 806 {
 807         struct rte_pci_device *pci_dev;
 808         struct ixgbe_hw *hw =
 809                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 810         struct ixgbe_vfta * shadow_vfta =
 811                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 812         struct ixgbe_hwstrip *hwstrip =
 813                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 814         struct ixgbe_dcb_config *dcb_config =
 815                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private);
 816         struct ixgbe_filter_info *filter_info =
 817                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private);
 818         uint32_t ctrl_ext;
 819         uint16_t csum;
 820         int diag, i;
 821
 822         PMD_INIT_FUNC_TRACE();
 823
 824         eth_dev->dev_ops = &ixgbe_eth_dev_ops;
 825         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
 826         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
 827
 828         /*
 829          * For secondary processes, we don't initialise any further as primary
 830          * has already done this work. Only check we don't need a different
 831          * RX and TX function.
 832          */
 833         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 834                 struct ixgbe_tx_queue *txq;
 835                 /* TX queue function in primary, set by last queue initialized
 836                  * Tx queue may not initialized by primary process */
 837                 if (eth_dev->data->tx_queues) {
 838                         txq = eth_dev->data->tx_queues[eth_dev->data->nb_tx_queues-1];
 839                         ixgbe_set_tx_function(eth_dev, txq);
 840                 } else {
 841                         /* Use default TX function if we get here */
 842                         PMD_INIT_LOG(INFO, "No TX queues configured yet. "
 843                                            "Using default TX function.");
 844                 }
 845
 846                 ixgbe_set_rx_function(eth_dev);
 847
 848                 return 0;
 849         }
 850         pci_dev = eth_dev->pci_dev;
 851
 852         /* Vendor and Device ID need to be set before init of shared code */
 853         hw->device_id = pci_dev->id.device_id;
 854         hw->vendor_id = pci_dev->id.vendor_id;
 855         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 856         hw->allow_unsupported_sfp = 1;
 857
 858         /* Initialize the shared code (base driver) */
 859 #ifdef RTE_NIC_BYPASS
 860         diag = ixgbe_bypass_init_shared_code(hw);
 861 #else
 862         diag = ixgbe_init_shared_code(hw);
 863 #endif /* RTE_NIC_BYPASS */
 864
 865         if (diag != IXGBE_SUCCESS) {
 866                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
 867                 return -EIO;
 868         }
 869
 870         /* pick up the PCI bus settings for reporting later */
 871         ixgbe_get_bus_info(hw);
 872
 873         /* Unlock any pending hardware semaphore */
 874         ixgbe_swfw_lock_reset(hw);
 875
 876         /* Initialize DCB configuration*/
 877         memset(dcb_config, 0, sizeof(struct ixgbe_dcb_config));
 878         ixgbe_dcb_init(hw,dcb_config);
 879         /* Get Hardware Flow Control setting */
 880         hw->fc.requested_mode = ixgbe_fc_full;
 881         hw->fc.current_mode = ixgbe_fc_full;
 882         hw->fc.pause_time = IXGBE_FC_PAUSE;
 883         for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
 884                 hw->fc.low_water[i] = IXGBE_FC_LO;
 885                 hw->fc.high_water[i] = IXGBE_FC_HI;
 886         }
 887         hw->fc.send_xon = 1;
 888
 889         /* Make sure we have a good EEPROM before we read from it */
 890         diag = ixgbe_validate_eeprom_checksum(hw, &csum);
 891         if (diag != IXGBE_SUCCESS) {
 892                 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag);
 893                 return -EIO;
 894         }
 895
 896 #ifdef RTE_NIC_BYPASS
 897         diag = ixgbe_bypass_init_hw(hw);
 898 #else
 899         diag = ixgbe_init_hw(hw);
 900 #endif /* RTE_NIC_BYPASS */
 901
 902         /*
 903          * Devices with copper phys will fail to initialise if ixgbe_init_hw()
 904          * is called too soon after the kernel driver unbinding/binding occurs.
 905          * The failure occurs in ixgbe_identify_phy_generic() for all devices,
 906          * but for non-copper devies, ixgbe_identify_sfp_module_generic() is
 907          * also called. See ixgbe_identify_phy_82599(). The reason for the
 908          * failure is not known, and only occuts when virtualisation features
 909          * are disabled in the bios. A delay of 100ms  was found to be enough by
 910          * trial-and-error, and is doubled to be safe.
 911          */
 912         if (diag && (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper)) {
 913                 rte_delay_ms(200);
 914                 diag = ixgbe_init_hw(hw);
 915         }
 916
 917         if (diag == IXGBE_ERR_EEPROM_VERSION) {
 918                 PMD_INIT_LOG(ERR, "This device is a pre-production adapter/"
 919                     "LOM.  Please be aware there may be issues associated "
 920                     "with your hardware.");
 921                 PMD_INIT_LOG(ERR, "If you are experiencing problems "
 922                     "please contact your Intel or hardware representative "
 923                     "who provided you with this hardware.");
 924         } else if (diag == IXGBE_ERR_SFP_NOT_SUPPORTED)
 925                 PMD_INIT_LOG(ERR, "Unsupported SFP+ Module");
 926         if (diag) {
 927                 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag);
 928                 return -EIO;
 929         }
 930
 931         /* Reset the hw statistics */
 932         ixgbe_dev_stats_reset(eth_dev);
 933
 934         /* disable interrupt */
 935         ixgbe_disable_intr(hw);
 936
 937         /* reset mappings for queue statistics hw counters*/
 938         ixgbe_reset_qstat_mappings(hw);
 939
 940         /* Allocate memory for storing MAC addresses */
 941         eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 942                         hw->mac.num_rar_entries, 0);
 943         if (eth_dev->data->mac_addrs == NULL) {
 944                 PMD_INIT_LOG(ERR,
 945                         "Failed to allocate %u bytes needed to store "
 946                         "MAC addresses",
 947                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 948                 return -ENOMEM;
 949         }
 950         /* Copy the permanent MAC address */
 951         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 952                         &eth_dev->data->mac_addrs[0]);
 953
 954         /* Allocate memory for storing hash filter MAC addresses */
 955         eth_dev->data->hash_mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 956                         IXGBE_VMDQ_NUM_UC_MAC, 0);
 957         if (eth_dev->data->hash_mac_addrs == NULL) {
 958                 PMD_INIT_LOG(ERR,
 959                         "Failed to allocate %d bytes needed to store MAC addresses",
 960                         ETHER_ADDR_LEN * IXGBE_VMDQ_NUM_UC_MAC);
 961                 return -ENOMEM;
 962         }
 963
 964         /* initialize the vfta */
 965         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 966
 967         /* initialize the hw strip bitmap*/
 968         memset(hwstrip, 0, sizeof(*hwstrip));
 969
 970         /* initialize PF if max_vfs not zero */
 971         ixgbe_pf_host_init(eth_dev);
 972
 973         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 974         /* let hardware know driver is loaded */
 975         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 976         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 977         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
 978         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 979         IXGBE_WRITE_FLUSH(hw);
 980
 981         if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
 982                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d, SFP+: %d",
 983                              (int) hw->mac.type, (int) hw->phy.type,
 984                              (int) hw->phy.sfp_type);
 985         else
 986                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d",
 987                              (int) hw->mac.type, (int) hw->phy.type);
 988
 989         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
 990                         eth_dev->data->port_id, pci_dev->id.vendor_id,
 991                         pci_dev->id.device_id);
 992
 993         rte_intr_callback_register(&(pci_dev->intr_handle),
 994                 ixgbe_dev_interrupt_handler, (void *)eth_dev);
 995
 996         /* enable uio intr after callback register */
 997         rte_intr_enable(&(pci_dev->intr_handle));
 998
 999         /* enable support intr */
1000         ixgbe_enable_intr(eth_dev);
1001
1002         /* initialize 5tuple filter list */
1003         TAILQ_INIT(&filter_info->fivetuple_list);
1004         memset(filter_info->fivetuple_mask, 0,
1005                 sizeof(uint32_t) * IXGBE_5TUPLE_ARRAY_SIZE);
1006
1007         return 0;
1008 }
1009
1010
1011 /*
1012  * Negotiate mailbox API version with the PF.
1013  * After reset API version is always set to the basic one (ixgbe_mbox_api_10).
1014  * Then we try to negotiate starting with the most recent one.
1015  * If all negotiation attempts fail, then we will proceed with
1016  * the default one (ixgbe_mbox_api_10).
1017  */
1018 static void
1019 ixgbevf_negotiate_api(struct ixgbe_hw *hw)
1020 {
1021         int32_t i;
1022
1023         /* start with highest supported, proceed down */
1024         static const enum ixgbe_pfvf_api_rev sup_ver[] = {
1025                 ixgbe_mbox_api_11,
1026                 ixgbe_mbox_api_10,
1027         };
1028
1029         for (i = 0;
1030                         i != RTE_DIM(sup_ver) &&
1031                         ixgbevf_negotiate_api_version(hw, sup_ver[i]) != 0;
1032                         i++)
1033                 ;
1034 }
1035
1036 static void
1037 generate_random_mac_addr(struct ether_addr *mac_addr)
1038 {
1039         uint64_t random;
1040
1041         /* Set Organizationally Unique Identifier (OUI) prefix. */
1042         mac_addr->addr_bytes[0] = 0x00;
1043         mac_addr->addr_bytes[1] = 0x09;
1044         mac_addr->addr_bytes[2] = 0xC0;
1045         /* Force indication of locally assigned MAC address. */
1046         mac_addr->addr_bytes[0] |= ETHER_LOCAL_ADMIN_ADDR;
1047         /* Generate the last 3 bytes of the MAC address with a random number. */
1048         random = rte_rand();
1049         memcpy(&mac_addr->addr_bytes[3], &random, 3);
1050 }
1051
1052 /*
1053  * Virtual Function device init
1054  */
1055 static int
1056 eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev)
1057 {
1058         int diag;
1059         uint32_t tc, tcs;
1060         struct rte_pci_device *pci_dev;
1061         struct ixgbe_hw *hw =
1062                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1063         struct ixgbe_vfta * shadow_vfta =
1064                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
1065         struct ixgbe_hwstrip *hwstrip =
1066                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
1067         struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr;
1068
1069         PMD_INIT_FUNC_TRACE();
1070
1071         eth_dev->dev_ops = &ixgbevf_eth_dev_ops;
1072         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
1073         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
1074
1075         /* for secondary processes, we don't initialise any further as primary
1076          * has already done this work. Only check we don't need a different
1077          * RX function */
1078         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
1079                 if (eth_dev->data->scattered_rx)
1080                         eth_dev->rx_pkt_burst = ixgbe_recv_pkts_lro_single_alloc;
1081                 return 0;
1082         }
1083
1084         pci_dev = eth_dev->pci_dev;
1085
1086         hw->device_id = pci_dev->id.device_id;
1087         hw->vendor_id = pci_dev->id.vendor_id;
1088         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
1089
1090         /* initialize the vfta */
1091         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
1092
1093         /* initialize the hw strip bitmap*/
1094         memset(hwstrip, 0, sizeof(*hwstrip));
1095
1096         /* Initialize the shared code (base driver) */
1097         diag = ixgbe_init_shared_code(hw);
1098         if (diag != IXGBE_SUCCESS) {
1099                 PMD_INIT_LOG(ERR, "Shared code init failed for ixgbevf: %d", diag);
1100                 return -EIO;
1101         }
1102
1103         /* init_mailbox_params */
1104         hw->mbx.ops.init_params(hw);
1105
1106         /* Reset the hw statistics */
1107         ixgbevf_dev_stats_reset(eth_dev);
1108
1109         /* Disable the interrupts for VF */
1110         ixgbevf_intr_disable(hw);
1111
1112         hw->mac.num_rar_entries = 128; /* The MAX of the underlying PF */
1113         diag = hw->mac.ops.reset_hw(hw);
1114
1115         /*
1116          * The VF reset operation returns the IXGBE_ERR_INVALID_MAC_ADDR when
1117          * the underlying PF driver has not assigned a MAC address to the VF.
1118          * In this case, assign a random MAC address.
1119          */
1120         if ((diag != IXGBE_SUCCESS) && (diag != IXGBE_ERR_INVALID_MAC_ADDR)) {
1121                 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
1122                 return (diag);
1123         }
1124
1125         /* negotiate mailbox API version to use with the PF. */
1126         ixgbevf_negotiate_api(hw);
1127
1128         /* Get Rx/Tx queue count via mailbox, which is ready after reset_hw */
1129         ixgbevf_get_queues(hw, &tcs, &tc);
1130
1131         /* Allocate memory for storing MAC addresses */
1132         eth_dev->data->mac_addrs = rte_zmalloc("ixgbevf", ETHER_ADDR_LEN *
1133                         hw->mac.num_rar_entries, 0);
1134         if (eth_dev->data->mac_addrs == NULL) {
1135                 PMD_INIT_LOG(ERR,
1136                         "Failed to allocate %u bytes needed to store "
1137                         "MAC addresses",
1138                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
1139                 return -ENOMEM;
1140         }
1141
1142         /* Generate a random MAC address, if none was assigned by PF. */
1143         if (is_zero_ether_addr(perm_addr)) {
1144                 generate_random_mac_addr(perm_addr);
1145                 diag = ixgbe_set_rar_vf(hw, 1, perm_addr->addr_bytes, 0, 1);
1146                 if (diag) {
1147                         rte_free(eth_dev->data->mac_addrs);
1148                         eth_dev->data->mac_addrs = NULL;
1149                         return diag;
1150                 }
1151                 PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF");
1152                 PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address "
1153                              "%02x:%02x:%02x:%02x:%02x:%02x",
1154                              perm_addr->addr_bytes[0],
1155                              perm_addr->addr_bytes[1],
1156                              perm_addr->addr_bytes[2],
1157                              perm_addr->addr_bytes[3],
1158                              perm_addr->addr_bytes[4],
1159                              perm_addr->addr_bytes[5]);
1160         }
1161
1162         /* Copy the permanent MAC address */
1163         ether_addr_copy(perm_addr, &eth_dev->data->mac_addrs[0]);
1164
1165         /* reset the hardware with the new settings */
1166         diag = hw->mac.ops.start_hw(hw);
1167         switch (diag) {
1168                 case  0:
1169                         break;
1170
1171                 default:
1172                         PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
1173                         return (-EIO);
1174         }
1175
1176         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x mac.type=%s",
1177                      eth_dev->data->port_id, pci_dev->id.vendor_id,
1178                      pci_dev->id.device_id, "ixgbe_mac_82599_vf");
1179
1180         return 0;
1181 }
1182
1183 static struct eth_driver rte_ixgbe_pmd = {
1184         .pci_drv = {
1185                 .name = "rte_ixgbe_pmd",
1186                 .id_table = pci_id_ixgbe_map,
1187                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
1188         },
1189         .eth_dev_init = eth_ixgbe_dev_init,
1190         .dev_private_size = sizeof(struct ixgbe_adapter),
1191 };
1192
1193 /*
1194  * virtual function driver struct
1195  */
1196 static struct eth_driver rte_ixgbevf_pmd = {
1197         .pci_drv = {
1198                 .name = "rte_ixgbevf_pmd",
1199                 .id_table = pci_id_ixgbevf_map,
1200                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
1201         },
1202         .eth_dev_init = eth_ixgbevf_dev_init,
1203         .dev_private_size = sizeof(struct ixgbe_adapter),
1204 };
1205
1206 /*
1207  * Driver initialization routine.
1208  * Invoked once at EAL init time.
1209  * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
1210  */
1211 static int
1212 rte_ixgbe_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
1213 {
1214         PMD_INIT_FUNC_TRACE();
1215
1216         rte_eth_driver_register(&rte_ixgbe_pmd);
1217         return 0;
1218 }
1219
1220 /*
1221  * VF Driver initialization routine.
1222  * Invoked one at EAL init time.
1223  * Register itself as the [Virtual Poll Mode] Driver of PCI niantic devices.
1224  */
1225 static int
1226 rte_ixgbevf_pmd_init(const char *name __rte_unused, const char *param __rte_unused)
1227 {
1228         PMD_INIT_FUNC_TRACE();
1229
1230         rte_eth_driver_register(&rte_ixgbevf_pmd);
1231         return (0);
1232 }
1233
1234 static int
1235 ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1236 {
1237         struct ixgbe_hw *hw =
1238                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1239         struct ixgbe_vfta * shadow_vfta =
1240                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1241         uint32_t vfta;
1242         uint32_t vid_idx;
1243         uint32_t vid_bit;
1244
1245         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
1246         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
1247         vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(vid_idx));
1248         if (on)
1249                 vfta |= vid_bit;
1250         else
1251                 vfta &= ~vid_bit;
1252         IXGBE_WRITE_REG(hw, IXGBE_VFTA(vid_idx), vfta);
1253
1254         /* update local VFTA copy */
1255         shadow_vfta->vfta[vid_idx] = vfta;
1256
1257         return 0;
1258 }
1259
1260 static void
1261 ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
1262 {
1263         if (on)
1264                 ixgbe_vlan_hw_strip_enable(dev, queue);
1265         else
1266                 ixgbe_vlan_hw_strip_disable(dev, queue);
1267 }
1268
1269 static void
1270 ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
1271 {
1272         struct ixgbe_hw *hw =
1273                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1274
1275         /* Only the high 16-bits is valid */
1276         IXGBE_WRITE_REG(hw, IXGBE_EXVET, tpid << 16);
1277 }
1278
1279 void
1280 ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev)
1281 {
1282         struct ixgbe_hw *hw =
1283                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1284         uint32_t vlnctrl;
1285
1286         PMD_INIT_FUNC_TRACE();
1287
1288         /* Filter Table Disable */
1289         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1290         vlnctrl &= ~IXGBE_VLNCTRL_VFE;
1291
1292         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
1293 }
1294
1295 void
1296 ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1297 {
1298         struct ixgbe_hw *hw =
1299                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1300         struct ixgbe_vfta * shadow_vfta =
1301                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1302         uint32_t vlnctrl;
1303         uint16_t i;
1304
1305         PMD_INIT_FUNC_TRACE();
1306
1307         /* Filter Table Enable */
1308         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1309         vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
1310         vlnctrl |= IXGBE_VLNCTRL_VFE;
1311
1312         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
1313
1314         /* write whatever is in local vfta copy */
1315         for (i = 0; i < IXGBE_VFTA_SIZE; i++)
1316                 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), shadow_vfta->vfta[i]);
1317 }
1318
1319 static void
1320 ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on)
1321 {
1322         struct ixgbe_hwstrip *hwstrip =
1323                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(dev->data->dev_private);
1324
1325         if(queue >= IXGBE_MAX_RX_QUEUE_NUM)
1326                 return;
1327
1328         if (on)
1329                 IXGBE_SET_HWSTRIP(hwstrip, queue);
1330         else
1331                 IXGBE_CLEAR_HWSTRIP(hwstrip, queue);
1332 }
1333
1334 static void
1335 ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue)
1336 {
1337         struct ixgbe_hw *hw =
1338                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1339         uint32_t ctrl;
1340
1341         PMD_INIT_FUNC_TRACE();
1342
1343         if (hw->mac.type == ixgbe_mac_82598EB) {
1344                 /* No queue level support */
1345                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
1346                 return;
1347         }
1348         else {
1349                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1350                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1351                 ctrl &= ~IXGBE_RXDCTL_VME;
1352                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
1353         }
1354         /* record those setting for HW strip per queue */
1355         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 0);
1356 }
1357
1358 static void
1359 ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue)
1360 {
1361         struct ixgbe_hw *hw =
1362                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1363         uint32_t ctrl;
1364
1365         PMD_INIT_FUNC_TRACE();
1366
1367         if (hw->mac.type == ixgbe_mac_82598EB) {
1368                 /* No queue level supported */
1369                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
1370                 return;
1371         }
1372         else {
1373                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1374                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1375                 ctrl |= IXGBE_RXDCTL_VME;
1376                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
1377         }
1378         /* record those setting for HW strip per queue */
1379         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 1);
1380 }
1381
1382 void
1383 ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev)
1384 {
1385         struct ixgbe_hw *hw =
1386                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1387         uint32_t ctrl;
1388         uint16_t i;
1389
1390         PMD_INIT_FUNC_TRACE();
1391
1392         if (hw->mac.type == ixgbe_mac_82598EB) {
1393                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1394                 ctrl &= ~IXGBE_VLNCTRL_VME;
1395                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1396         }
1397         else {
1398                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1399                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1400                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1401                         ctrl &= ~IXGBE_RXDCTL_VME;
1402                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1403
1404                         /* record those setting for HW strip per queue */
1405                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 0);
1406                 }
1407         }
1408 }
1409
1410 void
1411 ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev)
1412 {
1413         struct ixgbe_hw *hw =
1414                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1415         uint32_t ctrl;
1416         uint16_t i;
1417
1418         PMD_INIT_FUNC_TRACE();
1419
1420         if (hw->mac.type == ixgbe_mac_82598EB) {
1421                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1422                 ctrl |= IXGBE_VLNCTRL_VME;
1423                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1424         }
1425         else {
1426                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1427                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1428                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1429                         ctrl |= IXGBE_RXDCTL_VME;
1430                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1431
1432                         /* record those setting for HW strip per queue */
1433                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 1);
1434                 }
1435         }
1436 }
1437
1438 static void
1439 ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1440 {
1441         struct ixgbe_hw *hw =
1442                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1443         uint32_t ctrl;
1444
1445         PMD_INIT_FUNC_TRACE();
1446
1447         /* DMATXCTRL: Geric Double VLAN Disable */
1448         ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1449         ctrl &= ~IXGBE_DMATXCTL_GDV;
1450         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1451
1452         /* CTRL_EXT: Global Double VLAN Disable */
1453         ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1454         ctrl &= ~IXGBE_EXTENDED_VLAN;
1455         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1456
1457 }
1458
1459 static void
1460 ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1461 {
1462         struct ixgbe_hw *hw =
1463                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1464         uint32_t ctrl;
1465
1466         PMD_INIT_FUNC_TRACE();
1467
1468         /* DMATXCTRL: Geric Double VLAN Enable */
1469         ctrl  = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1470         ctrl |= IXGBE_DMATXCTL_GDV;
1471         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1472
1473         /* CTRL_EXT: Global Double VLAN Enable */
1474         ctrl  = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1475         ctrl |= IXGBE_EXTENDED_VLAN;
1476         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1477
1478         /*
1479          * VET EXT field in the EXVET register = 0x8100 by default
1480          * So no need to change. Same to VT field of DMATXCTL register
1481          */
1482 }
1483
1484 static void
1485 ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1486 {
1487         if(mask & ETH_VLAN_STRIP_MASK){
1488                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1489                         ixgbe_vlan_hw_strip_enable_all(dev);
1490                 else
1491                         ixgbe_vlan_hw_strip_disable_all(dev);
1492         }
1493
1494         if(mask & ETH_VLAN_FILTER_MASK){
1495                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1496                         ixgbe_vlan_hw_filter_enable(dev);
1497                 else
1498                         ixgbe_vlan_hw_filter_disable(dev);
1499         }
1500
1501         if(mask & ETH_VLAN_EXTEND_MASK){
1502                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1503                         ixgbe_vlan_hw_extend_enable(dev);
1504                 else
1505                         ixgbe_vlan_hw_extend_disable(dev);
1506         }
1507 }
1508
1509 static void
1510 ixgbe_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1511 {
1512         struct ixgbe_hw *hw =
1513                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1514         /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
1515         uint32_t vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1516         vlanctrl |= IXGBE_VLNCTRL_VFE ; /* enable vlan filters */
1517         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
1518 }
1519
1520 static int
1521 ixgbe_dev_configure(struct rte_eth_dev *dev)
1522 {
1523         struct ixgbe_interrupt *intr =
1524                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1525         struct ixgbe_adapter *adapter =
1526                 (struct ixgbe_adapter *)dev->data->dev_private;
1527
1528         PMD_INIT_FUNC_TRACE();
1529
1530         /* set flag to update link status after init */
1531         intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1532
1533         /*
1534          * Initialize to TRUE. If any of Rx queues doesn't meet the bulk
1535          * allocation or vector Rx preconditions we will reset it.
1536          */
1537         adapter->rx_bulk_alloc_allowed = true;
1538         adapter->rx_vec_allowed = true;
1539
1540         return 0;
1541 }
1542
1543 /*
1544  * Configure device link speed and setup link.
1545  * It returns 0 on success.
1546  */
1547 static int
1548 ixgbe_dev_start(struct rte_eth_dev *dev)
1549 {
1550         struct ixgbe_hw *hw =
1551                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1552         struct ixgbe_vf_info *vfinfo =
1553                 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private);
1554         int err, link_up = 0, negotiate = 0;
1555         uint32_t speed = 0;
1556         int mask = 0;
1557         int status;
1558         uint16_t vf, idx;
1559
1560         PMD_INIT_FUNC_TRACE();
1561
1562         /* IXGBE devices don't support half duplex */
1563         if ((dev->data->dev_conf.link_duplex != ETH_LINK_AUTONEG_DUPLEX) &&
1564                         (dev->data->dev_conf.link_duplex != ETH_LINK_FULL_DUPLEX)) {
1565                 PMD_INIT_LOG(ERR, "Invalid link_duplex (%hu) for port %hhu",
1566                              dev->data->dev_conf.link_duplex,
1567                              dev->data->port_id);
1568                 return -EINVAL;
1569         }
1570
1571         /* stop adapter */
1572         hw->adapter_stopped = FALSE;
1573         ixgbe_stop_adapter(hw);
1574
1575         /* reinitialize adapter
1576          * this calls reset and start */
1577         status = ixgbe_pf_reset_hw(hw);
1578         if (status != 0)
1579                 return -1;
1580         hw->mac.ops.start_hw(hw);
1581         hw->mac.get_link_status = true;
1582
1583         /* configure PF module if SRIOV enabled */
1584         ixgbe_pf_host_configure(dev);
1585
1586         /* initialize transmission unit */
1587         ixgbe_dev_tx_init(dev);
1588
1589         /* This can fail when allocating mbufs for descriptor rings */
1590         err = ixgbe_dev_rx_init(dev);
1591         if (err) {
1592                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
1593                 goto error;
1594         }
1595
1596         err = ixgbe_dev_rxtx_start(dev);
1597         if (err < 0) {
1598                 PMD_INIT_LOG(ERR, "Unable to start rxtx queues");
1599                 goto error;
1600         }
1601
1602         /* Skip link setup if loopback mode is enabled for 82599. */
1603         if (hw->mac.type == ixgbe_mac_82599EB &&
1604                         dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_82599_TX_RX)
1605                 goto skip_link_setup;
1606
1607         if (ixgbe_is_sfp(hw) && hw->phy.multispeed_fiber) {
1608                 err = hw->mac.ops.setup_sfp(hw);
1609                 if (err)
1610                         goto error;
1611         }
1612
1613         if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
1614                 /* Turn on the copper */
1615                 ixgbe_set_phy_power(hw, true);
1616         } else {
1617                 /* Turn on the laser */
1618                 ixgbe_enable_tx_laser(hw);
1619         }
1620
1621         err = ixgbe_check_link(hw, &speed, &link_up, 0);
1622         if (err)
1623                 goto error;
1624         dev->data->dev_link.link_status = link_up;
1625
1626         err = ixgbe_get_link_capabilities(hw, &speed, &negotiate);
1627         if (err)
1628                 goto error;
1629
1630         switch(dev->data->dev_conf.link_speed) {
1631         case ETH_LINK_SPEED_AUTONEG:
1632                 speed = (hw->mac.type != ixgbe_mac_82598EB) ?
1633                                 IXGBE_LINK_SPEED_82599_AUTONEG :
1634                                 IXGBE_LINK_SPEED_82598_AUTONEG;
1635                 break;
1636         case ETH_LINK_SPEED_100:
1637                 /*
1638                  * Invalid for 82598 but error will be detected by
1639                  * ixgbe_setup_link()
1640                  */
1641                 speed = IXGBE_LINK_SPEED_100_FULL;
1642                 break;
1643         case ETH_LINK_SPEED_1000:
1644                 speed = IXGBE_LINK_SPEED_1GB_FULL;
1645                 break;
1646         case ETH_LINK_SPEED_10000:
1647                 speed = IXGBE_LINK_SPEED_10GB_FULL;
1648                 break;
1649         default:
1650                 PMD_INIT_LOG(ERR, "Invalid link_speed (%hu) for port %hhu",
1651                              dev->data->dev_conf.link_speed,
1652                              dev->data->port_id);
1653                 goto error;
1654         }
1655
1656         err = ixgbe_setup_link(hw, speed, link_up);
1657         if (err)
1658                 goto error;
1659
1660 skip_link_setup:
1661
1662         /* check if lsc interrupt is enabled */
1663         if (dev->data->dev_conf.intr_conf.lsc != 0)
1664                 ixgbe_dev_lsc_interrupt_setup(dev);
1665
1666         /* resume enabled intr since hw reset */
1667         ixgbe_enable_intr(dev);
1668
1669         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
1670                 ETH_VLAN_EXTEND_MASK;
1671         ixgbe_vlan_offload_set(dev, mask);
1672
1673         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
1674                 /* Enable vlan filtering for VMDq */
1675                 ixgbe_vmdq_vlan_hw_filter_enable(dev);
1676         }
1677
1678         /* Configure DCB hw */
1679         ixgbe_configure_dcb(dev);
1680
1681         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) {
1682                 err = ixgbe_fdir_configure(dev);
1683                 if (err)
1684                         goto error;
1685         }
1686
1687         /* Restore vf rate limit */
1688         if (vfinfo != NULL) {
1689                 for (vf = 0; vf < dev->pci_dev->max_vfs; vf++)
1690                         for (idx = 0; idx < IXGBE_MAX_QUEUE_NUM_PER_VF; idx++)
1691                                 if (vfinfo[vf].tx_rate[idx] != 0)
1692                                         ixgbe_set_vf_rate_limit(dev, vf,
1693                                                 vfinfo[vf].tx_rate[idx],
1694                                                 1 << idx);
1695         }
1696
1697         ixgbe_restore_statistics_mapping(dev);
1698
1699         return (0);
1700
1701 error:
1702         PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err);
1703         ixgbe_dev_clear_queues(dev);
1704         return -EIO;
1705 }
1706
1707 /*
1708  * Stop device: disable rx and tx functions to allow for reconfiguring.
1709  */
1710 static void
1711 ixgbe_dev_stop(struct rte_eth_dev *dev)
1712 {
1713         struct rte_eth_link link;
1714         struct ixgbe_hw *hw =
1715                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1716         struct ixgbe_vf_info *vfinfo =
1717                 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private);
1718         struct ixgbe_filter_info *filter_info =
1719                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
1720         struct ixgbe_5tuple_filter *p_5tuple, *p_5tuple_next;
1721         int vf;
1722
1723         PMD_INIT_FUNC_TRACE();
1724
1725         /* disable interrupts */
1726         ixgbe_disable_intr(hw);
1727
1728         /* reset the NIC */
1729         ixgbe_pf_reset_hw(hw);
1730         hw->adapter_stopped = FALSE;
1731
1732         /* stop adapter */
1733         ixgbe_stop_adapter(hw);
1734
1735         for (vf = 0; vfinfo != NULL &&
1736                      vf < dev->pci_dev->max_vfs; vf++)
1737                 vfinfo[vf].clear_to_send = false;
1738
1739         if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
1740                 /* Turn off the copper */
1741                 ixgbe_set_phy_power(hw, false);
1742         } else {
1743                 /* Turn off the laser */
1744                 ixgbe_disable_tx_laser(hw);
1745         }
1746
1747         ixgbe_dev_clear_queues(dev);
1748
1749         /* Clear stored conf */
1750         dev->data->scattered_rx = 0;
1751         dev->data->lro = 0;
1752
1753         /* Clear recorded link status */
1754         memset(&link, 0, sizeof(link));
1755         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1756
1757         /* Remove all ntuple filters of the device */
1758         for (p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list);
1759              p_5tuple != NULL; p_5tuple = p_5tuple_next) {
1760                 p_5tuple_next = TAILQ_NEXT(p_5tuple, entries);
1761                 TAILQ_REMOVE(&filter_info->fivetuple_list,
1762                              p_5tuple, entries);
1763                 rte_free(p_5tuple);
1764         }
1765         memset(filter_info->fivetuple_mask, 0,
1766                 sizeof(uint32_t) * IXGBE_5TUPLE_ARRAY_SIZE);
1767
1768 }
1769
1770 /*
1771  * Set device link up: enable tx.
1772  */
1773 static int
1774 ixgbe_dev_set_link_up(struct rte_eth_dev *dev)
1775 {
1776         struct ixgbe_hw *hw =
1777                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1778         if (hw->mac.type == ixgbe_mac_82599EB) {
1779 #ifdef RTE_NIC_BYPASS
1780                 if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) {
1781                         /* Not suported in bypass mode */
1782                         PMD_INIT_LOG(ERR, "Set link up is not supported "
1783                                      "by device id 0x%x", hw->device_id);
1784                         return -ENOTSUP;
1785                 }
1786 #endif
1787         }
1788
1789         if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
1790                 /* Turn on the copper */
1791                 ixgbe_set_phy_power(hw, true);
1792         } else {
1793                 /* Turn on the laser */
1794                 ixgbe_enable_tx_laser(hw);
1795         }
1796
1797         return 0;
1798 }
1799
1800 /*
1801  * Set device link down: disable tx.
1802  */
1803 static int
1804 ixgbe_dev_set_link_down(struct rte_eth_dev *dev)
1805 {
1806         struct ixgbe_hw *hw =
1807                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1808         if (hw->mac.type == ixgbe_mac_82599EB) {
1809 #ifdef RTE_NIC_BYPASS
1810                 if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) {
1811                         /* Not suported in bypass mode */
1812                         PMD_INIT_LOG(ERR, "Set link down is not supported "
1813                                      "by device id 0x%x", hw->device_id);
1814                         return -ENOTSUP;
1815                 }
1816 #endif
1817         }
1818
1819         if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
1820                 /* Turn off the copper */
1821                 ixgbe_set_phy_power(hw, false);
1822         } else {
1823                 /* Turn off the laser */
1824                 ixgbe_disable_tx_laser(hw);
1825         }
1826
1827         return 0;
1828 }
1829
1830 /*
1831  * Reest and stop device.
1832  */
1833 static void
1834 ixgbe_dev_close(struct rte_eth_dev *dev)
1835 {
1836         struct ixgbe_hw *hw =
1837                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1838
1839         PMD_INIT_FUNC_TRACE();
1840
1841         ixgbe_pf_reset_hw(hw);
1842
1843         ixgbe_dev_stop(dev);
1844         hw->adapter_stopped = 1;
1845
1846         ixgbe_disable_pcie_master(hw);
1847
1848         /* reprogram the RAR[0] in case user changed it. */
1849         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
1850 }
1851
1852 static void
1853 ixgbe_read_stats_registers(struct ixgbe_hw *hw, struct ixgbe_hw_stats
1854                                                    *hw_stats, uint64_t *total_missed_rx,
1855                                                    uint64_t *total_qbrc, uint64_t *total_qprc,
1856                                                    uint64_t *rxnfgpc, uint64_t *txdgpc,
1857                                                    uint64_t *total_qprdc)
1858 {
1859         uint32_t bprc, lxon, lxoff, total;
1860         unsigned i;
1861
1862         hw_stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
1863         hw_stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
1864         hw_stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
1865         hw_stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
1866
1867         for (i = 0; i < 8; i++) {
1868                 uint32_t mp;
1869                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
1870                 /* global total per queue */
1871                 hw_stats->mpc[i] += mp;
1872                 /* Running comprehensive total for stats display */
1873                 *total_missed_rx += hw_stats->mpc[i];
1874                 if (hw->mac.type == ixgbe_mac_82598EB)
1875                         hw_stats->rnbc[i] +=
1876                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
1877                 hw_stats->pxontxc[i] +=
1878                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
1879                 hw_stats->pxonrxc[i] +=
1880                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
1881                 hw_stats->pxofftxc[i] +=
1882                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
1883                 hw_stats->pxoffrxc[i] +=
1884                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
1885                 hw_stats->pxon2offc[i] +=
1886                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
1887         }
1888         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1889                 hw_stats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
1890                 hw_stats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
1891                 hw_stats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
1892                 hw_stats->qbrc[i] +=
1893                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32);
1894                 hw_stats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
1895                 hw_stats->qbtc[i] +=
1896                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32);
1897                 *total_qprdc += hw_stats->qprdc[i] +=
1898                                 IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
1899
1900                 *total_qprc += hw_stats->qprc[i];
1901                 *total_qbrc += hw_stats->qbrc[i];
1902         }
1903         hw_stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
1904         hw_stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
1905         hw_stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
1906
1907         /* Note that gprc counts missed packets */
1908         hw_stats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
1909         *rxnfgpc += IXGBE_READ_REG(hw, IXGBE_RXNFGPC);
1910         *txdgpc += IXGBE_READ_REG(hw, IXGBE_TXDGPC);
1911
1912         if (hw->mac.type != ixgbe_mac_82598EB) {
1913                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
1914                 hw_stats->gorc += ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
1915                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
1916                 hw_stats->gotc += ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
1917                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
1918                 hw_stats->tor += ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
1919                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
1920                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
1921         } else {
1922                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
1923                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
1924                 /* 82598 only has a counter in the high register */
1925                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
1926                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
1927                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
1928         }
1929
1930         /*
1931          * Workaround: mprc hardware is incorrectly counting
1932          * broadcasts, so for now we subtract those.
1933          */
1934         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
1935         hw_stats->bprc += bprc;
1936         hw_stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
1937         if (hw->mac.type == ixgbe_mac_82598EB)
1938                 hw_stats->mprc -= bprc;
1939
1940         hw_stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
1941         hw_stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
1942         hw_stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
1943         hw_stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
1944         hw_stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
1945         hw_stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
1946
1947         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
1948         hw_stats->lxontxc += lxon;
1949         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
1950         hw_stats->lxofftxc += lxoff;
1951         total = lxon + lxoff;
1952
1953         hw_stats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
1954         hw_stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
1955         hw_stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
1956         hw_stats->gptc -= total;
1957         hw_stats->mptc -= total;
1958         hw_stats->ptc64 -= total;
1959         hw_stats->gotc -= total * ETHER_MIN_LEN;
1960
1961         hw_stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
1962         hw_stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
1963         hw_stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
1964         hw_stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
1965         hw_stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
1966         hw_stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
1967         hw_stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
1968         hw_stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
1969         hw_stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
1970         hw_stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
1971         hw_stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
1972         hw_stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
1973         hw_stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
1974         hw_stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
1975         hw_stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
1976         hw_stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);
1977         hw_stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
1978         hw_stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
1979         /* Only read FCOE on 82599 */
1980         if (hw->mac.type != ixgbe_mac_82598EB) {
1981                 hw_stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
1982                 hw_stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
1983                 hw_stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
1984                 hw_stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
1985                 hw_stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
1986         }
1987
1988         /* Flow Director Stats registers */
1989         hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
1990         hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
1991 }
1992
1993 /*
1994  * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c
1995  */
1996 static void
1997 ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1998 {
1999         struct ixgbe_hw *hw =
2000                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2001         struct ixgbe_hw_stats *hw_stats =
2002                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2003         uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc;
2004         uint64_t rxnfgpc, txdgpc;
2005         unsigned i;
2006
2007         total_missed_rx = 0;
2008         total_qbrc = 0;
2009         total_qprc = 0;
2010         total_qprdc = 0;
2011         rxnfgpc = 0;
2012         txdgpc = 0;
2013
2014         ixgbe_read_stats_registers(hw, hw_stats, &total_missed_rx, &total_qbrc,
2015                         &total_qprc, &rxnfgpc, &txdgpc, &total_qprdc);
2016
2017         if (stats == NULL)
2018                 return;
2019
2020         /* Fill out the rte_eth_stats statistics structure */
2021         stats->ipackets = total_qprc;
2022         stats->ibytes = total_qbrc;
2023         stats->opackets = hw_stats->gptc;
2024         stats->obytes = hw_stats->gotc;
2025         stats->imcasts = hw_stats->mprc;
2026
2027         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
2028                 stats->q_ipackets[i] = hw_stats->qprc[i];
2029                 stats->q_opackets[i] = hw_stats->qptc[i];
2030                 stats->q_ibytes[i] = hw_stats->qbrc[i];
2031                 stats->q_obytes[i] = hw_stats->qbtc[i];
2032                 stats->q_errors[i] = hw_stats->qprdc[i];
2033         }
2034
2035         /* Rx Errors */
2036         stats->ibadcrc  = hw_stats->crcerrs;
2037         stats->ibadlen  = hw_stats->rlec + hw_stats->ruc + hw_stats->roc;
2038         stats->imissed  = total_missed_rx;
2039         stats->ierrors  = stats->ibadcrc +
2040                           stats->ibadlen +
2041                           stats->imissed +
2042                           hw_stats->illerrc + hw_stats->errbc;
2043
2044         /* Tx Errors */
2045         /*txdgpc: packets that are DMA'ed*/
2046         /*gptc: packets that are sent*/
2047         stats->oerrors  = txdgpc - hw_stats->gptc;
2048
2049         /* XON/XOFF pause frames */
2050         stats->tx_pause_xon  = hw_stats->lxontxc;
2051         stats->rx_pause_xon  = hw_stats->lxonrxc;
2052         stats->tx_pause_xoff = hw_stats->lxofftxc;
2053         stats->rx_pause_xoff = hw_stats->lxoffrxc;
2054
2055         /* Flow Director Stats registers */
2056         stats->fdirmatch = hw_stats->fdirmatch;
2057         stats->fdirmiss = hw_stats->fdirmiss;
2058 }
2059
2060 static void
2061 ixgbe_dev_stats_reset(struct rte_eth_dev *dev)
2062 {
2063         struct ixgbe_hw_stats *stats =
2064                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2065
2066         /* HW registers are cleared on read */
2067         ixgbe_dev_stats_get(dev, NULL);
2068
2069         /* Reset software totals */
2070         memset(stats, 0, sizeof(*stats));
2071 }
2072
2073 static int
2074 ixgbe_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstats *xstats,
2075                                          unsigned n)
2076 {
2077         struct ixgbe_hw *hw =
2078                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2079         struct ixgbe_hw_stats *hw_stats =
2080                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2081         uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc;
2082         uint64_t rxnfgpc, txdgpc;
2083         unsigned i, count = IXGBE_NB_XSTATS;
2084
2085         if (n < count)
2086                 return count;
2087
2088         total_missed_rx = 0;
2089         total_qbrc = 0;
2090         total_qprc = 0;
2091         total_qprdc = 0;
2092         rxnfgpc = 0;
2093         txdgpc = 0;
2094
2095         ixgbe_read_stats_registers(hw, hw_stats, &total_missed_rx, &total_qbrc,
2096                                                            &total_qprc, &rxnfgpc, &txdgpc, &total_qprdc);
2097
2098         /* If this is a reset xstats is NULL, and we have cleared the
2099          * registers by reading them.
2100          */
2101         if (!xstats)
2102                 return 0;
2103
2104         /* Extended stats */
2105         for (i = 0; i < IXGBE_NB_XSTATS; i++) {
2106                 snprintf(xstats[i].name, sizeof(xstats[i].name),
2107                                 "%s", rte_ixgbe_stats_strings[i].name);
2108                 xstats[i].value = *(uint64_t *)(((char *)hw_stats) +
2109                                                         rte_ixgbe_stats_strings[i].offset);
2110         }
2111
2112         return count;
2113 }
2114
2115 static void
2116 ixgbe_dev_xstats_reset(struct rte_eth_dev *dev)
2117 {
2118         struct ixgbe_hw_stats *stats =
2119                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2120
2121         /* HW registers are cleared on read */
2122         ixgbe_dev_xstats_get(dev, NULL, IXGBE_NB_XSTATS);
2123
2124         /* Reset software totals */
2125         memset(stats, 0, sizeof(*stats));
2126 }
2127
2128 static void
2129 ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
2130 {
2131         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2132         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
2133                           IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2134
2135         /* Good Rx packet, include VF loopback */
2136         UPDATE_VF_STAT(IXGBE_VFGPRC,
2137             hw_stats->last_vfgprc, hw_stats->vfgprc);
2138
2139         /* Good Rx octets, include VF loopback */
2140         UPDATE_VF_STAT_36BIT(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
2141             hw_stats->last_vfgorc, hw_stats->vfgorc);
2142
2143         /* Good Tx packet, include VF loopback */
2144         UPDATE_VF_STAT(IXGBE_VFGPTC,
2145             hw_stats->last_vfgptc, hw_stats->vfgptc);
2146
2147         /* Good Tx octets, include VF loopback */
2148         UPDATE_VF_STAT_36BIT(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
2149             hw_stats->last_vfgotc, hw_stats->vfgotc);
2150
2151         /* Rx Multicst Packet */
2152         UPDATE_VF_STAT(IXGBE_VFMPRC,
2153             hw_stats->last_vfmprc, hw_stats->vfmprc);
2154
2155         if (stats == NULL)
2156                 return;
2157
2158         stats->ipackets = hw_stats->vfgprc;
2159         stats->ibytes = hw_stats->vfgorc;
2160         stats->opackets = hw_stats->vfgptc;
2161         stats->obytes = hw_stats->vfgotc;
2162         stats->imcasts = hw_stats->vfmprc;
2163 }
2164
2165 static void
2166 ixgbevf_dev_stats_reset(struct rte_eth_dev *dev)
2167 {
2168         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
2169                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2170
2171         /* Sync HW register to the last stats */
2172         ixgbevf_dev_stats_get(dev, NULL);
2173
2174         /* reset HW current stats*/
2175         hw_stats->vfgprc = 0;
2176         hw_stats->vfgorc = 0;
2177         hw_stats->vfgptc = 0;
2178         hw_stats->vfgotc = 0;
2179         hw_stats->vfmprc = 0;
2180
2181 }
2182
2183 static void
2184 ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
2185 {
2186         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2187
2188         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
2189         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
2190         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL register */
2191         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */
2192         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
2193         dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC;
2194         dev_info->max_vfs = dev->pci_dev->max_vfs;
2195         if (hw->mac.type == ixgbe_mac_82598EB)
2196                 dev_info->max_vmdq_pools = ETH_16_POOLS;
2197         else
2198                 dev_info->max_vmdq_pools = ETH_64_POOLS;
2199         dev_info->vmdq_queue_num = dev_info->max_rx_queues;
2200         dev_info->rx_offload_capa =
2201                 DEV_RX_OFFLOAD_VLAN_STRIP |
2202                 DEV_RX_OFFLOAD_IPV4_CKSUM |
2203                 DEV_RX_OFFLOAD_UDP_CKSUM  |
2204                 DEV_RX_OFFLOAD_TCP_CKSUM;
2205
2206         /*
2207          * RSC is only supported by 82599 and x540 PF devices in a non-SR-IOV
2208          * mode.
2209          */
2210         if ((hw->mac.type == ixgbe_mac_82599EB ||
2211              hw->mac.type == ixgbe_mac_X540) &&
2212             !RTE_ETH_DEV_SRIOV(dev).active)
2213                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_LRO;
2214
2215         dev_info->tx_offload_capa =
2216                 DEV_TX_OFFLOAD_VLAN_INSERT |
2217                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
2218                 DEV_TX_OFFLOAD_UDP_CKSUM   |
2219                 DEV_TX_OFFLOAD_TCP_CKSUM   |
2220                 DEV_TX_OFFLOAD_SCTP_CKSUM  |
2221                 DEV_TX_OFFLOAD_TCP_TSO;
2222
2223         dev_info->default_rxconf = (struct rte_eth_rxconf) {
2224                 .rx_thresh = {
2225                         .pthresh = IXGBE_DEFAULT_RX_PTHRESH,
2226                         .hthresh = IXGBE_DEFAULT_RX_HTHRESH,
2227                         .wthresh = IXGBE_DEFAULT_RX_WTHRESH,
2228                 },
2229                 .rx_free_thresh = IXGBE_DEFAULT_RX_FREE_THRESH,
2230                 .rx_drop_en = 0,
2231         };
2232
2233         dev_info->default_txconf = (struct rte_eth_txconf) {
2234                 .tx_thresh = {
2235                         .pthresh = IXGBE_DEFAULT_TX_PTHRESH,
2236                         .hthresh = IXGBE_DEFAULT_TX_HTHRESH,
2237                         .wthresh = IXGBE_DEFAULT_TX_WTHRESH,
2238                 },
2239                 .tx_free_thresh = IXGBE_DEFAULT_TX_FREE_THRESH,
2240                 .tx_rs_thresh = IXGBE_DEFAULT_TX_RSBIT_THRESH,
2241                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
2242                                 ETH_TXQ_FLAGS_NOOFFLOADS,
2243         };
2244         dev_info->hash_key_size = IXGBE_HKEY_MAX_INDEX * sizeof(uint32_t);
2245         dev_info->reta_size = ETH_RSS_RETA_SIZE_128;
2246         dev_info->flow_type_rss_offloads = IXGBE_RSS_OFFLOAD_ALL;
2247 }
2248
2249 static void
2250 ixgbevf_dev_info_get(struct rte_eth_dev *dev,
2251                      struct rte_eth_dev_info *dev_info)
2252 {
2253         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2254
2255         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
2256         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
2257         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL reg */
2258         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS reg */
2259         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
2260         dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC;
2261         dev_info->max_vfs = dev->pci_dev->max_vfs;
2262         if (hw->mac.type == ixgbe_mac_82598EB)
2263                 dev_info->max_vmdq_pools = ETH_16_POOLS;
2264         else
2265                 dev_info->max_vmdq_pools = ETH_64_POOLS;
2266         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
2267                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
2268                                 DEV_RX_OFFLOAD_UDP_CKSUM  |
2269                                 DEV_RX_OFFLOAD_TCP_CKSUM;
2270         dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
2271                                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
2272                                 DEV_TX_OFFLOAD_UDP_CKSUM   |
2273                                 DEV_TX_OFFLOAD_TCP_CKSUM   |
2274                                 DEV_TX_OFFLOAD_SCTP_CKSUM;
2275
2276         dev_info->default_rxconf = (struct rte_eth_rxconf) {
2277                 .rx_thresh = {
2278                         .pthresh = IXGBE_DEFAULT_RX_PTHRESH,
2279                         .hthresh = IXGBE_DEFAULT_RX_HTHRESH,
2280                         .wthresh = IXGBE_DEFAULT_RX_WTHRESH,
2281                 },
2282                 .rx_free_thresh = IXGBE_DEFAULT_RX_FREE_THRESH,
2283                 .rx_drop_en = 0,
2284         };
2285
2286         dev_info->default_txconf = (struct rte_eth_txconf) {
2287                 .tx_thresh = {
2288                         .pthresh = IXGBE_DEFAULT_TX_PTHRESH,
2289                         .hthresh = IXGBE_DEFAULT_TX_HTHRESH,
2290                         .wthresh = IXGBE_DEFAULT_TX_WTHRESH,
2291                 },
2292                 .tx_free_thresh = IXGBE_DEFAULT_TX_FREE_THRESH,
2293                 .tx_rs_thresh = IXGBE_DEFAULT_TX_RSBIT_THRESH,
2294                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
2295                                 ETH_TXQ_FLAGS_NOOFFLOADS,
2296         };
2297 }
2298
2299 /* return 0 means link status changed, -1 means not changed */
2300 static int
2301 ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
2302 {
2303         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2304         struct rte_eth_link link, old;
2305         ixgbe_link_speed link_speed = IXGBE_LINK_SPEED_UNKNOWN;
2306         int link_up;
2307         int diag;
2308
2309         link.link_status = 0;
2310         link.link_speed = 0;
2311         link.link_duplex = 0;
2312         memset(&old, 0, sizeof(old));
2313         rte_ixgbe_dev_atomic_read_link_status(dev, &old);
2314
2315         hw->mac.get_link_status = true;
2316
2317         /* check if it needs to wait to complete, if lsc interrupt is enabled */
2318         if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
2319                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 0);
2320         else
2321                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 1);
2322
2323         if (diag != 0) {
2324                 link.link_speed = ETH_LINK_SPEED_100;
2325                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
2326                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
2327                 if (link.link_status == old.link_status)
2328                         return -1;
2329                 return 0;
2330         }
2331
2332         if (link_up == 0) {
2333                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
2334                 if (link.link_status == old.link_status)
2335                         return -1;
2336                 return 0;
2337         }
2338         link.link_status = 1;
2339         link.link_duplex = ETH_LINK_FULL_DUPLEX;
2340
2341         switch (link_speed) {
2342         default:
2343         case IXGBE_LINK_SPEED_UNKNOWN:
2344                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
2345                 link.link_speed = ETH_LINK_SPEED_100;
2346                 break;
2347
2348         case IXGBE_LINK_SPEED_100_FULL:
2349                 link.link_speed = ETH_LINK_SPEED_100;
2350                 break;
2351
2352         case IXGBE_LINK_SPEED_1GB_FULL:
2353                 link.link_speed = ETH_LINK_SPEED_1000;
2354                 break;
2355
2356         case IXGBE_LINK_SPEED_10GB_FULL:
2357                 link.link_speed = ETH_LINK_SPEED_10000;
2358                 break;
2359         }
2360         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
2361
2362         if (link.link_status == old.link_status)
2363                 return -1;
2364
2365         return 0;
2366 }
2367
2368 static void
2369 ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev)
2370 {
2371         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2372         uint32_t fctrl;
2373
2374         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2375         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
2376         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2377 }
2378
2379 static void
2380 ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev)
2381 {
2382         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2383         uint32_t fctrl;
2384
2385         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2386         fctrl &= (~IXGBE_FCTRL_UPE);
2387         if (dev->data->all_multicast == 1)
2388                 fctrl |= IXGBE_FCTRL_MPE;
2389         else
2390                 fctrl &= (~IXGBE_FCTRL_MPE);
2391         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2392 }
2393
2394 static void
2395 ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev)
2396 {
2397         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2398         uint32_t fctrl;
2399
2400         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2401         fctrl |= IXGBE_FCTRL_MPE;
2402         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2403 }
2404
2405 static void
2406 ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev)
2407 {
2408         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2409         uint32_t fctrl;
2410
2411         if (dev->data->promiscuous == 1)
2412                 return; /* must remain in all_multicast mode */
2413
2414         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2415         fctrl &= (~IXGBE_FCTRL_MPE);
2416         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2417 }
2418
2419 /**
2420  * It clears the interrupt causes and enables the interrupt.
2421  * It will be called once only during nic initialized.
2422  *
2423  * @param dev
2424  *  Pointer to struct rte_eth_dev.
2425  *
2426  * @return
2427  *  - On success, zero.
2428  *  - On failure, a negative value.
2429  */
2430 static int
2431 ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev)
2432 {
2433         struct ixgbe_interrupt *intr =
2434                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2435
2436         ixgbe_dev_link_status_print(dev);
2437         intr->mask |= IXGBE_EICR_LSC;
2438
2439         return 0;
2440 }
2441
2442 /*
2443  * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update.
2444  *
2445  * @param dev
2446  *  Pointer to struct rte_eth_dev.
2447  *
2448  * @return
2449  *  - On success, zero.
2450  *  - On failure, a negative value.
2451  */
2452 static int
2453 ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
2454 {
2455         uint32_t eicr;
2456         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2457         struct ixgbe_interrupt *intr =
2458                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2459
2460         /* clear all cause mask */
2461         ixgbe_disable_intr(hw);
2462
2463         /* read-on-clear nic registers here */
2464         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
2465         PMD_DRV_LOG(INFO, "eicr %x", eicr);
2466
2467         intr->flags = 0;
2468         if (eicr & IXGBE_EICR_LSC) {
2469                 /* set flag for async link update */
2470                 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
2471         }
2472
2473         if (eicr & IXGBE_EICR_MAILBOX)
2474                 intr->flags |= IXGBE_FLAG_MAILBOX;
2475
2476         return 0;
2477 }
2478
2479 /**
2480  * It gets and then prints the link status.
2481  *
2482  * @param dev
2483  *  Pointer to struct rte_eth_dev.
2484  *
2485  * @return
2486  *  - On success, zero.
2487  *  - On failure, a negative value.
2488  */
2489 static void
2490 ixgbe_dev_link_status_print(struct rte_eth_dev *dev)
2491 {
2492         struct rte_eth_link link;
2493
2494         memset(&link, 0, sizeof(link));
2495         rte_ixgbe_dev_atomic_read_link_status(dev, &link);
2496         if (link.link_status) {
2497                 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
2498                                         (int)(dev->data->port_id),
2499                                         (unsigned)link.link_speed,
2500                         link.link_duplex == ETH_LINK_FULL_DUPLEX ?
2501                                         "full-duplex" : "half-duplex");
2502         } else {
2503                 PMD_INIT_LOG(INFO, " Port %d: Link Down",
2504                                 (int)(dev->data->port_id));
2505         }
2506         PMD_INIT_LOG(INFO, "PCI Address: %04d:%02d:%02d:%d",
2507                                 dev->pci_dev->addr.domain,
2508                                 dev->pci_dev->addr.bus,
2509                                 dev->pci_dev->addr.devid,
2510                                 dev->pci_dev->addr.function);
2511 }
2512
2513 /*
2514  * It executes link_update after knowing an interrupt occurred.
2515  *
2516  * @param dev
2517  *  Pointer to struct rte_eth_dev.
2518  *
2519  * @return
2520  *  - On success, zero.
2521  *  - On failure, a negative value.
2522  */
2523 static int
2524 ixgbe_dev_interrupt_action(struct rte_eth_dev *dev)
2525 {
2526         struct ixgbe_interrupt *intr =
2527                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2528         int64_t timeout;
2529         struct rte_eth_link link;
2530         int intr_enable_delay = false;
2531
2532         PMD_DRV_LOG(DEBUG, "intr action type %d", intr->flags);
2533
2534         if (intr->flags & IXGBE_FLAG_MAILBOX) {
2535                 ixgbe_pf_mbx_process(dev);
2536                 intr->flags &= ~IXGBE_FLAG_MAILBOX;
2537         }
2538
2539         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
2540                 /* get the link status before link update, for predicting later */
2541                 memset(&link, 0, sizeof(link));
2542                 rte_ixgbe_dev_atomic_read_link_status(dev, &link);
2543
2544                 ixgbe_dev_link_update(dev, 0);
2545
2546                 /* likely to up */
2547                 if (!link.link_status)
2548                         /* handle it 1 sec later, wait it being stable */
2549                         timeout = IXGBE_LINK_UP_CHECK_TIMEOUT;
2550                 /* likely to down */
2551                 else
2552                         /* handle it 4 sec later, wait it being stable */
2553                         timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT;
2554
2555                 ixgbe_dev_link_status_print(dev);
2556
2557                 intr_enable_delay = true;
2558         }
2559
2560         if (intr_enable_delay) {
2561                 if (rte_eal_alarm_set(timeout * 1000,
2562                                       ixgbe_dev_interrupt_delayed_handler, (void*)dev) < 0)
2563                         PMD_DRV_LOG(ERR, "Error setting alarm");
2564         } else {
2565                 PMD_DRV_LOG(DEBUG, "enable intr immediately");
2566                 ixgbe_enable_intr(dev);
2567                 rte_intr_enable(&(dev->pci_dev->intr_handle));
2568         }
2569
2570
2571         return 0;
2572 }
2573
2574 /**
2575  * Interrupt handler which shall be registered for alarm callback for delayed
2576  * handling specific interrupt to wait for the stable nic state. As the
2577  * NIC interrupt state is not stable for ixgbe after link is just down,
2578  * it needs to wait 4 seconds to get the stable status.
2579  *
2580  * @param handle
2581  *  Pointer to interrupt handle.
2582  * @param param
2583  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2584  *
2585  * @return
2586  *  void
2587  */
2588 static void
2589 ixgbe_dev_interrupt_delayed_handler(void *param)
2590 {
2591         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2592         struct ixgbe_interrupt *intr =
2593                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2594         struct ixgbe_hw *hw =
2595                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2596         uint32_t eicr;
2597
2598         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
2599         if (eicr & IXGBE_EICR_MAILBOX)
2600                 ixgbe_pf_mbx_process(dev);
2601
2602         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
2603                 ixgbe_dev_link_update(dev, 0);
2604                 intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
2605                 ixgbe_dev_link_status_print(dev);
2606                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
2607         }
2608
2609         PMD_DRV_LOG(DEBUG, "enable intr in delayed handler S[%08x]", eicr);
2610         ixgbe_enable_intr(dev);
2611         rte_intr_enable(&(dev->pci_dev->intr_handle));
2612 }
2613
2614 /**
2615  * Interrupt handler triggered by NIC  for handling
2616  * specific interrupt.
2617  *
2618  * @param handle
2619  *  Pointer to interrupt handle.
2620  * @param param
2621  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2622  *
2623  * @return
2624  *  void
2625  */
2626 static void
2627 ixgbe_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
2628                                                         void *param)
2629 {
2630         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2631         ixgbe_dev_interrupt_get_status(dev);
2632         ixgbe_dev_interrupt_action(dev);
2633 }
2634
2635 static int
2636 ixgbe_dev_led_on(struct rte_eth_dev *dev)
2637 {
2638         struct ixgbe_hw *hw;
2639
2640         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2641         return (ixgbe_led_on(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
2642 }
2643
2644 static int
2645 ixgbe_dev_led_off(struct rte_eth_dev *dev)
2646 {
2647         struct ixgbe_hw *hw;
2648
2649         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2650         return (ixgbe_led_off(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
2651 }
2652
2653 static int
2654 ixgbe_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2655 {
2656         struct ixgbe_hw *hw;
2657         uint32_t mflcn_reg;
2658         uint32_t fccfg_reg;
2659         int rx_pause;
2660         int tx_pause;
2661
2662         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2663
2664         fc_conf->pause_time = hw->fc.pause_time;
2665         fc_conf->high_water = hw->fc.high_water[0];
2666         fc_conf->low_water = hw->fc.low_water[0];
2667         fc_conf->send_xon = hw->fc.send_xon;
2668         fc_conf->autoneg = !hw->fc.disable_fc_autoneg;
2669
2670         /*
2671          * Return rx_pause status according to actual setting of
2672          * MFLCN register.
2673          */
2674         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
2675         if (mflcn_reg & (IXGBE_MFLCN_RPFCE | IXGBE_MFLCN_RFCE))
2676                 rx_pause = 1;
2677         else
2678                 rx_pause = 0;
2679
2680         /*
2681          * Return tx_pause status according to actual setting of
2682          * FCCFG register.
2683          */
2684         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
2685         if (fccfg_reg & (IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY))
2686                 tx_pause = 1;
2687         else
2688                 tx_pause = 0;
2689
2690         if (rx_pause && tx_pause)
2691                 fc_conf->mode = RTE_FC_FULL;
2692         else if (rx_pause)
2693                 fc_conf->mode = RTE_FC_RX_PAUSE;
2694         else if (tx_pause)
2695                 fc_conf->mode = RTE_FC_TX_PAUSE;
2696         else
2697                 fc_conf->mode = RTE_FC_NONE;
2698
2699         return 0;
2700 }
2701
2702 static int
2703 ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2704 {
2705         struct ixgbe_hw *hw;
2706         int err;
2707         uint32_t rx_buf_size;
2708         uint32_t max_high_water;
2709         uint32_t mflcn;
2710         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
2711                 ixgbe_fc_none,
2712                 ixgbe_fc_rx_pause,
2713                 ixgbe_fc_tx_pause,
2714                 ixgbe_fc_full
2715         };
2716
2717         PMD_INIT_FUNC_TRACE();
2718
2719         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2720         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0));
2721         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
2722
2723         /*
2724          * At least reserve one Ethernet frame for watermark
2725          * high_water/low_water in kilo bytes for ixgbe
2726          */
2727         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
2728         if ((fc_conf->high_water > max_high_water) ||
2729                 (fc_conf->high_water < fc_conf->low_water)) {
2730                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB");
2731                 PMD_INIT_LOG(ERR, "High_water must <= 0x%x", max_high_water);
2732                 return (-EINVAL);
2733         }
2734
2735         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[fc_conf->mode];
2736         hw->fc.pause_time     = fc_conf->pause_time;
2737         hw->fc.high_water[0]  = fc_conf->high_water;
2738         hw->fc.low_water[0]   = fc_conf->low_water;
2739         hw->fc.send_xon       = fc_conf->send_xon;
2740         hw->fc.disable_fc_autoneg = !fc_conf->autoneg;
2741
2742         err = ixgbe_fc_enable(hw);
2743
2744         /* Not negotiated is not an error case */
2745         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) {
2746
2747                 /* check if we want to forward MAC frames - driver doesn't have native
2748                  * capability to do that, so we'll write the registers ourselves */
2749
2750                 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN);
2751
2752                 /* set or clear MFLCN.PMCF bit depending on configuration */
2753                 if (fc_conf->mac_ctrl_frame_fwd != 0)
2754                         mflcn |= IXGBE_MFLCN_PMCF;
2755                 else
2756                         mflcn &= ~IXGBE_MFLCN_PMCF;
2757
2758                 IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn);
2759                 IXGBE_WRITE_FLUSH(hw);
2760
2761                 return 0;
2762         }
2763
2764         PMD_INIT_LOG(ERR, "ixgbe_fc_enable = 0x%x", err);
2765         return -EIO;
2766 }
2767
2768 /**
2769  *  ixgbe_pfc_enable_generic - Enable flow control
2770  *  @hw: pointer to hardware structure
2771  *  @tc_num: traffic class number
2772  *  Enable flow control according to the current settings.
2773  */
2774 static int
2775 ixgbe_dcb_pfc_enable_generic(struct ixgbe_hw *hw,uint8_t tc_num)
2776 {
2777         int ret_val = 0;
2778         uint32_t mflcn_reg, fccfg_reg;
2779         uint32_t reg;
2780         uint32_t fcrtl, fcrth;
2781         uint8_t i;
2782         uint8_t nb_rx_en;
2783
2784         /* Validate the water mark configuration */
2785         if (!hw->fc.pause_time) {
2786                 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2787                 goto out;
2788         }
2789
2790         /* Low water mark of zero causes XOFF floods */
2791         if (hw->fc.current_mode & ixgbe_fc_tx_pause) {
2792                  /* High/Low water can not be 0 */
2793                 if( (!hw->fc.high_water[tc_num])|| (!hw->fc.low_water[tc_num])) {
2794                         PMD_INIT_LOG(ERR, "Invalid water mark configuration");
2795                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2796                         goto out;
2797                 }
2798
2799                 if(hw->fc.low_water[tc_num] >= hw->fc.high_water[tc_num]) {
2800                         PMD_INIT_LOG(ERR, "Invalid water mark configuration");
2801                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2802                         goto out;
2803                 }
2804         }
2805         /* Negotiate the fc mode to use */
2806         ixgbe_fc_autoneg(hw);
2807
2808         /* Disable any previous flow control settings */
2809         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
2810         mflcn_reg &= ~(IXGBE_MFLCN_RPFCE_SHIFT | IXGBE_MFLCN_RFCE|IXGBE_MFLCN_RPFCE);
2811
2812         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
2813         fccfg_reg &= ~(IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY);
2814
2815         switch (hw->fc.current_mode) {
2816         case ixgbe_fc_none:
2817                 /*
2818                  * If the count of enabled RX Priority Flow control >1,
2819                  * and the TX pause can not be disabled
2820                  */
2821                 nb_rx_en = 0;
2822                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
2823                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
2824                         if (reg & IXGBE_FCRTH_FCEN)
2825                                 nb_rx_en++;
2826                 }
2827                 if (nb_rx_en > 1)
2828                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2829                 break;
2830         case ixgbe_fc_rx_pause:
2831                 /*
2832                  * Rx Flow control is enabled and Tx Flow control is
2833                  * disabled by software override. Since there really
2834                  * isn't a way to advertise that we are capable of RX
2835                  * Pause ONLY, we will advertise that we support both
2836                  * symmetric and asymmetric Rx PAUSE.  Later, we will
2837                  * disable the adapter's ability to send PAUSE frames.
2838                  */
2839                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
2840                 /*
2841                  * If the count of enabled RX Priority Flow control >1,
2842                  * and the TX pause can not be disabled
2843                  */
2844                 nb_rx_en = 0;
2845                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
2846                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
2847                         if (reg & IXGBE_FCRTH_FCEN)
2848                                 nb_rx_en++;
2849                 }
2850                 if (nb_rx_en > 1)
2851                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2852                 break;
2853         case ixgbe_fc_tx_pause:
2854                 /*
2855                  * Tx Flow control is enabled, and Rx Flow control is
2856                  * disabled by software override.
2857                  */
2858                 fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2859                 break;
2860         case ixgbe_fc_full:
2861                 /* Flow control (both Rx and Tx) is enabled by SW override. */
2862                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
2863                 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
2864                 break;
2865         default:
2866                 PMD_DRV_LOG(DEBUG, "Flow control param set incorrectly");
2867                 ret_val = IXGBE_ERR_CONFIG;
2868                 goto out;
2869                 break;
2870         }
2871
2872         /* Set 802.3x based flow control settings. */
2873         mflcn_reg |= IXGBE_MFLCN_DPF;
2874         IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg);
2875         IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg);
2876
2877         /* Set up and enable Rx high/low water mark thresholds, enable XON. */
2878         if ((hw->fc.current_mode & ixgbe_fc_tx_pause) &&
2879                 hw->fc.high_water[tc_num]) {
2880                 fcrtl = (hw->fc.low_water[tc_num] << 10) | IXGBE_FCRTL_XONE;
2881                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), fcrtl);
2882                 fcrth = (hw->fc.high_water[tc_num] << 10) | IXGBE_FCRTH_FCEN;
2883         } else {
2884                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), 0);
2885                 /*
2886                  * In order to prevent Tx hangs when the internal Tx
2887                  * switch is enabled we must set the high water mark
2888                  * to the maximum FCRTH value.  This allows the Tx
2889                  * switch to function even under heavy Rx workloads.
2890                  */
2891                 fcrth = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)) - 32;
2892         }
2893         IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(tc_num), fcrth);
2894
2895         /* Configure pause time (2 TCs per register) */
2896         reg = hw->fc.pause_time * 0x00010001;
2897         for (i = 0; i < (IXGBE_DCB_MAX_TRAFFIC_CLASS / 2); i++)
2898                 IXGBE_WRITE_REG(hw, IXGBE_FCTTV(i), reg);
2899
2900         /* Configure flow control refresh threshold value */
2901         IXGBE_WRITE_REG(hw, IXGBE_FCRTV, hw->fc.pause_time / 2);
2902
2903 out:
2904         return ret_val;
2905 }
2906
2907 static int
2908 ixgbe_dcb_pfc_enable(struct rte_eth_dev *dev,uint8_t tc_num)
2909 {
2910         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2911         int32_t ret_val = IXGBE_NOT_IMPLEMENTED;
2912
2913         if(hw->mac.type != ixgbe_mac_82598EB) {
2914                 ret_val = ixgbe_dcb_pfc_enable_generic(hw,tc_num);
2915         }
2916         return ret_val;
2917 }
2918
2919 static int
2920 ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf)
2921 {
2922         int err;
2923         uint32_t rx_buf_size;
2924         uint32_t max_high_water;
2925         uint8_t tc_num;
2926         uint8_t  map[IXGBE_DCB_MAX_USER_PRIORITY] = { 0 };
2927         struct ixgbe_hw *hw =
2928                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2929         struct ixgbe_dcb_config *dcb_config =
2930                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
2931
2932         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
2933                 ixgbe_fc_none,
2934                 ixgbe_fc_rx_pause,
2935                 ixgbe_fc_tx_pause,
2936                 ixgbe_fc_full
2937         };
2938
2939         PMD_INIT_FUNC_TRACE();
2940
2941         ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
2942         tc_num = map[pfc_conf->priority];
2943         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num));
2944         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
2945         /*
2946          * At least reserve one Ethernet frame for watermark
2947          * high_water/low_water in kilo bytes for ixgbe
2948          */
2949         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
2950         if ((pfc_conf->fc.high_water > max_high_water) ||
2951             (pfc_conf->fc.high_water <= pfc_conf->fc.low_water)) {
2952                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB");
2953                 PMD_INIT_LOG(ERR, "High_water must <= 0x%x", max_high_water);
2954                 return (-EINVAL);
2955         }
2956
2957         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[pfc_conf->fc.mode];
2958         hw->fc.pause_time = pfc_conf->fc.pause_time;
2959         hw->fc.send_xon = pfc_conf->fc.send_xon;
2960         hw->fc.low_water[tc_num] =  pfc_conf->fc.low_water;
2961         hw->fc.high_water[tc_num] = pfc_conf->fc.high_water;
2962
2963         err = ixgbe_dcb_pfc_enable(dev,tc_num);
2964
2965         /* Not negotiated is not an error case */
2966         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED))
2967                 return 0;
2968
2969         PMD_INIT_LOG(ERR, "ixgbe_dcb_pfc_enable = 0x%x", err);
2970         return -EIO;
2971 }
2972
2973 static int
2974 ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
2975                           struct rte_eth_rss_reta_entry64 *reta_conf,
2976                           uint16_t reta_size)
2977 {
2978         uint8_t i, j, mask;
2979         uint32_t reta, r;
2980         uint16_t idx, shift;
2981         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2982
2983         PMD_INIT_FUNC_TRACE();
2984         if (reta_size != ETH_RSS_RETA_SIZE_128) {
2985                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2986                         "(%d) doesn't match the number hardware can supported "
2987                         "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
2988                 return -EINVAL;
2989         }
2990
2991         for (i = 0; i < reta_size; i += IXGBE_4_BIT_WIDTH) {
2992                 idx = i / RTE_RETA_GROUP_SIZE;
2993                 shift = i % RTE_RETA_GROUP_SIZE;
2994                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2995                                                 IXGBE_4_BIT_MASK);
2996                 if (!mask)
2997                         continue;
2998                 if (mask == IXGBE_4_BIT_MASK)
2999                         r = 0;
3000                 else
3001                         r = IXGBE_READ_REG(hw, IXGBE_RETA(i >> 2));
3002                 for (j = 0, reta = 0; j < IXGBE_4_BIT_WIDTH; j++) {
3003                         if (mask & (0x1 << j))
3004                                 reta |= reta_conf[idx].reta[shift + j] <<
3005                                                         (CHAR_BIT * j);
3006                         else
3007                                 reta |= r & (IXGBE_8_BIT_MASK <<
3008                                                 (CHAR_BIT * j));
3009                 }
3010                 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
3011         }
3012
3013         return 0;
3014 }
3015
3016 static int
3017 ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
3018                          struct rte_eth_rss_reta_entry64 *reta_conf,
3019                          uint16_t reta_size)
3020 {
3021         uint8_t i, j, mask;
3022         uint32_t reta;
3023         uint16_t idx, shift;
3024         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3025
3026         PMD_INIT_FUNC_TRACE();
3027         if (reta_size != ETH_RSS_RETA_SIZE_128) {
3028                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
3029                         "(%d) doesn't match the number hardware can supported "
3030                                 "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
3031                 return -EINVAL;
3032         }
3033
3034         for (i = 0; i < ETH_RSS_RETA_SIZE_128; i += IXGBE_4_BIT_WIDTH) {
3035                 idx = i / RTE_RETA_GROUP_SIZE;
3036                 shift = i % RTE_RETA_GROUP_SIZE;
3037                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
3038                                                 IXGBE_4_BIT_MASK);
3039                 if (!mask)
3040                         continue;
3041
3042                 reta = IXGBE_READ_REG(hw, IXGBE_RETA(i >> 2));
3043                 for (j = 0; j < IXGBE_4_BIT_WIDTH; j++) {
3044                         if (mask & (0x1 << j))
3045                                 reta_conf[idx].reta[shift + j] =
3046                                         ((reta >> (CHAR_BIT * j)) &
3047                                                 IXGBE_8_BIT_MASK);
3048                 }
3049         }
3050
3051         return 0;
3052 }
3053
3054 static void
3055 ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
3056                                 uint32_t index, uint32_t pool)
3057 {
3058         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3059         uint32_t enable_addr = 1;
3060
3061         ixgbe_set_rar(hw, index, mac_addr->addr_bytes, pool, enable_addr);
3062 }
3063
3064 static void
3065 ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
3066 {
3067         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3068
3069         ixgbe_clear_rar(hw, index);
3070 }
3071
3072 static void
3073 ixgbe_set_default_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
3074 {
3075         ixgbe_remove_rar(dev, 0);
3076
3077         ixgbe_add_rar(dev, addr, 0, 0);
3078 }
3079
3080 static int
3081 ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
3082 {
3083         uint32_t hlreg0;
3084         uint32_t maxfrs;
3085         struct ixgbe_hw *hw;
3086         struct rte_eth_dev_info dev_info;
3087         uint32_t frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
3088
3089         ixgbe_dev_info_get(dev, &dev_info);
3090
3091         /* check that mtu is within the allowed range */
3092         if ((mtu < ETHER_MIN_MTU) || (frame_size > dev_info.max_rx_pktlen))
3093                 return -EINVAL;
3094
3095         /* refuse mtu that requires the support of scattered packets when this
3096          * feature has not been enabled before. */
3097         if (!dev->data->scattered_rx &&
3098             (frame_size + 2 * IXGBE_VLAN_TAG_SIZE >
3099              dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM))
3100                 return -EINVAL;
3101
3102         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3103         hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
3104
3105         /* switch to jumbo mode if needed */
3106         if (frame_size > ETHER_MAX_LEN) {
3107                 dev->data->dev_conf.rxmode.jumbo_frame = 1;
3108                 hlreg0 |= IXGBE_HLREG0_JUMBOEN;
3109         } else {
3110                 dev->data->dev_conf.rxmode.jumbo_frame = 0;
3111                 hlreg0 &= ~IXGBE_HLREG0_JUMBOEN;
3112         }
3113         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
3114
3115         /* update max frame size */
3116         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
3117
3118         maxfrs = IXGBE_READ_REG(hw, IXGBE_MAXFRS);
3119         maxfrs &= 0x0000FFFF;
3120         maxfrs |= (dev->data->dev_conf.rxmode.max_rx_pkt_len << 16);
3121         IXGBE_WRITE_REG(hw, IXGBE_MAXFRS, maxfrs);
3122
3123         return 0;
3124 }
3125
3126 /*
3127  * Virtual Function operations
3128  */
3129 static void
3130 ixgbevf_intr_disable(struct ixgbe_hw *hw)
3131 {
3132         PMD_INIT_FUNC_TRACE();
3133
3134         /* Clear interrupt mask to stop from interrupts being generated */
3135         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, IXGBE_VF_IRQ_CLEAR_MASK);
3136
3137         IXGBE_WRITE_FLUSH(hw);
3138 }
3139
3140 static int
3141 ixgbevf_dev_configure(struct rte_eth_dev *dev)
3142 {
3143         struct rte_eth_conf* conf = &dev->data->dev_conf;
3144         struct ixgbe_adapter *adapter =
3145                         (struct ixgbe_adapter *)dev->data->dev_private;
3146
3147         PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d",
3148                      dev->data->port_id);
3149
3150         /*
3151          * VF has no ability to enable/disable HW CRC
3152          * Keep the persistent behavior the same as Host PF
3153          */
3154 #ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC
3155         if (!conf->rxmode.hw_strip_crc) {
3156                 PMD_INIT_LOG(INFO, "VF can't disable HW CRC Strip");
3157                 conf->rxmode.hw_strip_crc = 1;
3158         }
3159 #else
3160         if (conf->rxmode.hw_strip_crc) {
3161                 PMD_INIT_LOG(INFO, "VF can't enable HW CRC Strip");
3162                 conf->rxmode.hw_strip_crc = 0;
3163         }
3164 #endif
3165
3166         /*
3167          * Initialize to TRUE. If any of Rx queues doesn't meet the bulk
3168          * allocation or vector Rx preconditions we will reset it.
3169          */
3170         adapter->rx_bulk_alloc_allowed = true;
3171         adapter->rx_vec_allowed = true;
3172
3173         return 0;
3174 }
3175
3176 static int
3177 ixgbevf_dev_start(struct rte_eth_dev *dev)
3178 {
3179         struct ixgbe_hw *hw =
3180                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3181         int err, mask = 0;
3182
3183         PMD_INIT_FUNC_TRACE();
3184
3185         hw->mac.ops.reset_hw(hw);
3186         hw->mac.get_link_status = true;
3187
3188         /* negotiate mailbox API version to use with the PF. */
3189         ixgbevf_negotiate_api(hw);
3190
3191         ixgbevf_dev_tx_init(dev);
3192
3193         /* This can fail when allocating mbufs for descriptor rings */
3194         err = ixgbevf_dev_rx_init(dev);
3195         if (err) {
3196                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware (%d)", err);
3197                 ixgbe_dev_clear_queues(dev);
3198                 return err;
3199         }
3200
3201         /* Set vfta */
3202         ixgbevf_set_vfta_all(dev,1);
3203
3204         /* Set HW strip */
3205         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
3206                 ETH_VLAN_EXTEND_MASK;
3207         ixgbevf_vlan_offload_set(dev, mask);
3208
3209         ixgbevf_dev_rxtx_start(dev);
3210
3211         return 0;
3212 }
3213
3214 static void
3215 ixgbevf_dev_stop(struct rte_eth_dev *dev)
3216 {
3217         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3218
3219         PMD_INIT_FUNC_TRACE();
3220
3221         hw->adapter_stopped = TRUE;
3222         ixgbe_stop_adapter(hw);
3223
3224         /*
3225           * Clear what we set, but we still keep shadow_vfta to
3226           * restore after device starts
3227           */
3228         ixgbevf_set_vfta_all(dev,0);
3229
3230         /* Clear stored conf */
3231         dev->data->scattered_rx = 0;
3232
3233         ixgbe_dev_clear_queues(dev);
3234 }
3235
3236 static void
3237 ixgbevf_dev_close(struct rte_eth_dev *dev)
3238 {
3239         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3240
3241         PMD_INIT_FUNC_TRACE();
3242
3243         ixgbe_reset_hw(hw);
3244
3245         ixgbevf_dev_stop(dev);
3246
3247         /* reprogram the RAR[0] in case user changed it. */
3248         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
3249 }
3250
3251 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on)
3252 {
3253         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3254         struct ixgbe_vfta * shadow_vfta =
3255                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
3256         int i = 0, j = 0, vfta = 0, mask = 1;
3257
3258         for (i = 0; i < IXGBE_VFTA_SIZE; i++){
3259                 vfta = shadow_vfta->vfta[i];
3260                 if(vfta){
3261                         mask = 1;
3262                         for (j = 0; j < 32; j++){
3263                                 if(vfta & mask)
3264                                         ixgbe_set_vfta(hw, (i<<5)+j, 0, on);
3265                                 mask<<=1;
3266                         }
3267                 }
3268         }
3269
3270 }
3271
3272 static int
3273 ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
3274 {
3275         struct ixgbe_hw *hw =
3276                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3277         struct ixgbe_vfta * shadow_vfta =
3278                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
3279         uint32_t vid_idx = 0;
3280         uint32_t vid_bit = 0;
3281         int ret = 0;
3282
3283         PMD_INIT_FUNC_TRACE();
3284
3285         /* vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf */
3286         ret = ixgbe_set_vfta(hw, vlan_id, 0, !!on);
3287         if(ret){
3288                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
3289                 return ret;
3290         }
3291         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
3292         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
3293
3294         /* Save what we set and retore it after device reset */
3295         if (on)
3296                 shadow_vfta->vfta[vid_idx] |= vid_bit;
3297         else
3298                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
3299
3300         return 0;
3301 }
3302
3303 static void
3304 ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
3305 {
3306         struct ixgbe_hw *hw =
3307                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3308         uint32_t ctrl;
3309
3310         PMD_INIT_FUNC_TRACE();
3311
3312         if(queue >= hw->mac.max_rx_queues)
3313                 return;
3314
3315         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
3316         if(on)
3317                 ctrl |= IXGBE_RXDCTL_VME;
3318         else
3319                 ctrl &= ~IXGBE_RXDCTL_VME;
3320         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
3321
3322         ixgbe_vlan_hw_strip_bitmap_set( dev, queue, on);
3323 }
3324
3325 static void
3326 ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
3327 {
3328         struct ixgbe_hw *hw =
3329                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3330         uint16_t i;
3331         int on = 0;
3332
3333         /* VF function only support hw strip feature, others are not support */
3334         if(mask & ETH_VLAN_STRIP_MASK){
3335                 on = !!(dev->data->dev_conf.rxmode.hw_vlan_strip);
3336
3337                 for(i=0; i < hw->mac.max_rx_queues; i++)
3338                         ixgbevf_vlan_strip_queue_set(dev,i,on);
3339         }
3340 }
3341
3342 static int
3343 ixgbe_vmdq_mode_check(struct ixgbe_hw *hw)
3344 {
3345         uint32_t reg_val;
3346
3347         /* we only need to do this if VMDq is enabled */
3348         reg_val = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
3349         if (!(reg_val & IXGBE_VT_CTL_VT_ENABLE)) {
3350                 PMD_INIT_LOG(ERR, "VMDq must be enabled for this setting");
3351                 return (-1);
3352         }
3353
3354         return 0;
3355 }
3356
3357 static uint32_t
3358 ixgbe_uta_vector(struct ixgbe_hw *hw, struct ether_addr* uc_addr)
3359 {
3360         uint32_t vector = 0;
3361         switch (hw->mac.mc_filter_type) {
3362         case 0:   /* use bits [47:36] of the address */
3363                 vector = ((uc_addr->addr_bytes[4] >> 4) |
3364                         (((uint16_t)uc_addr->addr_bytes[5]) << 4));
3365                 break;
3366         case 1:   /* use bits [46:35] of the address */
3367                 vector = ((uc_addr->addr_bytes[4] >> 3) |
3368                         (((uint16_t)uc_addr->addr_bytes[5]) << 5));
3369                 break;
3370         case 2:   /* use bits [45:34] of the address */
3371                 vector = ((uc_addr->addr_bytes[4] >> 2) |
3372                         (((uint16_t)uc_addr->addr_bytes[5]) << 6));
3373                 break;
3374         case 3:   /* use bits [43:32] of the address */
3375                 vector = ((uc_addr->addr_bytes[4]) |
3376                         (((uint16_t)uc_addr->addr_bytes[5]) << 8));
3377                 break;
3378         default:  /* Invalid mc_filter_type */
3379                 break;
3380         }
3381
3382         /* vector can only be 12-bits or boundary will be exceeded */
3383         vector &= 0xFFF;
3384         return vector;
3385 }
3386
3387 static int
3388 ixgbe_uc_hash_table_set(struct rte_eth_dev *dev,struct ether_addr* mac_addr,
3389                                uint8_t on)
3390 {
3391         uint32_t vector;
3392         uint32_t uta_idx;
3393         uint32_t reg_val;
3394         uint32_t uta_shift;
3395         uint32_t rc;
3396         const uint32_t ixgbe_uta_idx_mask = 0x7F;
3397         const uint32_t ixgbe_uta_bit_shift = 5;
3398         const uint32_t ixgbe_uta_bit_mask = (0x1 << ixgbe_uta_bit_shift) - 1;
3399         const uint32_t bit1 = 0x1;
3400
3401         struct ixgbe_hw *hw =
3402                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3403         struct ixgbe_uta_info *uta_info =
3404                 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private);
3405
3406         /* The UTA table only exists on 82599 hardware and newer */
3407         if (hw->mac.type < ixgbe_mac_82599EB)
3408                 return (-ENOTSUP);
3409
3410         vector = ixgbe_uta_vector(hw,mac_addr);
3411         uta_idx = (vector >> ixgbe_uta_bit_shift) & ixgbe_uta_idx_mask;
3412         uta_shift = vector & ixgbe_uta_bit_mask;
3413
3414         rc = ((uta_info->uta_shadow[uta_idx] >> uta_shift & bit1) != 0);
3415         if(rc == on)
3416                 return 0;
3417
3418         reg_val = IXGBE_READ_REG(hw, IXGBE_UTA(uta_idx));
3419         if (on) {
3420                 uta_info->uta_in_use++;
3421                 reg_val |= (bit1 << uta_shift);
3422                 uta_info->uta_shadow[uta_idx] |= (bit1 << uta_shift);
3423         } else {
3424                 uta_info->uta_in_use--;
3425                 reg_val &= ~(bit1 << uta_shift);
3426                 uta_info->uta_shadow[uta_idx] &= ~(bit1 << uta_shift);
3427         }
3428
3429         IXGBE_WRITE_REG(hw, IXGBE_UTA(uta_idx), reg_val);
3430
3431         if (uta_info->uta_in_use > 0)
3432                 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL,
3433                                 IXGBE_MCSTCTRL_MFE | hw->mac.mc_filter_type);
3434         else
3435                 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL,hw->mac.mc_filter_type);
3436
3437         return 0;
3438 }
3439
3440 static int
3441 ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on)
3442 {
3443         int i;
3444         struct ixgbe_hw *hw =
3445                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3446         struct ixgbe_uta_info *uta_info =
3447                 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private);
3448
3449         /* The UTA table only exists on 82599 hardware and newer */
3450         if (hw->mac.type < ixgbe_mac_82599EB)
3451                 return (-ENOTSUP);
3452
3453         if(on) {
3454                 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
3455                         uta_info->uta_shadow[i] = ~0;
3456                         IXGBE_WRITE_REG(hw, IXGBE_UTA(i), ~0);
3457                 }
3458         } else {
3459                 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
3460                         uta_info->uta_shadow[i] = 0;
3461                         IXGBE_WRITE_REG(hw, IXGBE_UTA(i), 0);
3462                 }
3463         }
3464         return 0;
3465
3466 }
3467
3468 uint32_t
3469 ixgbe_convert_vm_rx_mask_to_val(uint16_t rx_mask, uint32_t orig_val)
3470 {
3471         uint32_t new_val = orig_val;
3472
3473         if (rx_mask & ETH_VMDQ_ACCEPT_UNTAG)
3474                 new_val |= IXGBE_VMOLR_AUPE;
3475         if (rx_mask & ETH_VMDQ_ACCEPT_HASH_MC)
3476                 new_val |= IXGBE_VMOLR_ROMPE;
3477         if (rx_mask & ETH_VMDQ_ACCEPT_HASH_UC)
3478                 new_val |= IXGBE_VMOLR_ROPE;
3479         if (rx_mask & ETH_VMDQ_ACCEPT_BROADCAST)
3480                 new_val |= IXGBE_VMOLR_BAM;
3481         if (rx_mask & ETH_VMDQ_ACCEPT_MULTICAST)
3482                 new_val |= IXGBE_VMOLR_MPE;
3483
3484         return new_val;
3485 }
3486
3487 static int
3488 ixgbe_set_pool_rx_mode(struct rte_eth_dev *dev, uint16_t pool,
3489                                uint16_t rx_mask, uint8_t on)
3490 {
3491         int val = 0;
3492
3493         struct ixgbe_hw *hw =
3494                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3495         uint32_t vmolr = IXGBE_READ_REG(hw, IXGBE_VMOLR(pool));
3496
3497         if (hw->mac.type == ixgbe_mac_82598EB) {
3498                 PMD_INIT_LOG(ERR, "setting VF receive mode set should be done"
3499                              " on 82599 hardware and newer");
3500                 return (-ENOTSUP);
3501         }
3502         if (ixgbe_vmdq_mode_check(hw) < 0)
3503                 return (-ENOTSUP);
3504
3505         val = ixgbe_convert_vm_rx_mask_to_val(rx_mask, val);
3506
3507         if (on)
3508                 vmolr |= val;
3509         else
3510                 vmolr &= ~val;
3511
3512         IXGBE_WRITE_REG(hw, IXGBE_VMOLR(pool), vmolr);
3513
3514         return 0;
3515 }
3516
3517 static int
3518 ixgbe_set_pool_rx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on)
3519 {
3520         uint32_t reg,addr;
3521         uint32_t val;
3522         const uint8_t bit1 = 0x1;
3523
3524         struct ixgbe_hw *hw =
3525                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3526
3527         if (ixgbe_vmdq_mode_check(hw) < 0)
3528                 return (-ENOTSUP);
3529
3530         addr = IXGBE_VFRE(pool >= ETH_64_POOLS/2);
3531         reg = IXGBE_READ_REG(hw, addr);
3532         val = bit1 << pool;
3533
3534         if (on)
3535                 reg |= val;
3536         else
3537                 reg &= ~val;
3538
3539         IXGBE_WRITE_REG(hw, addr,reg);
3540
3541         return 0;
3542 }
3543
3544 static int
3545 ixgbe_set_pool_tx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on)
3546 {
3547         uint32_t reg,addr;
3548         uint32_t val;
3549         const uint8_t bit1 = 0x1;
3550
3551         struct ixgbe_hw *hw =
3552                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3553
3554         if (ixgbe_vmdq_mode_check(hw) < 0)
3555                 return (-ENOTSUP);
3556
3557         addr = IXGBE_VFTE(pool >= ETH_64_POOLS/2);
3558         reg = IXGBE_READ_REG(hw, addr);
3559         val = bit1 << pool;
3560
3561         if (on)
3562                 reg |= val;
3563         else
3564                 reg &= ~val;
3565
3566         IXGBE_WRITE_REG(hw, addr,reg);
3567
3568         return 0;
3569 }
3570
3571 static int
3572 ixgbe_set_pool_vlan_filter(struct rte_eth_dev *dev, uint16_t vlan,
3573                         uint64_t pool_mask, uint8_t vlan_on)
3574 {
3575         int ret = 0;
3576         uint16_t pool_idx;
3577         struct ixgbe_hw *hw =
3578                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3579
3580         if (ixgbe_vmdq_mode_check(hw) < 0)
3581                 return (-ENOTSUP);
3582         for (pool_idx = 0; pool_idx < ETH_64_POOLS; pool_idx++) {
3583                 if (pool_mask & ((uint64_t)(1ULL << pool_idx)))
3584                         ret = hw->mac.ops.set_vfta(hw,vlan,pool_idx,vlan_on);
3585                         if (ret < 0)
3586                                 return ret;
3587         }
3588
3589         return ret;
3590 }
3591
3592 #define IXGBE_MRCTL_VPME  0x01 /* Virtual Pool Mirroring. */
3593 #define IXGBE_MRCTL_UPME  0x02 /* Uplink Port Mirroring. */
3594 #define IXGBE_MRCTL_DPME  0x04 /* Downlink Port Mirroring. */
3595 #define IXGBE_MRCTL_VLME  0x08 /* VLAN Mirroring. */
3596 #define IXGBE_INVALID_MIRROR_TYPE(mirror_type) \
3597         ((mirror_type) & ~(uint8_t)(ETH_MIRROR_VIRTUAL_POOL_UP | \
3598         ETH_MIRROR_UPLINK_PORT | ETH_MIRROR_DOWNLINK_PORT | ETH_MIRROR_VLAN))
3599
3600 static int
3601 ixgbe_mirror_rule_set(struct rte_eth_dev *dev,
3602                         struct rte_eth_mirror_conf *mirror_conf,
3603                         uint8_t rule_id, uint8_t on)
3604 {
3605         uint32_t mr_ctl,vlvf;
3606         uint32_t mp_lsb = 0;
3607         uint32_t mv_msb = 0;
3608         uint32_t mv_lsb = 0;
3609         uint32_t mp_msb = 0;
3610         uint8_t i = 0;
3611         int reg_index = 0;
3612         uint64_t vlan_mask = 0;
3613
3614         const uint8_t pool_mask_offset = 32;
3615         const uint8_t vlan_mask_offset = 32;
3616         const uint8_t dst_pool_offset = 8;
3617         const uint8_t rule_mr_offset  = 4;
3618         const uint8_t mirror_rule_mask= 0x0F;
3619
3620         struct ixgbe_mirror_info *mr_info =
3621                         (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private));
3622         struct ixgbe_hw *hw =
3623                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3624         uint8_t mirror_type = 0;
3625
3626         if (ixgbe_vmdq_mode_check(hw) < 0)
3627                 return -ENOTSUP;
3628
3629         if (rule_id >= IXGBE_MAX_MIRROR_RULES)
3630                 return -EINVAL;
3631
3632         if (IXGBE_INVALID_MIRROR_TYPE(mirror_conf->rule_type)) {
3633                 PMD_DRV_LOG(ERR, "unsupported mirror type 0x%x.",
3634                         mirror_conf->rule_type);
3635                 return -EINVAL;
3636         }
3637
3638         if (mirror_conf->rule_type & ETH_MIRROR_VLAN) {
3639                 mirror_type |= IXGBE_MRCTL_VLME;
3640                 /* Check if vlan id is valid and find conresponding VLAN ID index in VLVF */
3641                 for (i = 0;i < IXGBE_VLVF_ENTRIES; i++) {
3642                         if (mirror_conf->vlan.vlan_mask & (1ULL << i)) {
3643                                 /* search vlan id related pool vlan filter index */
3644                                 reg_index = ixgbe_find_vlvf_slot(hw,
3645                                                 mirror_conf->vlan.vlan_id[i]);
3646                                 if(reg_index < 0)
3647                                         return -EINVAL;
3648                                 vlvf = IXGBE_READ_REG(hw, IXGBE_VLVF(reg_index));
3649                                 if ((vlvf & IXGBE_VLVF_VIEN) &&
3650                                     ((vlvf & IXGBE_VLVF_VLANID_MASK) ==
3651                                       mirror_conf->vlan.vlan_id[i]))
3652                                         vlan_mask |= (1ULL << reg_index);
3653                                 else
3654                                         return -EINVAL;
3655                         }
3656                 }
3657
3658                 if (on) {
3659                         mv_lsb = vlan_mask & 0xFFFFFFFF;
3660                         mv_msb = vlan_mask >> vlan_mask_offset;
3661
3662                         mr_info->mr_conf[rule_id].vlan.vlan_mask =
3663                                                 mirror_conf->vlan.vlan_mask;
3664                         for(i = 0 ;i < ETH_VMDQ_MAX_VLAN_FILTERS; i++) {
3665                                 if(mirror_conf->vlan.vlan_mask & (1ULL << i))
3666                                         mr_info->mr_conf[rule_id].vlan.vlan_id[i] =
3667                                                 mirror_conf->vlan.vlan_id[i];
3668                         }
3669                 } else {
3670                         mv_lsb = 0;
3671                         mv_msb = 0;
3672                         mr_info->mr_conf[rule_id].vlan.vlan_mask = 0;
3673                         for(i = 0 ;i < ETH_VMDQ_MAX_VLAN_FILTERS; i++)
3674                                 mr_info->mr_conf[rule_id].vlan.vlan_id[i] = 0;
3675                 }
3676         }
3677
3678         /*
3679          * if enable pool mirror, write related pool mask register,if disable
3680          * pool mirror, clear PFMRVM register
3681          */
3682         if (mirror_conf->rule_type & ETH_MIRROR_VIRTUAL_POOL_UP) {
3683                 mirror_type |= IXGBE_MRCTL_VPME;
3684                 if (on) {
3685                         mp_lsb = mirror_conf->pool_mask & 0xFFFFFFFF;
3686                         mp_msb = mirror_conf->pool_mask >> pool_mask_offset;
3687                         mr_info->mr_conf[rule_id].pool_mask =
3688                                         mirror_conf->pool_mask;
3689
3690                 } else {
3691                         mp_lsb = 0;
3692                         mp_msb = 0;
3693                         mr_info->mr_conf[rule_id].pool_mask = 0;
3694                 }
3695         }
3696         if (mirror_conf->rule_type & ETH_MIRROR_UPLINK_PORT)
3697                 mirror_type |= IXGBE_MRCTL_UPME;
3698         if (mirror_conf->rule_type & ETH_MIRROR_DOWNLINK_PORT)
3699                 mirror_type |= IXGBE_MRCTL_DPME;
3700
3701         /* read  mirror control register and recalculate it */
3702         mr_ctl = IXGBE_READ_REG(hw, IXGBE_MRCTL(rule_id));
3703
3704         if (on) {
3705                 mr_ctl |= mirror_type;
3706                 mr_ctl &= mirror_rule_mask;
3707                 mr_ctl |= mirror_conf->dst_pool << dst_pool_offset;
3708         } else
3709                 mr_ctl &= ~(mirror_conf->rule_type & mirror_rule_mask);
3710
3711         mr_info->mr_conf[rule_id].rule_type = mirror_conf->rule_type;
3712         mr_info->mr_conf[rule_id].dst_pool = mirror_conf->dst_pool;
3713
3714         /* write mirrror control  register */
3715         IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl);
3716
3717         /* write pool mirrror control  register */
3718         if (mirror_conf->rule_type == ETH_MIRROR_VIRTUAL_POOL_UP) {
3719                 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), mp_lsb);
3720                 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset),
3721                                 mp_msb);
3722         }
3723         /* write VLAN mirrror control  register */
3724         if (mirror_conf->rule_type == ETH_MIRROR_VLAN) {
3725                 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), mv_lsb);
3726                 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset),
3727                                 mv_msb);
3728         }
3729
3730         return 0;
3731 }
3732
3733 static int
3734 ixgbe_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t rule_id)
3735 {
3736         int mr_ctl = 0;
3737         uint32_t lsb_val = 0;
3738         uint32_t msb_val = 0;
3739         const uint8_t rule_mr_offset = 4;
3740
3741         struct ixgbe_hw *hw =
3742                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3743         struct ixgbe_mirror_info *mr_info =
3744                 (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private));
3745
3746         if (ixgbe_vmdq_mode_check(hw) < 0)
3747                 return (-ENOTSUP);
3748
3749         memset(&mr_info->mr_conf[rule_id], 0,
3750                 sizeof(struct rte_eth_mirror_conf));
3751
3752         /* clear PFVMCTL register */
3753         IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl);
3754
3755         /* clear pool mask register */
3756         IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), lsb_val);
3757         IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset), msb_val);
3758
3759         /* clear vlan mask register */
3760         IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), lsb_val);
3761         IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset), msb_val);
3762
3763         return 0;
3764 }
3765
3766 static int ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev,
3767         uint16_t queue_idx, uint16_t tx_rate)
3768 {
3769         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3770         uint32_t rf_dec, rf_int;
3771         uint32_t bcnrc_val;
3772         uint16_t link_speed = dev->data->dev_link.link_speed;
3773
3774         if (queue_idx >= hw->mac.max_tx_queues)
3775                 return -EINVAL;
3776
3777         if (tx_rate != 0) {
3778                 /* Calculate the rate factor values to set */
3779                 rf_int = (uint32_t)link_speed / (uint32_t)tx_rate;
3780                 rf_dec = (uint32_t)link_speed % (uint32_t)tx_rate;
3781                 rf_dec = (rf_dec << IXGBE_RTTBCNRC_RF_INT_SHIFT) / tx_rate;
3782
3783                 bcnrc_val = IXGBE_RTTBCNRC_RS_ENA;
3784                 bcnrc_val |= ((rf_int << IXGBE_RTTBCNRC_RF_INT_SHIFT) &
3785                                 IXGBE_RTTBCNRC_RF_INT_MASK_M);
3786                 bcnrc_val |= (rf_dec & IXGBE_RTTBCNRC_RF_DEC_MASK);
3787         } else {
3788                 bcnrc_val = 0;
3789         }
3790
3791         /*
3792          * Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
3793          * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported, otherwise
3794          * set as 0x4.
3795          */
3796         if ((dev->data->dev_conf.rxmode.jumbo_frame == 1) &&
3797                 (dev->data->dev_conf.rxmode.max_rx_pkt_len >=
3798                                 IXGBE_MAX_JUMBO_FRAME_SIZE))
3799                 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRM,
3800                         IXGBE_MMW_SIZE_JUMBO_FRAME);
3801         else
3802                 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRM,
3803                         IXGBE_MMW_SIZE_DEFAULT);
3804
3805         /* Set RTTBCNRC of queue X */
3806         IXGBE_WRITE_REG(hw, IXGBE_RTTDQSEL, queue_idx);
3807         IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRC, bcnrc_val);
3808         IXGBE_WRITE_FLUSH(hw);
3809
3810         return 0;
3811 }
3812
3813 static int ixgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf,
3814         uint16_t tx_rate, uint64_t q_msk)
3815 {
3816         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3817         struct ixgbe_vf_info *vfinfo =
3818                 *(IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private));
3819         uint8_t  nb_q_per_pool = RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool;
3820         uint32_t queue_stride =
3821                 IXGBE_MAX_RX_QUEUE_NUM / RTE_ETH_DEV_SRIOV(dev).active;
3822         uint32_t queue_idx = vf * queue_stride, idx = 0, vf_idx;
3823         uint32_t queue_end = queue_idx + nb_q_per_pool - 1;
3824         uint16_t total_rate = 0;
3825
3826         if (queue_end >= hw->mac.max_tx_queues)
3827                 return -EINVAL;
3828
3829         if (vfinfo != NULL) {
3830                 for (vf_idx = 0; vf_idx < dev->pci_dev->max_vfs; vf_idx++) {
3831                         if (vf_idx == vf)
3832                                 continue;
3833                         for (idx = 0; idx < RTE_DIM(vfinfo[vf_idx].tx_rate);
3834                                 idx++)
3835                                 total_rate += vfinfo[vf_idx].tx_rate[idx];
3836                 }
3837         } else
3838                 return -EINVAL;
3839
3840         /* Store tx_rate for this vf. */
3841         for (idx = 0; idx < nb_q_per_pool; idx++) {
3842                 if (((uint64_t)0x1 << idx) & q_msk) {
3843                         if (vfinfo[vf].tx_rate[idx] != tx_rate)
3844                                 vfinfo[vf].tx_rate[idx] = tx_rate;
3845                         total_rate += tx_rate;
3846                 }
3847         }
3848
3849         if (total_rate > dev->data->dev_link.link_speed) {
3850                 /*
3851                  * Reset stored TX rate of the VF if it causes exceed
3852                  * link speed.
3853                  */
3854                 memset(vfinfo[vf].tx_rate, 0, sizeof(vfinfo[vf].tx_rate));
3855                 return -EINVAL;
3856         }
3857
3858         /* Set RTTBCNRC of each queue/pool for vf X  */
3859         for (; queue_idx <= queue_end; queue_idx++) {
3860                 if (0x1 & q_msk)
3861                         ixgbe_set_queue_rate_limit(dev, queue_idx, tx_rate);
3862                 q_msk = q_msk >> 1;
3863         }
3864
3865         return 0;
3866 }
3867
3868 static void
3869 ixgbevf_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
3870                      __attribute__((unused)) uint32_t index,
3871                      __attribute__((unused)) uint32_t pool)
3872 {
3873         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3874         int diag;
3875
3876         /*
3877          * On a 82599 VF, adding again the same MAC addr is not an idempotent
3878          * operation. Trap this case to avoid exhausting the [very limited]
3879          * set of PF resources used to store VF MAC addresses.
3880          */
3881         if (memcmp(hw->mac.perm_addr, mac_addr, sizeof(struct ether_addr)) == 0)
3882                 return;
3883         diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes);
3884         if (diag == 0)
3885                 return;
3886         PMD_DRV_LOG(ERR, "Unable to add MAC address - diag=%d", diag);
3887 }
3888
3889 static void
3890 ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index)
3891 {
3892         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3893         struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr;
3894         struct ether_addr *mac_addr;
3895         uint32_t i;
3896         int diag;
3897
3898         /*
3899          * The IXGBE_VF_SET_MACVLAN command of the ixgbe-pf driver does
3900          * not support the deletion of a given MAC address.
3901          * Instead, it imposes to delete all MAC addresses, then to add again
3902          * all MAC addresses with the exception of the one to be deleted.
3903          */
3904         (void) ixgbevf_set_uc_addr_vf(hw, 0, NULL);
3905
3906         /*
3907          * Add again all MAC addresses, with the exception of the deleted one
3908          * and of the permanent MAC address.
3909          */
3910         for (i = 0, mac_addr = dev->data->mac_addrs;
3911              i < hw->mac.num_rar_entries; i++, mac_addr++) {
3912                 /* Skip the deleted MAC address */
3913                 if (i == index)
3914                         continue;
3915                 /* Skip NULL MAC addresses */
3916                 if (is_zero_ether_addr(mac_addr))
3917                         continue;
3918                 /* Skip the permanent MAC address */
3919                 if (memcmp(perm_addr, mac_addr, sizeof(struct ether_addr)) == 0)
3920                         continue;
3921                 diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes);
3922                 if (diag != 0)
3923                         PMD_DRV_LOG(ERR,
3924                                     "Adding again MAC address "
3925                                     "%02x:%02x:%02x:%02x:%02x:%02x failed "
3926                                     "diag=%d",
3927                                     mac_addr->addr_bytes[0],
3928                                     mac_addr->addr_bytes[1],
3929                                     mac_addr->addr_bytes[2],
3930                                     mac_addr->addr_bytes[3],
3931                                     mac_addr->addr_bytes[4],
3932                                     mac_addr->addr_bytes[5],
3933                                     diag);
3934         }
3935 }
3936
3937 static void
3938 ixgbevf_set_default_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
3939 {
3940         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3941
3942         hw->mac.ops.set_rar(hw, 0, (void *)addr, 0, 0);
3943 }
3944
3945 #define MAC_TYPE_FILTER_SUP(type)    do {\
3946         if ((type) != ixgbe_mac_82599EB && (type) != ixgbe_mac_X540 &&\
3947                 (type) != ixgbe_mac_X550)\
3948                 return -ENOTSUP;\
3949 } while (0)
3950
3951 static int
3952 ixgbe_syn_filter_set(struct rte_eth_dev *dev,
3953                         struct rte_eth_syn_filter *filter,
3954                         bool add)
3955 {
3956         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3957         uint32_t synqf;
3958
3959         if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM)
3960                 return -EINVAL;
3961
3962         synqf = IXGBE_READ_REG(hw, IXGBE_SYNQF);
3963
3964         if (add) {
3965                 if (synqf & IXGBE_SYN_FILTER_ENABLE)
3966                         return -EINVAL;
3967                 synqf = (uint32_t)(((filter->queue << IXGBE_SYN_FILTER_QUEUE_SHIFT) &
3968                         IXGBE_SYN_FILTER_QUEUE) | IXGBE_SYN_FILTER_ENABLE);
3969
3970                 if (filter->hig_pri)
3971                         synqf |= IXGBE_SYN_FILTER_SYNQFP;
3972                 else
3973                         synqf &= ~IXGBE_SYN_FILTER_SYNQFP;
3974         } else {
3975                 if (!(synqf & IXGBE_SYN_FILTER_ENABLE))
3976                         return -ENOENT;
3977                 synqf &= ~(IXGBE_SYN_FILTER_QUEUE | IXGBE_SYN_FILTER_ENABLE);
3978         }
3979         IXGBE_WRITE_REG(hw, IXGBE_SYNQF, synqf);
3980         IXGBE_WRITE_FLUSH(hw);
3981         return 0;
3982 }
3983
3984 static int
3985 ixgbe_syn_filter_get(struct rte_eth_dev *dev,
3986                         struct rte_eth_syn_filter *filter)
3987 {
3988         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3989         uint32_t synqf = IXGBE_READ_REG(hw, IXGBE_SYNQF);
3990
3991         if (synqf & IXGBE_SYN_FILTER_ENABLE) {
3992                 filter->hig_pri = (synqf & IXGBE_SYN_FILTER_SYNQFP) ? 1 : 0;
3993                 filter->queue = (uint16_t)((synqf & IXGBE_SYN_FILTER_QUEUE) >> 1);
3994                 return 0;
3995         }
3996         return -ENOENT;
3997 }
3998
3999 static int
4000 ixgbe_syn_filter_handle(struct rte_eth_dev *dev,
4001                         enum rte_filter_op filter_op,
4002                         void *arg)
4003 {
4004         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4005         int ret;
4006
4007         MAC_TYPE_FILTER_SUP(hw->mac.type);
4008
4009         if (filter_op == RTE_ETH_FILTER_NOP)
4010                 return 0;
4011
4012         if (arg == NULL) {
4013                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
4014                             filter_op);
4015                 return -EINVAL;
4016         }
4017
4018         switch (filter_op) {
4019         case RTE_ETH_FILTER_ADD:
4020                 ret = ixgbe_syn_filter_set(dev,
4021                                 (struct rte_eth_syn_filter *)arg,
4022                                 TRUE);
4023                 break;
4024         case RTE_ETH_FILTER_DELETE:
4025                 ret = ixgbe_syn_filter_set(dev,
4026                                 (struct rte_eth_syn_filter *)arg,
4027                                 FALSE);
4028                 break;
4029         case RTE_ETH_FILTER_GET:
4030                 ret = ixgbe_syn_filter_get(dev,
4031                                 (struct rte_eth_syn_filter *)arg);
4032                 break;
4033         default:
4034                 PMD_DRV_LOG(ERR, "unsupported operation %u\n", filter_op);
4035                 ret = -EINVAL;
4036                 break;
4037         }
4038
4039         return ret;
4040 }
4041
4042
4043 static inline enum ixgbe_5tuple_protocol
4044 convert_protocol_type(uint8_t protocol_value)
4045 {
4046         if (protocol_value == IPPROTO_TCP)
4047                 return IXGBE_FILTER_PROTOCOL_TCP;
4048         else if (protocol_value == IPPROTO_UDP)
4049                 return IXGBE_FILTER_PROTOCOL_UDP;
4050         else if (protocol_value == IPPROTO_SCTP)
4051                 return IXGBE_FILTER_PROTOCOL_SCTP;
4052         else
4053                 return IXGBE_FILTER_PROTOCOL_NONE;
4054 }
4055
4056 /*
4057  * add a 5tuple filter
4058  *
4059  * @param
4060  * dev: Pointer to struct rte_eth_dev.
4061  * index: the index the filter allocates.
4062  * filter: ponter to the filter that will be added.
4063  * rx_queue: the queue id the filter assigned to.
4064  *
4065  * @return
4066  *    - On success, zero.
4067  *    - On failure, a negative value.
4068  */
4069 static int
4070 ixgbe_add_5tuple_filter(struct rte_eth_dev *dev,
4071                         struct ixgbe_5tuple_filter *filter)
4072 {
4073         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4074         struct ixgbe_filter_info *filter_info =
4075                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4076         int i, idx, shift;
4077         uint32_t ftqf, sdpqf;
4078         uint32_t l34timir = 0;
4079         uint8_t mask = 0xff;
4080
4081         /*
4082          * look for an unused 5tuple filter index,
4083          * and insert the filter to list.
4084          */
4085         for (i = 0; i < IXGBE_MAX_FTQF_FILTERS; i++) {
4086                 idx = i / (sizeof(uint32_t) * NBBY);
4087                 shift = i % (sizeof(uint32_t) * NBBY);
4088                 if (!(filter_info->fivetuple_mask[idx] & (1 << shift))) {
4089                         filter_info->fivetuple_mask[idx] |= 1 << shift;
4090                         filter->index = i;
4091                         TAILQ_INSERT_TAIL(&filter_info->fivetuple_list,
4092                                           filter,
4093                                           entries);
4094                         break;
4095                 }
4096         }
4097         if (i >= IXGBE_MAX_FTQF_FILTERS) {
4098                 PMD_DRV_LOG(ERR, "5tuple filters are full.");
4099                 return -ENOSYS;
4100         }
4101
4102         sdpqf = (uint32_t)(filter->filter_info.dst_port <<
4103                                 IXGBE_SDPQF_DSTPORT_SHIFT);
4104         sdpqf = sdpqf | (filter->filter_info.src_port & IXGBE_SDPQF_SRCPORT);
4105
4106         ftqf = (uint32_t)(filter->filter_info.proto &
4107                 IXGBE_FTQF_PROTOCOL_MASK);
4108         ftqf |= (uint32_t)((filter->filter_info.priority &
4109                 IXGBE_FTQF_PRIORITY_MASK) << IXGBE_FTQF_PRIORITY_SHIFT);
4110         if (filter->filter_info.src_ip_mask == 0) /* 0 means compare. */
4111                 mask &= IXGBE_FTQF_SOURCE_ADDR_MASK;
4112         if (filter->filter_info.dst_ip_mask == 0)
4113                 mask &= IXGBE_FTQF_DEST_ADDR_MASK;
4114         if (filter->filter_info.src_port_mask == 0)
4115                 mask &= IXGBE_FTQF_SOURCE_PORT_MASK;
4116         if (filter->filter_info.dst_port_mask == 0)
4117                 mask &= IXGBE_FTQF_DEST_PORT_MASK;
4118         if (filter->filter_info.proto_mask == 0)
4119                 mask &= IXGBE_FTQF_PROTOCOL_COMP_MASK;
4120         ftqf |= mask << IXGBE_FTQF_5TUPLE_MASK_SHIFT;
4121         ftqf |= IXGBE_FTQF_POOL_MASK_EN;
4122         ftqf |= IXGBE_FTQF_QUEUE_ENABLE;
4123
4124         IXGBE_WRITE_REG(hw, IXGBE_DAQF(i), filter->filter_info.dst_ip);
4125         IXGBE_WRITE_REG(hw, IXGBE_SAQF(i), filter->filter_info.src_ip);
4126         IXGBE_WRITE_REG(hw, IXGBE_SDPQF(i), sdpqf);
4127         IXGBE_WRITE_REG(hw, IXGBE_FTQF(i), ftqf);
4128
4129         l34timir |= IXGBE_L34T_IMIR_RESERVE;
4130         l34timir |= (uint32_t)(filter->queue <<
4131                                 IXGBE_L34T_IMIR_QUEUE_SHIFT);
4132         IXGBE_WRITE_REG(hw, IXGBE_L34T_IMIR(i), l34timir);
4133         return 0;
4134 }
4135
4136 /*
4137  * remove a 5tuple filter
4138  *
4139  * @param
4140  * dev: Pointer to struct rte_eth_dev.
4141  * filter: the pointer of the filter will be removed.
4142  */
4143 static void
4144 ixgbe_remove_5tuple_filter(struct rte_eth_dev *dev,
4145                         struct ixgbe_5tuple_filter *filter)
4146 {
4147         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4148         struct ixgbe_filter_info *filter_info =
4149                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4150         uint16_t index = filter->index;
4151
4152         filter_info->fivetuple_mask[index / (sizeof(uint32_t) * NBBY)] &=
4153                                 ~(1 << (index % (sizeof(uint32_t) * NBBY)));
4154         TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries);
4155         rte_free(filter);
4156
4157         IXGBE_WRITE_REG(hw, IXGBE_DAQF(index), 0);
4158         IXGBE_WRITE_REG(hw, IXGBE_SAQF(index), 0);
4159         IXGBE_WRITE_REG(hw, IXGBE_SDPQF(index), 0);
4160         IXGBE_WRITE_REG(hw, IXGBE_FTQF(index), 0);
4161         IXGBE_WRITE_REG(hw, IXGBE_L34T_IMIR(index), 0);
4162 }
4163
4164 static int
4165 ixgbevf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
4166 {
4167         struct ixgbe_hw *hw;
4168         uint32_t max_frame = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
4169
4170         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4171
4172         if ((mtu < ETHER_MIN_MTU) || (max_frame > ETHER_MAX_JUMBO_FRAME_LEN))
4173                 return -EINVAL;
4174
4175         /* refuse mtu that requires the support of scattered packets when this
4176          * feature has not been enabled before. */
4177         if (!dev->data->scattered_rx &&
4178             (max_frame + 2 * IXGBE_VLAN_TAG_SIZE >
4179              dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM))
4180                 return -EINVAL;
4181
4182         /*
4183          * When supported by the underlying PF driver, use the IXGBE_VF_SET_MTU
4184          * request of the version 2.0 of the mailbox API.
4185          * For now, use the IXGBE_VF_SET_LPE request of the version 1.0
4186          * of the mailbox API.
4187          * This call to IXGBE_SET_LPE action won't work with ixgbe pf drivers
4188          * prior to 3.11.33 which contains the following change:
4189          * "ixgbe: Enable jumbo frames support w/ SR-IOV"
4190          */
4191         ixgbevf_rlpml_set_vf(hw, max_frame);
4192
4193         /* update max frame size */
4194         dev->data->dev_conf.rxmode.max_rx_pkt_len = max_frame;
4195         return 0;
4196 }
4197
4198 #define MAC_TYPE_FILTER_SUP_EXT(type)    do {\
4199         if ((type) != ixgbe_mac_82599EB && (type) != ixgbe_mac_X540)\
4200                 return -ENOTSUP;\
4201 } while (0)
4202
4203 static inline struct ixgbe_5tuple_filter *
4204 ixgbe_5tuple_filter_lookup(struct ixgbe_5tuple_filter_list *filter_list,
4205                         struct ixgbe_5tuple_filter_info *key)
4206 {
4207         struct ixgbe_5tuple_filter *it;
4208
4209         TAILQ_FOREACH(it, filter_list, entries) {
4210                 if (memcmp(key, &it->filter_info,
4211                         sizeof(struct ixgbe_5tuple_filter_info)) == 0) {
4212                         return it;
4213                 }
4214         }
4215         return NULL;
4216 }
4217
4218 /* translate elements in struct rte_eth_ntuple_filter to struct ixgbe_5tuple_filter_info*/
4219 static inline int
4220 ntuple_filter_to_5tuple(struct rte_eth_ntuple_filter *filter,
4221                         struct ixgbe_5tuple_filter_info *filter_info)
4222 {
4223         if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM ||
4224                 filter->priority > IXGBE_5TUPLE_MAX_PRI ||
4225                 filter->priority < IXGBE_5TUPLE_MIN_PRI)
4226                 return -EINVAL;
4227
4228         switch (filter->dst_ip_mask) {
4229         case UINT32_MAX:
4230                 filter_info->dst_ip_mask = 0;
4231                 filter_info->dst_ip = filter->dst_ip;
4232                 break;
4233         case 0:
4234                 filter_info->dst_ip_mask = 1;
4235                 break;
4236         default:
4237                 PMD_DRV_LOG(ERR, "invalid dst_ip mask.");
4238                 return -EINVAL;
4239         }
4240
4241         switch (filter->src_ip_mask) {
4242         case UINT32_MAX:
4243                 filter_info->src_ip_mask = 0;
4244                 filter_info->src_ip = filter->src_ip;
4245                 break;
4246         case 0:
4247                 filter_info->src_ip_mask = 1;
4248                 break;
4249         default:
4250                 PMD_DRV_LOG(ERR, "invalid src_ip mask.");
4251                 return -EINVAL;
4252         }
4253
4254         switch (filter->dst_port_mask) {
4255         case UINT16_MAX:
4256                 filter_info->dst_port_mask = 0;
4257                 filter_info->dst_port = filter->dst_port;
4258                 break;
4259         case 0:
4260                 filter_info->dst_port_mask = 1;
4261                 break;
4262         default:
4263                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
4264                 return -EINVAL;
4265         }
4266
4267         switch (filter->src_port_mask) {
4268         case UINT16_MAX:
4269                 filter_info->src_port_mask = 0;
4270                 filter_info->src_port = filter->src_port;
4271                 break;
4272         case 0:
4273                 filter_info->src_port_mask = 1;
4274                 break;
4275         default:
4276                 PMD_DRV_LOG(ERR, "invalid src_port mask.");
4277                 return -EINVAL;
4278         }
4279
4280         switch (filter->proto_mask) {
4281         case UINT8_MAX:
4282                 filter_info->proto_mask = 0;
4283                 filter_info->proto =
4284                         convert_protocol_type(filter->proto);
4285                 break;
4286         case 0:
4287                 filter_info->proto_mask = 1;
4288                 break;
4289         default:
4290                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
4291                 return -EINVAL;
4292         }
4293
4294         filter_info->priority = (uint8_t)filter->priority;
4295         return 0;
4296 }
4297
4298 /*
4299  * add or delete a ntuple filter
4300  *
4301  * @param
4302  * dev: Pointer to struct rte_eth_dev.
4303  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
4304  * add: if true, add filter, if false, remove filter
4305  *
4306  * @return
4307  *    - On success, zero.
4308  *    - On failure, a negative value.
4309  */
4310 static int
4311 ixgbe_add_del_ntuple_filter(struct rte_eth_dev *dev,
4312                         struct rte_eth_ntuple_filter *ntuple_filter,
4313                         bool add)
4314 {
4315         struct ixgbe_filter_info *filter_info =
4316                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4317         struct ixgbe_5tuple_filter_info filter_5tuple;
4318         struct ixgbe_5tuple_filter *filter;
4319         int ret;
4320
4321         if (ntuple_filter->flags != RTE_5TUPLE_FLAGS) {
4322                 PMD_DRV_LOG(ERR, "only 5tuple is supported.");
4323                 return -EINVAL;
4324         }
4325
4326         memset(&filter_5tuple, 0, sizeof(struct ixgbe_5tuple_filter_info));
4327         ret = ntuple_filter_to_5tuple(ntuple_filter, &filter_5tuple);
4328         if (ret < 0)
4329                 return ret;
4330
4331         filter = ixgbe_5tuple_filter_lookup(&filter_info->fivetuple_list,
4332                                          &filter_5tuple);
4333         if (filter != NULL && add) {
4334                 PMD_DRV_LOG(ERR, "filter exists.");
4335                 return -EEXIST;
4336         }
4337         if (filter == NULL && !add) {
4338                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
4339                 return -ENOENT;
4340         }
4341
4342         if (add) {
4343                 filter = rte_zmalloc("ixgbe_5tuple_filter",
4344                                 sizeof(struct ixgbe_5tuple_filter), 0);
4345                 if (filter == NULL)
4346                         return -ENOMEM;
4347                 (void)rte_memcpy(&filter->filter_info,
4348                                  &filter_5tuple,
4349                                  sizeof(struct ixgbe_5tuple_filter_info));
4350                 filter->queue = ntuple_filter->queue;
4351                 ret = ixgbe_add_5tuple_filter(dev, filter);
4352                 if (ret < 0) {
4353                         rte_free(filter);
4354                         return ret;
4355                 }
4356         } else
4357                 ixgbe_remove_5tuple_filter(dev, filter);
4358
4359         return 0;
4360 }
4361
4362 /*
4363  * get a ntuple filter
4364  *
4365  * @param
4366  * dev: Pointer to struct rte_eth_dev.
4367  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
4368  *
4369  * @return
4370  *    - On success, zero.
4371  *    - On failure, a negative value.
4372  */
4373 static int
4374 ixgbe_get_ntuple_filter(struct rte_eth_dev *dev,
4375                         struct rte_eth_ntuple_filter *ntuple_filter)
4376 {
4377         struct ixgbe_filter_info *filter_info =
4378                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4379         struct ixgbe_5tuple_filter_info filter_5tuple;
4380         struct ixgbe_5tuple_filter *filter;
4381         int ret;
4382
4383         if (ntuple_filter->flags != RTE_5TUPLE_FLAGS) {
4384                 PMD_DRV_LOG(ERR, "only 5tuple is supported.");
4385                 return -EINVAL;
4386         }
4387
4388         memset(&filter_5tuple, 0, sizeof(struct ixgbe_5tuple_filter_info));
4389         ret = ntuple_filter_to_5tuple(ntuple_filter, &filter_5tuple);
4390         if (ret < 0)
4391                 return ret;
4392
4393         filter = ixgbe_5tuple_filter_lookup(&filter_info->fivetuple_list,
4394                                          &filter_5tuple);
4395         if (filter == NULL) {
4396                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
4397                 return -ENOENT;
4398         }
4399         ntuple_filter->queue = filter->queue;
4400         return 0;
4401 }
4402
4403 /*
4404  * ixgbe_ntuple_filter_handle - Handle operations for ntuple filter.
4405  * @dev: pointer to rte_eth_dev structure
4406  * @filter_op:operation will be taken.
4407  * @arg: a pointer to specific structure corresponding to the filter_op
4408  *
4409  * @return
4410  *    - On success, zero.
4411  *    - On failure, a negative value.
4412  */
4413 static int
4414 ixgbe_ntuple_filter_handle(struct rte_eth_dev *dev,
4415                                 enum rte_filter_op filter_op,
4416                                 void *arg)
4417 {
4418         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4419         int ret;
4420
4421         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
4422
4423         if (filter_op == RTE_ETH_FILTER_NOP)
4424                 return 0;
4425
4426         if (arg == NULL) {
4427                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
4428                             filter_op);
4429                 return -EINVAL;
4430         }
4431
4432         switch (filter_op) {
4433         case RTE_ETH_FILTER_ADD:
4434                 ret = ixgbe_add_del_ntuple_filter(dev,
4435                         (struct rte_eth_ntuple_filter *)arg,
4436                         TRUE);
4437                 break;
4438         case RTE_ETH_FILTER_DELETE:
4439                 ret = ixgbe_add_del_ntuple_filter(dev,
4440                         (struct rte_eth_ntuple_filter *)arg,
4441                         FALSE);
4442                 break;
4443         case RTE_ETH_FILTER_GET:
4444                 ret = ixgbe_get_ntuple_filter(dev,
4445                         (struct rte_eth_ntuple_filter *)arg);
4446                 break;
4447         default:
4448                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
4449                 ret = -EINVAL;
4450                 break;
4451         }
4452         return ret;
4453 }
4454
4455 static inline int
4456 ixgbe_ethertype_filter_lookup(struct ixgbe_filter_info *filter_info,
4457                         uint16_t ethertype)
4458 {
4459         int i;
4460
4461         for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) {
4462                 if (filter_info->ethertype_filters[i] == ethertype &&
4463                     (filter_info->ethertype_mask & (1 << i)))
4464                         return i;
4465         }
4466         return -1;
4467 }
4468
4469 static inline int
4470 ixgbe_ethertype_filter_insert(struct ixgbe_filter_info *filter_info,
4471                         uint16_t ethertype)
4472 {
4473         int i;
4474
4475         for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) {
4476                 if (!(filter_info->ethertype_mask & (1 << i))) {
4477                         filter_info->ethertype_mask |= 1 << i;
4478                         filter_info->ethertype_filters[i] = ethertype;
4479                         return i;
4480                 }
4481         }
4482         return -1;
4483 }
4484
4485 static inline int
4486 ixgbe_ethertype_filter_remove(struct ixgbe_filter_info *filter_info,
4487                         uint8_t idx)
4488 {
4489         if (idx >= IXGBE_MAX_ETQF_FILTERS)
4490                 return -1;
4491         filter_info->ethertype_mask &= ~(1 << idx);
4492         filter_info->ethertype_filters[idx] = 0;
4493         return idx;
4494 }
4495
4496 static int
4497 ixgbe_add_del_ethertype_filter(struct rte_eth_dev *dev,
4498                         struct rte_eth_ethertype_filter *filter,
4499                         bool add)
4500 {
4501         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4502         struct ixgbe_filter_info *filter_info =
4503                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4504         uint32_t etqf = 0;
4505         uint32_t etqs = 0;
4506         int ret;
4507
4508         if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM)
4509                 return -EINVAL;
4510
4511         if (filter->ether_type == ETHER_TYPE_IPv4 ||
4512                 filter->ether_type == ETHER_TYPE_IPv6) {
4513                 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in"
4514                         " ethertype filter.", filter->ether_type);
4515                 return -EINVAL;
4516         }
4517
4518         if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) {
4519                 PMD_DRV_LOG(ERR, "mac compare is unsupported.");
4520                 return -EINVAL;
4521         }
4522         if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) {
4523                 PMD_DRV_LOG(ERR, "drop option is unsupported.");
4524                 return -EINVAL;
4525         }
4526
4527         ret = ixgbe_ethertype_filter_lookup(filter_info, filter->ether_type);
4528         if (ret >= 0 && add) {
4529                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.",
4530                             filter->ether_type);
4531                 return -EEXIST;
4532         }
4533         if (ret < 0 && !add) {
4534                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
4535                             filter->ether_type);
4536                 return -ENOENT;
4537         }
4538
4539         if (add) {
4540                 ret = ixgbe_ethertype_filter_insert(filter_info,
4541                         filter->ether_type);
4542                 if (ret < 0) {
4543                         PMD_DRV_LOG(ERR, "ethertype filters are full.");
4544                         return -ENOSYS;
4545                 }
4546                 etqf = IXGBE_ETQF_FILTER_EN;
4547                 etqf |= (uint32_t)filter->ether_type;
4548                 etqs |= (uint32_t)((filter->queue <<
4549                                     IXGBE_ETQS_RX_QUEUE_SHIFT) &
4550                                     IXGBE_ETQS_RX_QUEUE);
4551                 etqs |= IXGBE_ETQS_QUEUE_EN;
4552         } else {
4553                 ret = ixgbe_ethertype_filter_remove(filter_info, (uint8_t)ret);
4554                 if (ret < 0)
4555                         return -ENOSYS;
4556         }
4557         IXGBE_WRITE_REG(hw, IXGBE_ETQF(ret), etqf);
4558         IXGBE_WRITE_REG(hw, IXGBE_ETQS(ret), etqs);
4559         IXGBE_WRITE_FLUSH(hw);
4560
4561         return 0;
4562 }
4563
4564 static int
4565 ixgbe_get_ethertype_filter(struct rte_eth_dev *dev,
4566                         struct rte_eth_ethertype_filter *filter)
4567 {
4568         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4569         struct ixgbe_filter_info *filter_info =
4570                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4571         uint32_t etqf, etqs;
4572         int ret;
4573
4574         ret = ixgbe_ethertype_filter_lookup(filter_info, filter->ether_type);
4575         if (ret < 0) {
4576                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
4577                             filter->ether_type);
4578                 return -ENOENT;
4579         }
4580
4581         etqf = IXGBE_READ_REG(hw, IXGBE_ETQF(ret));
4582         if (etqf & IXGBE_ETQF_FILTER_EN) {
4583                 etqs = IXGBE_READ_REG(hw, IXGBE_ETQS(ret));
4584                 filter->ether_type = etqf & IXGBE_ETQF_ETHERTYPE;
4585                 filter->flags = 0;
4586                 filter->queue = (etqs & IXGBE_ETQS_RX_QUEUE) >>
4587                                IXGBE_ETQS_RX_QUEUE_SHIFT;
4588                 return 0;
4589         }
4590         return -ENOENT;
4591 }
4592
4593 /*
4594  * ixgbe_ethertype_filter_handle - Handle operations for ethertype filter.
4595  * @dev: pointer to rte_eth_dev structure
4596  * @filter_op:operation will be taken.
4597  * @arg: a pointer to specific structure corresponding to the filter_op
4598  */
4599 static int
4600 ixgbe_ethertype_filter_handle(struct rte_eth_dev *dev,
4601                                 enum rte_filter_op filter_op,
4602                                 void *arg)
4603 {
4604         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4605         int ret;
4606
4607         MAC_TYPE_FILTER_SUP(hw->mac.type);
4608
4609         if (filter_op == RTE_ETH_FILTER_NOP)
4610                 return 0;
4611
4612         if (arg == NULL) {
4613                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
4614                             filter_op);
4615                 return -EINVAL;
4616         }
4617
4618         switch (filter_op) {
4619         case RTE_ETH_FILTER_ADD:
4620                 ret = ixgbe_add_del_ethertype_filter(dev,
4621                         (struct rte_eth_ethertype_filter *)arg,
4622                         TRUE);
4623                 break;
4624         case RTE_ETH_FILTER_DELETE:
4625                 ret = ixgbe_add_del_ethertype_filter(dev,
4626                         (struct rte_eth_ethertype_filter *)arg,
4627                         FALSE);
4628                 break;
4629         case RTE_ETH_FILTER_GET:
4630                 ret = ixgbe_get_ethertype_filter(dev,
4631                         (struct rte_eth_ethertype_filter *)arg);
4632                 break;
4633         default:
4634                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
4635                 ret = -EINVAL;
4636                 break;
4637         }
4638         return ret;
4639 }
4640
4641 static int
4642 ixgbe_dev_filter_ctrl(struct rte_eth_dev *dev,
4643                      enum rte_filter_type filter_type,
4644                      enum rte_filter_op filter_op,
4645                      void *arg)
4646 {
4647         int ret = -EINVAL;
4648
4649         switch (filter_type) {
4650         case RTE_ETH_FILTER_NTUPLE:
4651                 ret = ixgbe_ntuple_filter_handle(dev, filter_op, arg);
4652                 break;
4653         case RTE_ETH_FILTER_ETHERTYPE:
4654                 ret = ixgbe_ethertype_filter_handle(dev, filter_op, arg);
4655                 break;
4656         case RTE_ETH_FILTER_SYN:
4657                 ret = ixgbe_syn_filter_handle(dev, filter_op, arg);
4658                 break;
4659         case RTE_ETH_FILTER_FDIR:
4660                 ret = ixgbe_fdir_ctrl_func(dev, filter_op, arg);
4661                 break;
4662         default:
4663                 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported",
4664                                                         filter_type);
4665                 break;
4666         }
4667
4668         return ret;
4669 }
4670
4671 static u8 *
4672 ixgbe_dev_addr_list_itr(__attribute__((unused)) struct ixgbe_hw *hw,
4673                         u8 **mc_addr_ptr, u32 *vmdq)
4674 {
4675         u8 *mc_addr;
4676
4677         *vmdq = 0;
4678         mc_addr = *mc_addr_ptr;
4679         *mc_addr_ptr = (mc_addr + sizeof(struct ether_addr));
4680         return mc_addr;
4681 }
4682
4683 static int
4684 ixgbe_dev_set_mc_addr_list(struct rte_eth_dev *dev,
4685                           struct ether_addr *mc_addr_set,
4686                           uint32_t nb_mc_addr)
4687 {
4688         struct ixgbe_hw *hw;
4689         u8 *mc_addr_list;
4690
4691         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4692         mc_addr_list = (u8 *)mc_addr_set;
4693         return ixgbe_update_mc_addr_list(hw, mc_addr_list, nb_mc_addr,
4694                                          ixgbe_dev_addr_list_itr, TRUE);
4695 }
4696
4697 static int
4698 ixgbe_timesync_enable(struct rte_eth_dev *dev)
4699 {
4700         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4701         uint32_t tsync_ctl;
4702         uint32_t tsauxc;
4703
4704         /* Enable system time for platforms where it isn't on by default. */
4705         tsauxc = IXGBE_READ_REG(hw, IXGBE_TSAUXC);
4706         tsauxc &= ~IXGBE_TSAUXC_DISABLE_SYSTIME;
4707         IXGBE_WRITE_REG(hw, IXGBE_TSAUXC, tsauxc);
4708
4709         /* Start incrementing the register used to timestamp PTP packets. */
4710         IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, IXGBE_TIMINCA_INIT);
4711
4712         /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
4713         IXGBE_WRITE_REG(hw, IXGBE_ETQF(IXGBE_ETQF_FILTER_1588),
4714                         (ETHER_TYPE_1588 |
4715                          IXGBE_ETQF_FILTER_EN |
4716                          IXGBE_ETQF_1588));
4717
4718         /* Enable timestamping of received PTP packets. */
4719         tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL);
4720         tsync_ctl |= IXGBE_TSYNCRXCTL_ENABLED;
4721         IXGBE_WRITE_REG(hw, IXGBE_TSYNCRXCTL, tsync_ctl);
4722
4723         /* Enable timestamping of transmitted PTP packets. */
4724         tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL);
4725         tsync_ctl |= IXGBE_TSYNCTXCTL_ENABLED;
4726         IXGBE_WRITE_REG(hw, IXGBE_TSYNCTXCTL, tsync_ctl);
4727
4728         return 0;
4729 }
4730
4731 static int
4732 ixgbe_timesync_disable(struct rte_eth_dev *dev)
4733 {
4734         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4735         uint32_t tsync_ctl;
4736
4737         /* Disable timestamping of transmitted PTP packets. */
4738         tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL);
4739         tsync_ctl &= ~IXGBE_TSYNCTXCTL_ENABLED;
4740         IXGBE_WRITE_REG(hw, IXGBE_TSYNCTXCTL, tsync_ctl);
4741
4742         /* Disable timestamping of received PTP packets. */
4743         tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL);
4744         tsync_ctl &= ~IXGBE_TSYNCRXCTL_ENABLED;
4745         IXGBE_WRITE_REG(hw, IXGBE_TSYNCRXCTL, tsync_ctl);
4746
4747         /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
4748         IXGBE_WRITE_REG(hw, IXGBE_ETQF(IXGBE_ETQF_FILTER_1588), 0);
4749
4750         /* Stop incrementating the System Time registers. */
4751         IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, 0);
4752
4753         return 0;
4754 }
4755
4756 static int
4757 ixgbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
4758                                  struct timespec *timestamp,
4759                                  uint32_t flags __rte_unused)
4760 {
4761         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4762         uint32_t tsync_rxctl;
4763         uint32_t rx_stmpl;
4764         uint32_t rx_stmph;
4765
4766         tsync_rxctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL);
4767         if ((tsync_rxctl & IXGBE_TSYNCRXCTL_VALID) == 0)
4768                 return -EINVAL;
4769
4770         rx_stmpl = IXGBE_READ_REG(hw, IXGBE_RXSTMPL);
4771         rx_stmph = IXGBE_READ_REG(hw, IXGBE_RXSTMPH);
4772
4773         timestamp->tv_sec = (uint64_t)(((uint64_t)rx_stmph << 32) | rx_stmpl);
4774         timestamp->tv_nsec = 0;
4775
4776         return  0;
4777 }
4778
4779 static int
4780 ixgbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
4781                                  struct timespec *timestamp)
4782 {
4783         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4784         uint32_t tsync_txctl;
4785         uint32_t tx_stmpl;
4786         uint32_t tx_stmph;
4787
4788         tsync_txctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL);
4789         if ((tsync_txctl & IXGBE_TSYNCTXCTL_VALID) == 0)
4790                 return -EINVAL;
4791
4792         tx_stmpl = IXGBE_READ_REG(hw, IXGBE_TXSTMPL);
4793         tx_stmph = IXGBE_READ_REG(hw, IXGBE_TXSTMPH);
4794
4795         timestamp->tv_sec = (uint64_t)(((uint64_t)tx_stmph << 32) | tx_stmpl);
4796         timestamp->tv_nsec = 0;
4797
4798         return  0;
4799 }
4800
4801 static int
4802 ixgbe_get_reg_length(struct rte_eth_dev *dev)
4803 {
4804         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4805         int count = 0;
4806         int g_ind = 0;
4807         const struct reg_info *reg_group;
4808         const struct reg_info **reg_set = (hw->mac.type == ixgbe_mac_82598EB) ?
4809                                     ixgbe_regs_mac_82598EB : ixgbe_regs_others;
4810
4811         while ((reg_group = reg_set[g_ind++]))
4812                 count += ixgbe_regs_group_count(reg_group);
4813
4814         return count;
4815 }
4816
4817 static int
4818 ixgbevf_get_reg_length(struct rte_eth_dev *dev __rte_unused)
4819 {
4820         int count = 0;
4821         int g_ind = 0;
4822         const struct reg_info *reg_group;
4823
4824         while ((reg_group = ixgbevf_regs[g_ind++]))
4825                 count += ixgbe_regs_group_count(reg_group);
4826
4827         return count;
4828 }
4829
4830 static int
4831 ixgbe_get_regs(struct rte_eth_dev *dev,
4832               struct rte_dev_reg_info *regs)
4833 {
4834         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4835         uint32_t *data = regs->data;
4836         int g_ind = 0;
4837         int count = 0;
4838         const struct reg_info *reg_group;
4839         const struct reg_info **reg_set = (hw->mac.type == ixgbe_mac_82598EB) ?
4840                                     ixgbe_regs_mac_82598EB : ixgbe_regs_others;
4841
4842         /* Support only full register dump */
4843         if ((regs->length == 0) ||
4844             (regs->length == (uint32_t)ixgbe_get_reg_length(dev))) {
4845                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
4846                         hw->device_id;
4847                 while ((reg_group = reg_set[g_ind++]))
4848                         count += ixgbe_read_regs_group(dev, &data[count],
4849                                 reg_group);
4850                 return 0;
4851         }
4852
4853         return -ENOTSUP;
4854 }
4855
4856 static int
4857 ixgbevf_get_regs(struct rte_eth_dev *dev,
4858                 struct rte_dev_reg_info *regs)
4859 {
4860         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4861         uint32_t *data = regs->data;
4862         int g_ind = 0;
4863         int count = 0;
4864         const struct reg_info *reg_group;
4865
4866         /* Support only full register dump */
4867         if ((regs->length == 0) ||
4868             (regs->length == (uint32_t)ixgbevf_get_reg_length(dev))) {
4869                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
4870                         hw->device_id;
4871                 while ((reg_group = ixgbevf_regs[g_ind++]))
4872                         count += ixgbe_read_regs_group(dev, &data[count],
4873                                                       reg_group);
4874                 return 0;
4875         }
4876
4877         return -ENOTSUP;
4878 }
4879
4880 static int
4881 ixgbe_get_eeprom_length(struct rte_eth_dev *dev)
4882 {
4883         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4884
4885         /* Return unit is byte count */
4886         return hw->eeprom.word_size * 2;
4887 }
4888
4889 static int
4890 ixgbe_get_eeprom(struct rte_eth_dev *dev,
4891                 struct rte_dev_eeprom_info *in_eeprom)
4892 {
4893         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4894         struct ixgbe_eeprom_info *eeprom = &hw->eeprom;
4895         uint16_t *data = in_eeprom->data;
4896         int first, length;
4897
4898         first = in_eeprom->offset >> 1;
4899         length = in_eeprom->length >> 1;
4900         if ((first >= hw->eeprom.word_size) ||
4901             ((first + length) >= hw->eeprom.word_size))
4902                 return -EINVAL;
4903
4904         in_eeprom->magic = hw->vendor_id | (hw->device_id << 16);
4905
4906         return eeprom->ops.read_buffer(hw, first, length, data);
4907 }
4908
4909 static int
4910 ixgbe_set_eeprom(struct rte_eth_dev *dev,
4911                 struct rte_dev_eeprom_info *in_eeprom)
4912 {
4913         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4914         struct ixgbe_eeprom_info *eeprom = &hw->eeprom;
4915         uint16_t *data = in_eeprom->data;
4916         int first, length;
4917
4918         first = in_eeprom->offset >> 1;
4919         length = in_eeprom->length >> 1;
4920         if ((first >= hw->eeprom.word_size) ||
4921             ((first + length) >= hw->eeprom.word_size))
4922                 return -EINVAL;
4923
4924         in_eeprom->magic = hw->vendor_id | (hw->device_id << 16);
4925
4926         return eeprom->ops.write_buffer(hw,  first, length, data);
4927 }
4928
4929 static struct rte_driver rte_ixgbe_driver = {
4930         .type = PMD_PDEV,
4931         .init = rte_ixgbe_pmd_init,
4932 };
4933
4934 static struct rte_driver rte_ixgbevf_driver = {
4935         .type = PMD_PDEV,
4936         .init = rte_ixgbevf_pmd_init,
4937 };
4938
4939 PMD_REGISTER_DRIVER(rte_ixgbe_driver);
4940 PMD_REGISTER_DRIVER(rte_ixgbevf_driver);