lib/librte_pmd_ixgbe/ixgbe_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <inttypes.h>
  42 #include <netinet/in.h>
  43 #include <rte_byteorder.h>
  44 #include <rte_common.h>
  45 #include <rte_cycles.h>
  46
  47 #include <rte_interrupts.h>
  48 #include <rte_log.h>
  49 #include <rte_debug.h>
  50 #include <rte_pci.h>
  51 #include <rte_atomic.h>
  52 #include <rte_branch_prediction.h>
  53 #include <rte_memory.h>
  54 #include <rte_memzone.h>
  55 #include <rte_tailq.h>
  56 #include <rte_eal.h>
  57 #include <rte_alarm.h>
  58 #include <rte_ether.h>
  59 #include <rte_ethdev.h>
  60 #include <rte_atomic.h>
  61 #include <rte_malloc.h>
  62 #include <rte_random.h>
  63 #include <rte_dev.h>
  64
  65 #include "ixgbe_logs.h"
  66 #include "ixgbe/ixgbe_api.h"
  67 #include "ixgbe/ixgbe_vf.h"
  68 #include "ixgbe/ixgbe_common.h"
  69 #include "ixgbe_ethdev.h"
  70 #include "ixgbe_bypass.h"
  71
  72 /*
  73  * High threshold controlling when to start sending XOFF frames. Must be at
  74  * least 8 bytes less than receive packet buffer size. This value is in units
  75  * of 1024 bytes.
  76  */
  77 #define IXGBE_FC_HI    0x80
  78
  79 /*
  80  * Low threshold controlling when to start sending XON frames. This value is
  81  * in units of 1024 bytes.
  82  */
  83 #define IXGBE_FC_LO    0x40
  84
  85 /* Timer value included in XOFF frames. */
  86 #define IXGBE_FC_PAUSE 0x680
  87
  88 #define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */
  89 #define IXGBE_LINK_UP_CHECK_TIMEOUT   1000 /* ms */
  90 #define IXGBE_VMDQ_NUM_UC_MAC         4096 /* Maximum nb. of UC MAC addr. */
  91
  92 #define IXGBE_MMW_SIZE_DEFAULT        0x4
  93 #define IXGBE_MMW_SIZE_JUMBO_FRAME    0x14
  94
  95 #define IXGBEVF_PMD_NAME "rte_ixgbevf_pmd" /* PMD name */
  96
  97 #define IXGBE_QUEUE_STAT_COUNTERS (sizeof(hw_stats->qprc) / sizeof(hw_stats->qprc[0]))
  98
  99 static int eth_ixgbe_dev_init(struct eth_driver *eth_drv,
 100                 struct rte_eth_dev *eth_dev);
 101 static int  ixgbe_dev_configure(struct rte_eth_dev *dev);
 102 static int  ixgbe_dev_start(struct rte_eth_dev *dev);
 103 static void ixgbe_dev_stop(struct rte_eth_dev *dev);
 104 static int  ixgbe_dev_set_link_up(struct rte_eth_dev *dev);
 105 static int  ixgbe_dev_set_link_down(struct rte_eth_dev *dev);
 106 static void ixgbe_dev_close(struct rte_eth_dev *dev);
 107 static void ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev);
 108 static void ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev);
 109 static void ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev);
 110 static void ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev);
 111 static int ixgbe_dev_link_update(struct rte_eth_dev *dev,
 112                                 int wait_to_complete);
 113 static void ixgbe_dev_stats_get(struct rte_eth_dev *dev,
 114                                 struct rte_eth_stats *stats);
 115 static void ixgbe_dev_stats_reset(struct rte_eth_dev *dev);
 116 static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 117                                              uint16_t queue_id,
 118                                              uint8_t stat_idx,
 119                                              uint8_t is_rx);
 120 static void ixgbe_dev_info_get(struct rte_eth_dev *dev,
 121                                 struct rte_eth_dev_info *dev_info);
 122 static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev,
 123                 uint16_t vlan_id, int on);
 124 static void ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 125 static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev,
 126                 uint16_t queue, bool on);
 127 static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue,
 128                 int on);
 129 static void ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 130 static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue);
 131 static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue);
 132 static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 133 static void ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 134
 135 static int ixgbe_dev_led_on(struct rte_eth_dev *dev);
 136 static int ixgbe_dev_led_off(struct rte_eth_dev *dev);
 137 static int ixgbe_flow_ctrl_get(struct rte_eth_dev *dev,
 138                                struct rte_eth_fc_conf *fc_conf);
 139 static int ixgbe_flow_ctrl_set(struct rte_eth_dev *dev,
 140                                struct rte_eth_fc_conf *fc_conf);
 141 static int ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 142                 struct rte_eth_pfc_conf *pfc_conf);
 143 static int ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
 144                 struct rte_eth_rss_reta *reta_conf);
 145 static int ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
 146                 struct rte_eth_rss_reta *reta_conf);
 147 static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev);
 148 static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev);
 149 static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev);
 150 static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev);
 151 static void ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle,
 152                 void *param);
 153 static void ixgbe_dev_interrupt_delayed_handler(void *param);
 154 static void ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 155                 uint32_t index, uint32_t pool);
 156 static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index);
 157 static void ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config);
 158
 159 /* For Virtual Function support */
 160 static int eth_ixgbevf_dev_init(struct eth_driver *eth_drv,
 161                 struct rte_eth_dev *eth_dev);
 162 static int  ixgbevf_dev_configure(struct rte_eth_dev *dev);
 163 static int  ixgbevf_dev_start(struct rte_eth_dev *dev);
 164 static void ixgbevf_dev_stop(struct rte_eth_dev *dev);
 165 static void ixgbevf_dev_close(struct rte_eth_dev *dev);
 166 static void ixgbevf_intr_disable(struct ixgbe_hw *hw);
 167 static void ixgbevf_dev_stats_get(struct rte_eth_dev *dev,
 168                 struct rte_eth_stats *stats);
 169 static void ixgbevf_dev_stats_reset(struct rte_eth_dev *dev);
 170 static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev,
 171                 uint16_t vlan_id, int on);
 172 static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev,
 173                 uint16_t queue, int on);
 174 static void ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 175 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 176
 177 /* For Eth VMDQ APIs support */
 178 static int ixgbe_uc_hash_table_set(struct rte_eth_dev *dev, struct
 179                 ether_addr* mac_addr,uint8_t on);
 180 static int ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev,uint8_t on);
 181 static int  ixgbe_set_pool_rx_mode(struct rte_eth_dev *dev,  uint16_t pool,
 182                 uint16_t rx_mask, uint8_t on);
 183 static int ixgbe_set_pool_rx(struct rte_eth_dev *dev,uint16_t pool,uint8_t on);
 184 static int ixgbe_set_pool_tx(struct rte_eth_dev *dev,uint16_t pool,uint8_t on);
 185 static int ixgbe_set_pool_vlan_filter(struct rte_eth_dev *dev, uint16_t vlan,
 186                 uint64_t pool_mask,uint8_t vlan_on);
 187 static int ixgbe_mirror_rule_set(struct rte_eth_dev *dev,
 188                 struct rte_eth_vmdq_mirror_conf *mirror_conf,
 189                 uint8_t rule_id, uint8_t on);
 190 static int ixgbe_mirror_rule_reset(struct rte_eth_dev *dev,
 191                 uint8_t rule_id);
 192
 193 static int ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev,
 194                 uint16_t queue_idx, uint16_t tx_rate);
 195 static int ixgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf,
 196                 uint16_t tx_rate, uint64_t q_msk);
 197
 198 static void ixgbevf_add_mac_addr(struct rte_eth_dev *dev,
 199                                  struct ether_addr *mac_addr,
 200                                  uint32_t index, uint32_t pool);
 201 static void ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index);
 202 static int ixgbe_add_syn_filter(struct rte_eth_dev *dev,
 203                         struct rte_syn_filter *filter, uint16_t rx_queue);
 204 static int ixgbe_remove_syn_filter(struct rte_eth_dev *dev);
 205 static int ixgbe_get_syn_filter(struct rte_eth_dev *dev,
 206                         struct rte_syn_filter *filter, uint16_t *rx_queue);
 207 static int ixgbe_add_ethertype_filter(struct rte_eth_dev *dev, uint16_t index,
 208                         struct rte_ethertype_filter *filter, uint16_t rx_queue);
 209 static int ixgbe_remove_ethertype_filter(struct rte_eth_dev *dev,
 210                         uint16_t index);
 211 static int ixgbe_get_ethertype_filter(struct rte_eth_dev *dev, uint16_t index,
 212                         struct rte_ethertype_filter *filter, uint16_t *rx_queue);
 213 static int ixgbe_add_5tuple_filter(struct rte_eth_dev *dev, uint16_t index,
 214                         struct rte_5tuple_filter *filter, uint16_t rx_queue);
 215 static int ixgbe_remove_5tuple_filter(struct rte_eth_dev *dev,
 216                         uint16_t index);
 217 static int ixgbe_get_5tuple_filter(struct rte_eth_dev *dev, uint16_t index,
 218                         struct rte_5tuple_filter *filter, uint16_t *rx_queue);
 219
 220 /*
 221  * Define VF Stats MACRO for Non "cleared on read" register
 222  */
 223 #define UPDATE_VF_STAT(reg, last, cur)                          \
 224 {                                                               \
 225         u32 latest = IXGBE_READ_REG(hw, reg);                   \
 226         cur += latest - last;                                   \
 227         last = latest;                                          \
 228 }
 229
 230 #define UPDATE_VF_STAT_36BIT(lsb, msb, last, cur)                \
 231 {                                                                \
 232         u64 new_lsb = IXGBE_READ_REG(hw, lsb);                   \
 233         u64 new_msb = IXGBE_READ_REG(hw, msb);                   \
 234         u64 latest = ((new_msb << 32) | new_lsb);                \
 235         cur += (0x1000000000LL + latest - last) & 0xFFFFFFFFFLL; \
 236         last = latest;                                           \
 237 }
 238
 239 #define IXGBE_SET_HWSTRIP(h, q) do{\
 240                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 241                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 242                 (h)->bitmap[idx] |= 1 << bit;\
 243         }while(0)
 244
 245 #define IXGBE_CLEAR_HWSTRIP(h, q) do{\
 246                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 247                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 248                 (h)->bitmap[idx] &= ~(1 << bit);\
 249         }while(0)
 250
 251 #define IXGBE_GET_HWSTRIP(h, q, r) do{\
 252                 uint32_t idx = (q) / (sizeof ((h)->bitmap[0]) * NBBY); \
 253                 uint32_t bit = (q) % (sizeof ((h)->bitmap[0]) * NBBY); \
 254                 (r) = (h)->bitmap[idx] >> bit & 1;\
 255         }while(0)
 256
 257 /*
 258  * The set of PCI devices this driver supports
 259  */
 260 static struct rte_pci_id pci_id_ixgbe_map[] = {
 261
 262 #define RTE_PCI_DEV_ID_DECL_IXGBE(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 263 #include "rte_pci_dev_ids.h"
 264
 265 { .vendor_id = 0, /* sentinel */ },
 266 };
 267
 268
 269 /*
 270  * The set of PCI devices this driver supports (for 82599 VF)
 271  */
 272 static struct rte_pci_id pci_id_ixgbevf_map[] = {
 273
 274 #define RTE_PCI_DEV_ID_DECL_IXGBEVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 275 #include "rte_pci_dev_ids.h"
 276 { .vendor_id = 0, /* sentinel */ },
 277
 278 };
 279
 280 static struct eth_dev_ops ixgbe_eth_dev_ops = {
 281         .dev_configure        = ixgbe_dev_configure,
 282         .dev_start            = ixgbe_dev_start,
 283         .dev_stop             = ixgbe_dev_stop,
 284         .dev_set_link_up    = ixgbe_dev_set_link_up,
 285         .dev_set_link_down  = ixgbe_dev_set_link_down,
 286         .dev_close            = ixgbe_dev_close,
 287         .promiscuous_enable   = ixgbe_dev_promiscuous_enable,
 288         .promiscuous_disable  = ixgbe_dev_promiscuous_disable,
 289         .allmulticast_enable  = ixgbe_dev_allmulticast_enable,
 290         .allmulticast_disable = ixgbe_dev_allmulticast_disable,
 291         .link_update          = ixgbe_dev_link_update,
 292         .stats_get            = ixgbe_dev_stats_get,
 293         .stats_reset          = ixgbe_dev_stats_reset,
 294         .queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set,
 295         .dev_infos_get        = ixgbe_dev_info_get,
 296         .vlan_filter_set      = ixgbe_vlan_filter_set,
 297         .vlan_tpid_set        = ixgbe_vlan_tpid_set,
 298         .vlan_offload_set     = ixgbe_vlan_offload_set,
 299         .vlan_strip_queue_set = ixgbe_vlan_strip_queue_set,
 300         .rx_queue_start       = ixgbe_dev_rx_queue_start,
 301         .rx_queue_stop        = ixgbe_dev_rx_queue_stop,
 302         .tx_queue_start       = ixgbe_dev_tx_queue_start,
 303         .tx_queue_stop        = ixgbe_dev_tx_queue_stop,
 304         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 305         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 306         .rx_queue_count       = ixgbe_dev_rx_queue_count,
 307         .rx_descriptor_done   = ixgbe_dev_rx_descriptor_done,
 308         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 309         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 310         .dev_led_on           = ixgbe_dev_led_on,
 311         .dev_led_off          = ixgbe_dev_led_off,
 312         .flow_ctrl_get        = ixgbe_flow_ctrl_get,
 313         .flow_ctrl_set        = ixgbe_flow_ctrl_set,
 314         .priority_flow_ctrl_set = ixgbe_priority_flow_ctrl_set,
 315         .mac_addr_add         = ixgbe_add_rar,
 316         .mac_addr_remove      = ixgbe_remove_rar,
 317         .uc_hash_table_set    = ixgbe_uc_hash_table_set,
 318         .uc_all_hash_table_set  = ixgbe_uc_all_hash_table_set,
 319         .mirror_rule_set      = ixgbe_mirror_rule_set,
 320         .mirror_rule_reset    = ixgbe_mirror_rule_reset,
 321         .set_vf_rx_mode       = ixgbe_set_pool_rx_mode,
 322         .set_vf_rx            = ixgbe_set_pool_rx,
 323         .set_vf_tx            = ixgbe_set_pool_tx,
 324         .set_vf_vlan_filter   = ixgbe_set_pool_vlan_filter,
 325         .set_queue_rate_limit = ixgbe_set_queue_rate_limit,
 326         .set_vf_rate_limit    = ixgbe_set_vf_rate_limit,
 327         .fdir_add_signature_filter    = ixgbe_fdir_add_signature_filter,
 328         .fdir_update_signature_filter = ixgbe_fdir_update_signature_filter,
 329         .fdir_remove_signature_filter = ixgbe_fdir_remove_signature_filter,
 330         .fdir_infos_get               = ixgbe_fdir_info_get,
 331         .fdir_add_perfect_filter      = ixgbe_fdir_add_perfect_filter,
 332         .fdir_update_perfect_filter   = ixgbe_fdir_update_perfect_filter,
 333         .fdir_remove_perfect_filter   = ixgbe_fdir_remove_perfect_filter,
 334         .fdir_set_masks               = ixgbe_fdir_set_masks,
 335         .reta_update          = ixgbe_dev_rss_reta_update,
 336         .reta_query           = ixgbe_dev_rss_reta_query,
 337 #ifdef RTE_NIC_BYPASS
 338         .bypass_init          = ixgbe_bypass_init,
 339         .bypass_state_set     = ixgbe_bypass_state_store,
 340         .bypass_state_show    = ixgbe_bypass_state_show,
 341         .bypass_event_set     = ixgbe_bypass_event_store,
 342         .bypass_event_show    = ixgbe_bypass_event_show,
 343         .bypass_wd_timeout_set  = ixgbe_bypass_wd_timeout_store,
 344         .bypass_wd_timeout_show = ixgbe_bypass_wd_timeout_show,
 345         .bypass_ver_show      = ixgbe_bypass_ver_show,
 346         .bypass_wd_reset      = ixgbe_bypass_wd_reset,
 347 #endif /* RTE_NIC_BYPASS */
 348         .rss_hash_update      = ixgbe_dev_rss_hash_update,
 349         .rss_hash_conf_get    = ixgbe_dev_rss_hash_conf_get,
 350         .add_syn_filter          = ixgbe_add_syn_filter,
 351         .remove_syn_filter       = ixgbe_remove_syn_filter,
 352         .get_syn_filter          = ixgbe_get_syn_filter,
 353         .add_ethertype_filter    = ixgbe_add_ethertype_filter,
 354         .remove_ethertype_filter = ixgbe_remove_ethertype_filter,
 355         .get_ethertype_filter    = ixgbe_get_ethertype_filter,
 356         .add_5tuple_filter       = ixgbe_add_5tuple_filter,
 357         .remove_5tuple_filter    = ixgbe_remove_5tuple_filter,
 358         .get_5tuple_filter       = ixgbe_get_5tuple_filter,
 359 };
 360
 361 /*
 362  * dev_ops for virtual function, bare necessities for basic vf
 363  * operation have been implemented
 364  */
 365 static struct eth_dev_ops ixgbevf_eth_dev_ops = {
 366
 367         .dev_configure        = ixgbevf_dev_configure,
 368         .dev_start            = ixgbevf_dev_start,
 369         .dev_stop             = ixgbevf_dev_stop,
 370         .link_update          = ixgbe_dev_link_update,
 371         .stats_get            = ixgbevf_dev_stats_get,
 372         .stats_reset          = ixgbevf_dev_stats_reset,
 373         .dev_close            = ixgbevf_dev_close,
 374         .dev_infos_get        = ixgbe_dev_info_get,
 375         .vlan_filter_set      = ixgbevf_vlan_filter_set,
 376         .vlan_strip_queue_set = ixgbevf_vlan_strip_queue_set,
 377         .vlan_offload_set     = ixgbevf_vlan_offload_set,
 378         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 379         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 380         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 381         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 382         .mac_addr_add         = ixgbevf_add_mac_addr,
 383         .mac_addr_remove      = ixgbevf_remove_mac_addr,
 384 };
 385
 386 /**
 387  * Atomically reads the link status information from global
 388  * structure rte_eth_dev.
 389  *
 390  * @param dev
 391  *   - Pointer to the structure rte_eth_dev to read from.
 392  *   - Pointer to the buffer to be saved with the link status.
 393  *
 394  * @return
 395  *   - On success, zero.
 396  *   - On failure, negative value.
 397  */
 398 static inline int
 399 rte_ixgbe_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 400                                 struct rte_eth_link *link)
 401 {
 402         struct rte_eth_link *dst = link;
 403         struct rte_eth_link *src = &(dev->data->dev_link);
 404
 405         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 406                                         *(uint64_t *)src) == 0)
 407                 return -1;
 408
 409         return 0;
 410 }
 411
 412 /**
 413  * Atomically writes the link status information into global
 414  * structure rte_eth_dev.
 415  *
 416  * @param dev
 417  *   - Pointer to the structure rte_eth_dev to read from.
 418  *   - Pointer to the buffer to be saved with the link status.
 419  *
 420  * @return
 421  *   - On success, zero.
 422  *   - On failure, negative value.
 423  */
 424 static inline int
 425 rte_ixgbe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 426                                 struct rte_eth_link *link)
 427 {
 428         struct rte_eth_link *dst = &(dev->data->dev_link);
 429         struct rte_eth_link *src = link;
 430
 431         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 432                                         *(uint64_t *)src) == 0)
 433                 return -1;
 434
 435         return 0;
 436 }
 437
 438 /*
 439  * This function is the same as ixgbe_is_sfp() in ixgbe/ixgbe.h.
 440  */
 441 static inline int
 442 ixgbe_is_sfp(struct ixgbe_hw *hw)
 443 {
 444         switch (hw->phy.type) {
 445         case ixgbe_phy_sfp_avago:
 446         case ixgbe_phy_sfp_ftl:
 447         case ixgbe_phy_sfp_intel:
 448         case ixgbe_phy_sfp_unknown:
 449         case ixgbe_phy_sfp_passive_tyco:
 450         case ixgbe_phy_sfp_passive_unknown:
 451                 return 1;
 452         default:
 453                 return 0;
 454         }
 455 }
 456
 457 static inline int32_t
 458 ixgbe_pf_reset_hw(struct ixgbe_hw *hw)
 459 {
 460         uint32_t ctrl_ext;
 461         int32_t status;
 462
 463         status = ixgbe_reset_hw(hw);
 464
 465         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 466         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 467         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
 468         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 469         IXGBE_WRITE_FLUSH(hw);
 470
 471         return status;
 472 }
 473
 474 static inline void
 475 ixgbe_enable_intr(struct rte_eth_dev *dev)
 476 {
 477         struct ixgbe_interrupt *intr =
 478                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 479         struct ixgbe_hw *hw =
 480                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 481
 482         IXGBE_WRITE_REG(hw, IXGBE_EIMS, intr->mask);
 483         IXGBE_WRITE_FLUSH(hw);
 484 }
 485
 486 /*
 487  * This function is based on ixgbe_disable_intr() in ixgbe/ixgbe.h.
 488  */
 489 static void
 490 ixgbe_disable_intr(struct ixgbe_hw *hw)
 491 {
 492         PMD_INIT_FUNC_TRACE();
 493
 494         if (hw->mac.type == ixgbe_mac_82598EB) {
 495                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ~0);
 496         } else {
 497                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xFFFF0000);
 498                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), ~0);
 499                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), ~0);
 500         }
 501         IXGBE_WRITE_FLUSH(hw);
 502 }
 503
 504 /*
 505  * This function resets queue statistics mapping registers.
 506  * From Niantic datasheet, Initialization of Statistics section:
 507  * "...if software requires the queue counters, the RQSMR and TQSM registers
 508  * must be re-programmed following a device reset.
 509  */
 510 static void
 511 ixgbe_reset_qstat_mappings(struct ixgbe_hw *hw)
 512 {
 513         uint32_t i;
 514
 515         for(i = 0; i != IXGBE_NB_STAT_MAPPING_REGS; i++) {
 516                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), 0);
 517                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), 0);
 518         }
 519 }
 520
 521
 522 static int
 523 ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 524                                   uint16_t queue_id,
 525                                   uint8_t stat_idx,
 526                                   uint8_t is_rx)
 527 {
 528 #define QSM_REG_NB_BITS_PER_QMAP_FIELD 8
 529 #define NB_QMAP_FIELDS_PER_QSM_REG 4
 530 #define QMAP_FIELD_RESERVED_BITS_MASK 0x0f
 531
 532         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 533         struct ixgbe_stat_mapping_registers *stat_mappings =
 534                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(eth_dev->data->dev_private);
 535         uint32_t qsmr_mask = 0;
 536         uint32_t clearing_mask = QMAP_FIELD_RESERVED_BITS_MASK;
 537         uint32_t q_map;
 538         uint8_t n, offset;
 539
 540         if ((hw->mac.type != ixgbe_mac_82599EB) && (hw->mac.type != ixgbe_mac_X540))
 541                 return -ENOSYS;
 542
 543         PMD_INIT_LOG(INFO, "Setting port %d, %s queue_id %d to stat index %d\n",
 544                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx);
 545
 546         n = (uint8_t)(queue_id / NB_QMAP_FIELDS_PER_QSM_REG);
 547         if (n >= IXGBE_NB_STAT_MAPPING_REGS) {
 548                 PMD_INIT_LOG(ERR, "Nb of stat mapping registers exceeded\n");
 549                 return -EIO;
 550         }
 551         offset = (uint8_t)(queue_id % NB_QMAP_FIELDS_PER_QSM_REG);
 552
 553         /* Now clear any previous stat_idx set */
 554         clearing_mask <<= (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 555         if (!is_rx)
 556                 stat_mappings->tqsm[n] &= ~clearing_mask;
 557         else
 558                 stat_mappings->rqsmr[n] &= ~clearing_mask;
 559
 560         q_map = (uint32_t)stat_idx;
 561         q_map &= QMAP_FIELD_RESERVED_BITS_MASK;
 562         qsmr_mask = q_map << (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 563         if (!is_rx)
 564                 stat_mappings->tqsm[n] |= qsmr_mask;
 565         else
 566                 stat_mappings->rqsmr[n] |= qsmr_mask;
 567
 568         PMD_INIT_LOG(INFO, "Set port %d, %s queue_id %d to stat index %d\n"
 569                      "%s[%d] = 0x%08x\n",
 570                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX", queue_id, stat_idx,
 571                      is_rx ? "RQSMR" : "TQSM",n, is_rx ? stat_mappings->rqsmr[n] : stat_mappings->tqsm[n]);
 572
 573         /* Now write the mapping in the appropriate register */
 574         if (is_rx) {
 575                 PMD_INIT_LOG(INFO, "Write 0x%x to RX IXGBE stat mapping reg:%d\n",
 576                              stat_mappings->rqsmr[n], n);
 577                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(n), stat_mappings->rqsmr[n]);
 578         }
 579         else {
 580                 PMD_INIT_LOG(INFO, "Write 0x%x to TX IXGBE stat mapping reg:%d\n",
 581                              stat_mappings->tqsm[n], n);
 582                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(n), stat_mappings->tqsm[n]);
 583         }
 584         return 0;
 585 }
 586
 587 static void
 588 ixgbe_restore_statistics_mapping(struct rte_eth_dev * dev)
 589 {
 590         struct ixgbe_stat_mapping_registers *stat_mappings =
 591                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(dev->data->dev_private);
 592         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 593         int i;
 594
 595         /* write whatever was in stat mapping table to the NIC */
 596         for (i = 0; i < IXGBE_NB_STAT_MAPPING_REGS; i++) {
 597                 /* rx */
 598                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), stat_mappings->rqsmr[i]);
 599
 600                 /* tx */
 601                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), stat_mappings->tqsm[i]);
 602         }
 603 }
 604
 605 static void
 606 ixgbe_dcb_init(struct ixgbe_hw *hw,struct ixgbe_dcb_config *dcb_config)
 607 {
 608         uint8_t i;
 609         struct ixgbe_dcb_tc_config *tc;
 610         uint8_t dcb_max_tc = IXGBE_DCB_MAX_TRAFFIC_CLASS;
 611
 612         dcb_config->num_tcs.pg_tcs = dcb_max_tc;
 613         dcb_config->num_tcs.pfc_tcs = dcb_max_tc;
 614         for (i = 0; i < dcb_max_tc; i++) {
 615                 tc = &dcb_config->tc_config[i];
 616                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_id = i;
 617                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
 618                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 619                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_id = i;
 620                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
 621                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 622                 tc->pfc = ixgbe_dcb_pfc_disabled;
 623         }
 624
 625         /* Initialize default user to priority mapping, UPx->TC0 */
 626         tc = &dcb_config->tc_config[0];
 627         tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
 628         tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
 629         for (i = 0; i< IXGBE_DCB_MAX_BW_GROUP; i++) {
 630                 dcb_config->bw_percentage[IXGBE_DCB_TX_CONFIG][i] = 100;
 631                 dcb_config->bw_percentage[IXGBE_DCB_RX_CONFIG][i] = 100;
 632         }
 633         dcb_config->rx_pba_cfg = ixgbe_dcb_pba_equal;
 634         dcb_config->pfc_mode_enable = false;
 635         dcb_config->vt_mode = true;
 636         dcb_config->round_robin_enable = false;
 637         /* support all DCB capabilities in 82599 */
 638         dcb_config->support.capabilities = 0xFF;
 639
 640         /*we only support 4 Tcs for X540*/
 641         if (hw->mac.type == ixgbe_mac_X540) {
 642                 dcb_config->num_tcs.pg_tcs = 4;
 643                 dcb_config->num_tcs.pfc_tcs = 4;
 644         }
 645 }
 646
 647 /*
 648  * Ensure that all locks are released before first NVM or PHY access
 649  */
 650 static void
 651 ixgbe_swfw_lock_reset(struct ixgbe_hw *hw)
 652 {
 653         uint16_t mask;
 654
 655         /*
 656          * Phy lock should not fail in this early stage. If this is the case,
 657          * it is due to an improper exit of the application.
 658          * So force the release of the faulty lock. Release of common lock
 659          * is done automatically by swfw_sync function.
 660          */
 661         mask = IXGBE_GSSR_PHY0_SM << hw->bus.func;
 662         if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) {
 663                    DEBUGOUT1("SWFW phy%d lock released", hw->bus.func);
 664         }
 665         ixgbe_release_swfw_semaphore(hw, mask);
 666
 667         /*
 668          * These ones are more tricky since they are common to all ports; but
 669          * swfw_sync retries last long enough (1s) to be almost sure that if
 670          * lock can not be taken it is due to an improper lock of the
 671          * semaphore.
 672          */
 673         mask = IXGBE_GSSR_EEP_SM | IXGBE_GSSR_MAC_CSR_SM | IXGBE_GSSR_SW_MNG_SM;
 674         if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) {
 675                    DEBUGOUT("SWFW common locks released");
 676         }
 677         ixgbe_release_swfw_semaphore(hw, mask);
 678 }
 679
 680 /*
 681  * This function is based on code in ixgbe_attach() in ixgbe/ixgbe.c.
 682  * It returns 0 on success.
 683  */
 684 static int
 685 eth_ixgbe_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 686                      struct rte_eth_dev *eth_dev)
 687 {
 688         struct rte_pci_device *pci_dev;
 689         struct ixgbe_hw *hw =
 690                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 691         struct ixgbe_vfta * shadow_vfta =
 692                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 693         struct ixgbe_hwstrip *hwstrip =
 694                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 695         struct ixgbe_dcb_config *dcb_config =
 696                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private);
 697         uint32_t ctrl_ext;
 698         uint16_t csum;
 699         int diag, i;
 700
 701         PMD_INIT_FUNC_TRACE();
 702
 703         eth_dev->dev_ops = &ixgbe_eth_dev_ops;
 704         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
 705         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
 706
 707         /* for secondary processes, we don't initialise any further as primary
 708          * has already done this work. Only check we don't need a different
 709          * RX function */
 710         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 711                 if (eth_dev->data->scattered_rx)
 712                         eth_dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
 713                 return 0;
 714         }
 715         pci_dev = eth_dev->pci_dev;
 716
 717         /* Vendor and Device ID need to be set before init of shared code */
 718         hw->device_id = pci_dev->id.device_id;
 719         hw->vendor_id = pci_dev->id.vendor_id;
 720         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 721 #ifdef RTE_LIBRTE_IXGBE_ALLOW_UNSUPPORTED_SFP
 722         hw->allow_unsupported_sfp = 1;
 723 #endif
 724
 725         /* Initialize the shared code */
 726 #ifdef RTE_NIC_BYPASS
 727         diag = ixgbe_bypass_init_shared_code(hw);
 728 #else
 729         diag = ixgbe_init_shared_code(hw);
 730 #endif /* RTE_NIC_BYPASS */
 731
 732         if (diag != IXGBE_SUCCESS) {
 733                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
 734                 return -EIO;
 735         }
 736
 737         /* pick up the PCI bus settings for reporting later */
 738         ixgbe_get_bus_info(hw);
 739
 740         /* Unlock any pending hardware semaphore */
 741         ixgbe_swfw_lock_reset(hw);
 742
 743         /* Initialize DCB configuration*/
 744         memset(dcb_config, 0, sizeof(struct ixgbe_dcb_config));
 745         ixgbe_dcb_init(hw,dcb_config);
 746         /* Get Hardware Flow Control setting */
 747         hw->fc.requested_mode = ixgbe_fc_full;
 748         hw->fc.current_mode = ixgbe_fc_full;
 749         hw->fc.pause_time = IXGBE_FC_PAUSE;
 750         for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
 751                 hw->fc.low_water[i] = IXGBE_FC_LO;
 752                 hw->fc.high_water[i] = IXGBE_FC_HI;
 753         }
 754         hw->fc.send_xon = 1;
 755
 756         /* Make sure we have a good EEPROM before we read from it */
 757         diag = ixgbe_validate_eeprom_checksum(hw, &csum);
 758         if (diag != IXGBE_SUCCESS) {
 759                 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag);
 760                 return -EIO;
 761         }
 762
 763 #ifdef RTE_NIC_BYPASS
 764         diag = ixgbe_bypass_init_hw(hw);
 765 #else
 766         diag = ixgbe_init_hw(hw);
 767 #endif /* RTE_NIC_BYPASS */
 768
 769         /*
 770          * Devices with copper phys will fail to initialise if ixgbe_init_hw()
 771          * is called too soon after the kernel driver unbinding/binding occurs.
 772          * The failure occurs in ixgbe_identify_phy_generic() for all devices,
 773          * but for non-copper devies, ixgbe_identify_sfp_module_generic() is
 774          * also called. See ixgbe_identify_phy_82599(). The reason for the
 775          * failure is not known, and only occuts when virtualisation features
 776          * are disabled in the bios. A delay of 100ms  was found to be enough by
 777          * trial-and-error, and is doubled to be safe.
 778          */
 779         if (diag && (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper)) {
 780                 rte_delay_ms(200);
 781                 diag = ixgbe_init_hw(hw);
 782         }
 783
 784         if (diag == IXGBE_ERR_EEPROM_VERSION) {
 785                 PMD_INIT_LOG(ERR, "This device is a pre-production adapter/"
 786                     "LOM.  Please be aware there may be issues associated "
 787                     "with your hardware.\n If you are experiencing problems "
 788                     "please contact your Intel or hardware representative "
 789                     "who provided you with this hardware.\n");
 790         } else if (diag == IXGBE_ERR_SFP_NOT_SUPPORTED)
 791                 PMD_INIT_LOG(ERR, "Unsupported SFP+ Module\n");
 792         if (diag) {
 793                 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag);
 794                 return -EIO;
 795         }
 796
 797         /* disable interrupt */
 798         ixgbe_disable_intr(hw);
 799
 800         /* reset mappings for queue statistics hw counters*/
 801         ixgbe_reset_qstat_mappings(hw);
 802
 803         /* Allocate memory for storing MAC addresses */
 804         eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 805                         hw->mac.num_rar_entries, 0);
 806         if (eth_dev->data->mac_addrs == NULL) {
 807                 PMD_INIT_LOG(ERR,
 808                         "Failed to allocate %u bytes needed to store "
 809                         "MAC addresses",
 810                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 811                 return -ENOMEM;
 812         }
 813         /* Copy the permanent MAC address */
 814         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 815                         &eth_dev->data->mac_addrs[0]);
 816
 817         /* Allocate memory for storing hash filter MAC addresses */
 818         eth_dev->data->hash_mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
 819                         IXGBE_VMDQ_NUM_UC_MAC, 0);
 820         if (eth_dev->data->hash_mac_addrs == NULL) {
 821                 PMD_INIT_LOG(ERR,
 822                         "Failed to allocate %d bytes needed to store MAC addresses",
 823                         ETHER_ADDR_LEN * IXGBE_VMDQ_NUM_UC_MAC);
 824                 return -ENOMEM;
 825         }
 826
 827         /* initialize the vfta */
 828         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 829
 830         /* initialize the hw strip bitmap*/
 831         memset(hwstrip, 0, sizeof(*hwstrip));
 832
 833         /* initialize PF if max_vfs not zero */
 834         ixgbe_pf_host_init(eth_dev);
 835
 836         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 837         /* let hardware know driver is loaded */
 838         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 839         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 840         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
 841         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 842         IXGBE_WRITE_FLUSH(hw);
 843
 844         if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
 845                 PMD_INIT_LOG(DEBUG,
 846                              "MAC: %d, PHY: %d, SFP+: %d<n",
 847                              (int) hw->mac.type, (int) hw->phy.type,
 848                              (int) hw->phy.sfp_type);
 849         else
 850                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d\n",
 851                              (int) hw->mac.type, (int) hw->phy.type);
 852
 853         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
 854                         eth_dev->data->port_id, pci_dev->id.vendor_id,
 855                         pci_dev->id.device_id);
 856
 857         rte_intr_callback_register(&(pci_dev->intr_handle),
 858                 ixgbe_dev_interrupt_handler, (void *)eth_dev);
 859
 860         /* enable uio intr after callback register */
 861         rte_intr_enable(&(pci_dev->intr_handle));
 862
 863         /* enable support intr */
 864         ixgbe_enable_intr(eth_dev);
 865
 866         return 0;
 867 }
 868
 869
 870 /*
 871  * Negotiate mailbox API version with the PF.
 872  * After reset API version is always set to the basic one (ixgbe_mbox_api_10).
 873  * Then we try to negotiate starting with the most recent one.
 874  * If all negotiation attempts fail, then we will proceed with
 875  * the default one (ixgbe_mbox_api_10).
 876  */
 877 static void
 878 ixgbevf_negotiate_api(struct ixgbe_hw *hw)
 879 {
 880         int32_t i;
 881
 882         /* start with highest supported, proceed down */
 883         static const enum ixgbe_pfvf_api_rev sup_ver[] = {
 884                 ixgbe_mbox_api_11,
 885                 ixgbe_mbox_api_10,
 886         };
 887
 888         for (i = 0;
 889                         i != RTE_DIM(sup_ver) &&
 890                         ixgbevf_negotiate_api_version(hw, sup_ver[i]) != 0;
 891                         i++)
 892                 ;
 893 }
 894
 895 static void
 896 generate_random_mac_addr(struct ether_addr *mac_addr)
 897 {
 898         uint64_t random;
 899
 900         /* Set Organizationally Unique Identifier (OUI) prefix. */
 901         mac_addr->addr_bytes[0] = 0x00;
 902         mac_addr->addr_bytes[1] = 0x09;
 903         mac_addr->addr_bytes[2] = 0xC0;
 904         /* Force indication of locally assigned MAC address. */
 905         mac_addr->addr_bytes[0] |= ETHER_LOCAL_ADMIN_ADDR;
 906         /* Generate the last 3 bytes of the MAC address with a random number. */
 907         random = rte_rand();
 908         memcpy(&mac_addr->addr_bytes[3], &random, 3);
 909 }
 910
 911 /*
 912  * Virtual Function device init
 913  */
 914 static int
 915 eth_ixgbevf_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 916                      struct rte_eth_dev *eth_dev)
 917 {
 918         int diag;
 919         uint32_t tc, tcs;
 920         struct rte_pci_device *pci_dev;
 921         struct ixgbe_hw *hw =
 922                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 923         struct ixgbe_vfta * shadow_vfta =
 924                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 925         struct ixgbe_hwstrip *hwstrip =
 926                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
 927         struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr;
 928
 929         PMD_INIT_LOG(DEBUG, "eth_ixgbevf_dev_init");
 930
 931         eth_dev->dev_ops = &ixgbevf_eth_dev_ops;
 932         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
 933         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
 934
 935         /* for secondary processes, we don't initialise any further as primary
 936          * has already done this work. Only check we don't need a different
 937          * RX function */
 938         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 939                 if (eth_dev->data->scattered_rx)
 940                         eth_dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
 941                 return 0;
 942         }
 943
 944         pci_dev = eth_dev->pci_dev;
 945
 946         hw->device_id = pci_dev->id.device_id;
 947         hw->vendor_id = pci_dev->id.vendor_id;
 948         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 949
 950         /* initialize the vfta */
 951         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 952
 953         /* initialize the hw strip bitmap*/
 954         memset(hwstrip, 0, sizeof(*hwstrip));
 955
 956         /* Initialize the shared code */
 957         diag = ixgbe_init_shared_code(hw);
 958         if (diag != IXGBE_SUCCESS) {
 959                 PMD_INIT_LOG(ERR, "Shared code init failed for ixgbevf: %d", diag);
 960                 return -EIO;
 961         }
 962
 963         /* init_mailbox_params */
 964         hw->mbx.ops.init_params(hw);
 965
 966         /* Disable the interrupts for VF */
 967         ixgbevf_intr_disable(hw);
 968
 969         hw->mac.num_rar_entries = 128; /* The MAX of the underlying PF */
 970         diag = hw->mac.ops.reset_hw(hw);
 971
 972         /*
 973          * The VF reset operation returns the IXGBE_ERR_INVALID_MAC_ADDR when
 974          * the underlying PF driver has not assigned a MAC address to the VF.
 975          * In this case, assign a random MAC address.
 976          */
 977         if ((diag != IXGBE_SUCCESS) && (diag != IXGBE_ERR_INVALID_MAC_ADDR)) {
 978                 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
 979                 return (diag);
 980         }
 981
 982         /* negotiate mailbox API version to use with the PF. */
 983         ixgbevf_negotiate_api(hw);
 984
 985         /* Get Rx/Tx queue count via mailbox, which is ready after reset_hw */
 986         ixgbevf_get_queues(hw, &tcs, &tc);
 987
 988         /* Allocate memory for storing MAC addresses */
 989         eth_dev->data->mac_addrs = rte_zmalloc("ixgbevf", ETHER_ADDR_LEN *
 990                         hw->mac.num_rar_entries, 0);
 991         if (eth_dev->data->mac_addrs == NULL) {
 992                 PMD_INIT_LOG(ERR,
 993                         "Failed to allocate %u bytes needed to store "
 994                         "MAC addresses",
 995                         ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 996                 return -ENOMEM;
 997         }
 998
 999         /* Generate a random MAC address, if none was assigned by PF. */
1000         if (is_zero_ether_addr(perm_addr)) {
1001                 generate_random_mac_addr(perm_addr);
1002                 diag = ixgbe_set_rar_vf(hw, 1, perm_addr->addr_bytes, 0, 1);
1003                 if (diag) {
1004                         rte_free(eth_dev->data->mac_addrs);
1005                         eth_dev->data->mac_addrs = NULL;
1006                         return diag;
1007                 }
1008                 RTE_LOG(INFO, PMD,
1009                         "\tVF MAC address not assigned by Host PF\n"
1010                         "\tAssign randomly generated MAC address "
1011                         "%02x:%02x:%02x:%02x:%02x:%02x\n",
1012                         perm_addr->addr_bytes[0],
1013                         perm_addr->addr_bytes[1],
1014                         perm_addr->addr_bytes[2],
1015                         perm_addr->addr_bytes[3],
1016                         perm_addr->addr_bytes[4],
1017                         perm_addr->addr_bytes[5]);
1018         }
1019
1020         /* Copy the permanent MAC address */
1021         ether_addr_copy(perm_addr, &eth_dev->data->mac_addrs[0]);
1022
1023         /* reset the hardware with the new settings */
1024         diag = hw->mac.ops.start_hw(hw);
1025         switch (diag) {
1026                 case  0:
1027                         break;
1028
1029                 default:
1030                         PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
1031                         return (-EIO);
1032         }
1033
1034         PMD_INIT_LOG(DEBUG, "\nport %d vendorID=0x%x deviceID=0x%x mac.type=%s\n",
1035                          eth_dev->data->port_id, pci_dev->id.vendor_id, pci_dev->id.device_id,
1036                          "ixgbe_mac_82599_vf");
1037
1038         return 0;
1039 }
1040
1041 static struct eth_driver rte_ixgbe_pmd = {
1042         {
1043                 .name = "rte_ixgbe_pmd",
1044                 .id_table = pci_id_ixgbe_map,
1045                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
1046         },
1047         .eth_dev_init = eth_ixgbe_dev_init,
1048         .dev_private_size = sizeof(struct ixgbe_adapter),
1049 };
1050
1051 /*
1052  * virtual function driver struct
1053  */
1054 static struct eth_driver rte_ixgbevf_pmd = {
1055         {
1056                 .name = "rte_ixgbevf_pmd",
1057                 .id_table = pci_id_ixgbevf_map,
1058                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
1059         },
1060         .eth_dev_init = eth_ixgbevf_dev_init,
1061         .dev_private_size = sizeof(struct ixgbe_adapter),
1062 };
1063
1064 /*
1065  * Driver initialization routine.
1066  * Invoked once at EAL init time.
1067  * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
1068  */
1069 static int
1070 rte_ixgbe_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
1071 {
1072         PMD_INIT_FUNC_TRACE();
1073
1074         rte_eth_driver_register(&rte_ixgbe_pmd);
1075         return 0;
1076 }
1077
1078 /*
1079  * VF Driver initialization routine.
1080  * Invoked one at EAL init time.
1081  * Register itself as the [Virtual Poll Mode] Driver of PCI niantic devices.
1082  */
1083 static int
1084 rte_ixgbevf_pmd_init(const char *name __rte_unused, const char *param __rte_unused)
1085 {
1086         DEBUGFUNC("rte_ixgbevf_pmd_init");
1087
1088         rte_eth_driver_register(&rte_ixgbevf_pmd);
1089         return (0);
1090 }
1091
1092 static int
1093 ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1094 {
1095         struct ixgbe_hw *hw =
1096                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1097         struct ixgbe_vfta * shadow_vfta =
1098                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1099         uint32_t vfta;
1100         uint32_t vid_idx;
1101         uint32_t vid_bit;
1102
1103         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
1104         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
1105         vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(vid_idx));
1106         if (on)
1107                 vfta |= vid_bit;
1108         else
1109                 vfta &= ~vid_bit;
1110         IXGBE_WRITE_REG(hw, IXGBE_VFTA(vid_idx), vfta);
1111
1112         /* update local VFTA copy */
1113         shadow_vfta->vfta[vid_idx] = vfta;
1114
1115         return 0;
1116 }
1117
1118 static void
1119 ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
1120 {
1121         if (on)
1122                 ixgbe_vlan_hw_strip_enable(dev, queue);
1123         else
1124                 ixgbe_vlan_hw_strip_disable(dev, queue);
1125 }
1126
1127 static void
1128 ixgbe_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
1129 {
1130         struct ixgbe_hw *hw =
1131                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1132
1133         /* Only the high 16-bits is valid */
1134         IXGBE_WRITE_REG(hw, IXGBE_EXVET, tpid << 16);
1135 }
1136
1137 void
1138 ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev)
1139 {
1140         struct ixgbe_hw *hw =
1141                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1142         uint32_t vlnctrl;
1143
1144         PMD_INIT_FUNC_TRACE();
1145
1146         /* Filter Table Disable */
1147         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1148         vlnctrl &= ~IXGBE_VLNCTRL_VFE;
1149
1150         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
1151 }
1152
1153 void
1154 ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1155 {
1156         struct ixgbe_hw *hw =
1157                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1158         struct ixgbe_vfta * shadow_vfta =
1159                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1160         uint32_t vlnctrl;
1161         uint16_t i;
1162
1163         PMD_INIT_FUNC_TRACE();
1164
1165         /* Filter Table Enable */
1166         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1167         vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
1168         vlnctrl |= IXGBE_VLNCTRL_VFE;
1169
1170         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
1171
1172         /* write whatever is in local vfta copy */
1173         for (i = 0; i < IXGBE_VFTA_SIZE; i++)
1174                 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), shadow_vfta->vfta[i]);
1175 }
1176
1177 static void
1178 ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on)
1179 {
1180         struct ixgbe_hwstrip *hwstrip =
1181                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(dev->data->dev_private);
1182
1183         if(queue >= IXGBE_MAX_RX_QUEUE_NUM)
1184                 return;
1185
1186         if (on)
1187                 IXGBE_SET_HWSTRIP(hwstrip, queue);
1188         else
1189                 IXGBE_CLEAR_HWSTRIP(hwstrip, queue);
1190 }
1191
1192 static void
1193 ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue)
1194 {
1195         struct ixgbe_hw *hw =
1196                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1197         uint32_t ctrl;
1198
1199         PMD_INIT_FUNC_TRACE();
1200
1201         if (hw->mac.type == ixgbe_mac_82598EB) {
1202                 /* No queue level support */
1203                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
1204                 return;
1205         }
1206         else {
1207                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1208                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1209                 ctrl &= ~IXGBE_RXDCTL_VME;
1210                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
1211         }
1212         /* record those setting for HW strip per queue */
1213         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 0);
1214 }
1215
1216 static void
1217 ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue)
1218 {
1219         struct ixgbe_hw *hw =
1220                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1221         uint32_t ctrl;
1222
1223         PMD_INIT_FUNC_TRACE();
1224
1225         if (hw->mac.type == ixgbe_mac_82598EB) {
1226                 /* No queue level supported */
1227                 PMD_INIT_LOG(INFO, "82598EB not support queue level hw strip");
1228                 return;
1229         }
1230         else {
1231                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1232                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1233                 ctrl |= IXGBE_RXDCTL_VME;
1234                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
1235         }
1236         /* record those setting for HW strip per queue */
1237         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 1);
1238 }
1239
1240 void
1241 ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev)
1242 {
1243         struct ixgbe_hw *hw =
1244                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1245         uint32_t ctrl;
1246         uint16_t i;
1247
1248         PMD_INIT_FUNC_TRACE();
1249
1250         if (hw->mac.type == ixgbe_mac_82598EB) {
1251                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1252                 ctrl &= ~IXGBE_VLNCTRL_VME;
1253                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1254         }
1255         else {
1256                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1257                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1258                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1259                         ctrl &= ~IXGBE_RXDCTL_VME;
1260                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1261
1262                         /* record those setting for HW strip per queue */
1263                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 0);
1264                 }
1265         }
1266 }
1267
1268 void
1269 ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev)
1270 {
1271         struct ixgbe_hw *hw =
1272                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1273         uint32_t ctrl;
1274         uint16_t i;
1275
1276         PMD_INIT_FUNC_TRACE();
1277
1278         if (hw->mac.type == ixgbe_mac_82598EB) {
1279                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1280                 ctrl |= IXGBE_VLNCTRL_VME;
1281                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1282         }
1283         else {
1284                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1285                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1286                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1287                         ctrl |= IXGBE_RXDCTL_VME;
1288                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1289
1290                         /* record those setting for HW strip per queue */
1291                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 1);
1292                 }
1293         }
1294 }
1295
1296 static void
1297 ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1298 {
1299         struct ixgbe_hw *hw =
1300                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1301         uint32_t ctrl;
1302
1303         PMD_INIT_FUNC_TRACE();
1304
1305         /* DMATXCTRL: Geric Double VLAN Disable */
1306         ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1307         ctrl &= ~IXGBE_DMATXCTL_GDV;
1308         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1309
1310         /* CTRL_EXT: Global Double VLAN Disable */
1311         ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1312         ctrl &= ~IXGBE_EXTENDED_VLAN;
1313         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1314
1315 }
1316
1317 static void
1318 ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1319 {
1320         struct ixgbe_hw *hw =
1321                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1322         uint32_t ctrl;
1323
1324         PMD_INIT_FUNC_TRACE();
1325
1326         /* DMATXCTRL: Geric Double VLAN Enable */
1327         ctrl  = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1328         ctrl |= IXGBE_DMATXCTL_GDV;
1329         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1330
1331         /* CTRL_EXT: Global Double VLAN Enable */
1332         ctrl  = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1333         ctrl |= IXGBE_EXTENDED_VLAN;
1334         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1335
1336         /*
1337          * VET EXT field in the EXVET register = 0x8100 by default
1338          * So no need to change. Same to VT field of DMATXCTL register
1339          */
1340 }
1341
1342 static void
1343 ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1344 {
1345         if(mask & ETH_VLAN_STRIP_MASK){
1346                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1347                         ixgbe_vlan_hw_strip_enable_all(dev);
1348                 else
1349                         ixgbe_vlan_hw_strip_disable_all(dev);
1350         }
1351
1352         if(mask & ETH_VLAN_FILTER_MASK){
1353                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1354                         ixgbe_vlan_hw_filter_enable(dev);
1355                 else
1356                         ixgbe_vlan_hw_filter_disable(dev);
1357         }
1358
1359         if(mask & ETH_VLAN_EXTEND_MASK){
1360                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1361                         ixgbe_vlan_hw_extend_enable(dev);
1362                 else
1363                         ixgbe_vlan_hw_extend_disable(dev);
1364         }
1365 }
1366
1367 static void
1368 ixgbe_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1369 {
1370         struct ixgbe_hw *hw =
1371                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1372         /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
1373         uint32_t vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1374         vlanctrl |= IXGBE_VLNCTRL_VFE ; /* enable vlan filters */
1375         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
1376 }
1377
1378 static int
1379 ixgbe_dev_configure(struct rte_eth_dev *dev)
1380 {
1381         struct ixgbe_interrupt *intr =
1382                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1383
1384         PMD_INIT_FUNC_TRACE();
1385
1386         /* set flag to update link status after init */
1387         intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1388
1389         return 0;
1390 }
1391
1392 /*
1393  * Configure device link speed and setup link.
1394  * It returns 0 on success.
1395  */
1396 static int
1397 ixgbe_dev_start(struct rte_eth_dev *dev)
1398 {
1399         struct ixgbe_hw *hw =
1400                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1401         struct ixgbe_vf_info *vfinfo =
1402                 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private);
1403         int err, link_up = 0, negotiate = 0;
1404         uint32_t speed = 0;
1405         int mask = 0;
1406         int status;
1407         uint16_t vf, idx;
1408
1409         PMD_INIT_FUNC_TRACE();
1410
1411         /* IXGBE devices don't support half duplex */
1412         if ((dev->data->dev_conf.link_duplex != ETH_LINK_AUTONEG_DUPLEX) &&
1413                         (dev->data->dev_conf.link_duplex != ETH_LINK_FULL_DUPLEX)) {
1414                 PMD_INIT_LOG(ERR, "Invalid link_duplex (%hu) for port %hhu\n",
1415                                 dev->data->dev_conf.link_duplex,
1416                                 dev->data->port_id);
1417                 return -EINVAL;
1418         }
1419
1420         /* stop adapter */
1421         hw->adapter_stopped = FALSE;
1422         ixgbe_stop_adapter(hw);
1423
1424         /* reinitialize adapter
1425          * this calls reset and start */
1426         status = ixgbe_pf_reset_hw(hw);
1427         if (status != 0)
1428                 return -1;
1429         hw->mac.ops.start_hw(hw);
1430
1431         /* configure PF module if SRIOV enabled */
1432         ixgbe_pf_host_configure(dev);
1433
1434         /* initialize transmission unit */
1435         ixgbe_dev_tx_init(dev);
1436
1437         /* This can fail when allocating mbufs for descriptor rings */
1438         err = ixgbe_dev_rx_init(dev);
1439         if (err) {
1440                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware\n");
1441                 goto error;
1442         }
1443
1444         ixgbe_dev_rxtx_start(dev);
1445
1446         if (ixgbe_is_sfp(hw) && hw->phy.multispeed_fiber) {
1447                 err = hw->mac.ops.setup_sfp(hw);
1448                 if (err)
1449                         goto error;
1450         }
1451
1452         /* Turn on the laser */
1453         ixgbe_enable_tx_laser(hw);
1454
1455         /* Skip link setup if loopback mode is enabled for 82599. */
1456         if (hw->mac.type == ixgbe_mac_82599EB &&
1457                         dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_82599_TX_RX)
1458                 goto skip_link_setup;
1459
1460         err = ixgbe_check_link(hw, &speed, &link_up, 0);
1461         if (err)
1462                 goto error;
1463         err = ixgbe_get_link_capabilities(hw, &speed, &negotiate);
1464         if (err)
1465                 goto error;
1466
1467         switch(dev->data->dev_conf.link_speed) {
1468         case ETH_LINK_SPEED_AUTONEG:
1469                 speed = (hw->mac.type != ixgbe_mac_82598EB) ?
1470                                 IXGBE_LINK_SPEED_82599_AUTONEG :
1471                                 IXGBE_LINK_SPEED_82598_AUTONEG;
1472                 break;
1473         case ETH_LINK_SPEED_100:
1474                 /*
1475                  * Invalid for 82598 but error will be detected by
1476                  * ixgbe_setup_link()
1477                  */
1478                 speed = IXGBE_LINK_SPEED_100_FULL;
1479                 break;
1480         case ETH_LINK_SPEED_1000:
1481                 speed = IXGBE_LINK_SPEED_1GB_FULL;
1482                 break;
1483         case ETH_LINK_SPEED_10000:
1484                 speed = IXGBE_LINK_SPEED_10GB_FULL;
1485                 break;
1486         default:
1487                 PMD_INIT_LOG(ERR, "Invalid link_speed (%hu) for port %hhu\n",
1488                                 dev->data->dev_conf.link_speed,
1489                                 dev->data->port_id);
1490                 goto error;
1491         }
1492
1493         err = ixgbe_setup_link(hw, speed, negotiate, link_up);
1494         if (err)
1495                 goto error;
1496
1497 skip_link_setup:
1498
1499         /* check if lsc interrupt is enabled */
1500         if (dev->data->dev_conf.intr_conf.lsc != 0)
1501                 ixgbe_dev_lsc_interrupt_setup(dev);
1502
1503         /* resume enabled intr since hw reset */
1504         ixgbe_enable_intr(dev);
1505
1506         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
1507                 ETH_VLAN_EXTEND_MASK;
1508         ixgbe_vlan_offload_set(dev, mask);
1509
1510         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
1511                 /* Enable vlan filtering for VMDq */
1512                 ixgbe_vmdq_vlan_hw_filter_enable(dev);
1513         }
1514
1515         /* Configure DCB hw */
1516         ixgbe_configure_dcb(dev);
1517
1518         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) {
1519                 err = ixgbe_fdir_configure(dev);
1520                 if (err)
1521                         goto error;
1522         }
1523
1524         /* Restore vf rate limit */
1525         if (vfinfo != NULL) {
1526                 for (vf = 0; vf < dev->pci_dev->max_vfs; vf++)
1527                         for (idx = 0; idx < IXGBE_MAX_QUEUE_NUM_PER_VF; idx++)
1528                                 if (vfinfo[vf].tx_rate[idx] != 0)
1529                                         ixgbe_set_vf_rate_limit(dev, vf,
1530                                                 vfinfo[vf].tx_rate[idx],
1531                                                 1 << idx);
1532         }
1533
1534         ixgbe_restore_statistics_mapping(dev);
1535
1536         return (0);
1537
1538 error:
1539         PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err);
1540         ixgbe_dev_clear_queues(dev);
1541         return -EIO;
1542 }
1543
1544 /*
1545  * Stop device: disable rx and tx functions to allow for reconfiguring.
1546  */
1547 static void
1548 ixgbe_dev_stop(struct rte_eth_dev *dev)
1549 {
1550         struct rte_eth_link link;
1551         struct ixgbe_hw *hw =
1552                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1553         struct ixgbe_vf_info *vfinfo =
1554                 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private);
1555         int vf;
1556
1557         PMD_INIT_FUNC_TRACE();
1558
1559         /* disable interrupts */
1560         ixgbe_disable_intr(hw);
1561
1562         /* reset the NIC */
1563         ixgbe_pf_reset_hw(hw);
1564         hw->adapter_stopped = FALSE;
1565
1566         /* stop adapter */
1567         ixgbe_stop_adapter(hw);
1568
1569         for (vf = 0; vfinfo != NULL &&
1570                      vf < dev->pci_dev->max_vfs; vf++)
1571                 vfinfo[vf].clear_to_send = false;
1572
1573         /* Turn off the laser */
1574         ixgbe_disable_tx_laser(hw);
1575
1576         ixgbe_dev_clear_queues(dev);
1577
1578         /* Clear recorded link status */
1579         memset(&link, 0, sizeof(link));
1580         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1581 }
1582
1583 /*
1584  * Set device link up: enable tx laser.
1585  */
1586 static int
1587 ixgbe_dev_set_link_up(struct rte_eth_dev *dev)
1588 {
1589         struct ixgbe_hw *hw =
1590                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1591         if (hw->mac.type == ixgbe_mac_82599EB) {
1592 #ifdef RTE_NIC_BYPASS
1593                 if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) {
1594                         /* Not suported in bypass mode */
1595                         PMD_INIT_LOG(ERR,
1596                                 "\nSet link up is not supported "
1597                                 "by device id 0x%x\n",
1598                                 hw->device_id);
1599                         return -ENOTSUP;
1600                 }
1601 #endif
1602                 /* Turn on the laser */
1603                 ixgbe_enable_tx_laser(hw);
1604                 return 0;
1605         }
1606
1607         PMD_INIT_LOG(ERR, "\nSet link up is not supported by device id 0x%x\n",
1608                 hw->device_id);
1609         return -ENOTSUP;
1610 }
1611
1612 /*
1613  * Set device link down: disable tx laser.
1614  */
1615 static int
1616 ixgbe_dev_set_link_down(struct rte_eth_dev *dev)
1617 {
1618         struct ixgbe_hw *hw =
1619                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1620         if (hw->mac.type == ixgbe_mac_82599EB) {
1621 #ifdef RTE_NIC_BYPASS
1622                 if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) {
1623                         /* Not suported in bypass mode */
1624                         PMD_INIT_LOG(ERR,
1625                                 "\nSet link down is not supported "
1626                                 "by device id 0x%x\n",
1627                                  hw->device_id);
1628                         return -ENOTSUP;
1629                 }
1630 #endif
1631                 /* Turn off the laser */
1632                 ixgbe_disable_tx_laser(hw);
1633                 return 0;
1634         }
1635
1636         PMD_INIT_LOG(ERR,
1637                 "\nSet link down is not supported by device id 0x%x\n",
1638                  hw->device_id);
1639         return -ENOTSUP;
1640 }
1641
1642 /*
1643  * Reest and stop device.
1644  */
1645 static void
1646 ixgbe_dev_close(struct rte_eth_dev *dev)
1647 {
1648         struct ixgbe_hw *hw =
1649                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1650
1651         PMD_INIT_FUNC_TRACE();
1652
1653         ixgbe_pf_reset_hw(hw);
1654
1655         ixgbe_dev_stop(dev);
1656         hw->adapter_stopped = 1;
1657
1658         ixgbe_disable_pcie_master(hw);
1659
1660         /* reprogram the RAR[0] in case user changed it. */
1661         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
1662 }
1663
1664 /*
1665  * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c
1666  */
1667 static void
1668 ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1669 {
1670         struct ixgbe_hw *hw =
1671                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1672         struct ixgbe_hw_stats *hw_stats =
1673                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1674         uint32_t bprc, lxon, lxoff, total;
1675         uint64_t total_missed_rx, total_qbrc, total_qprc;
1676         unsigned i;
1677
1678         total_missed_rx = 0;
1679         total_qbrc = 0;
1680         total_qprc = 0;
1681
1682         hw_stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
1683         hw_stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
1684         hw_stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
1685         hw_stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
1686
1687         for (i = 0; i < 8; i++) {
1688                 uint32_t mp;
1689                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
1690                 /* global total per queue */
1691                 hw_stats->mpc[i] += mp;
1692                 /* Running comprehensive total for stats display */
1693                 total_missed_rx += hw_stats->mpc[i];
1694                 if (hw->mac.type == ixgbe_mac_82598EB)
1695                         hw_stats->rnbc[i] +=
1696                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
1697                 hw_stats->pxontxc[i] +=
1698                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
1699                 hw_stats->pxonrxc[i] +=
1700                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
1701                 hw_stats->pxofftxc[i] +=
1702                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
1703                 hw_stats->pxoffrxc[i] +=
1704                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
1705                 hw_stats->pxon2offc[i] +=
1706                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
1707         }
1708         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1709                 hw_stats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
1710                 hw_stats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
1711                 hw_stats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
1712                 hw_stats->qbrc[i] +=
1713                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32);
1714                 hw_stats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
1715                 hw_stats->qbtc[i] +=
1716                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32);
1717                 hw_stats->qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
1718
1719                 total_qprc += hw_stats->qprc[i];
1720                 total_qbrc += hw_stats->qbrc[i];
1721         }
1722         hw_stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
1723         hw_stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
1724         hw_stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
1725
1726         /* Note that gprc counts missed packets */
1727         hw_stats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
1728
1729         if (hw->mac.type != ixgbe_mac_82598EB) {
1730                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
1731                 hw_stats->gorc += ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
1732                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
1733                 hw_stats->gotc += ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
1734                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
1735                 hw_stats->tor += ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
1736                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
1737                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
1738         } else {
1739                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
1740                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
1741                 /* 82598 only has a counter in the high register */
1742                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
1743                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
1744                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
1745         }
1746
1747         /*
1748          * Workaround: mprc hardware is incorrectly counting
1749          * broadcasts, so for now we subtract those.
1750          */
1751         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
1752         hw_stats->bprc += bprc;
1753         hw_stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
1754         if (hw->mac.type == ixgbe_mac_82598EB)
1755                 hw_stats->mprc -= bprc;
1756
1757         hw_stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
1758         hw_stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
1759         hw_stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
1760         hw_stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
1761         hw_stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
1762         hw_stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
1763
1764         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
1765         hw_stats->lxontxc += lxon;
1766         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
1767         hw_stats->lxofftxc += lxoff;
1768         total = lxon + lxoff;
1769
1770         hw_stats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
1771         hw_stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
1772         hw_stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
1773         hw_stats->gptc -= total;
1774         hw_stats->mptc -= total;
1775         hw_stats->ptc64 -= total;
1776         hw_stats->gotc -= total * ETHER_MIN_LEN;
1777
1778         hw_stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
1779         hw_stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
1780         hw_stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
1781         hw_stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
1782         hw_stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
1783         hw_stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
1784         hw_stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
1785         hw_stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
1786         hw_stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
1787         hw_stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
1788         hw_stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
1789         hw_stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
1790         hw_stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
1791         hw_stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
1792         hw_stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
1793         hw_stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);
1794         hw_stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
1795         hw_stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
1796         /* Only read FCOE on 82599 */
1797         if (hw->mac.type != ixgbe_mac_82598EB) {
1798                 hw_stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
1799                 hw_stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
1800                 hw_stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
1801                 hw_stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
1802                 hw_stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
1803         }
1804
1805         if (stats == NULL)
1806                 return;
1807
1808         /* Fill out the rte_eth_stats statistics structure */
1809         stats->ipackets = total_qprc;
1810         stats->ibytes = total_qbrc;
1811         stats->opackets = hw_stats->gptc;
1812         stats->obytes = hw_stats->gotc;
1813         stats->imcasts = hw_stats->mprc;
1814
1815         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
1816                 stats->q_ipackets[i] = hw_stats->qprc[i];
1817                 stats->q_opackets[i] = hw_stats->qptc[i];
1818                 stats->q_ibytes[i] = hw_stats->qbrc[i];
1819                 stats->q_obytes[i] = hw_stats->qbtc[i];
1820                 stats->q_errors[i] = hw_stats->qprdc[i];
1821         }
1822
1823         /* Rx Errors */
1824         stats->ibadcrc  = hw_stats->crcerrs;
1825         stats->ibadlen  = hw_stats->rlec + hw_stats->ruc + hw_stats->roc;
1826         stats->imissed  = total_missed_rx;
1827         stats->ierrors  = stats->ibadcrc +
1828                           stats->ibadlen +
1829                           stats->imissed +
1830                           hw_stats->illerrc + hw_stats->errbc;
1831
1832         /* Tx Errors */
1833         stats->oerrors  = 0;
1834
1835         /* XON/XOFF pause frames */
1836         stats->tx_pause_xon  = hw_stats->lxontxc;
1837         stats->rx_pause_xon  = hw_stats->lxonrxc;
1838         stats->tx_pause_xoff = hw_stats->lxofftxc;
1839         stats->rx_pause_xoff = hw_stats->lxoffrxc;
1840
1841         /* Flow Director Stats registers */
1842         hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
1843         hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
1844         stats->fdirmatch = hw_stats->fdirmatch;
1845         stats->fdirmiss = hw_stats->fdirmiss;
1846 }
1847
1848 static void
1849 ixgbe_dev_stats_reset(struct rte_eth_dev *dev)
1850 {
1851         struct ixgbe_hw_stats *stats =
1852                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1853
1854         /* HW registers are cleared on read */
1855         ixgbe_dev_stats_get(dev, NULL);
1856
1857         /* Reset software totals */
1858         memset(stats, 0, sizeof(*stats));
1859 }
1860
1861 static void
1862 ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1863 {
1864         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1865         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1866                           IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1867
1868         /* Good Rx packet, include VF loopback */
1869         UPDATE_VF_STAT(IXGBE_VFGPRC,
1870             hw_stats->last_vfgprc, hw_stats->vfgprc);
1871
1872         /* Good Rx octets, include VF loopback */
1873         UPDATE_VF_STAT_36BIT(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
1874             hw_stats->last_vfgorc, hw_stats->vfgorc);
1875
1876         /* Good Tx packet, include VF loopback */
1877         UPDATE_VF_STAT(IXGBE_VFGPTC,
1878             hw_stats->last_vfgptc, hw_stats->vfgptc);
1879
1880         /* Good Tx octets, include VF loopback */
1881         UPDATE_VF_STAT_36BIT(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
1882             hw_stats->last_vfgotc, hw_stats->vfgotc);
1883
1884         /* Rx Multicst Packet */
1885         UPDATE_VF_STAT(IXGBE_VFMPRC,
1886             hw_stats->last_vfmprc, hw_stats->vfmprc);
1887
1888         if (stats == NULL)
1889                 return;
1890
1891         memset(stats, 0, sizeof(*stats));
1892         stats->ipackets = hw_stats->vfgprc;
1893         stats->ibytes = hw_stats->vfgorc;
1894         stats->opackets = hw_stats->vfgptc;
1895         stats->obytes = hw_stats->vfgotc;
1896         stats->imcasts = hw_stats->vfmprc;
1897 }
1898
1899 static void
1900 ixgbevf_dev_stats_reset(struct rte_eth_dev *dev)
1901 {
1902         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats*)
1903                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1904
1905         /* Sync HW register to the last stats */
1906         ixgbevf_dev_stats_get(dev, NULL);
1907
1908         /* reset HW current stats*/
1909         hw_stats->vfgprc = 0;
1910         hw_stats->vfgorc = 0;
1911         hw_stats->vfgptc = 0;
1912         hw_stats->vfgotc = 0;
1913         hw_stats->vfmprc = 0;
1914
1915 }
1916
1917 static void
1918 ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1919 {
1920         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1921
1922         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
1923         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
1924         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL register */
1925         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */
1926         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
1927         dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC;
1928         dev_info->max_vfs = dev->pci_dev->max_vfs;
1929         if (hw->mac.type == ixgbe_mac_82598EB)
1930                 dev_info->max_vmdq_pools = ETH_16_POOLS;
1931         else
1932                 dev_info->max_vmdq_pools = ETH_64_POOLS;
1933         dev_info->rx_offload_capa =
1934                 DEV_RX_OFFLOAD_VLAN_STRIP |
1935                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1936                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1937                 DEV_RX_OFFLOAD_TCP_CKSUM;
1938         dev_info->tx_offload_capa =
1939                 DEV_TX_OFFLOAD_VLAN_INSERT |
1940                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1941                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1942                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1943                 DEV_TX_OFFLOAD_SCTP_CKSUM;
1944 }
1945
1946 /* return 0 means link status changed, -1 means not changed */
1947 static int
1948 ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1949 {
1950         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1951         struct rte_eth_link link, old;
1952         ixgbe_link_speed link_speed;
1953         int link_up;
1954         int diag;
1955
1956         link.link_status = 0;
1957         link.link_speed = 0;
1958         link.link_duplex = 0;
1959         memset(&old, 0, sizeof(old));
1960         rte_ixgbe_dev_atomic_read_link_status(dev, &old);
1961
1962         /* check if it needs to wait to complete, if lsc interrupt is enabled */
1963         if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
1964                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 0);
1965         else
1966                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 1);
1967         if (diag != 0) {
1968                 link.link_speed = ETH_LINK_SPEED_100;
1969                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1970                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1971                 if (link.link_status == old.link_status)
1972                         return -1;
1973                 return 0;
1974         }
1975
1976         if (link_up == 0) {
1977                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
1978                 if (link.link_status == old.link_status)
1979                         return -1;
1980                 return 0;
1981         }
1982         link.link_status = 1;
1983         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1984
1985         switch (link_speed) {
1986         default:
1987         case IXGBE_LINK_SPEED_UNKNOWN:
1988                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1989                 link.link_speed = ETH_LINK_SPEED_100;
1990                 break;
1991
1992         case IXGBE_LINK_SPEED_100_FULL:
1993                 link.link_speed = ETH_LINK_SPEED_100;
1994                 break;
1995
1996         case IXGBE_LINK_SPEED_1GB_FULL:
1997                 link.link_speed = ETH_LINK_SPEED_1000;
1998                 break;
1999
2000         case IXGBE_LINK_SPEED_10GB_FULL:
2001                 link.link_speed = ETH_LINK_SPEED_10000;
2002                 break;
2003         }
2004         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
2005
2006         if (link.link_status == old.link_status)
2007                 return -1;
2008
2009         return 0;
2010 }
2011
2012 static void
2013 ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev)
2014 {
2015         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2016         uint32_t fctrl;
2017
2018         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2019         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
2020         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2021 }
2022
2023 static void
2024 ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev)
2025 {
2026         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2027         uint32_t fctrl;
2028
2029         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2030         fctrl &= (~IXGBE_FCTRL_UPE);
2031         if (dev->data->all_multicast == 1)
2032                 fctrl |= IXGBE_FCTRL_MPE;
2033         else
2034                 fctrl &= (~IXGBE_FCTRL_MPE);
2035         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2036 }
2037
2038 static void
2039 ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev)
2040 {
2041         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2042         uint32_t fctrl;
2043
2044         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2045         fctrl |= IXGBE_FCTRL_MPE;
2046         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2047 }
2048
2049 static void
2050 ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev)
2051 {
2052         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2053         uint32_t fctrl;
2054
2055         if (dev->data->promiscuous == 1)
2056                 return; /* must remain in all_multicast mode */
2057
2058         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2059         fctrl &= (~IXGBE_FCTRL_MPE);
2060         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2061 }
2062
2063 /**
2064  * It clears the interrupt causes and enables the interrupt.
2065  * It will be called once only during nic initialized.
2066  *
2067  * @param dev
2068  *  Pointer to struct rte_eth_dev.
2069  *
2070  * @return
2071  *  - On success, zero.
2072  *  - On failure, a negative value.
2073  */
2074 static int
2075 ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev)
2076 {
2077         struct ixgbe_interrupt *intr =
2078                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2079
2080         ixgbe_dev_link_status_print(dev);
2081         intr->mask |= IXGBE_EICR_LSC;
2082
2083         return 0;
2084 }
2085
2086 /*
2087  * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update.
2088  *
2089  * @param dev
2090  *  Pointer to struct rte_eth_dev.
2091  *
2092  * @return
2093  *  - On success, zero.
2094  *  - On failure, a negative value.
2095  */
2096 static int
2097 ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
2098 {
2099         uint32_t eicr;
2100         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2101         struct ixgbe_interrupt *intr =
2102                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2103
2104         /* clear all cause mask */
2105         ixgbe_disable_intr(hw);
2106
2107         /* read-on-clear nic registers here */
2108         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
2109         PMD_DRV_LOG(INFO, "eicr %x", eicr);
2110
2111         intr->flags = 0;
2112         if (eicr & IXGBE_EICR_LSC) {
2113                 /* set flag for async link update */
2114                 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
2115         }
2116
2117         if (eicr & IXGBE_EICR_MAILBOX)
2118                 intr->flags |= IXGBE_FLAG_MAILBOX;
2119
2120         return 0;
2121 }
2122
2123 /**
2124  * It gets and then prints the link status.
2125  *
2126  * @param dev
2127  *  Pointer to struct rte_eth_dev.
2128  *
2129  * @return
2130  *  - On success, zero.
2131  *  - On failure, a negative value.
2132  */
2133 static void
2134 ixgbe_dev_link_status_print(struct rte_eth_dev *dev)
2135 {
2136         struct rte_eth_link link;
2137
2138         memset(&link, 0, sizeof(link));
2139         rte_ixgbe_dev_atomic_read_link_status(dev, &link);
2140         if (link.link_status) {
2141                 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
2142                                         (int)(dev->data->port_id),
2143                                         (unsigned)link.link_speed,
2144                         link.link_duplex == ETH_LINK_FULL_DUPLEX ?
2145                                         "full-duplex" : "half-duplex");
2146         } else {
2147                 PMD_INIT_LOG(INFO, " Port %d: Link Down",
2148                                 (int)(dev->data->port_id));
2149         }
2150         PMD_INIT_LOG(INFO, "PCI Address: %04d:%02d:%02d:%d",
2151                                 dev->pci_dev->addr.domain,
2152                                 dev->pci_dev->addr.bus,
2153                                 dev->pci_dev->addr.devid,
2154                                 dev->pci_dev->addr.function);
2155 }
2156
2157 /*
2158  * It executes link_update after knowing an interrupt occurred.
2159  *
2160  * @param dev
2161  *  Pointer to struct rte_eth_dev.
2162  *
2163  * @return
2164  *  - On success, zero.
2165  *  - On failure, a negative value.
2166  */
2167 static int
2168 ixgbe_dev_interrupt_action(struct rte_eth_dev *dev)
2169 {
2170         struct ixgbe_interrupt *intr =
2171                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2172         int64_t timeout;
2173         struct rte_eth_link link;
2174         int intr_enable_delay = false;
2175
2176         PMD_DRV_LOG(DEBUG, "intr action type %d\n", intr->flags);
2177
2178         if (intr->flags & IXGBE_FLAG_MAILBOX) {
2179                 ixgbe_pf_mbx_process(dev);
2180                 intr->flags &= ~IXGBE_FLAG_MAILBOX;
2181         }
2182
2183         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
2184                 /* get the link status before link update, for predicting later */
2185                 memset(&link, 0, sizeof(link));
2186                 rte_ixgbe_dev_atomic_read_link_status(dev, &link);
2187
2188                 ixgbe_dev_link_update(dev, 0);
2189
2190                 /* likely to up */
2191                 if (!link.link_status)
2192                         /* handle it 1 sec later, wait it being stable */
2193                         timeout = IXGBE_LINK_UP_CHECK_TIMEOUT;
2194                 /* likely to down */
2195                 else
2196                         /* handle it 4 sec later, wait it being stable */
2197                         timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT;
2198
2199                 ixgbe_dev_link_status_print(dev);
2200
2201                 intr_enable_delay = true;
2202         }
2203
2204         if (intr_enable_delay) {
2205                 if (rte_eal_alarm_set(timeout * 1000,
2206                                       ixgbe_dev_interrupt_delayed_handler, (void*)dev) < 0)
2207                         PMD_DRV_LOG(ERR, "Error setting alarm");
2208         } else {
2209                 PMD_DRV_LOG(DEBUG, "enable intr immediately");
2210                 ixgbe_enable_intr(dev);
2211                 rte_intr_enable(&(dev->pci_dev->intr_handle));
2212         }
2213
2214
2215         return 0;
2216 }
2217
2218 /**
2219  * Interrupt handler which shall be registered for alarm callback for delayed
2220  * handling specific interrupt to wait for the stable nic state. As the
2221  * NIC interrupt state is not stable for ixgbe after link is just down,
2222  * it needs to wait 4 seconds to get the stable status.
2223  *
2224  * @param handle
2225  *  Pointer to interrupt handle.
2226  * @param param
2227  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2228  *
2229  * @return
2230  *  void
2231  */
2232 static void
2233 ixgbe_dev_interrupt_delayed_handler(void *param)
2234 {
2235         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2236         struct ixgbe_interrupt *intr =
2237                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2238         struct ixgbe_hw *hw =
2239                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2240         uint32_t eicr;
2241
2242         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
2243         if (eicr & IXGBE_EICR_MAILBOX)
2244                 ixgbe_pf_mbx_process(dev);
2245
2246         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
2247                 ixgbe_dev_link_update(dev, 0);
2248                 intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
2249                 ixgbe_dev_link_status_print(dev);
2250                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
2251         }
2252
2253         PMD_DRV_LOG(DEBUG, "enable intr in delayed handler S[%08x]\n", eicr);
2254         ixgbe_enable_intr(dev);
2255         rte_intr_enable(&(dev->pci_dev->intr_handle));
2256 }
2257
2258 /**
2259  * Interrupt handler triggered by NIC  for handling
2260  * specific interrupt.
2261  *
2262  * @param handle
2263  *  Pointer to interrupt handle.
2264  * @param param
2265  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2266  *
2267  * @return
2268  *  void
2269  */
2270 static void
2271 ixgbe_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
2272                                                         void *param)
2273 {
2274         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2275         ixgbe_dev_interrupt_get_status(dev);
2276         ixgbe_dev_interrupt_action(dev);
2277 }
2278
2279 static int
2280 ixgbe_dev_led_on(struct rte_eth_dev *dev)
2281 {
2282         struct ixgbe_hw *hw;
2283
2284         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2285         return (ixgbe_led_on(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
2286 }
2287
2288 static int
2289 ixgbe_dev_led_off(struct rte_eth_dev *dev)
2290 {
2291         struct ixgbe_hw *hw;
2292
2293         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2294         return (ixgbe_led_off(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP);
2295 }
2296
2297 static int
2298 ixgbe_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2299 {
2300         struct ixgbe_hw *hw;
2301         uint32_t mflcn_reg;
2302         uint32_t fccfg_reg;
2303         int rx_pause;
2304         int tx_pause;
2305
2306         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2307
2308         fc_conf->pause_time = hw->fc.pause_time;
2309         fc_conf->high_water = hw->fc.high_water[0];
2310         fc_conf->low_water = hw->fc.low_water[0];
2311         fc_conf->send_xon = hw->fc.send_xon;
2312         fc_conf->autoneg = !hw->fc.disable_fc_autoneg;
2313
2314         /*
2315          * Return rx_pause status according to actual setting of
2316          * MFLCN register.
2317          */
2318         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
2319         if (mflcn_reg & (IXGBE_MFLCN_RPFCE | IXGBE_MFLCN_RFCE))
2320                 rx_pause = 1;
2321         else
2322                 rx_pause = 0;
2323
2324         /*
2325          * Return tx_pause status according to actual setting of
2326          * FCCFG register.
2327          */
2328         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
2329         if (fccfg_reg & (IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY))
2330                 tx_pause = 1;
2331         else
2332                 tx_pause = 0;
2333
2334         if (rx_pause && tx_pause)
2335                 fc_conf->mode = RTE_FC_FULL;
2336         else if (rx_pause)
2337                 fc_conf->mode = RTE_FC_RX_PAUSE;
2338         else if (tx_pause)
2339                 fc_conf->mode = RTE_FC_TX_PAUSE;
2340         else
2341                 fc_conf->mode = RTE_FC_NONE;
2342
2343         return 0;
2344 }
2345
2346 static int
2347 ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2348 {
2349         struct ixgbe_hw *hw;
2350         int err;
2351         uint32_t rx_buf_size;
2352         uint32_t max_high_water;
2353         uint32_t mflcn;
2354         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
2355                 ixgbe_fc_none,
2356                 ixgbe_fc_rx_pause,
2357                 ixgbe_fc_tx_pause,
2358                 ixgbe_fc_full
2359         };
2360
2361         PMD_INIT_FUNC_TRACE();
2362
2363         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2364         if (fc_conf->autoneg != !hw->fc.disable_fc_autoneg)
2365                 return -ENOTSUP;
2366         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0));
2367         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
2368
2369         /*
2370          * At least reserve one Ethernet frame for watermark
2371          * high_water/low_water in kilo bytes for ixgbe
2372          */
2373         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
2374         if ((fc_conf->high_water > max_high_water) ||
2375                 (fc_conf->high_water < fc_conf->low_water)) {
2376                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
2377                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
2378                 return (-EINVAL);
2379         }
2380
2381         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[fc_conf->mode];
2382         hw->fc.pause_time     = fc_conf->pause_time;
2383         hw->fc.high_water[0]  = fc_conf->high_water;
2384         hw->fc.low_water[0]   = fc_conf->low_water;
2385         hw->fc.send_xon       = fc_conf->send_xon;
2386
2387         err = ixgbe_fc_enable(hw);
2388
2389         /* Not negotiated is not an error case */
2390         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) {
2391
2392                 /* check if we want to forward MAC frames - driver doesn't have native
2393                  * capability to do that, so we'll write the registers ourselves */
2394
2395                 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN);
2396
2397                 /* set or clear MFLCN.PMCF bit depending on configuration */
2398                 if (fc_conf->mac_ctrl_frame_fwd != 0)
2399                         mflcn |= IXGBE_MFLCN_PMCF;
2400                 else
2401                         mflcn &= ~IXGBE_MFLCN_PMCF;
2402
2403                 IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn);
2404                 IXGBE_WRITE_FLUSH(hw);
2405
2406                 return 0;
2407         }
2408
2409         PMD_INIT_LOG(ERR, "ixgbe_fc_enable = 0x%x \n", err);
2410         return -EIO;
2411 }
2412
2413 /**
2414  *  ixgbe_pfc_enable_generic - Enable flow control
2415  *  @hw: pointer to hardware structure
2416  *  @tc_num: traffic class number
2417  *  Enable flow control according to the current settings.
2418  */
2419 static int
2420 ixgbe_dcb_pfc_enable_generic(struct ixgbe_hw *hw,uint8_t tc_num)
2421 {
2422         int ret_val = 0;
2423         uint32_t mflcn_reg, fccfg_reg;
2424         uint32_t reg;
2425         uint32_t fcrtl, fcrth;
2426         uint8_t i;
2427         uint8_t nb_rx_en;
2428
2429         /* Validate the water mark configuration */
2430         if (!hw->fc.pause_time) {
2431                 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2432                 goto out;
2433         }
2434
2435         /* Low water mark of zero causes XOFF floods */
2436         if (hw->fc.current_mode & ixgbe_fc_tx_pause) {
2437                  /* High/Low water can not be 0 */
2438                 if( (!hw->fc.high_water[tc_num])|| (!hw->fc.low_water[tc_num])) {
2439                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
2440                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2441                         goto out;
2442                 }
2443
2444                 if(hw->fc.low_water[tc_num] >= hw->fc.high_water[tc_num]) {
2445                         PMD_INIT_LOG(ERR,"Invalid water mark configuration\n");
2446                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
2447                         goto out;
2448                 }
2449         }
2450         /* Negotiate the fc mode to use */
2451         ixgbe_fc_autoneg(hw);
2452
2453         /* Disable any previous flow control settings */
2454         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
2455         mflcn_reg &= ~(IXGBE_MFLCN_RPFCE_SHIFT | IXGBE_MFLCN_RFCE|IXGBE_MFLCN_RPFCE);
2456
2457         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
2458         fccfg_reg &= ~(IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY);
2459
2460         switch (hw->fc.current_mode) {
2461         case ixgbe_fc_none:
2462                 /*
2463                  * If the count of enabled RX Priority Flow control >1,
2464                  * and the TX pause can not be disabled
2465                  */
2466                 nb_rx_en = 0;
2467                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
2468                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
2469                         if (reg & IXGBE_FCRTH_FCEN)
2470                                 nb_rx_en++;
2471                 }
2472                 if (nb_rx_en > 1)
2473                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2474                 break;
2475         case ixgbe_fc_rx_pause:
2476                 /*
2477                  * Rx Flow control is enabled and Tx Flow control is
2478                  * disabled by software override. Since there really
2479                  * isn't a way to advertise that we are capable of RX
2480                  * Pause ONLY, we will advertise that we support both
2481                  * symmetric and asymmetric Rx PAUSE.  Later, we will
2482                  * disable the adapter's ability to send PAUSE frames.
2483                  */
2484                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
2485                 /*
2486                  * If the count of enabled RX Priority Flow control >1,
2487                  * and the TX pause can not be disabled
2488                  */
2489                 nb_rx_en = 0;
2490                 for (i =0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
2491                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
2492                         if (reg & IXGBE_FCRTH_FCEN)
2493                                 nb_rx_en++;
2494                 }
2495                 if (nb_rx_en > 1)
2496                         fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2497                 break;
2498         case ixgbe_fc_tx_pause:
2499                 /*
2500                  * Tx Flow control is enabled, and Rx Flow control is
2501                  * disabled by software override.
2502                  */
2503                 fccfg_reg |=IXGBE_FCCFG_TFCE_PRIORITY;
2504                 break;
2505         case ixgbe_fc_full:
2506                 /* Flow control (both Rx and Tx) is enabled by SW override. */
2507                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
2508                 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
2509                 break;
2510         default:
2511                 DEBUGOUT("Flow control param set incorrectly\n");
2512                 ret_val = IXGBE_ERR_CONFIG;
2513                 goto out;
2514                 break;
2515         }
2516
2517         /* Set 802.3x based flow control settings. */
2518         mflcn_reg |= IXGBE_MFLCN_DPF;
2519         IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg);
2520         IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg);
2521
2522         /* Set up and enable Rx high/low water mark thresholds, enable XON. */
2523         if ((hw->fc.current_mode & ixgbe_fc_tx_pause) &&
2524                 hw->fc.high_water[tc_num]) {
2525                 fcrtl = (hw->fc.low_water[tc_num] << 10) | IXGBE_FCRTL_XONE;
2526                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), fcrtl);
2527                 fcrth = (hw->fc.high_water[tc_num] << 10) | IXGBE_FCRTH_FCEN;
2528         } else {
2529                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), 0);
2530                 /*
2531                  * In order to prevent Tx hangs when the internal Tx
2532                  * switch is enabled we must set the high water mark
2533                  * to the maximum FCRTH value.  This allows the Tx
2534                  * switch to function even under heavy Rx workloads.
2535                  */
2536                 fcrth = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)) - 32;
2537         }
2538         IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(tc_num), fcrth);
2539
2540         /* Configure pause time (2 TCs per register) */
2541         reg = hw->fc.pause_time * 0x00010001;
2542         for (i = 0; i < (IXGBE_DCB_MAX_TRAFFIC_CLASS / 2); i++)
2543                 IXGBE_WRITE_REG(hw, IXGBE_FCTTV(i), reg);
2544
2545         /* Configure flow control refresh threshold value */
2546         IXGBE_WRITE_REG(hw, IXGBE_FCRTV, hw->fc.pause_time / 2);
2547
2548 out:
2549         return ret_val;
2550 }
2551
2552 static int
2553 ixgbe_dcb_pfc_enable(struct rte_eth_dev *dev,uint8_t tc_num)
2554 {
2555         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2556         int32_t ret_val = IXGBE_NOT_IMPLEMENTED;
2557
2558         if(hw->mac.type != ixgbe_mac_82598EB) {
2559                 ret_val = ixgbe_dcb_pfc_enable_generic(hw,tc_num);
2560         }
2561         return ret_val;
2562 }
2563
2564 static int
2565 ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf)
2566 {
2567         int err;
2568         uint32_t rx_buf_size;
2569         uint32_t max_high_water;
2570         uint8_t tc_num;
2571         uint8_t  map[IXGBE_DCB_MAX_USER_PRIORITY] = { 0 };
2572         struct ixgbe_hw *hw =
2573                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2574         struct ixgbe_dcb_config *dcb_config =
2575                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
2576
2577         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
2578                 ixgbe_fc_none,
2579                 ixgbe_fc_rx_pause,
2580                 ixgbe_fc_tx_pause,
2581                 ixgbe_fc_full
2582         };
2583
2584         PMD_INIT_FUNC_TRACE();
2585
2586         ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
2587         tc_num = map[pfc_conf->priority];
2588         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num));
2589         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
2590         /*
2591          * At least reserve one Ethernet frame for watermark
2592          * high_water/low_water in kilo bytes for ixgbe
2593          */
2594         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
2595         if ((pfc_conf->fc.high_water > max_high_water) ||
2596                 (pfc_conf->fc.high_water <= pfc_conf->fc.low_water)) {
2597                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB\n");
2598                 PMD_INIT_LOG(ERR, "High_water must <=  0x%x\n", max_high_water);
2599                 return (-EINVAL);
2600         }
2601
2602         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[pfc_conf->fc.mode];
2603         hw->fc.pause_time = pfc_conf->fc.pause_time;
2604         hw->fc.send_xon = pfc_conf->fc.send_xon;
2605         hw->fc.low_water[tc_num] =  pfc_conf->fc.low_water;
2606         hw->fc.high_water[tc_num] = pfc_conf->fc.high_water;
2607
2608         err = ixgbe_dcb_pfc_enable(dev,tc_num);
2609
2610         /* Not negotiated is not an error case */
2611         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED))
2612                 return 0;
2613
2614         PMD_INIT_LOG(ERR, "ixgbe_dcb_pfc_enable = 0x%x \n", err);
2615         return -EIO;
2616 }
2617
2618 static int
2619 ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
2620                                 struct rte_eth_rss_reta *reta_conf)
2621 {
2622         uint8_t i,j,mask;
2623         uint32_t reta;
2624         struct ixgbe_hw *hw =
2625                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2626
2627         PMD_INIT_FUNC_TRACE();
2628         /*
2629         * Update Redirection Table RETA[n],n=0...31,The redirection table has
2630         * 128-entries in 32 registers
2631          */
2632         for(i = 0; i < ETH_RSS_RETA_NUM_ENTRIES; i += 4) {
2633                 if (i < ETH_RSS_RETA_NUM_ENTRIES/2)
2634                         mask = (uint8_t)((reta_conf->mask_lo >> i) & 0xF);
2635                 else
2636                         mask = (uint8_t)((reta_conf->mask_hi >>
2637                                 (i - ETH_RSS_RETA_NUM_ENTRIES/2)) & 0xF);
2638                 if (mask != 0) {
2639                         reta = 0;
2640                         if (mask != 0xF)
2641                                 reta = IXGBE_READ_REG(hw,IXGBE_RETA(i >> 2));
2642
2643                         for (j = 0; j < 4; j++) {
2644                                 if (mask & (0x1 << j)) {
2645                                         if (mask != 0xF)
2646                                                 reta &= ~(0xFF << 8 * j);
2647                                         reta |= reta_conf->reta[i + j] << 8*j;
2648                                 }
2649                         }
2650                         IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2),reta);
2651                 }
2652         }
2653
2654         return 0;
2655 }
2656
2657 static int
2658 ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
2659                                 struct rte_eth_rss_reta *reta_conf)
2660 {
2661         uint8_t i,j,mask;
2662         uint32_t reta;
2663         struct ixgbe_hw *hw =
2664                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2665
2666         PMD_INIT_FUNC_TRACE();
2667         /*
2668          * Read Redirection Table RETA[n],n=0...31,The redirection table has
2669          * 128-entries in 32 registers
2670          */
2671         for(i = 0; i < ETH_RSS_RETA_NUM_ENTRIES; i += 4) {
2672                 if (i < ETH_RSS_RETA_NUM_ENTRIES/2)
2673                         mask = (uint8_t)((reta_conf->mask_lo >> i) & 0xF);
2674                 else
2675                         mask = (uint8_t)((reta_conf->mask_hi >>
2676                                 (i - ETH_RSS_RETA_NUM_ENTRIES/2)) & 0xF);
2677
2678                 if (mask != 0) {
2679                         reta = IXGBE_READ_REG(hw,IXGBE_RETA(i >> 2));
2680                         for (j = 0; j < 4; j++) {
2681                                 if (mask & (0x1 << j))
2682                                         reta_conf->reta[i + j] =
2683                                                 (uint8_t)((reta >> 8 * j) & 0xFF);
2684                         }
2685                 }
2686         }
2687
2688         return 0;
2689 }
2690
2691 static void
2692 ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
2693                                 uint32_t index, uint32_t pool)
2694 {
2695         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2696         uint32_t enable_addr = 1;
2697
2698         ixgbe_set_rar(hw, index, mac_addr->addr_bytes, pool, enable_addr);
2699 }
2700
2701 static void
2702 ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
2703 {
2704         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2705
2706         ixgbe_clear_rar(hw, index);
2707 }
2708
2709 /*
2710  * Virtual Function operations
2711  */
2712 static void
2713 ixgbevf_intr_disable(struct ixgbe_hw *hw)
2714 {
2715         PMD_INIT_LOG(DEBUG, "ixgbevf_intr_disable");
2716
2717         /* Clear interrupt mask to stop from interrupts being generated */
2718         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, IXGBE_VF_IRQ_CLEAR_MASK);
2719
2720         IXGBE_WRITE_FLUSH(hw);
2721 }
2722
2723 static int
2724 ixgbevf_dev_configure(struct rte_eth_dev *dev)
2725 {
2726         struct rte_eth_conf* conf = &dev->data->dev_conf;
2727
2728         PMD_INIT_LOG(DEBUG, "\nConfigured Virtual Function port id: %d\n",
2729                 dev->data->port_id);
2730
2731         /*
2732          * VF has no ability to enable/disable HW CRC
2733          * Keep the persistent behavior the same as Host PF
2734          */
2735 #ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC
2736         if (!conf->rxmode.hw_strip_crc) {
2737                 PMD_INIT_LOG(INFO, "VF can't disable HW CRC Strip\n");
2738                 conf->rxmode.hw_strip_crc = 1;
2739         }
2740 #else
2741         if (conf->rxmode.hw_strip_crc) {
2742                 PMD_INIT_LOG(INFO, "VF can't enable HW CRC Strip\n");
2743                 conf->rxmode.hw_strip_crc = 0;
2744         }
2745 #endif
2746
2747         return 0;
2748 }
2749
2750 static int
2751 ixgbevf_dev_start(struct rte_eth_dev *dev)
2752 {
2753         struct ixgbe_hw *hw =
2754                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2755         int err, mask = 0;
2756
2757         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_start");
2758
2759         hw->mac.ops.reset_hw(hw);
2760
2761         /* negotiate mailbox API version to use with the PF. */
2762         ixgbevf_negotiate_api(hw);
2763
2764         ixgbevf_dev_tx_init(dev);
2765
2766         /* This can fail when allocating mbufs for descriptor rings */
2767         err = ixgbevf_dev_rx_init(dev);
2768         if (err) {
2769                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware (%d)\n", err);
2770                 ixgbe_dev_clear_queues(dev);
2771                 return err;
2772         }
2773
2774         /* Set vfta */
2775         ixgbevf_set_vfta_all(dev,1);
2776
2777         /* Set HW strip */
2778         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
2779                 ETH_VLAN_EXTEND_MASK;
2780         ixgbevf_vlan_offload_set(dev, mask);
2781
2782         ixgbevf_dev_rxtx_start(dev);
2783
2784         return 0;
2785 }
2786
2787 static void
2788 ixgbevf_dev_stop(struct rte_eth_dev *dev)
2789 {
2790         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2791
2792         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_stop");
2793
2794         hw->adapter_stopped = TRUE;
2795         ixgbe_stop_adapter(hw);
2796
2797         /*
2798           * Clear what we set, but we still keep shadow_vfta to
2799           * restore after device starts
2800           */
2801         ixgbevf_set_vfta_all(dev,0);
2802
2803         ixgbe_dev_clear_queues(dev);
2804 }
2805
2806 static void
2807 ixgbevf_dev_close(struct rte_eth_dev *dev)
2808 {
2809         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2810
2811         PMD_INIT_LOG(DEBUG, "ixgbevf_dev_close");
2812
2813         ixgbe_reset_hw(hw);
2814
2815         ixgbevf_dev_stop(dev);
2816
2817         /* reprogram the RAR[0] in case user changed it. */
2818         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
2819 }
2820
2821 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on)
2822 {
2823         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2824         struct ixgbe_vfta * shadow_vfta =
2825                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2826         int i = 0, j = 0, vfta = 0, mask = 1;
2827
2828         for (i = 0; i < IXGBE_VFTA_SIZE; i++){
2829                 vfta = shadow_vfta->vfta[i];
2830                 if(vfta){
2831                         mask = 1;
2832                         for (j = 0; j < 32; j++){
2833                                 if(vfta & mask)
2834                                         ixgbe_set_vfta(hw, (i<<5)+j, 0, on);
2835                                 mask<<=1;
2836                         }
2837                 }
2838         }
2839
2840 }
2841
2842 static int
2843 ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2844 {
2845         struct ixgbe_hw *hw =
2846                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2847         struct ixgbe_vfta * shadow_vfta =
2848                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2849         uint32_t vid_idx = 0;
2850         uint32_t vid_bit = 0;
2851         int ret = 0;
2852
2853         PMD_INIT_FUNC_TRACE();
2854
2855         /* vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf */
2856         ret = ixgbe_set_vfta(hw, vlan_id, 0, !!on);
2857         if(ret){
2858                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
2859                 return ret;
2860         }
2861         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
2862         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
2863
2864         /* Save what we set and retore it after device reset */
2865         if (on)
2866                 shadow_vfta->vfta[vid_idx] |= vid_bit;
2867         else
2868                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
2869
2870         return 0;
2871 }
2872
2873 static void
2874 ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
2875 {
2876         struct ixgbe_hw *hw =
2877                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2878         uint32_t ctrl;
2879
2880         PMD_INIT_FUNC_TRACE();
2881
2882         if(queue >= hw->mac.max_rx_queues)
2883                 return;
2884
2885         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
2886         if(on)
2887                 ctrl |= IXGBE_RXDCTL_VME;
2888         else
2889                 ctrl &= ~IXGBE_RXDCTL_VME;
2890         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
2891
2892         ixgbe_vlan_hw_strip_bitmap_set( dev, queue, on);
2893 }
2894
2895 static void
2896 ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2897 {
2898         struct ixgbe_hw *hw =
2899                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2900         uint16_t i;
2901         int on = 0;
2902
2903         /* VF function only support hw strip feature, others are not support */
2904         if(mask & ETH_VLAN_STRIP_MASK){
2905                 on = !!(dev->data->dev_conf.rxmode.hw_vlan_strip);
2906
2907                 for(i=0; i < hw->mac.max_rx_queues; i++)
2908                         ixgbevf_vlan_strip_queue_set(dev,i,on);
2909         }
2910 }
2911
2912 static int
2913 ixgbe_vmdq_mode_check(struct ixgbe_hw *hw)
2914 {
2915         uint32_t reg_val;
2916
2917         /* we only need to do this if VMDq is enabled */
2918         reg_val = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
2919         if (!(reg_val & IXGBE_VT_CTL_VT_ENABLE)) {
2920                 PMD_INIT_LOG(ERR, "VMDq must be enabled for this setting\n");
2921                 return (-1);
2922         }
2923
2924         return 0;
2925 }
2926
2927 static uint32_t
2928 ixgbe_uta_vector(struct ixgbe_hw *hw, struct ether_addr* uc_addr)
2929 {
2930         uint32_t vector = 0;
2931         switch (hw->mac.mc_filter_type) {
2932         case 0:   /* use bits [47:36] of the address */
2933                 vector = ((uc_addr->addr_bytes[4] >> 4) |
2934                         (((uint16_t)uc_addr->addr_bytes[5]) << 4));
2935                 break;
2936         case 1:   /* use bits [46:35] of the address */
2937                 vector = ((uc_addr->addr_bytes[4] >> 3) |
2938                         (((uint16_t)uc_addr->addr_bytes[5]) << 5));
2939                 break;
2940         case 2:   /* use bits [45:34] of the address */
2941                 vector = ((uc_addr->addr_bytes[4] >> 2) |
2942                         (((uint16_t)uc_addr->addr_bytes[5]) << 6));
2943                 break;
2944         case 3:   /* use bits [43:32] of the address */
2945                 vector = ((uc_addr->addr_bytes[4]) |
2946                         (((uint16_t)uc_addr->addr_bytes[5]) << 8));
2947                 break;
2948         default:  /* Invalid mc_filter_type */
2949                 break;
2950         }
2951
2952         /* vector can only be 12-bits or boundary will be exceeded */
2953         vector &= 0xFFF;
2954         return vector;
2955 }
2956
2957 static int
2958 ixgbe_uc_hash_table_set(struct rte_eth_dev *dev,struct ether_addr* mac_addr,
2959                                uint8_t on)
2960 {
2961         uint32_t vector;
2962         uint32_t uta_idx;
2963         uint32_t reg_val;
2964         uint32_t uta_shift;
2965         uint32_t rc;
2966         const uint32_t ixgbe_uta_idx_mask = 0x7F;
2967         const uint32_t ixgbe_uta_bit_shift = 5;
2968         const uint32_t ixgbe_uta_bit_mask = (0x1 << ixgbe_uta_bit_shift) - 1;
2969         const uint32_t bit1 = 0x1;
2970
2971         struct ixgbe_hw *hw =
2972                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2973         struct ixgbe_uta_info *uta_info =
2974                 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private);
2975
2976         /* The UTA table only exists on 82599 hardware and newer */
2977         if (hw->mac.type < ixgbe_mac_82599EB)
2978                 return (-ENOTSUP);
2979
2980         vector = ixgbe_uta_vector(hw,mac_addr);
2981         uta_idx = (vector >> ixgbe_uta_bit_shift) & ixgbe_uta_idx_mask;
2982         uta_shift = vector & ixgbe_uta_bit_mask;
2983
2984         rc = ((uta_info->uta_shadow[uta_idx] >> uta_shift & bit1) != 0);
2985         if(rc == on)
2986                 return 0;
2987
2988         reg_val = IXGBE_READ_REG(hw, IXGBE_UTA(uta_idx));
2989         if (on) {
2990                 uta_info->uta_in_use++;
2991                 reg_val |= (bit1 << uta_shift);
2992                 uta_info->uta_shadow[uta_idx] |= (bit1 << uta_shift);
2993         } else {
2994                 uta_info->uta_in_use--;
2995                 reg_val &= ~(bit1 << uta_shift);
2996                 uta_info->uta_shadow[uta_idx] &= ~(bit1 << uta_shift);
2997         }
2998
2999         IXGBE_WRITE_REG(hw, IXGBE_UTA(uta_idx), reg_val);
3000
3001         if (uta_info->uta_in_use > 0)
3002                 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL,
3003                                 IXGBE_MCSTCTRL_MFE | hw->mac.mc_filter_type);
3004         else
3005                 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL,hw->mac.mc_filter_type);
3006
3007         return 0;
3008 }
3009
3010 static int
3011 ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on)
3012 {
3013         int i;
3014         struct ixgbe_hw *hw =
3015                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3016         struct ixgbe_uta_info *uta_info =
3017                 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private);
3018
3019         /* The UTA table only exists on 82599 hardware and newer */
3020         if (hw->mac.type < ixgbe_mac_82599EB)
3021                 return (-ENOTSUP);
3022
3023         if(on) {
3024                 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
3025                         uta_info->uta_shadow[i] = ~0;
3026                         IXGBE_WRITE_REG(hw, IXGBE_UTA(i), ~0);
3027                 }
3028         } else {
3029                 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
3030                         uta_info->uta_shadow[i] = 0;
3031                         IXGBE_WRITE_REG(hw, IXGBE_UTA(i), 0);
3032                 }
3033         }
3034         return 0;
3035
3036 }
3037 static int
3038 ixgbe_set_pool_rx_mode(struct rte_eth_dev *dev, uint16_t pool,
3039                                uint16_t rx_mask, uint8_t on)
3040 {
3041         int val = 0;
3042
3043         struct ixgbe_hw *hw =
3044                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3045         uint32_t vmolr = IXGBE_READ_REG(hw, IXGBE_VMOLR(pool));
3046
3047         if (hw->mac.type == ixgbe_mac_82598EB) {
3048                 PMD_INIT_LOG(ERR, "setting VF receive mode set should be done"
3049                         " on 82599 hardware and newer\n");
3050                 return (-ENOTSUP);
3051         }
3052         if (ixgbe_vmdq_mode_check(hw) < 0)
3053                 return (-ENOTSUP);
3054
3055         if (rx_mask & ETH_VMDQ_ACCEPT_UNTAG )
3056                 val |= IXGBE_VMOLR_AUPE;
3057         if (rx_mask & ETH_VMDQ_ACCEPT_HASH_MC )
3058                 val |= IXGBE_VMOLR_ROMPE;
3059         if (rx_mask & ETH_VMDQ_ACCEPT_HASH_UC)
3060                 val |= IXGBE_VMOLR_ROPE;
3061         if (rx_mask & ETH_VMDQ_ACCEPT_BROADCAST)
3062                 val |= IXGBE_VMOLR_BAM;
3063         if (rx_mask & ETH_VMDQ_ACCEPT_MULTICAST)
3064                 val |= IXGBE_VMOLR_MPE;
3065
3066         if (on)
3067                 vmolr |= val;
3068         else
3069                 vmolr &= ~val;
3070
3071         IXGBE_WRITE_REG(hw, IXGBE_VMOLR(pool), vmolr);
3072
3073         return 0;
3074 }
3075
3076 static int
3077 ixgbe_set_pool_rx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on)
3078 {
3079         uint32_t reg,addr;
3080         uint32_t val;
3081         const uint8_t bit1 = 0x1;
3082
3083         struct ixgbe_hw *hw =
3084                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3085
3086         if (ixgbe_vmdq_mode_check(hw) < 0)
3087                 return (-ENOTSUP);
3088
3089         addr = IXGBE_VFRE(pool >= ETH_64_POOLS/2);
3090         reg = IXGBE_READ_REG(hw, addr);
3091         val = bit1 << pool;
3092
3093         if (on)
3094                 reg |= val;
3095         else
3096                 reg &= ~val;
3097
3098         IXGBE_WRITE_REG(hw, addr,reg);
3099
3100         return 0;
3101 }
3102
3103 static int
3104 ixgbe_set_pool_tx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on)
3105 {
3106         uint32_t reg,addr;
3107         uint32_t val;
3108         const uint8_t bit1 = 0x1;
3109
3110         struct ixgbe_hw *hw =
3111                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3112
3113         if (ixgbe_vmdq_mode_check(hw) < 0)
3114                 return (-ENOTSUP);
3115
3116         addr = IXGBE_VFTE(pool >= ETH_64_POOLS/2);
3117         reg = IXGBE_READ_REG(hw, addr);
3118         val = bit1 << pool;
3119
3120         if (on)
3121                 reg |= val;
3122         else
3123                 reg &= ~val;
3124
3125         IXGBE_WRITE_REG(hw, addr,reg);
3126
3127         return 0;
3128 }
3129
3130 static int
3131 ixgbe_set_pool_vlan_filter(struct rte_eth_dev *dev, uint16_t vlan,
3132                         uint64_t pool_mask, uint8_t vlan_on)
3133 {
3134         int ret = 0;
3135         uint16_t pool_idx;
3136         struct ixgbe_hw *hw =
3137                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3138
3139         if (ixgbe_vmdq_mode_check(hw) < 0)
3140                 return (-ENOTSUP);
3141         for (pool_idx = 0; pool_idx < ETH_64_POOLS; pool_idx++) {
3142                 if (pool_mask & ((uint64_t)(1ULL << pool_idx)))
3143                         ret = hw->mac.ops.set_vfta(hw,vlan,pool_idx,vlan_on);
3144                         if (ret < 0)
3145                                 return ret;
3146         }
3147
3148         return ret;
3149 }
3150
3151 static int
3152 ixgbe_mirror_rule_set(struct rte_eth_dev *dev,
3153                         struct rte_eth_vmdq_mirror_conf *mirror_conf,
3154                         uint8_t rule_id, uint8_t on)
3155 {
3156         uint32_t mr_ctl,vlvf;
3157         uint32_t mp_lsb = 0;
3158         uint32_t mv_msb = 0;
3159         uint32_t mv_lsb = 0;
3160         uint32_t mp_msb = 0;
3161         uint8_t i = 0;
3162         int reg_index = 0;
3163         uint64_t vlan_mask = 0;
3164
3165         const uint8_t pool_mask_offset = 32;
3166         const uint8_t vlan_mask_offset = 32;
3167         const uint8_t dst_pool_offset = 8;
3168         const uint8_t rule_mr_offset  = 4;
3169         const uint8_t mirror_rule_mask= 0x0F;
3170
3171         struct ixgbe_mirror_info *mr_info =
3172                         (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private));
3173         struct ixgbe_hw *hw =
3174                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3175
3176         if (ixgbe_vmdq_mode_check(hw) < 0)
3177                 return (-ENOTSUP);
3178
3179         /* Check if vlan mask is valid */
3180         if ((mirror_conf->rule_type_mask & ETH_VMDQ_VLAN_MIRROR) && (on)) {
3181                 if (mirror_conf->vlan.vlan_mask == 0)
3182                         return (-EINVAL);
3183         }
3184
3185         /* Check if vlan id is valid and find conresponding VLAN ID index in VLVF */
3186         if (mirror_conf->rule_type_mask & ETH_VMDQ_VLAN_MIRROR) {
3187                 for (i = 0;i < IXGBE_VLVF_ENTRIES; i++) {
3188                         if (mirror_conf->vlan.vlan_mask & (1ULL << i)) {
3189                                 /* search vlan id related pool vlan filter index */
3190                                 reg_index = ixgbe_find_vlvf_slot(hw,
3191                                                 mirror_conf->vlan.vlan_id[i]);
3192                                 if(reg_index < 0)
3193                                         return (-EINVAL);
3194                                 vlvf = IXGBE_READ_REG(hw, IXGBE_VLVF(reg_index));
3195                                 if ((vlvf & IXGBE_VLVF_VIEN) &&
3196                                         ((vlvf & IXGBE_VLVF_VLANID_MASK)
3197                                                 == mirror_conf->vlan.vlan_id[i]))
3198                                         vlan_mask |= (1ULL << reg_index);
3199                                 else
3200                                         return (-EINVAL);
3201                         }
3202                 }
3203
3204                 if (on) {
3205                         mv_lsb = vlan_mask & 0xFFFFFFFF;
3206                         mv_msb = vlan_mask >> vlan_mask_offset;
3207
3208                         mr_info->mr_conf[rule_id].vlan.vlan_mask =
3209                                                 mirror_conf->vlan.vlan_mask;
3210                         for(i = 0 ;i < ETH_VMDQ_MAX_VLAN_FILTERS; i++) {
3211                                 if(mirror_conf->vlan.vlan_mask & (1ULL << i))
3212                                         mr_info->mr_conf[rule_id].vlan.vlan_id[i] =
3213                                                 mirror_conf->vlan.vlan_id[i];
3214                         }
3215                 } else {
3216                         mv_lsb = 0;
3217                         mv_msb = 0;
3218                         mr_info->mr_conf[rule_id].vlan.vlan_mask = 0;
3219                         for(i = 0 ;i < ETH_VMDQ_MAX_VLAN_FILTERS; i++)
3220                                 mr_info->mr_conf[rule_id].vlan.vlan_id[i] = 0;
3221                 }
3222         }
3223
3224         /*
3225          * if enable pool mirror, write related pool mask register,if disable
3226          * pool mirror, clear PFMRVM register
3227          */
3228         if (mirror_conf->rule_type_mask & ETH_VMDQ_POOL_MIRROR) {
3229                 if (on) {
3230                         mp_lsb = mirror_conf->pool_mask & 0xFFFFFFFF;
3231                         mp_msb = mirror_conf->pool_mask >> pool_mask_offset;
3232                         mr_info->mr_conf[rule_id].pool_mask =
3233                                         mirror_conf->pool_mask;
3234
3235                 } else {
3236                         mp_lsb = 0;
3237                         mp_msb = 0;
3238                         mr_info->mr_conf[rule_id].pool_mask = 0;
3239                 }
3240         }
3241
3242         /* read  mirror control register and recalculate it */
3243         mr_ctl = IXGBE_READ_REG(hw,IXGBE_MRCTL(rule_id));
3244
3245         if (on) {
3246                 mr_ctl |= mirror_conf->rule_type_mask;
3247                 mr_ctl &= mirror_rule_mask;
3248                 mr_ctl |= mirror_conf->dst_pool << dst_pool_offset;
3249         } else
3250                 mr_ctl &= ~(mirror_conf->rule_type_mask & mirror_rule_mask);
3251
3252         mr_info->mr_conf[rule_id].rule_type_mask = (uint8_t)(mr_ctl & mirror_rule_mask);
3253         mr_info->mr_conf[rule_id].dst_pool = mirror_conf->dst_pool;
3254
3255         /* write mirrror control  register */
3256         IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl);
3257
3258         /* write pool mirrror control  register */
3259         if (mirror_conf->rule_type_mask & ETH_VMDQ_POOL_MIRROR) {
3260                 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), mp_lsb);
3261                 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset),
3262                                 mp_msb);
3263         }
3264         /* write VLAN mirrror control  register */
3265         if (mirror_conf->rule_type_mask & ETH_VMDQ_VLAN_MIRROR) {
3266                 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), mv_lsb);
3267                 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset),
3268                                 mv_msb);
3269         }
3270
3271         return 0;
3272 }
3273
3274 static int
3275 ixgbe_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t rule_id)
3276 {
3277         int mr_ctl = 0;
3278         uint32_t lsb_val = 0;
3279         uint32_t msb_val = 0;
3280         const uint8_t rule_mr_offset = 4;
3281
3282         struct ixgbe_hw *hw =
3283                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3284         struct ixgbe_mirror_info *mr_info =
3285                 (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private));
3286
3287         if (ixgbe_vmdq_mode_check(hw) < 0)
3288                 return (-ENOTSUP);
3289
3290         memset(&mr_info->mr_conf[rule_id], 0,
3291                 sizeof(struct rte_eth_vmdq_mirror_conf));
3292
3293         /* clear PFVMCTL register */
3294         IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl);
3295
3296         /* clear pool mask register */
3297         IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), lsb_val);
3298         IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset), msb_val);
3299
3300         /* clear vlan mask register */
3301         IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), lsb_val);
3302         IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset), msb_val);
3303
3304         return 0;
3305 }
3306
3307 static int ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev,
3308         uint16_t queue_idx, uint16_t tx_rate)
3309 {
3310         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3311         uint32_t rf_dec, rf_int;
3312         uint32_t bcnrc_val;
3313         uint16_t link_speed = dev->data->dev_link.link_speed;
3314
3315         if (queue_idx >= hw->mac.max_tx_queues)
3316                 return -EINVAL;
3317
3318         if (tx_rate != 0) {
3319                 /* Calculate the rate factor values to set */
3320                 rf_int = (uint32_t)link_speed / (uint32_t)tx_rate;
3321                 rf_dec = (uint32_t)link_speed % (uint32_t)tx_rate;
3322                 rf_dec = (rf_dec << IXGBE_RTTBCNRC_RF_INT_SHIFT) / tx_rate;
3323
3324                 bcnrc_val = IXGBE_RTTBCNRC_RS_ENA;
3325                 bcnrc_val |= ((rf_int << IXGBE_RTTBCNRC_RF_INT_SHIFT) &
3326                                 IXGBE_RTTBCNRC_RF_INT_MASK_M);
3327                 bcnrc_val |= (rf_dec & IXGBE_RTTBCNRC_RF_DEC_MASK);
3328         } else {
3329                 bcnrc_val = 0;
3330         }
3331
3332         /*
3333          * Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
3334          * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported, otherwise
3335          * set as 0x4.
3336          */
3337         if ((dev->data->dev_conf.rxmode.jumbo_frame == 1) &&
3338                 (dev->data->dev_conf.rxmode.max_rx_pkt_len >=
3339                                 IXGBE_MAX_JUMBO_FRAME_SIZE))
3340                 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRM,
3341                         IXGBE_MMW_SIZE_JUMBO_FRAME);
3342         else
3343                 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRM,
3344                         IXGBE_MMW_SIZE_DEFAULT);
3345
3346         /* Set RTTBCNRC of queue X */
3347         IXGBE_WRITE_REG(hw, IXGBE_RTTDQSEL, queue_idx);
3348         IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRC, bcnrc_val);
3349         IXGBE_WRITE_FLUSH(hw);
3350
3351         return 0;
3352 }
3353
3354 static int ixgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf,
3355         uint16_t tx_rate, uint64_t q_msk)
3356 {
3357         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3358         struct ixgbe_vf_info *vfinfo =
3359                 *(IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private));
3360         uint8_t  nb_q_per_pool = RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool;
3361         uint32_t queue_stride =
3362                 IXGBE_MAX_RX_QUEUE_NUM / RTE_ETH_DEV_SRIOV(dev).active;
3363         uint32_t queue_idx = vf * queue_stride, idx = 0, vf_idx;
3364         uint32_t queue_end = queue_idx + nb_q_per_pool - 1;
3365         uint16_t total_rate = 0;
3366
3367         if (queue_end >= hw->mac.max_tx_queues)
3368                 return -EINVAL;
3369
3370         if (vfinfo != NULL) {
3371                 for (vf_idx = 0; vf_idx < dev->pci_dev->max_vfs; vf_idx++) {
3372                         if (vf_idx == vf)
3373                                 continue;
3374                         for (idx = 0; idx < RTE_DIM(vfinfo[vf_idx].tx_rate);
3375                                 idx++)
3376                                 total_rate += vfinfo[vf_idx].tx_rate[idx];
3377                 }
3378         } else
3379                 return -EINVAL;
3380
3381         /* Store tx_rate for this vf. */
3382         for (idx = 0; idx < nb_q_per_pool; idx++) {
3383                 if (((uint64_t)0x1 << idx) & q_msk) {
3384                         if (vfinfo[vf].tx_rate[idx] != tx_rate)
3385                                 vfinfo[vf].tx_rate[idx] = tx_rate;
3386                         total_rate += tx_rate;
3387                 }
3388         }
3389
3390         if (total_rate > dev->data->dev_link.link_speed) {
3391                 /*
3392                  * Reset stored TX rate of the VF if it causes exceed
3393                  * link speed.
3394                  */
3395                 memset(vfinfo[vf].tx_rate, 0, sizeof(vfinfo[vf].tx_rate));
3396                 return -EINVAL;
3397         }
3398
3399         /* Set RTTBCNRC of each queue/pool for vf X  */
3400         for (; queue_idx <= queue_end; queue_idx++) {
3401                 if (0x1 & q_msk)
3402                         ixgbe_set_queue_rate_limit(dev, queue_idx, tx_rate);
3403                 q_msk = q_msk >> 1;
3404         }
3405
3406         return 0;
3407 }
3408
3409 static void
3410 ixgbevf_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
3411                      __attribute__((unused)) uint32_t index,
3412                      __attribute__((unused)) uint32_t pool)
3413 {
3414         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3415         int diag;
3416
3417         /*
3418          * On a 82599 VF, adding again the same MAC addr is not an idempotent
3419          * operation. Trap this case to avoid exhausting the [very limited]
3420          * set of PF resources used to store VF MAC addresses.
3421          */
3422         if (memcmp(hw->mac.perm_addr, mac_addr, sizeof(struct ether_addr)) == 0)
3423                 return;
3424         diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes);
3425         if (diag == 0)
3426                 return;
3427         PMD_DRV_LOG(ERR, "Unable to add MAC address - diag=%d", diag);
3428 }
3429
3430 static void
3431 ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index)
3432 {
3433         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3434         struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr;
3435         struct ether_addr *mac_addr;
3436         uint32_t i;
3437         int diag;
3438
3439         /*
3440          * The IXGBE_VF_SET_MACVLAN command of the ixgbe-pf driver does
3441          * not support the deletion of a given MAC address.
3442          * Instead, it imposes to delete all MAC addresses, then to add again
3443          * all MAC addresses with the exception of the one to be deleted.
3444          */
3445         (void) ixgbevf_set_uc_addr_vf(hw, 0, NULL);
3446
3447         /*
3448          * Add again all MAC addresses, with the exception of the deleted one
3449          * and of the permanent MAC address.
3450          */
3451         for (i = 0, mac_addr = dev->data->mac_addrs;
3452              i < hw->mac.num_rar_entries; i++, mac_addr++) {
3453                 /* Skip the deleted MAC address */
3454                 if (i == index)
3455                         continue;
3456                 /* Skip NULL MAC addresses */
3457                 if (is_zero_ether_addr(mac_addr))
3458                         continue;
3459                 /* Skip the permanent MAC address */
3460                 if (memcmp(perm_addr, mac_addr, sizeof(struct ether_addr)) == 0)
3461                         continue;
3462                 diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes);
3463                 if (diag != 0)
3464                         PMD_DRV_LOG(ERR,
3465                                     "Adding again MAC address "
3466                                     "%02x:%02x:%02x:%02x:%02x:%02x failed "
3467                                     "diag=%d",
3468                                     mac_addr->addr_bytes[0],
3469                                     mac_addr->addr_bytes[1],
3470                                     mac_addr->addr_bytes[2],
3471                                     mac_addr->addr_bytes[3],
3472                                     mac_addr->addr_bytes[4],
3473                                     mac_addr->addr_bytes[5],
3474                                     diag);
3475         }
3476 }
3477
3478 /*
3479  * add syn filter
3480  *
3481  * @param
3482  * dev: Pointer to struct rte_eth_dev.
3483  * filter: ponter to the filter that will be added.
3484  * rx_queue: the queue id the filter assigned to.
3485  *
3486  * @return
3487  *    - On success, zero.
3488  *    - On failure, a negative value.
3489  */
3490 static int
3491 ixgbe_add_syn_filter(struct rte_eth_dev *dev,
3492                         struct rte_syn_filter *filter, uint16_t rx_queue)
3493 {
3494         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3495         uint32_t synqf;
3496
3497         if (hw->mac.type != ixgbe_mac_82599EB)
3498                 return -ENOSYS;
3499
3500         if (rx_queue >= IXGBE_MAX_RX_QUEUE_NUM)
3501                 return -EINVAL;
3502
3503         synqf = IXGBE_READ_REG(hw, IXGBE_SYNQF);
3504
3505         if (synqf & IXGBE_SYN_FILTER_ENABLE)
3506                 return -EINVAL;
3507
3508         synqf = (uint32_t)(((rx_queue << IXGBE_SYN_FILTER_QUEUE_SHIFT) &
3509                 IXGBE_SYN_FILTER_QUEUE) | IXGBE_SYN_FILTER_ENABLE);
3510
3511         if (filter->hig_pri)
3512                 synqf |= IXGBE_SYN_FILTER_SYNQFP;
3513         else
3514                 synqf &= ~IXGBE_SYN_FILTER_SYNQFP;
3515
3516         IXGBE_WRITE_REG(hw, IXGBE_SYNQF, synqf);
3517         return 0;
3518 }
3519
3520 /*
3521  * remove syn filter
3522  *
3523  * @param
3524  * dev: Pointer to struct rte_eth_dev.
3525  *
3526  * @return
3527  *    - On success, zero.
3528  *    - On failure, a negative value.
3529  */
3530 static int
3531 ixgbe_remove_syn_filter(struct rte_eth_dev *dev)
3532 {
3533         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3534         uint32_t synqf;
3535
3536         if (hw->mac.type != ixgbe_mac_82599EB)
3537                 return -ENOSYS;
3538
3539         synqf = IXGBE_READ_REG(hw, IXGBE_SYNQF);
3540
3541         synqf &= ~(IXGBE_SYN_FILTER_QUEUE | IXGBE_SYN_FILTER_ENABLE);
3542
3543         IXGBE_WRITE_REG(hw, IXGBE_SYNQF, synqf);
3544         return 0;
3545 }
3546
3547 /*
3548  * get the syn filter's info
3549  *
3550  * @param
3551  * dev: Pointer to struct rte_eth_dev.
3552  * filter: ponter to the filter that returns.
3553  * *rx_queue: pointer to the queue id the filter assigned to.
3554  *
3555  * @return
3556  *    - On success, zero.
3557  *    - On failure, a negative value.
3558  */
3559 static int
3560 ixgbe_get_syn_filter(struct rte_eth_dev *dev,
3561                         struct rte_syn_filter *filter, uint16_t *rx_queue)
3562
3563 {
3564         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3565         uint32_t synqf;
3566
3567         if (hw->mac.type != ixgbe_mac_82599EB)
3568                 return -ENOSYS;
3569
3570         synqf = IXGBE_READ_REG(hw, IXGBE_SYNQF);
3571         if (synqf & IXGBE_SYN_FILTER_ENABLE) {
3572                 filter->hig_pri = (synqf & IXGBE_SYN_FILTER_SYNQFP) ? 1 : 0;
3573                 *rx_queue = (uint16_t)((synqf & IXGBE_SYN_FILTER_QUEUE) >> 1);
3574                 return 0;
3575         }
3576         return -ENOENT;
3577 }
3578
3579 /*
3580  * add an ethertype filter
3581  *
3582  * @param
3583  * dev: Pointer to struct rte_eth_dev.
3584  * index: the index the filter allocates.
3585  * filter: ponter to the filter that will be added.
3586  * rx_queue: the queue id the filter assigned to.
3587  *
3588  * @return
3589  *    - On success, zero.
3590  *    - On failure, a negative value.
3591  */
3592 static int
3593 ixgbe_add_ethertype_filter(struct rte_eth_dev *dev,
3594                         uint16_t index, struct rte_ethertype_filter *filter,
3595                         uint16_t rx_queue)
3596 {
3597         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3598         uint32_t etqf, etqs = 0;
3599
3600         if (hw->mac.type != ixgbe_mac_82599EB)
3601                 return -ENOSYS;
3602
3603         if (index >= IXGBE_MAX_ETQF_FILTERS ||
3604                 rx_queue >= IXGBE_MAX_RX_QUEUE_NUM)
3605                 return -EINVAL;
3606
3607         etqf = IXGBE_READ_REG(hw, IXGBE_ETQF(index));
3608         if (etqf & IXGBE_ETQF_FILTER_EN)
3609                 return -EINVAL;  /* filter index is in use. */
3610
3611         etqf = 0;
3612         etqf |= IXGBE_ETQF_FILTER_EN;
3613         etqf |= (uint32_t)filter->ethertype;
3614
3615         if (filter->priority_en) {
3616                 if (filter->priority > IXGBE_ETQF_MAX_PRI)
3617                         return -EINVAL;
3618                 etqf |= (uint32_t)((filter->priority << IXGBE_ETQF_SHIFT) & IXGBE_ETQF_UP);
3619                 etqf |= IXGBE_ETQF_UP_EN;
3620         }
3621         etqs |= (uint32_t)((rx_queue << IXGBE_ETQS_RX_QUEUE_SHIFT) & IXGBE_ETQS_RX_QUEUE);
3622         etqs |= IXGBE_ETQS_QUEUE_EN;
3623
3624         IXGBE_WRITE_REG(hw, IXGBE_ETQF(index), etqf);
3625         IXGBE_WRITE_REG(hw, IXGBE_ETQS(index), etqs);
3626         return 0;
3627 }
3628
3629 /*
3630  * remove an ethertype filter
3631  *
3632  * @param
3633  * dev: Pointer to struct rte_eth_dev.
3634  * index: the index the filter allocates.
3635  *
3636  * @return
3637  *    - On success, zero.
3638  *    - On failure, a negative value.
3639  */
3640 static int
3641 ixgbe_remove_ethertype_filter(struct rte_eth_dev *dev,
3642                         uint16_t index)
3643 {
3644         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3645
3646         if (hw->mac.type != ixgbe_mac_82599EB)
3647                 return -ENOSYS;
3648
3649         if (index >= IXGBE_MAX_ETQF_FILTERS)
3650                 return -EINVAL;
3651
3652         IXGBE_WRITE_REG(hw, IXGBE_ETQF(index), 0);
3653         IXGBE_WRITE_REG(hw, IXGBE_ETQS(index), 0);
3654
3655         return 0;
3656 }
3657
3658 /*
3659  * get an ethertype filter
3660  *
3661  * @param
3662  * dev: Pointer to struct rte_eth_dev.
3663  * index: the index the filter allocates.
3664  * filter: ponter to the filter that will be gotten.
3665  * *rx_queue: the ponited of the queue id the filter assigned to.
3666  *
3667  * @return
3668  *    - On success, zero.
3669  *    - On failure, a negative value.
3670  */
3671 static int
3672 ixgbe_get_ethertype_filter(struct rte_eth_dev *dev,
3673                         uint16_t index, struct rte_ethertype_filter *filter,
3674                         uint16_t *rx_queue)
3675 {
3676         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3677         uint32_t etqf, etqs;
3678
3679         if (hw->mac.type != ixgbe_mac_82599EB)
3680                 return -ENOSYS;
3681
3682         if (index >= IXGBE_MAX_ETQF_FILTERS)
3683                 return -EINVAL;
3684
3685         etqf = IXGBE_READ_REG(hw, IXGBE_ETQF(index));
3686         etqs = IXGBE_READ_REG(hw, IXGBE_ETQS(index));
3687         if (etqf & IXGBE_ETQF_FILTER_EN) {
3688                 filter->ethertype = etqf & IXGBE_ETQF_ETHERTYPE;
3689                 filter->priority_en = (etqf & IXGBE_ETQF_UP_EN) ? 1 : 0;
3690                 if (filter->priority_en)
3691                         filter->priority = (etqf & IXGBE_ETQF_UP) >> 16;
3692                 *rx_queue = (etqs & IXGBE_ETQS_RX_QUEUE) >> IXGBE_ETQS_RX_QUEUE_SHIFT;
3693                 return 0;
3694         }
3695         return -ENOENT;
3696 }
3697
3698 static inline enum ixgbe_5tuple_protocol
3699 convert_protocol_type(uint8_t protocol_value)
3700 {
3701         if (protocol_value == IPPROTO_TCP)
3702                 return IXGBE_FILTER_PROTOCOL_TCP;
3703         else if (protocol_value == IPPROTO_UDP)
3704                 return IXGBE_FILTER_PROTOCOL_UDP;
3705         else if (protocol_value == IPPROTO_SCTP)
3706                 return IXGBE_FILTER_PROTOCOL_SCTP;
3707         else
3708                 return IXGBE_FILTER_PROTOCOL_NONE;
3709 }
3710
3711 static inline uint8_t
3712 revert_protocol_type(enum ixgbe_5tuple_protocol protocol)
3713 {
3714         if (protocol == IXGBE_FILTER_PROTOCOL_TCP)
3715                 return IPPROTO_TCP;
3716         else if (protocol == IXGBE_FILTER_PROTOCOL_UDP)
3717                 return IPPROTO_UDP;
3718         else if (protocol == IXGBE_FILTER_PROTOCOL_SCTP)
3719                 return IPPROTO_SCTP;
3720         else
3721                 return 0;
3722 }
3723
3724 /*
3725  * add a 5tuple filter
3726  *
3727  * @param
3728  * dev: Pointer to struct rte_eth_dev.
3729  * index: the index the filter allocates.
3730  * filter: ponter to the filter that will be added.
3731  * rx_queue: the queue id the filter assigned to.
3732  *
3733  * @return
3734  *    - On success, zero.
3735  *    - On failure, a negative value.
3736  */
3737 static int
3738 ixgbe_add_5tuple_filter(struct rte_eth_dev *dev, uint16_t index,
3739                         struct rte_5tuple_filter *filter, uint16_t rx_queue)
3740 {
3741         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3742         uint32_t ftqf, sdpqf = 0;
3743         uint32_t l34timir = 0;
3744         uint8_t mask = 0xff;
3745
3746         if (hw->mac.type != ixgbe_mac_82599EB)
3747                 return -ENOSYS;
3748
3749         if (index >= IXGBE_MAX_FTQF_FILTERS ||
3750                 rx_queue >= IXGBE_MAX_RX_QUEUE_NUM ||
3751                 filter->priority > IXGBE_5TUPLE_MAX_PRI ||
3752                 filter->priority < IXGBE_5TUPLE_MIN_PRI)
3753                 return -EINVAL;  /* filter index is out of range. */
3754
3755         if (filter->tcp_flags) {
3756                 PMD_INIT_LOG(INFO, "82599EB not tcp flags in 5tuple");
3757                 return -EINVAL;
3758         }
3759
3760         ftqf = IXGBE_READ_REG(hw, IXGBE_FTQF(index));
3761         if (ftqf & IXGBE_FTQF_QUEUE_ENABLE)
3762                 return -EINVAL;  /* filter index is in use. */
3763
3764         ftqf = 0;
3765         sdpqf = (uint32_t)(filter->dst_port << IXGBE_SDPQF_DSTPORT_SHIFT);
3766         sdpqf = sdpqf | (filter->src_port & IXGBE_SDPQF_SRCPORT);
3767
3768         ftqf |= (uint32_t)(convert_protocol_type(filter->protocol) &
3769                 IXGBE_FTQF_PROTOCOL_MASK);
3770         ftqf |= (uint32_t)((filter->priority & IXGBE_FTQF_PRIORITY_MASK) <<
3771                 IXGBE_FTQF_PRIORITY_SHIFT);
3772         if (filter->src_ip_mask == 0) /* 0 means compare. */
3773                 mask &= IXGBE_FTQF_SOURCE_ADDR_MASK;
3774         if (filter->dst_ip_mask == 0)
3775                 mask &= IXGBE_FTQF_DEST_ADDR_MASK;
3776         if (filter->src_port_mask == 0)
3777                 mask &= IXGBE_FTQF_SOURCE_PORT_MASK;
3778         if (filter->dst_port_mask == 0)
3779                 mask &= IXGBE_FTQF_DEST_PORT_MASK;
3780         if (filter->protocol_mask == 0)
3781                 mask &= IXGBE_FTQF_PROTOCOL_COMP_MASK;
3782         ftqf |= mask << IXGBE_FTQF_5TUPLE_MASK_SHIFT;
3783         ftqf |= IXGBE_FTQF_POOL_MASK_EN;
3784         ftqf |= IXGBE_FTQF_QUEUE_ENABLE;
3785
3786         IXGBE_WRITE_REG(hw, IXGBE_DAQF(index), filter->dst_ip);
3787         IXGBE_WRITE_REG(hw, IXGBE_SAQF(index), filter->src_ip);
3788         IXGBE_WRITE_REG(hw, IXGBE_SDPQF(index), sdpqf);
3789         IXGBE_WRITE_REG(hw, IXGBE_FTQF(index), ftqf);
3790
3791         l34timir |= IXGBE_L34T_IMIR_RESERVE;
3792         l34timir |= (uint32_t)(rx_queue << IXGBE_L34T_IMIR_QUEUE_SHIFT);
3793         IXGBE_WRITE_REG(hw, IXGBE_L34T_IMIR(index), l34timir);
3794         return 0;
3795 }
3796
3797 /*
3798  * remove a 5tuple filter
3799  *
3800  * @param
3801  * dev: Pointer to struct rte_eth_dev.
3802  * index: the index the filter allocates.
3803  *
3804  * @return
3805  *    - On success, zero.
3806  *    - On failure, a negative value.
3807  */
3808 static int
3809 ixgbe_remove_5tuple_filter(struct rte_eth_dev *dev,
3810                         uint16_t index)
3811 {
3812         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3813
3814         if (hw->mac.type != ixgbe_mac_82599EB)
3815                 return -ENOSYS;
3816
3817         if (index >= IXGBE_MAX_FTQF_FILTERS)
3818                 return -EINVAL;  /* filter index is out of range. */
3819
3820         IXGBE_WRITE_REG(hw, IXGBE_DAQF(index), 0);
3821         IXGBE_WRITE_REG(hw, IXGBE_SAQF(index), 0);
3822         IXGBE_WRITE_REG(hw, IXGBE_SDPQF(index), 0);
3823         IXGBE_WRITE_REG(hw, IXGBE_FTQF(index), 0);
3824         IXGBE_WRITE_REG(hw, IXGBE_L34T_IMIR(index), 0);
3825         return 0;
3826 }
3827
3828 /*
3829  * get a 5tuple filter
3830  *
3831  * @param
3832  * dev: Pointer to struct rte_eth_dev.
3833  * index: the index the filter allocates
3834  * filter: ponter to the filter that returns.
3835  * *rx_queue: pointer of the queue id the filter assigned to.
3836  *
3837  * @return
3838  *    - On success, zero.
3839  *    - On failure, a negative value.
3840  */
3841 static int
3842 ixgbe_get_5tuple_filter(struct rte_eth_dev *dev, uint16_t index,
3843                         struct rte_5tuple_filter *filter, uint16_t *rx_queue)
3844 {
3845         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3846         uint32_t sdpqf, ftqf, l34timir;
3847         uint8_t mask;
3848         enum ixgbe_5tuple_protocol proto;
3849
3850         if (hw->mac.type != ixgbe_mac_82599EB)
3851                 return -ENOSYS;
3852
3853         if (index >= IXGBE_MAX_FTQF_FILTERS)
3854                 return -EINVAL;  /* filter index is out of range. */
3855
3856         ftqf = IXGBE_READ_REG(hw, IXGBE_FTQF(index));
3857         if (ftqf & IXGBE_FTQF_QUEUE_ENABLE) {
3858                 proto = (enum ixgbe_5tuple_protocol)(ftqf & IXGBE_FTQF_PROTOCOL_MASK);
3859                 filter->protocol = revert_protocol_type(proto);
3860                 filter->priority = (ftqf >> IXGBE_FTQF_PRIORITY_SHIFT) &
3861                                         IXGBE_FTQF_PRIORITY_MASK;
3862                 mask = (uint8_t)((ftqf >> IXGBE_FTQF_5TUPLE_MASK_SHIFT) &
3863                                         IXGBE_FTQF_5TUPLE_MASK_MASK);
3864                 filter->src_ip_mask =
3865                         (mask & IXGBE_FTQF_SOURCE_ADDR_MASK) ? 1 : 0;
3866                 filter->dst_ip_mask =
3867                         (mask & IXGBE_FTQF_DEST_ADDR_MASK) ? 1 : 0;
3868                 filter->src_port_mask =
3869                         (mask & IXGBE_FTQF_SOURCE_PORT_MASK) ? 1 : 0;
3870                 filter->dst_port_mask =
3871                         (mask & IXGBE_FTQF_DEST_PORT_MASK) ? 1 : 0;
3872                 filter->protocol_mask =
3873                         (mask & IXGBE_FTQF_PROTOCOL_COMP_MASK) ? 1 : 0;
3874
3875                 sdpqf = IXGBE_READ_REG(hw, IXGBE_SDPQF(index));
3876                 filter->dst_port = (sdpqf & IXGBE_SDPQF_DSTPORT) >>
3877                                         IXGBE_SDPQF_DSTPORT_SHIFT;
3878                 filter->src_port = sdpqf & IXGBE_SDPQF_SRCPORT;
3879                 filter->dst_ip = IXGBE_READ_REG(hw, IXGBE_DAQF(index));
3880                 filter->src_ip = IXGBE_READ_REG(hw, IXGBE_SAQF(index));
3881
3882                 l34timir = IXGBE_READ_REG(hw, IXGBE_L34T_IMIR(index));
3883                 *rx_queue = (l34timir & IXGBE_L34T_IMIR_QUEUE) >>
3884                                         IXGBE_L34T_IMIR_QUEUE_SHIFT;
3885                 return 0;
3886         }
3887         return -ENOENT;
3888 }
3889
3890 static struct rte_driver rte_ixgbe_driver = {
3891         .type = PMD_PDEV,
3892         .init = rte_ixgbe_pmd_init,
3893 };
3894
3895 static struct rte_driver rte_ixgbevf_driver = {
3896         .type = PMD_PDEV,
3897         .init = rte_ixgbevf_pmd_init,
3898 };
3899
3900 PMD_REGISTER_DRIVER(rte_ixgbe_driver);
3901 PMD_REGISTER_DRIVER(rte_ixgbevf_driver);