drivers/net/ixgbe/ixgbe_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <inttypes.h>
  42 #include <netinet/in.h>
  43 #include <rte_byteorder.h>
  44 #include <rte_common.h>
  45 #include <rte_cycles.h>
  46
  47 #include <rte_interrupts.h>
  48 #include <rte_log.h>
  49 #include <rte_debug.h>
  50 #include <rte_pci.h>
  51 #include <rte_atomic.h>
  52 #include <rte_branch_prediction.h>
  53 #include <rte_memory.h>
  54 #include <rte_memzone.h>
  55 #include <rte_eal.h>
  56 #include <rte_alarm.h>
  57 #include <rte_ether.h>
  58 #include <rte_ethdev.h>
  59 #include <rte_atomic.h>
  60 #include <rte_malloc.h>
  61 #include <rte_random.h>
  62 #include <rte_dev.h>
  63
  64 #include "ixgbe_logs.h"
  65 #include "base/ixgbe_api.h"
  66 #include "base/ixgbe_vf.h"
  67 #include "base/ixgbe_common.h"
  68 #include "ixgbe_ethdev.h"
  69 #include "ixgbe_bypass.h"
  70 #include "ixgbe_rxtx.h"
  71 #include "base/ixgbe_type.h"
  72 #include "base/ixgbe_phy.h"
  73 #include "ixgbe_regs.h"
  74
  75 /*
  76  * High threshold controlling when to start sending XOFF frames. Must be at
  77  * least 8 bytes less than receive packet buffer size. This value is in units
  78  * of 1024 bytes.
  79  */
  80 #define IXGBE_FC_HI    0x80
  81
  82 /*
  83  * Low threshold controlling when to start sending XON frames. This value is
  84  * in units of 1024 bytes.
  85  */
  86 #define IXGBE_FC_LO    0x40
  87
  88 /* Default minimum inter-interrupt interval for EITR configuration */
  89 #define IXGBE_MIN_INTER_INTERRUPT_INTERVAL_DEFAULT    0x79E
  90
  91 /* Timer value included in XOFF frames. */
  92 #define IXGBE_FC_PAUSE 0x680
  93
  94 #define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */
  95 #define IXGBE_LINK_UP_CHECK_TIMEOUT   1000 /* ms */
  96 #define IXGBE_VMDQ_NUM_UC_MAC         4096 /* Maximum nb. of UC MAC addr. */
  97
  98 #define IXGBE_MMW_SIZE_DEFAULT        0x4
  99 #define IXGBE_MMW_SIZE_JUMBO_FRAME    0x14
 100 #define IXGBE_MAX_RING_DESC           4096 /* replicate define from rxtx */
 101
 102 /*
 103  *  Default values for RX/TX configuration
 104  */
 105 #define IXGBE_DEFAULT_RX_FREE_THRESH  32
 106 #define IXGBE_DEFAULT_RX_PTHRESH      8
 107 #define IXGBE_DEFAULT_RX_HTHRESH      8
 108 #define IXGBE_DEFAULT_RX_WTHRESH      0
 109
 110 #define IXGBE_DEFAULT_TX_FREE_THRESH  32
 111 #define IXGBE_DEFAULT_TX_PTHRESH      32
 112 #define IXGBE_DEFAULT_TX_HTHRESH      0
 113 #define IXGBE_DEFAULT_TX_WTHRESH      0
 114 #define IXGBE_DEFAULT_TX_RSBIT_THRESH 32
 115
 116 /* Bit shift and mask */
 117 #define IXGBE_4_BIT_WIDTH  (CHAR_BIT / 2)
 118 #define IXGBE_4_BIT_MASK   RTE_LEN2MASK(IXGBE_4_BIT_WIDTH, uint8_t)
 119 #define IXGBE_8_BIT_WIDTH  CHAR_BIT
 120 #define IXGBE_8_BIT_MASK   UINT8_MAX
 121
 122 #define IXGBEVF_PMD_NAME "rte_ixgbevf_pmd" /* PMD name */
 123
 124 #define IXGBE_QUEUE_STAT_COUNTERS (sizeof(hw_stats->qprc) / sizeof(hw_stats->qprc[0]))
 125
 126 #define IXGBE_HKEY_MAX_INDEX 10
 127
 128 /* Additional timesync values. */
 129 #define NSEC_PER_SEC             1000000000L
 130 #define IXGBE_INCVAL_10GB        0x66666666
 131 #define IXGBE_INCVAL_1GB         0x40000000
 132 #define IXGBE_INCVAL_100         0x50000000
 133 #define IXGBE_INCVAL_SHIFT_10GB  28
 134 #define IXGBE_INCVAL_SHIFT_1GB   24
 135 #define IXGBE_INCVAL_SHIFT_100   21
 136 #define IXGBE_INCVAL_SHIFT_82599 7
 137 #define IXGBE_INCPER_SHIFT_82599 24
 138
 139 #define IXGBE_CYCLECOUNTER_MASK   0xffffffffffffffffULL
 140
 141 #define IXGBE_VT_CTL_POOLING_MODE_MASK         0x00030000
 142 #define IXGBE_VT_CTL_POOLING_MODE_ETAG         0x00010000
 143 #define DEFAULT_ETAG_ETYPE                     0x893f
 144 #define IXGBE_ETAG_ETYPE                       0x00005084
 145 #define IXGBE_ETAG_ETYPE_MASK                  0x0000ffff
 146 #define IXGBE_ETAG_ETYPE_VALID                 0x80000000
 147 #define IXGBE_RAH_ADTYPE                       0x40000000
 148 #define IXGBE_RAL_ETAG_FILTER_MASK             0x00003fff
 149 #define IXGBE_VMVIR_TAGA_MASK                  0x18000000
 150 #define IXGBE_VMVIR_TAGA_ETAG_INSERT           0x08000000
 151 #define IXGBE_VMTIR(_i) (0x00017000 + ((_i) * 4)) /* 64 of these (0-63) */
 152 #define IXGBE_QDE_STRIP_TAG                    0x00000004
 153 #define IXGBE_VTEICR_MASK                      0x07
 154
 155 enum ixgbevf_xcast_modes {
 156         IXGBEVF_XCAST_MODE_NONE = 0,
 157         IXGBEVF_XCAST_MODE_MULTI,
 158         IXGBEVF_XCAST_MODE_ALLMULTI,
 159 };
 160
 161 static int eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev);
 162 static int eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev);
 163 static int  ixgbe_dev_configure(struct rte_eth_dev *dev);
 164 static int  ixgbe_dev_start(struct rte_eth_dev *dev);
 165 static void ixgbe_dev_stop(struct rte_eth_dev *dev);
 166 static int  ixgbe_dev_set_link_up(struct rte_eth_dev *dev);
 167 static int  ixgbe_dev_set_link_down(struct rte_eth_dev *dev);
 168 static void ixgbe_dev_close(struct rte_eth_dev *dev);
 169 static void ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev);
 170 static void ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev);
 171 static void ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev);
 172 static void ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev);
 173 static int ixgbe_dev_link_update(struct rte_eth_dev *dev,
 174                                 int wait_to_complete);
 175 static void ixgbe_dev_stats_get(struct rte_eth_dev *dev,
 176                                 struct rte_eth_stats *stats);
 177 static int ixgbe_dev_xstats_get(struct rte_eth_dev *dev,
 178                                 struct rte_eth_xstat *xstats, unsigned n);
 179 static int ixgbevf_dev_xstats_get(struct rte_eth_dev *dev,
 180                                   struct rte_eth_xstat *xstats, unsigned n);
 181 static void ixgbe_dev_stats_reset(struct rte_eth_dev *dev);
 182 static void ixgbe_dev_xstats_reset(struct rte_eth_dev *dev);
 183 static int ixgbe_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
 184         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit);
 185 static int ixgbevf_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
 186         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit);
 187 static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 188                                              uint16_t queue_id,
 189                                              uint8_t stat_idx,
 190                                              uint8_t is_rx);
 191 static void ixgbe_dev_info_get(struct rte_eth_dev *dev,
 192                                struct rte_eth_dev_info *dev_info);
 193 static const uint32_t *ixgbe_dev_supported_ptypes_get(struct rte_eth_dev *dev);
 194 static void ixgbevf_dev_info_get(struct rte_eth_dev *dev,
 195                                  struct rte_eth_dev_info *dev_info);
 196 static int ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 197
 198 static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev,
 199                 uint16_t vlan_id, int on);
 200 static int ixgbe_vlan_tpid_set(struct rte_eth_dev *dev,
 201                                enum rte_vlan_type vlan_type,
 202                                uint16_t tpid_id);
 203 static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev,
 204                 uint16_t queue, bool on);
 205 static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue,
 206                 int on);
 207 static void ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 208 static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue);
 209 static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue);
 210 static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 211 static void ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 212
 213 static int ixgbe_dev_led_on(struct rte_eth_dev *dev);
 214 static int ixgbe_dev_led_off(struct rte_eth_dev *dev);
 215 static int ixgbe_flow_ctrl_get(struct rte_eth_dev *dev,
 216                                struct rte_eth_fc_conf *fc_conf);
 217 static int ixgbe_flow_ctrl_set(struct rte_eth_dev *dev,
 218                                struct rte_eth_fc_conf *fc_conf);
 219 static int ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev,
 220                 struct rte_eth_pfc_conf *pfc_conf);
 221 static int ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
 222                         struct rte_eth_rss_reta_entry64 *reta_conf,
 223                         uint16_t reta_size);
 224 static int ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
 225                         struct rte_eth_rss_reta_entry64 *reta_conf,
 226                         uint16_t reta_size);
 227 static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev);
 228 static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev);
 229 static int ixgbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 230 static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev);
 231 static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev);
 232 static void ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle,
 233                 void *param);
 234 static void ixgbe_dev_interrupt_delayed_handler(void *param);
 235 static void ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 236                 uint32_t index, uint32_t pool);
 237 static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index);
 238 static void ixgbe_set_default_mac_addr(struct rte_eth_dev *dev,
 239                                            struct ether_addr *mac_addr);
 240 static void ixgbe_dcb_init(struct ixgbe_hw *hw, struct ixgbe_dcb_config *dcb_config);
 241
 242 /* For Virtual Function support */
 243 static int eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev);
 244 static int eth_ixgbevf_dev_uninit(struct rte_eth_dev *eth_dev);
 245 static int  ixgbevf_dev_configure(struct rte_eth_dev *dev);
 246 static int  ixgbevf_dev_start(struct rte_eth_dev *dev);
 247 static void ixgbevf_dev_stop(struct rte_eth_dev *dev);
 248 static void ixgbevf_dev_close(struct rte_eth_dev *dev);
 249 static void ixgbevf_intr_disable(struct ixgbe_hw *hw);
 250 static void ixgbevf_intr_enable(struct ixgbe_hw *hw);
 251 static void ixgbevf_dev_stats_get(struct rte_eth_dev *dev,
 252                 struct rte_eth_stats *stats);
 253 static void ixgbevf_dev_stats_reset(struct rte_eth_dev *dev);
 254 static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev,
 255                 uint16_t vlan_id, int on);
 256 static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev,
 257                 uint16_t queue, int on);
 258 static void ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 259 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 260 static int ixgbevf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev,
 261                                             uint16_t queue_id);
 262 static int ixgbevf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev,
 263                                              uint16_t queue_id);
 264 static void ixgbevf_set_ivar_map(struct ixgbe_hw *hw, int8_t direction,
 265                                  uint8_t queue, uint8_t msix_vector);
 266 static void ixgbevf_configure_msix(struct rte_eth_dev *dev);
 267 static void ixgbevf_dev_allmulticast_enable(struct rte_eth_dev *dev);
 268 static void ixgbevf_dev_allmulticast_disable(struct rte_eth_dev *dev);
 269
 270 /* For Eth VMDQ APIs support */
 271 static int ixgbe_uc_hash_table_set(struct rte_eth_dev *dev, struct
 272                 ether_addr * mac_addr, uint8_t on);
 273 static int ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on);
 274 static int  ixgbe_set_pool_rx_mode(struct rte_eth_dev *dev,  uint16_t pool,
 275                 uint16_t rx_mask, uint8_t on);
 276 static int ixgbe_set_pool_rx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on);
 277 static int ixgbe_set_pool_tx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on);
 278 static int ixgbe_set_pool_vlan_filter(struct rte_eth_dev *dev, uint16_t vlan,
 279                 uint64_t pool_mask, uint8_t vlan_on);
 280 static int ixgbe_mirror_rule_set(struct rte_eth_dev *dev,
 281                 struct rte_eth_mirror_conf *mirror_conf,
 282                 uint8_t rule_id, uint8_t on);
 283 static int ixgbe_mirror_rule_reset(struct rte_eth_dev *dev,
 284                 uint8_t rule_id);
 285 static int ixgbe_dev_rx_queue_intr_enable(struct rte_eth_dev *dev,
 286                                           uint16_t queue_id);
 287 static int ixgbe_dev_rx_queue_intr_disable(struct rte_eth_dev *dev,
 288                                            uint16_t queue_id);
 289 static void ixgbe_set_ivar_map(struct ixgbe_hw *hw, int8_t direction,
 290                                uint8_t queue, uint8_t msix_vector);
 291 static void ixgbe_configure_msix(struct rte_eth_dev *dev);
 292
 293 static int ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev,
 294                 uint16_t queue_idx, uint16_t tx_rate);
 295 static int ixgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf,
 296                 uint16_t tx_rate, uint64_t q_msk);
 297
 298 static void ixgbevf_add_mac_addr(struct rte_eth_dev *dev,
 299                                  struct ether_addr *mac_addr,
 300                                  uint32_t index, uint32_t pool);
 301 static void ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index);
 302 static void ixgbevf_set_default_mac_addr(struct rte_eth_dev *dev,
 303                                              struct ether_addr *mac_addr);
 304 static int ixgbe_syn_filter_set(struct rte_eth_dev *dev,
 305                         struct rte_eth_syn_filter *filter,
 306                         bool add);
 307 static int ixgbe_syn_filter_get(struct rte_eth_dev *dev,
 308                         struct rte_eth_syn_filter *filter);
 309 static int ixgbe_syn_filter_handle(struct rte_eth_dev *dev,
 310                         enum rte_filter_op filter_op,
 311                         void *arg);
 312 static int ixgbe_add_5tuple_filter(struct rte_eth_dev *dev,
 313                         struct ixgbe_5tuple_filter *filter);
 314 static void ixgbe_remove_5tuple_filter(struct rte_eth_dev *dev,
 315                         struct ixgbe_5tuple_filter *filter);
 316 static int ixgbe_add_del_ntuple_filter(struct rte_eth_dev *dev,
 317                         struct rte_eth_ntuple_filter *filter,
 318                         bool add);
 319 static int ixgbe_ntuple_filter_handle(struct rte_eth_dev *dev,
 320                                 enum rte_filter_op filter_op,
 321                                 void *arg);
 322 static int ixgbe_get_ntuple_filter(struct rte_eth_dev *dev,
 323                         struct rte_eth_ntuple_filter *filter);
 324 static int ixgbe_add_del_ethertype_filter(struct rte_eth_dev *dev,
 325                         struct rte_eth_ethertype_filter *filter,
 326                         bool add);
 327 static int ixgbe_ethertype_filter_handle(struct rte_eth_dev *dev,
 328                                 enum rte_filter_op filter_op,
 329                                 void *arg);
 330 static int ixgbe_get_ethertype_filter(struct rte_eth_dev *dev,
 331                         struct rte_eth_ethertype_filter *filter);
 332 static int ixgbe_dev_filter_ctrl(struct rte_eth_dev *dev,
 333                      enum rte_filter_type filter_type,
 334                      enum rte_filter_op filter_op,
 335                      void *arg);
 336 static int ixgbevf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu);
 337
 338 static int ixgbe_dev_set_mc_addr_list(struct rte_eth_dev *dev,
 339                                       struct ether_addr *mc_addr_set,
 340                                       uint32_t nb_mc_addr);
 341 static int ixgbe_dev_get_dcb_info(struct rte_eth_dev *dev,
 342                                    struct rte_eth_dcb_info *dcb_info);
 343
 344 static int ixgbe_get_reg_length(struct rte_eth_dev *dev);
 345 static int ixgbe_get_regs(struct rte_eth_dev *dev,
 346                             struct rte_dev_reg_info *regs);
 347 static int ixgbe_get_eeprom_length(struct rte_eth_dev *dev);
 348 static int ixgbe_get_eeprom(struct rte_eth_dev *dev,
 349                                 struct rte_dev_eeprom_info *eeprom);
 350 static int ixgbe_set_eeprom(struct rte_eth_dev *dev,
 351                                 struct rte_dev_eeprom_info *eeprom);
 352
 353 static int ixgbevf_get_reg_length(struct rte_eth_dev *dev);
 354 static int ixgbevf_get_regs(struct rte_eth_dev *dev,
 355                                 struct rte_dev_reg_info *regs);
 356
 357 static int ixgbe_timesync_enable(struct rte_eth_dev *dev);
 358 static int ixgbe_timesync_disable(struct rte_eth_dev *dev);
 359 static int ixgbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
 360                                             struct timespec *timestamp,
 361                                             uint32_t flags);
 362 static int ixgbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
 363                                             struct timespec *timestamp);
 364 static int ixgbe_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta);
 365 static int ixgbe_timesync_read_time(struct rte_eth_dev *dev,
 366                                    struct timespec *timestamp);
 367 static int ixgbe_timesync_write_time(struct rte_eth_dev *dev,
 368                                    const struct timespec *timestamp);
 369 static void ixgbevf_dev_interrupt_handler(struct rte_intr_handle *handle,
 370                                           void *param);
 371
 372 static int ixgbe_dev_l2_tunnel_eth_type_conf
 373         (struct rte_eth_dev *dev, struct rte_eth_l2_tunnel_conf *l2_tunnel);
 374 static int ixgbe_dev_l2_tunnel_offload_set
 375         (struct rte_eth_dev *dev,
 376          struct rte_eth_l2_tunnel_conf *l2_tunnel,
 377          uint32_t mask,
 378          uint8_t en);
 379 static int ixgbe_dev_l2_tunnel_filter_handle(struct rte_eth_dev *dev,
 380                                              enum rte_filter_op filter_op,
 381                                              void *arg);
 382
 383 static int ixgbe_dev_udp_tunnel_port_add(struct rte_eth_dev *dev,
 384                                          struct rte_eth_udp_tunnel *udp_tunnel);
 385 static int ixgbe_dev_udp_tunnel_port_del(struct rte_eth_dev *dev,
 386                                          struct rte_eth_udp_tunnel *udp_tunnel);
 387
 388 /*
 389  * Define VF Stats MACRO for Non "cleared on read" register
 390  */
 391 #define UPDATE_VF_STAT(reg, last, cur)                          \
 392 {                                                               \
 393         uint32_t latest = IXGBE_READ_REG(hw, reg);              \
 394         cur += (latest - last) & UINT_MAX;                      \
 395         last = latest;                                          \
 396 }
 397
 398 #define UPDATE_VF_STAT_36BIT(lsb, msb, last, cur)                \
 399 {                                                                \
 400         u64 new_lsb = IXGBE_READ_REG(hw, lsb);                   \
 401         u64 new_msb = IXGBE_READ_REG(hw, msb);                   \
 402         u64 latest = ((new_msb << 32) | new_lsb);                \
 403         cur += (0x1000000000LL + latest - last) & 0xFFFFFFFFFLL; \
 404         last = latest;                                           \
 405 }
 406
 407 #define IXGBE_SET_HWSTRIP(h, q) do {\
 408                 uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \
 409                 uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \
 410                 (h)->bitmap[idx] |= 1 << bit;\
 411         } while (0)
 412
 413 #define IXGBE_CLEAR_HWSTRIP(h, q) do {\
 414                 uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \
 415                 uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \
 416                 (h)->bitmap[idx] &= ~(1 << bit);\
 417         } while (0)
 418
 419 #define IXGBE_GET_HWSTRIP(h, q, r) do {\
 420                 uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \
 421                 uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \
 422                 (r) = (h)->bitmap[idx] >> bit & 1;\
 423         } while (0)
 424
 425 /*
 426  * The set of PCI devices this driver supports
 427  */
 428 static const struct rte_pci_id pci_id_ixgbe_map[] = {
 429
 430 #define RTE_PCI_DEV_ID_DECL_IXGBE(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 431 #include "rte_pci_dev_ids.h"
 432
 433 { .vendor_id = 0, /* sentinel */ },
 434 };
 435
 436
 437 /*
 438  * The set of PCI devices this driver supports (for 82599 VF)
 439  */
 440 static const struct rte_pci_id pci_id_ixgbevf_map[] = {
 441
 442 #define RTE_PCI_DEV_ID_DECL_IXGBEVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 443 #include "rte_pci_dev_ids.h"
 444 { .vendor_id = 0, /* sentinel */ },
 445
 446 };
 447
 448 static const struct rte_eth_desc_lim rx_desc_lim = {
 449         .nb_max = IXGBE_MAX_RING_DESC,
 450         .nb_min = IXGBE_MIN_RING_DESC,
 451         .nb_align = IXGBE_RXD_ALIGN,
 452 };
 453
 454 static const struct rte_eth_desc_lim tx_desc_lim = {
 455         .nb_max = IXGBE_MAX_RING_DESC,
 456         .nb_min = IXGBE_MIN_RING_DESC,
 457         .nb_align = IXGBE_TXD_ALIGN,
 458 };
 459
 460 static const struct eth_dev_ops ixgbe_eth_dev_ops = {
 461         .dev_configure        = ixgbe_dev_configure,
 462         .dev_start            = ixgbe_dev_start,
 463         .dev_stop             = ixgbe_dev_stop,
 464         .dev_set_link_up    = ixgbe_dev_set_link_up,
 465         .dev_set_link_down  = ixgbe_dev_set_link_down,
 466         .dev_close            = ixgbe_dev_close,
 467         .promiscuous_enable   = ixgbe_dev_promiscuous_enable,
 468         .promiscuous_disable  = ixgbe_dev_promiscuous_disable,
 469         .allmulticast_enable  = ixgbe_dev_allmulticast_enable,
 470         .allmulticast_disable = ixgbe_dev_allmulticast_disable,
 471         .link_update          = ixgbe_dev_link_update,
 472         .stats_get            = ixgbe_dev_stats_get,
 473         .xstats_get           = ixgbe_dev_xstats_get,
 474         .stats_reset          = ixgbe_dev_stats_reset,
 475         .xstats_reset         = ixgbe_dev_xstats_reset,
 476         .xstats_get_names     = ixgbe_dev_xstats_get_names,
 477         .queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set,
 478         .dev_infos_get        = ixgbe_dev_info_get,
 479         .dev_supported_ptypes_get = ixgbe_dev_supported_ptypes_get,
 480         .mtu_set              = ixgbe_dev_mtu_set,
 481         .vlan_filter_set      = ixgbe_vlan_filter_set,
 482         .vlan_tpid_set        = ixgbe_vlan_tpid_set,
 483         .vlan_offload_set     = ixgbe_vlan_offload_set,
 484         .vlan_strip_queue_set = ixgbe_vlan_strip_queue_set,
 485         .rx_queue_start       = ixgbe_dev_rx_queue_start,
 486         .rx_queue_stop        = ixgbe_dev_rx_queue_stop,
 487         .tx_queue_start       = ixgbe_dev_tx_queue_start,
 488         .tx_queue_stop        = ixgbe_dev_tx_queue_stop,
 489         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 490         .rx_queue_intr_enable = ixgbe_dev_rx_queue_intr_enable,
 491         .rx_queue_intr_disable = ixgbe_dev_rx_queue_intr_disable,
 492         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 493         .rx_queue_count       = ixgbe_dev_rx_queue_count,
 494         .rx_descriptor_done   = ixgbe_dev_rx_descriptor_done,
 495         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 496         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 497         .dev_led_on           = ixgbe_dev_led_on,
 498         .dev_led_off          = ixgbe_dev_led_off,
 499         .flow_ctrl_get        = ixgbe_flow_ctrl_get,
 500         .flow_ctrl_set        = ixgbe_flow_ctrl_set,
 501         .priority_flow_ctrl_set = ixgbe_priority_flow_ctrl_set,
 502         .mac_addr_add         = ixgbe_add_rar,
 503         .mac_addr_remove      = ixgbe_remove_rar,
 504         .mac_addr_set         = ixgbe_set_default_mac_addr,
 505         .uc_hash_table_set    = ixgbe_uc_hash_table_set,
 506         .uc_all_hash_table_set  = ixgbe_uc_all_hash_table_set,
 507         .mirror_rule_set      = ixgbe_mirror_rule_set,
 508         .mirror_rule_reset    = ixgbe_mirror_rule_reset,
 509         .set_vf_rx_mode       = ixgbe_set_pool_rx_mode,
 510         .set_vf_rx            = ixgbe_set_pool_rx,
 511         .set_vf_tx            = ixgbe_set_pool_tx,
 512         .set_vf_vlan_filter   = ixgbe_set_pool_vlan_filter,
 513         .set_queue_rate_limit = ixgbe_set_queue_rate_limit,
 514         .set_vf_rate_limit    = ixgbe_set_vf_rate_limit,
 515         .reta_update          = ixgbe_dev_rss_reta_update,
 516         .reta_query           = ixgbe_dev_rss_reta_query,
 517 #ifdef RTE_NIC_BYPASS
 518         .bypass_init          = ixgbe_bypass_init,
 519         .bypass_state_set     = ixgbe_bypass_state_store,
 520         .bypass_state_show    = ixgbe_bypass_state_show,
 521         .bypass_event_set     = ixgbe_bypass_event_store,
 522         .bypass_event_show    = ixgbe_bypass_event_show,
 523         .bypass_wd_timeout_set  = ixgbe_bypass_wd_timeout_store,
 524         .bypass_wd_timeout_show = ixgbe_bypass_wd_timeout_show,
 525         .bypass_ver_show      = ixgbe_bypass_ver_show,
 526         .bypass_wd_reset      = ixgbe_bypass_wd_reset,
 527 #endif /* RTE_NIC_BYPASS */
 528         .rss_hash_update      = ixgbe_dev_rss_hash_update,
 529         .rss_hash_conf_get    = ixgbe_dev_rss_hash_conf_get,
 530         .filter_ctrl          = ixgbe_dev_filter_ctrl,
 531         .set_mc_addr_list     = ixgbe_dev_set_mc_addr_list,
 532         .rxq_info_get         = ixgbe_rxq_info_get,
 533         .txq_info_get         = ixgbe_txq_info_get,
 534         .timesync_enable      = ixgbe_timesync_enable,
 535         .timesync_disable     = ixgbe_timesync_disable,
 536         .timesync_read_rx_timestamp = ixgbe_timesync_read_rx_timestamp,
 537         .timesync_read_tx_timestamp = ixgbe_timesync_read_tx_timestamp,
 538         .get_reg_length       = ixgbe_get_reg_length,
 539         .get_reg              = ixgbe_get_regs,
 540         .get_eeprom_length    = ixgbe_get_eeprom_length,
 541         .get_eeprom           = ixgbe_get_eeprom,
 542         .set_eeprom           = ixgbe_set_eeprom,
 543         .get_dcb_info         = ixgbe_dev_get_dcb_info,
 544         .timesync_adjust_time = ixgbe_timesync_adjust_time,
 545         .timesync_read_time   = ixgbe_timesync_read_time,
 546         .timesync_write_time  = ixgbe_timesync_write_time,
 547         .l2_tunnel_eth_type_conf = ixgbe_dev_l2_tunnel_eth_type_conf,
 548         .l2_tunnel_offload_set   = ixgbe_dev_l2_tunnel_offload_set,
 549         .udp_tunnel_port_add  = ixgbe_dev_udp_tunnel_port_add,
 550         .udp_tunnel_port_del  = ixgbe_dev_udp_tunnel_port_del,
 551 };
 552
 553 /*
 554  * dev_ops for virtual function, bare necessities for basic vf
 555  * operation have been implemented
 556  */
 557 static const struct eth_dev_ops ixgbevf_eth_dev_ops = {
 558         .dev_configure        = ixgbevf_dev_configure,
 559         .dev_start            = ixgbevf_dev_start,
 560         .dev_stop             = ixgbevf_dev_stop,
 561         .link_update          = ixgbe_dev_link_update,
 562         .stats_get            = ixgbevf_dev_stats_get,
 563         .xstats_get           = ixgbevf_dev_xstats_get,
 564         .stats_reset          = ixgbevf_dev_stats_reset,
 565         .xstats_reset         = ixgbevf_dev_stats_reset,
 566         .xstats_get_names     = ixgbevf_dev_xstats_get_names,
 567         .dev_close            = ixgbevf_dev_close,
 568         .allmulticast_enable  = ixgbevf_dev_allmulticast_enable,
 569         .allmulticast_disable = ixgbevf_dev_allmulticast_disable,
 570         .dev_infos_get        = ixgbevf_dev_info_get,
 571         .dev_supported_ptypes_get = ixgbe_dev_supported_ptypes_get,
 572         .mtu_set              = ixgbevf_dev_set_mtu,
 573         .vlan_filter_set      = ixgbevf_vlan_filter_set,
 574         .vlan_strip_queue_set = ixgbevf_vlan_strip_queue_set,
 575         .vlan_offload_set     = ixgbevf_vlan_offload_set,
 576         .rx_queue_setup       = ixgbe_dev_rx_queue_setup,
 577         .rx_queue_release     = ixgbe_dev_rx_queue_release,
 578         .rx_descriptor_done   = ixgbe_dev_rx_descriptor_done,
 579         .tx_queue_setup       = ixgbe_dev_tx_queue_setup,
 580         .tx_queue_release     = ixgbe_dev_tx_queue_release,
 581         .rx_queue_intr_enable = ixgbevf_dev_rx_queue_intr_enable,
 582         .rx_queue_intr_disable = ixgbevf_dev_rx_queue_intr_disable,
 583         .mac_addr_add         = ixgbevf_add_mac_addr,
 584         .mac_addr_remove      = ixgbevf_remove_mac_addr,
 585         .set_mc_addr_list     = ixgbe_dev_set_mc_addr_list,
 586         .rxq_info_get         = ixgbe_rxq_info_get,
 587         .txq_info_get         = ixgbe_txq_info_get,
 588         .mac_addr_set         = ixgbevf_set_default_mac_addr,
 589         .get_reg_length       = ixgbevf_get_reg_length,
 590         .get_reg              = ixgbevf_get_regs,
 591         .reta_update          = ixgbe_dev_rss_reta_update,
 592         .reta_query           = ixgbe_dev_rss_reta_query,
 593         .rss_hash_update      = ixgbe_dev_rss_hash_update,
 594         .rss_hash_conf_get    = ixgbe_dev_rss_hash_conf_get,
 595 };
 596
 597 /* store statistics names and its offset in stats structure */
 598 struct rte_ixgbe_xstats_name_off {
 599         char name[RTE_ETH_XSTATS_NAME_SIZE];
 600         unsigned offset;
 601 };
 602
 603 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_stats_strings[] = {
 604         {"rx_crc_errors", offsetof(struct ixgbe_hw_stats, crcerrs)},
 605         {"rx_illegal_byte_errors", offsetof(struct ixgbe_hw_stats, illerrc)},
 606         {"rx_error_bytes", offsetof(struct ixgbe_hw_stats, errbc)},
 607         {"mac_local_errors", offsetof(struct ixgbe_hw_stats, mlfc)},
 608         {"mac_remote_errors", offsetof(struct ixgbe_hw_stats, mrfc)},
 609         {"rx_length_errors", offsetof(struct ixgbe_hw_stats, rlec)},
 610         {"tx_xon_packets", offsetof(struct ixgbe_hw_stats, lxontxc)},
 611         {"rx_xon_packets", offsetof(struct ixgbe_hw_stats, lxonrxc)},
 612         {"tx_xoff_packets", offsetof(struct ixgbe_hw_stats, lxofftxc)},
 613         {"rx_xoff_packets", offsetof(struct ixgbe_hw_stats, lxoffrxc)},
 614         {"rx_size_64_packets", offsetof(struct ixgbe_hw_stats, prc64)},
 615         {"rx_size_65_to_127_packets", offsetof(struct ixgbe_hw_stats, prc127)},
 616         {"rx_size_128_to_255_packets", offsetof(struct ixgbe_hw_stats, prc255)},
 617         {"rx_size_256_to_511_packets", offsetof(struct ixgbe_hw_stats, prc511)},
 618         {"rx_size_512_to_1023_packets", offsetof(struct ixgbe_hw_stats,
 619                 prc1023)},
 620         {"rx_size_1024_to_max_packets", offsetof(struct ixgbe_hw_stats,
 621                 prc1522)},
 622         {"rx_broadcast_packets", offsetof(struct ixgbe_hw_stats, bprc)},
 623         {"rx_multicast_packets", offsetof(struct ixgbe_hw_stats, mprc)},
 624         {"rx_fragment_errors", offsetof(struct ixgbe_hw_stats, rfc)},
 625         {"rx_undersize_errors", offsetof(struct ixgbe_hw_stats, ruc)},
 626         {"rx_oversize_errors", offsetof(struct ixgbe_hw_stats, roc)},
 627         {"rx_jabber_errors", offsetof(struct ixgbe_hw_stats, rjc)},
 628         {"rx_management_packets", offsetof(struct ixgbe_hw_stats, mngprc)},
 629         {"rx_management_dropped", offsetof(struct ixgbe_hw_stats, mngpdc)},
 630         {"tx_management_packets", offsetof(struct ixgbe_hw_stats, mngptc)},
 631         {"rx_total_packets", offsetof(struct ixgbe_hw_stats, tpr)},
 632         {"rx_total_bytes", offsetof(struct ixgbe_hw_stats, tor)},
 633         {"tx_total_packets", offsetof(struct ixgbe_hw_stats, tpt)},
 634         {"tx_size_64_packets", offsetof(struct ixgbe_hw_stats, ptc64)},
 635         {"tx_size_65_to_127_packets", offsetof(struct ixgbe_hw_stats, ptc127)},
 636         {"tx_size_128_to_255_packets", offsetof(struct ixgbe_hw_stats, ptc255)},
 637         {"tx_size_256_to_511_packets", offsetof(struct ixgbe_hw_stats, ptc511)},
 638         {"tx_size_512_to_1023_packets", offsetof(struct ixgbe_hw_stats,
 639                 ptc1023)},
 640         {"tx_size_1024_to_max_packets", offsetof(struct ixgbe_hw_stats,
 641                 ptc1522)},
 642         {"tx_multicast_packets", offsetof(struct ixgbe_hw_stats, mptc)},
 643         {"tx_broadcast_packets", offsetof(struct ixgbe_hw_stats, bptc)},
 644         {"rx_mac_short_packet_dropped", offsetof(struct ixgbe_hw_stats, mspdc)},
 645         {"rx_l3_l4_xsum_error", offsetof(struct ixgbe_hw_stats, xec)},
 646
 647         {"flow_director_added_filters", offsetof(struct ixgbe_hw_stats,
 648                 fdirustat_add)},
 649         {"flow_director_removed_filters", offsetof(struct ixgbe_hw_stats,
 650                 fdirustat_remove)},
 651         {"flow_director_filter_add_errors", offsetof(struct ixgbe_hw_stats,
 652                 fdirfstat_fadd)},
 653         {"flow_director_filter_remove_errors", offsetof(struct ixgbe_hw_stats,
 654                 fdirfstat_fremove)},
 655         {"flow_director_matched_filters", offsetof(struct ixgbe_hw_stats,
 656                 fdirmatch)},
 657         {"flow_director_missed_filters", offsetof(struct ixgbe_hw_stats,
 658                 fdirmiss)},
 659
 660         {"rx_fcoe_crc_errors", offsetof(struct ixgbe_hw_stats, fccrc)},
 661         {"rx_fcoe_dropped", offsetof(struct ixgbe_hw_stats, fcoerpdc)},
 662         {"rx_fcoe_mbuf_allocation_errors", offsetof(struct ixgbe_hw_stats,
 663                 fclast)},
 664         {"rx_fcoe_packets", offsetof(struct ixgbe_hw_stats, fcoeprc)},
 665         {"tx_fcoe_packets", offsetof(struct ixgbe_hw_stats, fcoeptc)},
 666         {"rx_fcoe_bytes", offsetof(struct ixgbe_hw_stats, fcoedwrc)},
 667         {"tx_fcoe_bytes", offsetof(struct ixgbe_hw_stats, fcoedwtc)},
 668         {"rx_fcoe_no_direct_data_placement", offsetof(struct ixgbe_hw_stats,
 669                 fcoe_noddp)},
 670         {"rx_fcoe_no_direct_data_placement_ext_buff",
 671                 offsetof(struct ixgbe_hw_stats, fcoe_noddp_ext_buff)},
 672
 673         {"tx_flow_control_xon_packets", offsetof(struct ixgbe_hw_stats,
 674                 lxontxc)},
 675         {"rx_flow_control_xon_packets", offsetof(struct ixgbe_hw_stats,
 676                 lxonrxc)},
 677         {"tx_flow_control_xoff_packets", offsetof(struct ixgbe_hw_stats,
 678                 lxofftxc)},
 679         {"rx_flow_control_xoff_packets", offsetof(struct ixgbe_hw_stats,
 680                 lxoffrxc)},
 681         {"rx_total_missed_packets", offsetof(struct ixgbe_hw_stats, mpctotal)},
 682 };
 683
 684 #define IXGBE_NB_HW_STATS (sizeof(rte_ixgbe_stats_strings) / \
 685                            sizeof(rte_ixgbe_stats_strings[0]))
 686
 687 /* Per-queue statistics */
 688 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_rxq_strings[] = {
 689         {"mbuf_allocation_errors", offsetof(struct ixgbe_hw_stats, rnbc)},
 690         {"dropped", offsetof(struct ixgbe_hw_stats, mpc)},
 691         {"xon_packets", offsetof(struct ixgbe_hw_stats, pxonrxc)},
 692         {"xoff_packets", offsetof(struct ixgbe_hw_stats, pxoffrxc)},
 693 };
 694
 695 #define IXGBE_NB_RXQ_PRIO_STATS (sizeof(rte_ixgbe_rxq_strings) / \
 696                            sizeof(rte_ixgbe_rxq_strings[0]))
 697 #define IXGBE_NB_RXQ_PRIO_VALUES 8
 698
 699 static const struct rte_ixgbe_xstats_name_off rte_ixgbe_txq_strings[] = {
 700         {"xon_packets", offsetof(struct ixgbe_hw_stats, pxontxc)},
 701         {"xoff_packets", offsetof(struct ixgbe_hw_stats, pxofftxc)},
 702         {"xon_to_xoff_packets", offsetof(struct ixgbe_hw_stats,
 703                 pxon2offc)},
 704 };
 705
 706 #define IXGBE_NB_TXQ_PRIO_STATS (sizeof(rte_ixgbe_txq_strings) / \
 707                            sizeof(rte_ixgbe_txq_strings[0]))
 708 #define IXGBE_NB_TXQ_PRIO_VALUES 8
 709
 710 static const struct rte_ixgbe_xstats_name_off rte_ixgbevf_stats_strings[] = {
 711         {"rx_multicast_packets", offsetof(struct ixgbevf_hw_stats, vfmprc)},
 712 };
 713
 714 #define IXGBEVF_NB_XSTATS (sizeof(rte_ixgbevf_stats_strings) /  \
 715                 sizeof(rte_ixgbevf_stats_strings[0]))
 716
 717 /**
 718  * Atomically reads the link status information from global
 719  * structure rte_eth_dev.
 720  *
 721  * @param dev
 722  *   - Pointer to the structure rte_eth_dev to read from.
 723  *   - Pointer to the buffer to be saved with the link status.
 724  *
 725  * @return
 726  *   - On success, zero.
 727  *   - On failure, negative value.
 728  */
 729 static inline int
 730 rte_ixgbe_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 731                                 struct rte_eth_link *link)
 732 {
 733         struct rte_eth_link *dst = link;
 734         struct rte_eth_link *src = &(dev->data->dev_link);
 735
 736         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 737                                         *(uint64_t *)src) == 0)
 738                 return -1;
 739
 740         return 0;
 741 }
 742
 743 /**
 744  * Atomically writes the link status information into global
 745  * structure rte_eth_dev.
 746  *
 747  * @param dev
 748  *   - Pointer to the structure rte_eth_dev to read from.
 749  *   - Pointer to the buffer to be saved with the link status.
 750  *
 751  * @return
 752  *   - On success, zero.
 753  *   - On failure, negative value.
 754  */
 755 static inline int
 756 rte_ixgbe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 757                                 struct rte_eth_link *link)
 758 {
 759         struct rte_eth_link *dst = &(dev->data->dev_link);
 760         struct rte_eth_link *src = link;
 761
 762         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 763                                         *(uint64_t *)src) == 0)
 764                 return -1;
 765
 766         return 0;
 767 }
 768
 769 /*
 770  * This function is the same as ixgbe_is_sfp() in base/ixgbe.h.
 771  */
 772 static inline int
 773 ixgbe_is_sfp(struct ixgbe_hw *hw)
 774 {
 775         switch (hw->phy.type) {
 776         case ixgbe_phy_sfp_avago:
 777         case ixgbe_phy_sfp_ftl:
 778         case ixgbe_phy_sfp_intel:
 779         case ixgbe_phy_sfp_unknown:
 780         case ixgbe_phy_sfp_passive_tyco:
 781         case ixgbe_phy_sfp_passive_unknown:
 782                 return 1;
 783         default:
 784                 return 0;
 785         }
 786 }
 787
 788 static inline int32_t
 789 ixgbe_pf_reset_hw(struct ixgbe_hw *hw)
 790 {
 791         uint32_t ctrl_ext;
 792         int32_t status;
 793
 794         status = ixgbe_reset_hw(hw);
 795
 796         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 797         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 798         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
 799         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 800         IXGBE_WRITE_FLUSH(hw);
 801
 802         return status;
 803 }
 804
 805 static inline void
 806 ixgbe_enable_intr(struct rte_eth_dev *dev)
 807 {
 808         struct ixgbe_interrupt *intr =
 809                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 810         struct ixgbe_hw *hw =
 811                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 812
 813         IXGBE_WRITE_REG(hw, IXGBE_EIMS, intr->mask);
 814         IXGBE_WRITE_FLUSH(hw);
 815 }
 816
 817 /*
 818  * This function is based on ixgbe_disable_intr() in base/ixgbe.h.
 819  */
 820 static void
 821 ixgbe_disable_intr(struct ixgbe_hw *hw)
 822 {
 823         PMD_INIT_FUNC_TRACE();
 824
 825         if (hw->mac.type == ixgbe_mac_82598EB) {
 826                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ~0);
 827         } else {
 828                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xFFFF0000);
 829                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), ~0);
 830                 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), ~0);
 831         }
 832         IXGBE_WRITE_FLUSH(hw);
 833 }
 834
 835 /*
 836  * This function resets queue statistics mapping registers.
 837  * From Niantic datasheet, Initialization of Statistics section:
 838  * "...if software requires the queue counters, the RQSMR and TQSM registers
 839  * must be re-programmed following a device reset.
 840  */
 841 static void
 842 ixgbe_reset_qstat_mappings(struct ixgbe_hw *hw)
 843 {
 844         uint32_t i;
 845
 846         for (i = 0; i != IXGBE_NB_STAT_MAPPING_REGS; i++) {
 847                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), 0);
 848                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), 0);
 849         }
 850 }
 851
 852
 853 static int
 854 ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
 855                                   uint16_t queue_id,
 856                                   uint8_t stat_idx,
 857                                   uint8_t is_rx)
 858 {
 859 #define QSM_REG_NB_BITS_PER_QMAP_FIELD 8
 860 #define NB_QMAP_FIELDS_PER_QSM_REG 4
 861 #define QMAP_FIELD_RESERVED_BITS_MASK 0x0f
 862
 863         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 864         struct ixgbe_stat_mapping_registers *stat_mappings =
 865                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(eth_dev->data->dev_private);
 866         uint32_t qsmr_mask = 0;
 867         uint32_t clearing_mask = QMAP_FIELD_RESERVED_BITS_MASK;
 868         uint32_t q_map;
 869         uint8_t n, offset;
 870
 871         if ((hw->mac.type != ixgbe_mac_82599EB) &&
 872                 (hw->mac.type != ixgbe_mac_X540) &&
 873                 (hw->mac.type != ixgbe_mac_X550) &&
 874                 (hw->mac.type != ixgbe_mac_X550EM_x) &&
 875                 (hw->mac.type != ixgbe_mac_X550EM_a))
 876                 return -ENOSYS;
 877
 878         PMD_INIT_LOG(DEBUG, "Setting port %d, %s queue_id %d to stat index %d",
 879                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX",
 880                      queue_id, stat_idx);
 881
 882         n = (uint8_t)(queue_id / NB_QMAP_FIELDS_PER_QSM_REG);
 883         if (n >= IXGBE_NB_STAT_MAPPING_REGS) {
 884                 PMD_INIT_LOG(ERR, "Nb of stat mapping registers exceeded");
 885                 return -EIO;
 886         }
 887         offset = (uint8_t)(queue_id % NB_QMAP_FIELDS_PER_QSM_REG);
 888
 889         /* Now clear any previous stat_idx set */
 890         clearing_mask <<= (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 891         if (!is_rx)
 892                 stat_mappings->tqsm[n] &= ~clearing_mask;
 893         else
 894                 stat_mappings->rqsmr[n] &= ~clearing_mask;
 895
 896         q_map = (uint32_t)stat_idx;
 897         q_map &= QMAP_FIELD_RESERVED_BITS_MASK;
 898         qsmr_mask = q_map << (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
 899         if (!is_rx)
 900                 stat_mappings->tqsm[n] |= qsmr_mask;
 901         else
 902                 stat_mappings->rqsmr[n] |= qsmr_mask;
 903
 904         PMD_INIT_LOG(DEBUG, "Set port %d, %s queue_id %d to stat index %d",
 905                      (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX",
 906                      queue_id, stat_idx);
 907         PMD_INIT_LOG(DEBUG, "%s[%d] = 0x%08x", is_rx ? "RQSMR" : "TQSM", n,
 908                      is_rx ? stat_mappings->rqsmr[n] : stat_mappings->tqsm[n]);
 909
 910         /* Now write the mapping in the appropriate register */
 911         if (is_rx) {
 912                 PMD_INIT_LOG(DEBUG, "Write 0x%x to RX IXGBE stat mapping reg:%d",
 913                              stat_mappings->rqsmr[n], n);
 914                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(n), stat_mappings->rqsmr[n]);
 915         } else {
 916                 PMD_INIT_LOG(DEBUG, "Write 0x%x to TX IXGBE stat mapping reg:%d",
 917                              stat_mappings->tqsm[n], n);
 918                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(n), stat_mappings->tqsm[n]);
 919         }
 920         return 0;
 921 }
 922
 923 static void
 924 ixgbe_restore_statistics_mapping(struct rte_eth_dev *dev)
 925 {
 926         struct ixgbe_stat_mapping_registers *stat_mappings =
 927                 IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(dev->data->dev_private);
 928         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 929         int i;
 930
 931         /* write whatever was in stat mapping table to the NIC */
 932         for (i = 0; i < IXGBE_NB_STAT_MAPPING_REGS; i++) {
 933                 /* rx */
 934                 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), stat_mappings->rqsmr[i]);
 935
 936                 /* tx */
 937                 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), stat_mappings->tqsm[i]);
 938         }
 939 }
 940
 941 static void
 942 ixgbe_dcb_init(struct ixgbe_hw *hw, struct ixgbe_dcb_config *dcb_config)
 943 {
 944         uint8_t i;
 945         struct ixgbe_dcb_tc_config *tc;
 946         uint8_t dcb_max_tc = IXGBE_DCB_MAX_TRAFFIC_CLASS;
 947
 948         dcb_config->num_tcs.pg_tcs = dcb_max_tc;
 949         dcb_config->num_tcs.pfc_tcs = dcb_max_tc;
 950         for (i = 0; i < dcb_max_tc; i++) {
 951                 tc = &dcb_config->tc_config[i];
 952                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_id = i;
 953                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
 954                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 955                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_id = i;
 956                 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
 957                                  (uint8_t)(100/dcb_max_tc + (i & 1));
 958                 tc->pfc = ixgbe_dcb_pfc_disabled;
 959         }
 960
 961         /* Initialize default user to priority mapping, UPx->TC0 */
 962         tc = &dcb_config->tc_config[0];
 963         tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
 964         tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
 965         for (i = 0; i < IXGBE_DCB_MAX_BW_GROUP; i++) {
 966                 dcb_config->bw_percentage[IXGBE_DCB_TX_CONFIG][i] = 100;
 967                 dcb_config->bw_percentage[IXGBE_DCB_RX_CONFIG][i] = 100;
 968         }
 969         dcb_config->rx_pba_cfg = ixgbe_dcb_pba_equal;
 970         dcb_config->pfc_mode_enable = false;
 971         dcb_config->vt_mode = true;
 972         dcb_config->round_robin_enable = false;
 973         /* support all DCB capabilities in 82599 */
 974         dcb_config->support.capabilities = 0xFF;
 975
 976         /*we only support 4 Tcs for X540, X550 */
 977         if (hw->mac.type == ixgbe_mac_X540 ||
 978                 hw->mac.type == ixgbe_mac_X550 ||
 979                 hw->mac.type == ixgbe_mac_X550EM_x ||
 980                 hw->mac.type == ixgbe_mac_X550EM_a) {
 981                 dcb_config->num_tcs.pg_tcs = 4;
 982                 dcb_config->num_tcs.pfc_tcs = 4;
 983         }
 984 }
 985
 986 /*
 987  * Ensure that all locks are released before first NVM or PHY access
 988  */
 989 static void
 990 ixgbe_swfw_lock_reset(struct ixgbe_hw *hw)
 991 {
 992         uint16_t mask;
 993
 994         /*
 995          * Phy lock should not fail in this early stage. If this is the case,
 996          * it is due to an improper exit of the application.
 997          * So force the release of the faulty lock. Release of common lock
 998          * is done automatically by swfw_sync function.
 999          */
1000         mask = IXGBE_GSSR_PHY0_SM << hw->bus.func;
1001         if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) {
1002                 PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released", hw->bus.func);
1003         }
1004         ixgbe_release_swfw_semaphore(hw, mask);
1005
1006         /*
1007          * These ones are more tricky since they are common to all ports; but
1008          * swfw_sync retries last long enough (1s) to be almost sure that if
1009          * lock can not be taken it is due to an improper lock of the
1010          * semaphore.
1011          */
1012         mask = IXGBE_GSSR_EEP_SM | IXGBE_GSSR_MAC_CSR_SM | IXGBE_GSSR_SW_MNG_SM;
1013         if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) {
1014                 PMD_DRV_LOG(DEBUG, "SWFW common locks released");
1015         }
1016         ixgbe_release_swfw_semaphore(hw, mask);
1017 }
1018
1019 /*
1020  * This function is based on code in ixgbe_attach() in base/ixgbe.c.
1021  * It returns 0 on success.
1022  */
1023 static int
1024 eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev)
1025 {
1026         struct rte_pci_device *pci_dev;
1027         struct ixgbe_hw *hw =
1028                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1029         struct ixgbe_vfta *shadow_vfta =
1030                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
1031         struct ixgbe_hwstrip *hwstrip =
1032                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
1033         struct ixgbe_dcb_config *dcb_config =
1034                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private);
1035         struct ixgbe_filter_info *filter_info =
1036                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private);
1037         uint32_t ctrl_ext;
1038         uint16_t csum;
1039         int diag, i;
1040
1041         PMD_INIT_FUNC_TRACE();
1042
1043         eth_dev->dev_ops = &ixgbe_eth_dev_ops;
1044         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
1045         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
1046
1047         /*
1048          * For secondary processes, we don't initialise any further as primary
1049          * has already done this work. Only check we don't need a different
1050          * RX and TX function.
1051          */
1052         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
1053                 struct ixgbe_tx_queue *txq;
1054                 /* TX queue function in primary, set by last queue initialized
1055                  * Tx queue may not initialized by primary process
1056                  */
1057                 if (eth_dev->data->tx_queues) {
1058                         txq = eth_dev->data->tx_queues[eth_dev->data->nb_tx_queues-1];
1059                         ixgbe_set_tx_function(eth_dev, txq);
1060                 } else {
1061                         /* Use default TX function if we get here */
1062                         PMD_INIT_LOG(NOTICE, "No TX queues configured yet. "
1063                                      "Using default TX function.");
1064                 }
1065
1066                 ixgbe_set_rx_function(eth_dev);
1067
1068                 return 0;
1069         }
1070         pci_dev = eth_dev->pci_dev;
1071
1072         rte_eth_copy_pci_info(eth_dev, pci_dev);
1073
1074         /* Vendor and Device ID need to be set before init of shared code */
1075         hw->device_id = pci_dev->id.device_id;
1076         hw->vendor_id = pci_dev->id.vendor_id;
1077         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
1078         hw->allow_unsupported_sfp = 1;
1079
1080         /* Initialize the shared code (base driver) */
1081 #ifdef RTE_NIC_BYPASS
1082         diag = ixgbe_bypass_init_shared_code(hw);
1083 #else
1084         diag = ixgbe_init_shared_code(hw);
1085 #endif /* RTE_NIC_BYPASS */
1086
1087         if (diag != IXGBE_SUCCESS) {
1088                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
1089                 return -EIO;
1090         }
1091
1092         /* pick up the PCI bus settings for reporting later */
1093         ixgbe_get_bus_info(hw);
1094
1095         /* Unlock any pending hardware semaphore */
1096         ixgbe_swfw_lock_reset(hw);
1097
1098         /* Initialize DCB configuration*/
1099         memset(dcb_config, 0, sizeof(struct ixgbe_dcb_config));
1100         ixgbe_dcb_init(hw, dcb_config);
1101         /* Get Hardware Flow Control setting */
1102         hw->fc.requested_mode = ixgbe_fc_full;
1103         hw->fc.current_mode = ixgbe_fc_full;
1104         hw->fc.pause_time = IXGBE_FC_PAUSE;
1105         for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
1106                 hw->fc.low_water[i] = IXGBE_FC_LO;
1107                 hw->fc.high_water[i] = IXGBE_FC_HI;
1108         }
1109         hw->fc.send_xon = 1;
1110
1111         /* Make sure we have a good EEPROM before we read from it */
1112         diag = ixgbe_validate_eeprom_checksum(hw, &csum);
1113         if (diag != IXGBE_SUCCESS) {
1114                 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag);
1115                 return -EIO;
1116         }
1117
1118 #ifdef RTE_NIC_BYPASS
1119         diag = ixgbe_bypass_init_hw(hw);
1120 #else
1121         diag = ixgbe_init_hw(hw);
1122 #endif /* RTE_NIC_BYPASS */
1123
1124         /*
1125          * Devices with copper phys will fail to initialise if ixgbe_init_hw()
1126          * is called too soon after the kernel driver unbinding/binding occurs.
1127          * The failure occurs in ixgbe_identify_phy_generic() for all devices,
1128          * but for non-copper devies, ixgbe_identify_sfp_module_generic() is
1129          * also called. See ixgbe_identify_phy_82599(). The reason for the
1130          * failure is not known, and only occuts when virtualisation features
1131          * are disabled in the bios. A delay of 100ms  was found to be enough by
1132          * trial-and-error, and is doubled to be safe.
1133          */
1134         if (diag && (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper)) {
1135                 rte_delay_ms(200);
1136                 diag = ixgbe_init_hw(hw);
1137         }
1138
1139         if (diag == IXGBE_ERR_EEPROM_VERSION) {
1140                 PMD_INIT_LOG(ERR, "This device is a pre-production adapter/"
1141                              "LOM.  Please be aware there may be issues associated "
1142                              "with your hardware.");
1143                 PMD_INIT_LOG(ERR, "If you are experiencing problems "
1144                              "please contact your Intel or hardware representative "
1145                              "who provided you with this hardware.");
1146         } else if (diag == IXGBE_ERR_SFP_NOT_SUPPORTED)
1147                 PMD_INIT_LOG(ERR, "Unsupported SFP+ Module");
1148         if (diag) {
1149                 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag);
1150                 return -EIO;
1151         }
1152
1153         /* Reset the hw statistics */
1154         ixgbe_dev_stats_reset(eth_dev);
1155
1156         /* disable interrupt */
1157         ixgbe_disable_intr(hw);
1158
1159         /* reset mappings for queue statistics hw counters*/
1160         ixgbe_reset_qstat_mappings(hw);
1161
1162         /* Allocate memory for storing MAC addresses */
1163         eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
1164                                                hw->mac.num_rar_entries, 0);
1165         if (eth_dev->data->mac_addrs == NULL) {
1166                 PMD_INIT_LOG(ERR,
1167                              "Failed to allocate %u bytes needed to store "
1168                              "MAC addresses",
1169                              ETHER_ADDR_LEN * hw->mac.num_rar_entries);
1170                 return -ENOMEM;
1171         }
1172         /* Copy the permanent MAC address */
1173         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
1174                         &eth_dev->data->mac_addrs[0]);
1175
1176         /* Allocate memory for storing hash filter MAC addresses */
1177         eth_dev->data->hash_mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
1178                                                     IXGBE_VMDQ_NUM_UC_MAC, 0);
1179         if (eth_dev->data->hash_mac_addrs == NULL) {
1180                 PMD_INIT_LOG(ERR,
1181                              "Failed to allocate %d bytes needed to store MAC addresses",
1182                              ETHER_ADDR_LEN * IXGBE_VMDQ_NUM_UC_MAC);
1183                 return -ENOMEM;
1184         }
1185
1186         /* initialize the vfta */
1187         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
1188
1189         /* initialize the hw strip bitmap*/
1190         memset(hwstrip, 0, sizeof(*hwstrip));
1191
1192         /* initialize PF if max_vfs not zero */
1193         ixgbe_pf_host_init(eth_dev);
1194
1195         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1196         /* let hardware know driver is loaded */
1197         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
1198         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
1199         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
1200         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
1201         IXGBE_WRITE_FLUSH(hw);
1202
1203         if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
1204                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d, SFP+: %d",
1205                              (int) hw->mac.type, (int) hw->phy.type,
1206                              (int) hw->phy.sfp_type);
1207         else
1208                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d",
1209                              (int) hw->mac.type, (int) hw->phy.type);
1210
1211         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
1212                      eth_dev->data->port_id, pci_dev->id.vendor_id,
1213                      pci_dev->id.device_id);
1214
1215         rte_intr_callback_register(&pci_dev->intr_handle,
1216                                    ixgbe_dev_interrupt_handler,
1217                                    (void *)eth_dev);
1218
1219         /* enable uio/vfio intr/eventfd mapping */
1220         rte_intr_enable(&pci_dev->intr_handle);
1221
1222         /* enable support intr */
1223         ixgbe_enable_intr(eth_dev);
1224
1225         /* initialize 5tuple filter list */
1226         TAILQ_INIT(&filter_info->fivetuple_list);
1227         memset(filter_info->fivetuple_mask, 0,
1228                sizeof(uint32_t) * IXGBE_5TUPLE_ARRAY_SIZE);
1229
1230         return 0;
1231 }
1232
1233 static int
1234 eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev)
1235 {
1236         struct rte_pci_device *pci_dev;
1237         struct ixgbe_hw *hw;
1238
1239         PMD_INIT_FUNC_TRACE();
1240
1241         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1242                 return -EPERM;
1243
1244         hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1245         pci_dev = eth_dev->pci_dev;
1246
1247         if (hw->adapter_stopped == 0)
1248                 ixgbe_dev_close(eth_dev);
1249
1250         eth_dev->dev_ops = NULL;
1251         eth_dev->rx_pkt_burst = NULL;
1252         eth_dev->tx_pkt_burst = NULL;
1253
1254         /* Unlock any pending hardware semaphore */
1255         ixgbe_swfw_lock_reset(hw);
1256
1257         /* disable uio intr before callback unregister */
1258         rte_intr_disable(&(pci_dev->intr_handle));
1259         rte_intr_callback_unregister(&(pci_dev->intr_handle),
1260                 ixgbe_dev_interrupt_handler, (void *)eth_dev);
1261
1262         /* uninitialize PF if max_vfs not zero */
1263         ixgbe_pf_host_uninit(eth_dev);
1264
1265         rte_free(eth_dev->data->mac_addrs);
1266         eth_dev->data->mac_addrs = NULL;
1267
1268         rte_free(eth_dev->data->hash_mac_addrs);
1269         eth_dev->data->hash_mac_addrs = NULL;
1270
1271         return 0;
1272 }
1273
1274 /*
1275  * Negotiate mailbox API version with the PF.
1276  * After reset API version is always set to the basic one (ixgbe_mbox_api_10).
1277  * Then we try to negotiate starting with the most recent one.
1278  * If all negotiation attempts fail, then we will proceed with
1279  * the default one (ixgbe_mbox_api_10).
1280  */
1281 static void
1282 ixgbevf_negotiate_api(struct ixgbe_hw *hw)
1283 {
1284         int32_t i;
1285
1286         /* start with highest supported, proceed down */
1287         static const enum ixgbe_pfvf_api_rev sup_ver[] = {
1288                 ixgbe_mbox_api_12,
1289                 ixgbe_mbox_api_11,
1290                 ixgbe_mbox_api_10,
1291         };
1292
1293         for (i = 0;
1294                         i != RTE_DIM(sup_ver) &&
1295                         ixgbevf_negotiate_api_version(hw, sup_ver[i]) != 0;
1296                         i++)
1297                 ;
1298 }
1299
1300 static void
1301 generate_random_mac_addr(struct ether_addr *mac_addr)
1302 {
1303         uint64_t random;
1304
1305         /* Set Organizationally Unique Identifier (OUI) prefix. */
1306         mac_addr->addr_bytes[0] = 0x00;
1307         mac_addr->addr_bytes[1] = 0x09;
1308         mac_addr->addr_bytes[2] = 0xC0;
1309         /* Force indication of locally assigned MAC address. */
1310         mac_addr->addr_bytes[0] |= ETHER_LOCAL_ADMIN_ADDR;
1311         /* Generate the last 3 bytes of the MAC address with a random number. */
1312         random = rte_rand();
1313         memcpy(&mac_addr->addr_bytes[3], &random, 3);
1314 }
1315
1316 /*
1317  * Virtual Function device init
1318  */
1319 static int
1320 eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev)
1321 {
1322         int diag;
1323         uint32_t tc, tcs;
1324         struct rte_pci_device *pci_dev;
1325         struct ixgbe_hw *hw =
1326                 IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1327         struct ixgbe_vfta *shadow_vfta =
1328                 IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
1329         struct ixgbe_hwstrip *hwstrip =
1330                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
1331         struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr;
1332
1333         PMD_INIT_FUNC_TRACE();
1334
1335         eth_dev->dev_ops = &ixgbevf_eth_dev_ops;
1336         eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
1337         eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
1338
1339         /* for secondary processes, we don't initialise any further as primary
1340          * has already done this work. Only check we don't need a different
1341          * RX function
1342          */
1343         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
1344                 struct ixgbe_tx_queue *txq;
1345                 /* TX queue function in primary, set by last queue initialized
1346                  * Tx queue may not initialized by primary process
1347                  */
1348                 if (eth_dev->data->tx_queues) {
1349                         txq = eth_dev->data->tx_queues[eth_dev->data->nb_tx_queues - 1];
1350                         ixgbe_set_tx_function(eth_dev, txq);
1351                 } else {
1352                         /* Use default TX function if we get here */
1353                         PMD_INIT_LOG(NOTICE,
1354                                      "No TX queues configured yet. Using default TX function.");
1355                 }
1356
1357                 ixgbe_set_rx_function(eth_dev);
1358
1359                 return 0;
1360         }
1361
1362         pci_dev = eth_dev->pci_dev;
1363
1364         rte_eth_copy_pci_info(eth_dev, pci_dev);
1365
1366         hw->device_id = pci_dev->id.device_id;
1367         hw->vendor_id = pci_dev->id.vendor_id;
1368         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
1369
1370         /* initialize the vfta */
1371         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
1372
1373         /* initialize the hw strip bitmap*/
1374         memset(hwstrip, 0, sizeof(*hwstrip));
1375
1376         /* Initialize the shared code (base driver) */
1377         diag = ixgbe_init_shared_code(hw);
1378         if (diag != IXGBE_SUCCESS) {
1379                 PMD_INIT_LOG(ERR, "Shared code init failed for ixgbevf: %d", diag);
1380                 return -EIO;
1381         }
1382
1383         /* init_mailbox_params */
1384         hw->mbx.ops.init_params(hw);
1385
1386         /* Reset the hw statistics */
1387         ixgbevf_dev_stats_reset(eth_dev);
1388
1389         /* Disable the interrupts for VF */
1390         ixgbevf_intr_disable(hw);
1391
1392         hw->mac.num_rar_entries = 128; /* The MAX of the underlying PF */
1393         diag = hw->mac.ops.reset_hw(hw);
1394
1395         /*
1396          * The VF reset operation returns the IXGBE_ERR_INVALID_MAC_ADDR when
1397          * the underlying PF driver has not assigned a MAC address to the VF.
1398          * In this case, assign a random MAC address.
1399          */
1400         if ((diag != IXGBE_SUCCESS) && (diag != IXGBE_ERR_INVALID_MAC_ADDR)) {
1401                 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
1402                 return diag;
1403         }
1404
1405         /* negotiate mailbox API version to use with the PF. */
1406         ixgbevf_negotiate_api(hw);
1407
1408         /* Get Rx/Tx queue count via mailbox, which is ready after reset_hw */
1409         ixgbevf_get_queues(hw, &tcs, &tc);
1410
1411         /* Allocate memory for storing MAC addresses */
1412         eth_dev->data->mac_addrs = rte_zmalloc("ixgbevf", ETHER_ADDR_LEN *
1413                                                hw->mac.num_rar_entries, 0);
1414         if (eth_dev->data->mac_addrs == NULL) {
1415                 PMD_INIT_LOG(ERR,
1416                              "Failed to allocate %u bytes needed to store "
1417                              "MAC addresses",
1418                              ETHER_ADDR_LEN * hw->mac.num_rar_entries);
1419                 return -ENOMEM;
1420         }
1421
1422         /* Generate a random MAC address, if none was assigned by PF. */
1423         if (is_zero_ether_addr(perm_addr)) {
1424                 generate_random_mac_addr(perm_addr);
1425                 diag = ixgbe_set_rar_vf(hw, 1, perm_addr->addr_bytes, 0, 1);
1426                 if (diag) {
1427                         rte_free(eth_dev->data->mac_addrs);
1428                         eth_dev->data->mac_addrs = NULL;
1429                         return diag;
1430                 }
1431                 PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF");
1432                 PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address "
1433                              "%02x:%02x:%02x:%02x:%02x:%02x",
1434                              perm_addr->addr_bytes[0],
1435                              perm_addr->addr_bytes[1],
1436                              perm_addr->addr_bytes[2],
1437                              perm_addr->addr_bytes[3],
1438                              perm_addr->addr_bytes[4],
1439                              perm_addr->addr_bytes[5]);
1440         }
1441
1442         /* Copy the permanent MAC address */
1443         ether_addr_copy(perm_addr, &eth_dev->data->mac_addrs[0]);
1444
1445         /* reset the hardware with the new settings */
1446         diag = hw->mac.ops.start_hw(hw);
1447         switch (diag) {
1448         case  0:
1449                 break;
1450
1451         default:
1452                 PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
1453                 return -EIO;
1454         }
1455
1456         rte_intr_callback_register(&pci_dev->intr_handle,
1457                                    ixgbevf_dev_interrupt_handler,
1458                                    (void *)eth_dev);
1459         rte_intr_enable(&pci_dev->intr_handle);
1460         ixgbevf_intr_enable(hw);
1461
1462         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x mac.type=%s",
1463                      eth_dev->data->port_id, pci_dev->id.vendor_id,
1464                      pci_dev->id.device_id, "ixgbe_mac_82599_vf");
1465
1466         return 0;
1467 }
1468
1469 /* Virtual Function device uninit */
1470
1471 static int
1472 eth_ixgbevf_dev_uninit(struct rte_eth_dev *eth_dev)
1473 {
1474         struct ixgbe_hw *hw;
1475         struct rte_pci_device *pci_dev = eth_dev->pci_dev;
1476
1477         PMD_INIT_FUNC_TRACE();
1478
1479         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1480                 return -EPERM;
1481
1482         hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1483
1484         if (hw->adapter_stopped == 0)
1485                 ixgbevf_dev_close(eth_dev);
1486
1487         eth_dev->dev_ops = NULL;
1488         eth_dev->rx_pkt_burst = NULL;
1489         eth_dev->tx_pkt_burst = NULL;
1490
1491         /* Disable the interrupts for VF */
1492         ixgbevf_intr_disable(hw);
1493
1494         rte_free(eth_dev->data->mac_addrs);
1495         eth_dev->data->mac_addrs = NULL;
1496
1497         rte_intr_disable(&pci_dev->intr_handle);
1498         rte_intr_callback_unregister(&pci_dev->intr_handle,
1499                                      ixgbevf_dev_interrupt_handler,
1500                                      (void *)eth_dev);
1501
1502         return 0;
1503 }
1504
1505 static struct eth_driver rte_ixgbe_pmd = {
1506         .pci_drv = {
1507                 .name = "rte_ixgbe_pmd",
1508                 .id_table = pci_id_ixgbe_map,
1509                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
1510                         RTE_PCI_DRV_DETACHABLE,
1511         },
1512         .eth_dev_init = eth_ixgbe_dev_init,
1513         .eth_dev_uninit = eth_ixgbe_dev_uninit,
1514         .dev_private_size = sizeof(struct ixgbe_adapter),
1515 };
1516
1517 /*
1518  * virtual function driver struct
1519  */
1520 static struct eth_driver rte_ixgbevf_pmd = {
1521         .pci_drv = {
1522                 .name = "rte_ixgbevf_pmd",
1523                 .id_table = pci_id_ixgbevf_map,
1524                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
1525         },
1526         .eth_dev_init = eth_ixgbevf_dev_init,
1527         .eth_dev_uninit = eth_ixgbevf_dev_uninit,
1528         .dev_private_size = sizeof(struct ixgbe_adapter),
1529 };
1530
1531 /*
1532  * Driver initialization routine.
1533  * Invoked once at EAL init time.
1534  * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
1535  */
1536 static int
1537 rte_ixgbe_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
1538 {
1539         PMD_INIT_FUNC_TRACE();
1540
1541         rte_eth_driver_register(&rte_ixgbe_pmd);
1542         return 0;
1543 }
1544
1545 /*
1546  * VF Driver initialization routine.
1547  * Invoked one at EAL init time.
1548  * Register itself as the [Virtual Poll Mode] Driver of PCI niantic devices.
1549  */
1550 static int
1551 rte_ixgbevf_pmd_init(const char *name __rte_unused, const char *param __rte_unused)
1552 {
1553         PMD_INIT_FUNC_TRACE();
1554
1555         rte_eth_driver_register(&rte_ixgbevf_pmd);
1556         return 0;
1557 }
1558
1559 static int
1560 ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1561 {
1562         struct ixgbe_hw *hw =
1563                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1564         struct ixgbe_vfta *shadow_vfta =
1565                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1566         uint32_t vfta;
1567         uint32_t vid_idx;
1568         uint32_t vid_bit;
1569
1570         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
1571         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
1572         vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(vid_idx));
1573         if (on)
1574                 vfta |= vid_bit;
1575         else
1576                 vfta &= ~vid_bit;
1577         IXGBE_WRITE_REG(hw, IXGBE_VFTA(vid_idx), vfta);
1578
1579         /* update local VFTA copy */
1580         shadow_vfta->vfta[vid_idx] = vfta;
1581
1582         return 0;
1583 }
1584
1585 static void
1586 ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
1587 {
1588         if (on)
1589                 ixgbe_vlan_hw_strip_enable(dev, queue);
1590         else
1591                 ixgbe_vlan_hw_strip_disable(dev, queue);
1592 }
1593
1594 static int
1595 ixgbe_vlan_tpid_set(struct rte_eth_dev *dev,
1596                     enum rte_vlan_type vlan_type,
1597                     uint16_t tpid)
1598 {
1599         struct ixgbe_hw *hw =
1600                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1601         int ret = 0;
1602
1603         switch (vlan_type) {
1604         case ETH_VLAN_TYPE_INNER:
1605                 /* Only the high 16-bits is valid */
1606                 IXGBE_WRITE_REG(hw, IXGBE_EXVET, tpid << 16);
1607                 break;
1608         default:
1609                 ret = -EINVAL;
1610                 PMD_DRV_LOG(ERR, "Unsupported vlan type %d\n", vlan_type);
1611                 break;
1612         }
1613
1614         return ret;
1615 }
1616
1617 void
1618 ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev)
1619 {
1620         struct ixgbe_hw *hw =
1621                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1622         uint32_t vlnctrl;
1623
1624         PMD_INIT_FUNC_TRACE();
1625
1626         /* Filter Table Disable */
1627         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1628         vlnctrl &= ~IXGBE_VLNCTRL_VFE;
1629
1630         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
1631 }
1632
1633 void
1634 ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1635 {
1636         struct ixgbe_hw *hw =
1637                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1638         struct ixgbe_vfta *shadow_vfta =
1639                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1640         uint32_t vlnctrl;
1641         uint16_t i;
1642
1643         PMD_INIT_FUNC_TRACE();
1644
1645         /* Filter Table Enable */
1646         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1647         vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
1648         vlnctrl |= IXGBE_VLNCTRL_VFE;
1649
1650         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
1651
1652         /* write whatever is in local vfta copy */
1653         for (i = 0; i < IXGBE_VFTA_SIZE; i++)
1654                 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), shadow_vfta->vfta[i]);
1655 }
1656
1657 static void
1658 ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on)
1659 {
1660         struct ixgbe_hwstrip *hwstrip =
1661                 IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(dev->data->dev_private);
1662         struct ixgbe_rx_queue *rxq;
1663
1664         if (queue >= IXGBE_MAX_RX_QUEUE_NUM)
1665                 return;
1666
1667         if (on)
1668                 IXGBE_SET_HWSTRIP(hwstrip, queue);
1669         else
1670                 IXGBE_CLEAR_HWSTRIP(hwstrip, queue);
1671
1672         if (queue >= dev->data->nb_rx_queues)
1673                 return;
1674
1675         rxq = dev->data->rx_queues[queue];
1676
1677         if (on)
1678                 rxq->vlan_flags = PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED;
1679         else
1680                 rxq->vlan_flags = PKT_RX_VLAN_PKT;
1681 }
1682
1683 static void
1684 ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue)
1685 {
1686         struct ixgbe_hw *hw =
1687                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1688         uint32_t ctrl;
1689
1690         PMD_INIT_FUNC_TRACE();
1691
1692         if (hw->mac.type == ixgbe_mac_82598EB) {
1693                 /* No queue level support */
1694                 PMD_INIT_LOG(NOTICE, "82598EB not support queue level hw strip");
1695                 return;
1696         }
1697
1698         /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1699         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1700         ctrl &= ~IXGBE_RXDCTL_VME;
1701         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
1702
1703         /* record those setting for HW strip per queue */
1704         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 0);
1705 }
1706
1707 static void
1708 ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue)
1709 {
1710         struct ixgbe_hw *hw =
1711                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1712         uint32_t ctrl;
1713
1714         PMD_INIT_FUNC_TRACE();
1715
1716         if (hw->mac.type == ixgbe_mac_82598EB) {
1717                 /* No queue level supported */
1718                 PMD_INIT_LOG(NOTICE, "82598EB not support queue level hw strip");
1719                 return;
1720         }
1721
1722         /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1723         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1724         ctrl |= IXGBE_RXDCTL_VME;
1725         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
1726
1727         /* record those setting for HW strip per queue */
1728         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 1);
1729 }
1730
1731 void
1732 ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev)
1733 {
1734         struct ixgbe_hw *hw =
1735                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1736         uint32_t ctrl;
1737         uint16_t i;
1738
1739         PMD_INIT_FUNC_TRACE();
1740
1741         if (hw->mac.type == ixgbe_mac_82598EB) {
1742                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1743                 ctrl &= ~IXGBE_VLNCTRL_VME;
1744                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1745         } else {
1746                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1747                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1748                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1749                         ctrl &= ~IXGBE_RXDCTL_VME;
1750                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1751
1752                         /* record those setting for HW strip per queue */
1753                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 0);
1754                 }
1755         }
1756 }
1757
1758 void
1759 ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev)
1760 {
1761         struct ixgbe_hw *hw =
1762                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1763         uint32_t ctrl;
1764         uint16_t i;
1765
1766         PMD_INIT_FUNC_TRACE();
1767
1768         if (hw->mac.type == ixgbe_mac_82598EB) {
1769                 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1770                 ctrl |= IXGBE_VLNCTRL_VME;
1771                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1772         } else {
1773                 /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1774                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1775                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1776                         ctrl |= IXGBE_RXDCTL_VME;
1777                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
1778
1779                         /* record those setting for HW strip per queue */
1780                         ixgbe_vlan_hw_strip_bitmap_set(dev, i, 1);
1781                 }
1782         }
1783 }
1784
1785 static void
1786 ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1787 {
1788         struct ixgbe_hw *hw =
1789                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1790         uint32_t ctrl;
1791
1792         PMD_INIT_FUNC_TRACE();
1793
1794         /* DMATXCTRL: Geric Double VLAN Disable */
1795         ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1796         ctrl &= ~IXGBE_DMATXCTL_GDV;
1797         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1798
1799         /* CTRL_EXT: Global Double VLAN Disable */
1800         ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1801         ctrl &= ~IXGBE_EXTENDED_VLAN;
1802         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1803
1804 }
1805
1806 static void
1807 ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1808 {
1809         struct ixgbe_hw *hw =
1810                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1811         uint32_t ctrl;
1812
1813         PMD_INIT_FUNC_TRACE();
1814
1815         /* DMATXCTRL: Geric Double VLAN Enable */
1816         ctrl  = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1817         ctrl |= IXGBE_DMATXCTL_GDV;
1818         IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
1819
1820         /* CTRL_EXT: Global Double VLAN Enable */
1821         ctrl  = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1822         ctrl |= IXGBE_EXTENDED_VLAN;
1823         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
1824
1825         /* Clear pooling mode of PFVTCTL. It's required by X550. */
1826         if (hw->mac.type == ixgbe_mac_X550 ||
1827             hw->mac.type == ixgbe_mac_X550EM_x ||
1828             hw->mac.type == ixgbe_mac_X550EM_a) {
1829                 ctrl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
1830                 ctrl &= ~IXGBE_VT_CTL_POOLING_MODE_MASK;
1831                 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, ctrl);
1832         }
1833
1834         /*
1835          * VET EXT field in the EXVET register = 0x8100 by default
1836          * So no need to change. Same to VT field of DMATXCTL register
1837          */
1838 }
1839
1840 static void
1841 ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1842 {
1843         if (mask & ETH_VLAN_STRIP_MASK) {
1844                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1845                         ixgbe_vlan_hw_strip_enable_all(dev);
1846                 else
1847                         ixgbe_vlan_hw_strip_disable_all(dev);
1848         }
1849
1850         if (mask & ETH_VLAN_FILTER_MASK) {
1851                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1852                         ixgbe_vlan_hw_filter_enable(dev);
1853                 else
1854                         ixgbe_vlan_hw_filter_disable(dev);
1855         }
1856
1857         if (mask & ETH_VLAN_EXTEND_MASK) {
1858                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1859                         ixgbe_vlan_hw_extend_enable(dev);
1860                 else
1861                         ixgbe_vlan_hw_extend_disable(dev);
1862         }
1863 }
1864
1865 static void
1866 ixgbe_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1867 {
1868         struct ixgbe_hw *hw =
1869                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1870         /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
1871         uint32_t vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1872
1873         vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */
1874         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
1875 }
1876
1877 static int
1878 ixgbe_check_vf_rss_rxq_num(struct rte_eth_dev *dev, uint16_t nb_rx_q)
1879 {
1880         switch (nb_rx_q) {
1881         case 1:
1882         case 2:
1883                 RTE_ETH_DEV_SRIOV(dev).active = ETH_64_POOLS;
1884                 break;
1885         case 4:
1886                 RTE_ETH_DEV_SRIOV(dev).active = ETH_32_POOLS;
1887                 break;
1888         default:
1889                 return -EINVAL;
1890         }
1891
1892         RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = nb_rx_q;
1893         RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx = dev->pci_dev->max_vfs * nb_rx_q;
1894
1895         return 0;
1896 }
1897
1898 static int
1899 ixgbe_check_mq_mode(struct rte_eth_dev *dev)
1900 {
1901         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
1902         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1903         uint16_t nb_rx_q = dev->data->nb_rx_queues;
1904         uint16_t nb_tx_q = dev->data->nb_tx_queues;
1905
1906         if (RTE_ETH_DEV_SRIOV(dev).active != 0) {
1907                 /* check multi-queue mode */
1908                 switch (dev_conf->rxmode.mq_mode) {
1909                 case ETH_MQ_RX_VMDQ_DCB:
1910                 case ETH_MQ_RX_VMDQ_DCB_RSS:
1911                         /* DCB/RSS VMDQ in SRIOV mode, not implement yet */
1912                         PMD_INIT_LOG(ERR, "SRIOV active,"
1913                                         " unsupported mq_mode rx %d.",
1914                                         dev_conf->rxmode.mq_mode);
1915                         return -EINVAL;
1916                 case ETH_MQ_RX_RSS:
1917                 case ETH_MQ_RX_VMDQ_RSS:
1918                         dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_RSS;
1919                         if (nb_rx_q <= RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool)
1920                                 if (ixgbe_check_vf_rss_rxq_num(dev, nb_rx_q)) {
1921                                         PMD_INIT_LOG(ERR, "SRIOV is active,"
1922                                                 " invalid queue number"
1923                                                 " for VMDQ RSS, allowed"
1924                                                 " value are 1, 2 or 4.");
1925                                         return -EINVAL;
1926                                 }
1927                         break;
1928                 case ETH_MQ_RX_VMDQ_ONLY:
1929                 case ETH_MQ_RX_NONE:
1930                         /* if nothing mq mode configure, use default scheme */
1931                         dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY;
1932                         if (RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool > 1)
1933                                 RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 1;
1934                         break;
1935                 default: /* ETH_MQ_RX_DCB, ETH_MQ_RX_DCB_RSS or ETH_MQ_TX_DCB*/
1936                         /* SRIOV only works in VMDq enable mode */
1937                         PMD_INIT_LOG(ERR, "SRIOV is active,"
1938                                         " wrong mq_mode rx %d.",
1939                                         dev_conf->rxmode.mq_mode);
1940                         return -EINVAL;
1941                 }
1942
1943                 switch (dev_conf->txmode.mq_mode) {
1944                 case ETH_MQ_TX_VMDQ_DCB:
1945                         /* DCB VMDQ in SRIOV mode, not implement yet */
1946                         PMD_INIT_LOG(ERR, "SRIOV is active,"
1947                                         " unsupported VMDQ mq_mode tx %d.",
1948                                         dev_conf->txmode.mq_mode);
1949                         return -EINVAL;
1950                 default: /* ETH_MQ_TX_VMDQ_ONLY or ETH_MQ_TX_NONE */
1951                         dev->data->dev_conf.txmode.mq_mode = ETH_MQ_TX_VMDQ_ONLY;
1952                         break;
1953                 }
1954
1955                 /* check valid queue number */
1956                 if ((nb_rx_q > RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool) ||
1957                     (nb_tx_q > RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool)) {
1958                         PMD_INIT_LOG(ERR, "SRIOV is active,"
1959                                         " nb_rx_q=%d nb_tx_q=%d queue number"
1960                                         " must be less than or equal to %d.",
1961                                         nb_rx_q, nb_tx_q,
1962                                         RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool);
1963                         return -EINVAL;
1964                 }
1965         } else {
1966                 if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_VMDQ_DCB_RSS) {
1967                         PMD_INIT_LOG(ERR, "VMDQ+DCB+RSS mq_mode is"
1968                                           " not supported.");
1969                         return -EINVAL;
1970                 }
1971                 /* check configuration for vmdb+dcb mode */
1972                 if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_VMDQ_DCB) {
1973                         const struct rte_eth_vmdq_dcb_conf *conf;
1974
1975                         if (nb_rx_q != IXGBE_VMDQ_DCB_NB_QUEUES) {
1976                                 PMD_INIT_LOG(ERR, "VMDQ+DCB, nb_rx_q != %d.",
1977                                                 IXGBE_VMDQ_DCB_NB_QUEUES);
1978                                 return -EINVAL;
1979                         }
1980                         conf = &dev_conf->rx_adv_conf.vmdq_dcb_conf;
1981                         if (!(conf->nb_queue_pools == ETH_16_POOLS ||
1982                                conf->nb_queue_pools == ETH_32_POOLS)) {
1983                                 PMD_INIT_LOG(ERR, "VMDQ+DCB selected,"
1984                                                 " nb_queue_pools must be %d or %d.",
1985                                                 ETH_16_POOLS, ETH_32_POOLS);
1986                                 return -EINVAL;
1987                         }
1988                 }
1989                 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_VMDQ_DCB) {
1990                         const struct rte_eth_vmdq_dcb_tx_conf *conf;
1991
1992                         if (nb_tx_q != IXGBE_VMDQ_DCB_NB_QUEUES) {
1993                                 PMD_INIT_LOG(ERR, "VMDQ+DCB, nb_tx_q != %d",
1994                                                  IXGBE_VMDQ_DCB_NB_QUEUES);
1995                                 return -EINVAL;
1996                         }
1997                         conf = &dev_conf->tx_adv_conf.vmdq_dcb_tx_conf;
1998                         if (!(conf->nb_queue_pools == ETH_16_POOLS ||
1999                                conf->nb_queue_pools == ETH_32_POOLS)) {
2000                                 PMD_INIT_LOG(ERR, "VMDQ+DCB selected,"
2001                                                 " nb_queue_pools != %d and"
2002                                                 " nb_queue_pools != %d.",
2003                                                 ETH_16_POOLS, ETH_32_POOLS);
2004                                 return -EINVAL;
2005                         }
2006                 }
2007
2008                 /* For DCB mode check our configuration before we go further */
2009                 if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_DCB) {
2010                         const struct rte_eth_dcb_rx_conf *conf;
2011
2012                         if (nb_rx_q != IXGBE_DCB_NB_QUEUES) {
2013                                 PMD_INIT_LOG(ERR, "DCB selected, nb_rx_q != %d.",
2014                                                  IXGBE_DCB_NB_QUEUES);
2015                                 return -EINVAL;
2016                         }
2017                         conf = &dev_conf->rx_adv_conf.dcb_rx_conf;
2018                         if (!(conf->nb_tcs == ETH_4_TCS ||
2019                                conf->nb_tcs == ETH_8_TCS)) {
2020                                 PMD_INIT_LOG(ERR, "DCB selected, nb_tcs != %d"
2021                                                 " and nb_tcs != %d.",
2022                                                 ETH_4_TCS, ETH_8_TCS);
2023                                 return -EINVAL;
2024                         }
2025                 }
2026
2027                 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_DCB) {
2028                         const struct rte_eth_dcb_tx_conf *conf;
2029
2030                         if (nb_tx_q != IXGBE_DCB_NB_QUEUES) {
2031                                 PMD_INIT_LOG(ERR, "DCB, nb_tx_q != %d.",
2032                                                  IXGBE_DCB_NB_QUEUES);
2033                                 return -EINVAL;
2034                         }
2035                         conf = &dev_conf->tx_adv_conf.dcb_tx_conf;
2036                         if (!(conf->nb_tcs == ETH_4_TCS ||
2037                                conf->nb_tcs == ETH_8_TCS)) {
2038                                 PMD_INIT_LOG(ERR, "DCB selected, nb_tcs != %d"
2039                                                 " and nb_tcs != %d.",
2040                                                 ETH_4_TCS, ETH_8_TCS);
2041                                 return -EINVAL;
2042                         }
2043                 }
2044
2045                 /*
2046                  * When DCB/VT is off, maximum number of queues changes,
2047                  * except for 82598EB, which remains constant.
2048                  */
2049                 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_NONE &&
2050                                 hw->mac.type != ixgbe_mac_82598EB) {
2051                         if (nb_tx_q > IXGBE_NONE_MODE_TX_NB_QUEUES) {
2052                                 PMD_INIT_LOG(ERR,
2053                                              "Neither VT nor DCB are enabled, "
2054                                              "nb_tx_q > %d.",
2055                                              IXGBE_NONE_MODE_TX_NB_QUEUES);
2056                                 return -EINVAL;
2057                         }
2058                 }
2059         }
2060         return 0;
2061 }
2062
2063 static int
2064 ixgbe_dev_configure(struct rte_eth_dev *dev)
2065 {
2066         struct ixgbe_interrupt *intr =
2067                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2068         struct ixgbe_adapter *adapter =
2069                 (struct ixgbe_adapter *)dev->data->dev_private;
2070         int ret;
2071
2072         PMD_INIT_FUNC_TRACE();
2073         /* multipe queue mode checking */
2074         ret  = ixgbe_check_mq_mode(dev);
2075         if (ret != 0) {
2076                 PMD_DRV_LOG(ERR, "ixgbe_check_mq_mode fails with %d.",
2077                             ret);
2078                 return ret;
2079         }
2080
2081         /* set flag to update link status after init */
2082         intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
2083
2084         /*
2085          * Initialize to TRUE. If any of Rx queues doesn't meet the bulk
2086          * allocation or vector Rx preconditions we will reset it.
2087          */
2088         adapter->rx_bulk_alloc_allowed = true;
2089         adapter->rx_vec_allowed = true;
2090
2091         return 0;
2092 }
2093
2094 static void
2095 ixgbe_dev_phy_intr_setup(struct rte_eth_dev *dev)
2096 {
2097         struct ixgbe_hw *hw =
2098                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2099         struct ixgbe_interrupt *intr =
2100                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2101         uint32_t gpie;
2102
2103         /* only set up it on X550EM_X */
2104         if (hw->mac.type == ixgbe_mac_X550EM_x) {
2105                 gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
2106                 gpie |= IXGBE_SDP0_GPIEN_X550EM_x;
2107                 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
2108                 if (hw->phy.type == ixgbe_phy_x550em_ext_t)
2109                         intr->mask |= IXGBE_EICR_GPI_SDP0_X550EM_x;
2110         }
2111 }
2112
2113 /*
2114  * Configure device link speed and setup link.
2115  * It returns 0 on success.
2116  */
2117 static int
2118 ixgbe_dev_start(struct rte_eth_dev *dev)
2119 {
2120         struct ixgbe_hw *hw =
2121                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2122         struct ixgbe_vf_info *vfinfo =
2123                 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private);
2124         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
2125         uint32_t intr_vector = 0;
2126         int err, link_up = 0, negotiate = 0;
2127         uint32_t speed = 0;
2128         int mask = 0;
2129         int status;
2130         uint16_t vf, idx;
2131         uint32_t *link_speeds;
2132
2133         PMD_INIT_FUNC_TRACE();
2134
2135         /* IXGBE devices don't support:
2136         *    - half duplex (checked afterwards for valid speeds)
2137         *    - fixed speed: TODO implement
2138         */
2139         if (dev->data->dev_conf.link_speeds & ETH_LINK_SPEED_FIXED) {
2140                 PMD_INIT_LOG(ERR, "Invalid link_speeds for port %hhu; fix speed not supported",
2141                              dev->data->port_id);
2142                 return -EINVAL;
2143         }
2144
2145         /* disable uio/vfio intr/eventfd mapping */
2146         rte_intr_disable(intr_handle);
2147
2148         /* stop adapter */
2149         hw->adapter_stopped = 0;
2150         ixgbe_stop_adapter(hw);
2151
2152         /* reinitialize adapter
2153          * this calls reset and start
2154          */
2155         status = ixgbe_pf_reset_hw(hw);
2156         if (status != 0)
2157                 return -1;
2158         hw->mac.ops.start_hw(hw);
2159         hw->mac.get_link_status = true;
2160
2161         /* configure PF module if SRIOV enabled */
2162         ixgbe_pf_host_configure(dev);
2163
2164         ixgbe_dev_phy_intr_setup(dev);
2165
2166         /* check and configure queue intr-vector mapping */
2167         if ((rte_intr_cap_multiple(intr_handle) ||
2168              !RTE_ETH_DEV_SRIOV(dev).active) &&
2169             dev->data->dev_conf.intr_conf.rxq != 0) {
2170                 intr_vector = dev->data->nb_rx_queues;
2171                 if (intr_vector > IXGBE_MAX_INTR_QUEUE_NUM) {
2172                         PMD_INIT_LOG(ERR, "At most %d intr queues supported",
2173                                         IXGBE_MAX_INTR_QUEUE_NUM);
2174                         return -ENOTSUP;
2175                 }
2176                 if (rte_intr_efd_enable(intr_handle, intr_vector))
2177                         return -1;
2178         }
2179
2180         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
2181                 intr_handle->intr_vec =
2182                         rte_zmalloc("intr_vec",
2183                                     dev->data->nb_rx_queues * sizeof(int), 0);
2184                 if (intr_handle->intr_vec == NULL) {
2185                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2186                                      " intr_vec\n", dev->data->nb_rx_queues);
2187                         return -ENOMEM;
2188                 }
2189         }
2190
2191         /* confiugre msix for sleep until rx interrupt */
2192         ixgbe_configure_msix(dev);
2193
2194         /* initialize transmission unit */
2195         ixgbe_dev_tx_init(dev);
2196
2197         /* This can fail when allocating mbufs for descriptor rings */
2198         err = ixgbe_dev_rx_init(dev);
2199         if (err) {
2200                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
2201                 goto error;
2202         }
2203
2204         err = ixgbe_dev_rxtx_start(dev);
2205         if (err < 0) {
2206                 PMD_INIT_LOG(ERR, "Unable to start rxtx queues");
2207                 goto error;
2208         }
2209
2210         /* Skip link setup if loopback mode is enabled for 82599. */
2211         if (hw->mac.type == ixgbe_mac_82599EB &&
2212                         dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_82599_TX_RX)
2213                 goto skip_link_setup;
2214
2215         if (ixgbe_is_sfp(hw) && hw->phy.multispeed_fiber) {
2216                 err = hw->mac.ops.setup_sfp(hw);
2217                 if (err)
2218                         goto error;
2219         }
2220
2221         if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
2222                 /* Turn on the copper */
2223                 ixgbe_set_phy_power(hw, true);
2224         } else {
2225                 /* Turn on the laser */
2226                 ixgbe_enable_tx_laser(hw);
2227         }
2228
2229         err = ixgbe_check_link(hw, &speed, &link_up, 0);
2230         if (err)
2231                 goto error;
2232         dev->data->dev_link.link_status = link_up;
2233
2234         err = ixgbe_get_link_capabilities(hw, &speed, &negotiate);
2235         if (err)
2236                 goto error;
2237
2238         link_speeds = &dev->data->dev_conf.link_speeds;
2239         if (*link_speeds & ~(ETH_LINK_SPEED_100M | ETH_LINK_SPEED_1G |
2240                         ETH_LINK_SPEED_10G)) {
2241                 PMD_INIT_LOG(ERR, "Invalid link setting");
2242                 goto error;
2243         }
2244
2245         speed = 0x0;
2246         if (*link_speeds == ETH_LINK_SPEED_AUTONEG) {
2247                 speed = (hw->mac.type != ixgbe_mac_82598EB) ?
2248                                 IXGBE_LINK_SPEED_82599_AUTONEG :
2249                                 IXGBE_LINK_SPEED_82598_AUTONEG;
2250         } else {
2251                 if (*link_speeds & ETH_LINK_SPEED_10G)
2252                         speed |= IXGBE_LINK_SPEED_10GB_FULL;
2253                 if (*link_speeds & ETH_LINK_SPEED_1G)
2254                         speed |= IXGBE_LINK_SPEED_1GB_FULL;
2255                 if (*link_speeds & ETH_LINK_SPEED_100M)
2256                         speed |= IXGBE_LINK_SPEED_100_FULL;
2257         }
2258
2259         err = ixgbe_setup_link(hw, speed, link_up);
2260         if (err)
2261                 goto error;
2262
2263 skip_link_setup:
2264
2265         if (rte_intr_allow_others(intr_handle)) {
2266                 /* check if lsc interrupt is enabled */
2267                 if (dev->data->dev_conf.intr_conf.lsc != 0)
2268                         ixgbe_dev_lsc_interrupt_setup(dev);
2269         } else {
2270                 rte_intr_callback_unregister(intr_handle,
2271                                              ixgbe_dev_interrupt_handler,
2272                                              (void *)dev);
2273                 if (dev->data->dev_conf.intr_conf.lsc != 0)
2274                         PMD_INIT_LOG(INFO, "lsc won't enable because of"
2275                                      " no intr multiplex\n");
2276         }
2277
2278         /* check if rxq interrupt is enabled */
2279         if (dev->data->dev_conf.intr_conf.rxq != 0 &&
2280             rte_intr_dp_is_en(intr_handle))
2281                 ixgbe_dev_rxq_interrupt_setup(dev);
2282
2283         /* enable uio/vfio intr/eventfd mapping */
2284         rte_intr_enable(intr_handle);
2285
2286         /* resume enabled intr since hw reset */
2287         ixgbe_enable_intr(dev);
2288
2289         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK |
2290                 ETH_VLAN_EXTEND_MASK;
2291         ixgbe_vlan_offload_set(dev, mask);
2292
2293         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
2294                 /* Enable vlan filtering for VMDq */
2295                 ixgbe_vmdq_vlan_hw_filter_enable(dev);
2296         }
2297
2298         /* Configure DCB hw */
2299         ixgbe_configure_dcb(dev);
2300
2301         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) {
2302                 err = ixgbe_fdir_configure(dev);
2303                 if (err)
2304                         goto error;
2305         }
2306
2307         /* Restore vf rate limit */
2308         if (vfinfo != NULL) {
2309                 for (vf = 0; vf < dev->pci_dev->max_vfs; vf++)
2310                         for (idx = 0; idx < IXGBE_MAX_QUEUE_NUM_PER_VF; idx++)
2311                                 if (vfinfo[vf].tx_rate[idx] != 0)
2312                                         ixgbe_set_vf_rate_limit(dev, vf,
2313                                                 vfinfo[vf].tx_rate[idx],
2314                                                 1 << idx);
2315         }
2316
2317         ixgbe_restore_statistics_mapping(dev);
2318
2319         return 0;
2320
2321 error:
2322         PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err);
2323         ixgbe_dev_clear_queues(dev);
2324         return -EIO;
2325 }
2326
2327 /*
2328  * Stop device: disable rx and tx functions to allow for reconfiguring.
2329  */
2330 static void
2331 ixgbe_dev_stop(struct rte_eth_dev *dev)
2332 {
2333         struct rte_eth_link link;
2334         struct ixgbe_hw *hw =
2335                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2336         struct ixgbe_vf_info *vfinfo =
2337                 *IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private);
2338         struct ixgbe_filter_info *filter_info =
2339                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
2340         struct ixgbe_5tuple_filter *p_5tuple, *p_5tuple_next;
2341         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
2342         int vf;
2343
2344         PMD_INIT_FUNC_TRACE();
2345
2346         /* disable interrupts */
2347         ixgbe_disable_intr(hw);
2348
2349         /* reset the NIC */
2350         ixgbe_pf_reset_hw(hw);
2351         hw->adapter_stopped = 0;
2352
2353         /* stop adapter */
2354         ixgbe_stop_adapter(hw);
2355
2356         for (vf = 0; vfinfo != NULL &&
2357                      vf < dev->pci_dev->max_vfs; vf++)
2358                 vfinfo[vf].clear_to_send = false;
2359
2360         if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
2361                 /* Turn off the copper */
2362                 ixgbe_set_phy_power(hw, false);
2363         } else {
2364                 /* Turn off the laser */
2365                 ixgbe_disable_tx_laser(hw);
2366         }
2367
2368         ixgbe_dev_clear_queues(dev);
2369
2370         /* Clear stored conf */
2371         dev->data->scattered_rx = 0;
2372         dev->data->lro = 0;
2373
2374         /* Clear recorded link status */
2375         memset(&link, 0, sizeof(link));
2376         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
2377
2378         /* Remove all ntuple filters of the device */
2379         for (p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list);
2380              p_5tuple != NULL; p_5tuple = p_5tuple_next) {
2381                 p_5tuple_next = TAILQ_NEXT(p_5tuple, entries);
2382                 TAILQ_REMOVE(&filter_info->fivetuple_list,
2383                              p_5tuple, entries);
2384                 rte_free(p_5tuple);
2385         }
2386         memset(filter_info->fivetuple_mask, 0,
2387                 sizeof(uint32_t) * IXGBE_5TUPLE_ARRAY_SIZE);
2388
2389         if (!rte_intr_allow_others(intr_handle))
2390                 /* resume to the default handler */
2391                 rte_intr_callback_register(intr_handle,
2392                                            ixgbe_dev_interrupt_handler,
2393                                            (void *)dev);
2394
2395         /* Clean datapath event and queue/vec mapping */
2396         rte_intr_efd_disable(intr_handle);
2397         if (intr_handle->intr_vec != NULL) {
2398                 rte_free(intr_handle->intr_vec);
2399                 intr_handle->intr_vec = NULL;
2400         }
2401 }
2402
2403 /*
2404  * Set device link up: enable tx.
2405  */
2406 static int
2407 ixgbe_dev_set_link_up(struct rte_eth_dev *dev)
2408 {
2409         struct ixgbe_hw *hw =
2410                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2411         if (hw->mac.type == ixgbe_mac_82599EB) {
2412 #ifdef RTE_NIC_BYPASS
2413                 if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) {
2414                         /* Not suported in bypass mode */
2415                         PMD_INIT_LOG(ERR, "Set link up is not supported "
2416                                      "by device id 0x%x", hw->device_id);
2417                         return -ENOTSUP;
2418                 }
2419 #endif
2420         }
2421
2422         if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
2423                 /* Turn on the copper */
2424                 ixgbe_set_phy_power(hw, true);
2425         } else {
2426                 /* Turn on the laser */
2427                 ixgbe_enable_tx_laser(hw);
2428         }
2429
2430         return 0;
2431 }
2432
2433 /*
2434  * Set device link down: disable tx.
2435  */
2436 static int
2437 ixgbe_dev_set_link_down(struct rte_eth_dev *dev)
2438 {
2439         struct ixgbe_hw *hw =
2440                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2441         if (hw->mac.type == ixgbe_mac_82599EB) {
2442 #ifdef RTE_NIC_BYPASS
2443                 if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) {
2444                         /* Not suported in bypass mode */
2445                         PMD_INIT_LOG(ERR, "Set link down is not supported "
2446                                      "by device id 0x%x", hw->device_id);
2447                         return -ENOTSUP;
2448                 }
2449 #endif
2450         }
2451
2452         if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
2453                 /* Turn off the copper */
2454                 ixgbe_set_phy_power(hw, false);
2455         } else {
2456                 /* Turn off the laser */
2457                 ixgbe_disable_tx_laser(hw);
2458         }
2459
2460         return 0;
2461 }
2462
2463 /*
2464  * Reest and stop device.
2465  */
2466 static void
2467 ixgbe_dev_close(struct rte_eth_dev *dev)
2468 {
2469         struct ixgbe_hw *hw =
2470                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2471
2472         PMD_INIT_FUNC_TRACE();
2473
2474         ixgbe_pf_reset_hw(hw);
2475
2476         ixgbe_dev_stop(dev);
2477         hw->adapter_stopped = 1;
2478
2479         ixgbe_dev_free_queues(dev);
2480
2481         ixgbe_disable_pcie_master(hw);
2482
2483         /* reprogram the RAR[0] in case user changed it. */
2484         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
2485 }
2486
2487 static void
2488 ixgbe_read_stats_registers(struct ixgbe_hw *hw,
2489                            struct ixgbe_hw_stats *hw_stats,
2490                            uint64_t *total_missed_rx, uint64_t *total_qbrc,
2491                            uint64_t *total_qprc, uint64_t *total_qprdc)
2492 {
2493         uint32_t bprc, lxon, lxoff, total;
2494         uint32_t delta_gprc = 0;
2495         unsigned i;
2496         /* Workaround for RX byte count not including CRC bytes when CRC
2497 +        * strip is enabled. CRC bytes are removed from counters when crc_strip
2498          * is disabled.
2499 +        */
2500         int crc_strip = (IXGBE_READ_REG(hw, IXGBE_HLREG0) &
2501                         IXGBE_HLREG0_RXCRCSTRP);
2502
2503         hw_stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
2504         hw_stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
2505         hw_stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
2506         hw_stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
2507
2508         for (i = 0; i < 8; i++) {
2509                 uint32_t mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
2510
2511                 /* global total per queue */
2512                 hw_stats->mpc[i] += mp;
2513                 /* Running comprehensive total for stats display */
2514                 *total_missed_rx += hw_stats->mpc[i];
2515                 if (hw->mac.type == ixgbe_mac_82598EB) {
2516                         hw_stats->rnbc[i] +=
2517                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
2518                         hw_stats->pxonrxc[i] +=
2519                                 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
2520                         hw_stats->pxoffrxc[i] +=
2521                                 IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
2522                 } else {
2523                         hw_stats->pxonrxc[i] +=
2524                                 IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
2525                         hw_stats->pxoffrxc[i] +=
2526                                 IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i));
2527                         hw_stats->pxon2offc[i] +=
2528                                 IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
2529                 }
2530                 hw_stats->pxontxc[i] +=
2531                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
2532                 hw_stats->pxofftxc[i] +=
2533                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
2534         }
2535         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
2536                 uint32_t delta_qprc = IXGBE_READ_REG(hw, IXGBE_QPRC(i));
2537                 uint32_t delta_qptc = IXGBE_READ_REG(hw, IXGBE_QPTC(i));
2538                 uint32_t delta_qprdc = IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
2539
2540                 delta_gprc += delta_qprc;
2541
2542                 hw_stats->qprc[i] += delta_qprc;
2543                 hw_stats->qptc[i] += delta_qptc;
2544
2545                 hw_stats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
2546                 hw_stats->qbrc[i] +=
2547                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32);
2548                 if (crc_strip == 0)
2549                         hw_stats->qbrc[i] -= delta_qprc * ETHER_CRC_LEN;
2550
2551                 hw_stats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
2552                 hw_stats->qbtc[i] +=
2553                     ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32);
2554
2555                 hw_stats->qprdc[i] += delta_qprdc;
2556                 *total_qprdc += hw_stats->qprdc[i];
2557
2558                 *total_qprc += hw_stats->qprc[i];
2559                 *total_qbrc += hw_stats->qbrc[i];
2560         }
2561         hw_stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
2562         hw_stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
2563         hw_stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
2564
2565         /*
2566          * An errata states that gprc actually counts good + missed packets:
2567          * Workaround to set gprc to summated queue packet receives
2568          */
2569         hw_stats->gprc = *total_qprc;
2570
2571         if (hw->mac.type != ixgbe_mac_82598EB) {
2572                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
2573                 hw_stats->gorc += ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
2574                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
2575                 hw_stats->gotc += ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
2576                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
2577                 hw_stats->tor += ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
2578                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
2579                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
2580         } else {
2581                 hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
2582                 hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
2583                 /* 82598 only has a counter in the high register */
2584                 hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
2585                 hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
2586                 hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
2587         }
2588         uint64_t old_tpr = hw_stats->tpr;
2589
2590         hw_stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
2591         hw_stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
2592
2593         if (crc_strip == 0)
2594                 hw_stats->gorc -= delta_gprc * ETHER_CRC_LEN;
2595
2596         uint64_t delta_gptc = IXGBE_READ_REG(hw, IXGBE_GPTC);
2597         hw_stats->gptc += delta_gptc;
2598         hw_stats->gotc -= delta_gptc * ETHER_CRC_LEN;
2599         hw_stats->tor -= (hw_stats->tpr - old_tpr) * ETHER_CRC_LEN;
2600
2601         /*
2602          * Workaround: mprc hardware is incorrectly counting
2603          * broadcasts, so for now we subtract those.
2604          */
2605         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
2606         hw_stats->bprc += bprc;
2607         hw_stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
2608         if (hw->mac.type == ixgbe_mac_82598EB)
2609                 hw_stats->mprc -= bprc;
2610
2611         hw_stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
2612         hw_stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
2613         hw_stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
2614         hw_stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
2615         hw_stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
2616         hw_stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
2617
2618         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
2619         hw_stats->lxontxc += lxon;
2620         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
2621         hw_stats->lxofftxc += lxoff;
2622         total = lxon + lxoff;
2623
2624         hw_stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
2625         hw_stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
2626         hw_stats->gptc -= total;
2627         hw_stats->mptc -= total;
2628         hw_stats->ptc64 -= total;
2629         hw_stats->gotc -= total * ETHER_MIN_LEN;
2630
2631         hw_stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
2632         hw_stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
2633         hw_stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
2634         hw_stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
2635         hw_stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
2636         hw_stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
2637         hw_stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
2638         hw_stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
2639         hw_stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
2640         hw_stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
2641         hw_stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
2642         hw_stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
2643         hw_stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
2644         hw_stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);
2645         hw_stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
2646         hw_stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
2647         /* Only read FCOE on 82599 */
2648         if (hw->mac.type != ixgbe_mac_82598EB) {
2649                 hw_stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
2650                 hw_stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
2651                 hw_stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
2652                 hw_stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
2653                 hw_stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
2654         }
2655
2656         /* Flow Director Stats registers */
2657         hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
2658         hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
2659 }
2660
2661 /*
2662  * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c
2663  */
2664 static void
2665 ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
2666 {
2667         struct ixgbe_hw *hw =
2668                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2669         struct ixgbe_hw_stats *hw_stats =
2670                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2671         uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc;
2672         unsigned i;
2673
2674         total_missed_rx = 0;
2675         total_qbrc = 0;
2676         total_qprc = 0;
2677         total_qprdc = 0;
2678
2679         ixgbe_read_stats_registers(hw, hw_stats, &total_missed_rx, &total_qbrc,
2680                         &total_qprc, &total_qprdc);
2681
2682         if (stats == NULL)
2683                 return;
2684
2685         /* Fill out the rte_eth_stats statistics structure */
2686         stats->ipackets = total_qprc;
2687         stats->ibytes = total_qbrc;
2688         stats->opackets = hw_stats->gptc;
2689         stats->obytes = hw_stats->gotc;
2690
2691         for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
2692                 stats->q_ipackets[i] = hw_stats->qprc[i];
2693                 stats->q_opackets[i] = hw_stats->qptc[i];
2694                 stats->q_ibytes[i] = hw_stats->qbrc[i];
2695                 stats->q_obytes[i] = hw_stats->qbtc[i];
2696                 stats->q_errors[i] = hw_stats->qprdc[i];
2697         }
2698
2699         /* Rx Errors */
2700         stats->imissed  = total_missed_rx;
2701         stats->ierrors  = hw_stats->crcerrs +
2702                           hw_stats->mspdc +
2703                           hw_stats->rlec +
2704                           hw_stats->ruc +
2705                           hw_stats->roc +
2706                           hw_stats->illerrc +
2707                           hw_stats->errbc +
2708                           hw_stats->rfc +
2709                           hw_stats->fccrc +
2710                           hw_stats->fclast;
2711
2712         /* Tx Errors */
2713         stats->oerrors  = 0;
2714 }
2715
2716 static void
2717 ixgbe_dev_stats_reset(struct rte_eth_dev *dev)
2718 {
2719         struct ixgbe_hw_stats *stats =
2720                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2721
2722         /* HW registers are cleared on read */
2723         ixgbe_dev_stats_get(dev, NULL);
2724
2725         /* Reset software totals */
2726         memset(stats, 0, sizeof(*stats));
2727 }
2728
2729 /* This function calculates the number of xstats based on the current config */
2730 static unsigned
2731 ixgbe_xstats_calc_num(void) {
2732         return IXGBE_NB_HW_STATS +
2733                 (IXGBE_NB_RXQ_PRIO_STATS * IXGBE_NB_RXQ_PRIO_VALUES) +
2734                 (IXGBE_NB_TXQ_PRIO_STATS * IXGBE_NB_TXQ_PRIO_VALUES);
2735 }
2736
2737 static int ixgbe_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
2738         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
2739 {
2740         const unsigned cnt_stats = ixgbe_xstats_calc_num();
2741         unsigned stat, i, count;
2742
2743         if (xstats_names != NULL) {
2744                 count = 0;
2745
2746                 /* Note: limit >= cnt_stats checked upstream
2747                  * in rte_eth_xstats_names()
2748                  */
2749
2750                 /* Extended stats from ixgbe_hw_stats */
2751                 for (i = 0; i < IXGBE_NB_HW_STATS; i++) {
2752                         xstats_names[count].id = count;
2753                         snprintf(xstats_names[count].name,
2754                                 sizeof(xstats_names[count].name),
2755                                 "%s",
2756                                 rte_ixgbe_stats_strings[i].name);
2757                         count++;
2758                 }
2759
2760                 /* RX Priority Stats */
2761                 for (stat = 0; stat < IXGBE_NB_RXQ_PRIO_STATS; stat++) {
2762                         for (i = 0; i < IXGBE_NB_RXQ_PRIO_VALUES; i++) {
2763                                 xstats_names[count].id = count;
2764                                 snprintf(xstats_names[count].name,
2765                                         sizeof(xstats_names[count].name),
2766                                         "rx_priority%u_%s", i,
2767                                         rte_ixgbe_rxq_strings[stat].name);
2768                                 count++;
2769                         }
2770                 }
2771
2772                 /* TX Priority Stats */
2773                 for (stat = 0; stat < IXGBE_NB_TXQ_PRIO_STATS; stat++) {
2774                         for (i = 0; i < IXGBE_NB_TXQ_PRIO_VALUES; i++) {
2775                                 xstats_names[count].id = count;
2776                                 snprintf(xstats_names[count].name,
2777                                         sizeof(xstats_names[count].name),
2778                                         "tx_priority%u_%s", i,
2779                                         rte_ixgbe_txq_strings[stat].name);
2780                                 count++;
2781                         }
2782                 }
2783         }
2784         return cnt_stats;
2785 }
2786
2787 static int ixgbevf_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
2788         struct rte_eth_xstat_name *xstats_names, unsigned limit)
2789 {
2790         unsigned i;
2791
2792         if (limit < IXGBEVF_NB_XSTATS && xstats_names != NULL)
2793                 return -ENOMEM;
2794
2795         if (xstats_names != NULL)
2796                 for (i = 0; i < IXGBEVF_NB_XSTATS; i++)
2797                         snprintf(xstats_names[i].name,
2798                                 sizeof(xstats_names[i].name),
2799                                 "%s", rte_ixgbevf_stats_strings[i].name);
2800         return IXGBEVF_NB_XSTATS;
2801 }
2802
2803 static int
2804 ixgbe_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
2805                                          unsigned n)
2806 {
2807         struct ixgbe_hw *hw =
2808                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2809         struct ixgbe_hw_stats *hw_stats =
2810                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2811         uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc;
2812         unsigned i, stat, count = 0;
2813
2814         count = ixgbe_xstats_calc_num();
2815
2816         if (n < count)
2817                 return count;
2818
2819         total_missed_rx = 0;
2820         total_qbrc = 0;
2821         total_qprc = 0;
2822         total_qprdc = 0;
2823
2824         ixgbe_read_stats_registers(hw, hw_stats, &total_missed_rx, &total_qbrc,
2825                                    &total_qprc, &total_qprdc);
2826
2827         /* If this is a reset xstats is NULL, and we have cleared the
2828          * registers by reading them.
2829          */
2830         if (!xstats)
2831                 return 0;
2832
2833         /* Extended stats from ixgbe_hw_stats */
2834         count = 0;
2835         for (i = 0; i < IXGBE_NB_HW_STATS; i++) {
2836                 xstats[count].id = count;
2837                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
2838                                 rte_ixgbe_stats_strings[i].offset);
2839                 count++;
2840         }
2841
2842         /* RX Priority Stats */
2843         for (stat = 0; stat < IXGBE_NB_RXQ_PRIO_STATS; stat++) {
2844                 for (i = 0; i < IXGBE_NB_RXQ_PRIO_VALUES; i++) {
2845                         xstats[count].id = count;
2846                         xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
2847                                         rte_ixgbe_rxq_strings[stat].offset +
2848                                         (sizeof(uint64_t) * i));
2849                         count++;
2850                 }
2851         }
2852
2853         /* TX Priority Stats */
2854         for (stat = 0; stat < IXGBE_NB_TXQ_PRIO_STATS; stat++) {
2855                 for (i = 0; i < IXGBE_NB_TXQ_PRIO_VALUES; i++) {
2856                         xstats[count].id = count;
2857                         xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
2858                                         rte_ixgbe_txq_strings[stat].offset +
2859                                         (sizeof(uint64_t) * i));
2860                         count++;
2861                 }
2862         }
2863         return count;
2864 }
2865
2866 static void
2867 ixgbe_dev_xstats_reset(struct rte_eth_dev *dev)
2868 {
2869         struct ixgbe_hw_stats *stats =
2870                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2871
2872         unsigned count = ixgbe_xstats_calc_num();
2873
2874         /* HW registers are cleared on read */
2875         ixgbe_dev_xstats_get(dev, NULL, count);
2876
2877         /* Reset software totals */
2878         memset(stats, 0, sizeof(*stats));
2879 }
2880
2881 static void
2882 ixgbevf_update_stats(struct rte_eth_dev *dev)
2883 {
2884         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2885         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *)
2886                           IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2887
2888         /* Good Rx packet, include VF loopback */
2889         UPDATE_VF_STAT(IXGBE_VFGPRC,
2890             hw_stats->last_vfgprc, hw_stats->vfgprc);
2891
2892         /* Good Rx octets, include VF loopback */
2893         UPDATE_VF_STAT_36BIT(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
2894             hw_stats->last_vfgorc, hw_stats->vfgorc);
2895
2896         /* Good Tx packet, include VF loopback */
2897         UPDATE_VF_STAT(IXGBE_VFGPTC,
2898             hw_stats->last_vfgptc, hw_stats->vfgptc);
2899
2900         /* Good Tx octets, include VF loopback */
2901         UPDATE_VF_STAT_36BIT(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
2902             hw_stats->last_vfgotc, hw_stats->vfgotc);
2903
2904         /* Rx Multicst Packet */
2905         UPDATE_VF_STAT(IXGBE_VFMPRC,
2906             hw_stats->last_vfmprc, hw_stats->vfmprc);
2907 }
2908
2909 static int
2910 ixgbevf_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
2911                        unsigned n)
2912 {
2913         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *)
2914                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2915         unsigned i;
2916
2917         if (n < IXGBEVF_NB_XSTATS)
2918                 return IXGBEVF_NB_XSTATS;
2919
2920         ixgbevf_update_stats(dev);
2921
2922         if (!xstats)
2923                 return 0;
2924
2925         /* Extended stats */
2926         for (i = 0; i < IXGBEVF_NB_XSTATS; i++) {
2927                 xstats[i].id = i;
2928                 xstats[i].value = *(uint64_t *)(((char *)hw_stats) +
2929                         rte_ixgbevf_stats_strings[i].offset);
2930         }
2931
2932         return IXGBEVF_NB_XSTATS;
2933 }
2934
2935 static void
2936 ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
2937 {
2938         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *)
2939                           IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2940
2941         ixgbevf_update_stats(dev);
2942
2943         if (stats == NULL)
2944                 return;
2945
2946         stats->ipackets = hw_stats->vfgprc;
2947         stats->ibytes = hw_stats->vfgorc;
2948         stats->opackets = hw_stats->vfgptc;
2949         stats->obytes = hw_stats->vfgotc;
2950 }
2951
2952 static void
2953 ixgbevf_dev_stats_reset(struct rte_eth_dev *dev)
2954 {
2955         struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *)
2956                         IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2957
2958         /* Sync HW register to the last stats */
2959         ixgbevf_dev_stats_get(dev, NULL);
2960
2961         /* reset HW current stats*/
2962         hw_stats->vfgprc = 0;
2963         hw_stats->vfgorc = 0;
2964         hw_stats->vfgptc = 0;
2965         hw_stats->vfgotc = 0;
2966 }
2967
2968 static void
2969 ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
2970 {
2971         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2972         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
2973
2974         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
2975         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
2976         if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
2977                 /*
2978                  * When DCB/VT is off, maximum number of queues changes,
2979                  * except for 82598EB, which remains constant.
2980                  */
2981                 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_NONE &&
2982                                 hw->mac.type != ixgbe_mac_82598EB)
2983                         dev_info->max_tx_queues = IXGBE_NONE_MODE_TX_NB_QUEUES;
2984         }
2985         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL register */
2986         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */
2987         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
2988         dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC;
2989         dev_info->max_vfs = dev->pci_dev->max_vfs;
2990         if (hw->mac.type == ixgbe_mac_82598EB)
2991                 dev_info->max_vmdq_pools = ETH_16_POOLS;
2992         else
2993                 dev_info->max_vmdq_pools = ETH_64_POOLS;
2994         dev_info->vmdq_queue_num = dev_info->max_rx_queues;
2995         dev_info->rx_offload_capa =
2996                 DEV_RX_OFFLOAD_VLAN_STRIP |
2997                 DEV_RX_OFFLOAD_IPV4_CKSUM |
2998                 DEV_RX_OFFLOAD_UDP_CKSUM  |
2999                 DEV_RX_OFFLOAD_TCP_CKSUM;
3000
3001         /*
3002          * RSC is only supported by 82599 and x540 PF devices in a non-SR-IOV
3003          * mode.
3004          */
3005         if ((hw->mac.type == ixgbe_mac_82599EB ||
3006              hw->mac.type == ixgbe_mac_X540) &&
3007             !RTE_ETH_DEV_SRIOV(dev).active)
3008                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_LRO;
3009
3010         if (hw->mac.type == ixgbe_mac_X550 ||
3011             hw->mac.type == ixgbe_mac_X550EM_x ||
3012             hw->mac.type == ixgbe_mac_X550EM_a)
3013                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM;
3014
3015         dev_info->tx_offload_capa =
3016                 DEV_TX_OFFLOAD_VLAN_INSERT |
3017                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
3018                 DEV_TX_OFFLOAD_UDP_CKSUM   |
3019                 DEV_TX_OFFLOAD_TCP_CKSUM   |
3020                 DEV_TX_OFFLOAD_SCTP_CKSUM  |
3021                 DEV_TX_OFFLOAD_TCP_TSO;
3022
3023         if (hw->mac.type == ixgbe_mac_X550 ||
3024             hw->mac.type == ixgbe_mac_X550EM_x ||
3025             hw->mac.type == ixgbe_mac_X550EM_a)
3026                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
3027
3028         dev_info->default_rxconf = (struct rte_eth_rxconf) {
3029                 .rx_thresh = {
3030                         .pthresh = IXGBE_DEFAULT_RX_PTHRESH,
3031                         .hthresh = IXGBE_DEFAULT_RX_HTHRESH,
3032                         .wthresh = IXGBE_DEFAULT_RX_WTHRESH,
3033                 },
3034                 .rx_free_thresh = IXGBE_DEFAULT_RX_FREE_THRESH,
3035                 .rx_drop_en = 0,
3036         };
3037
3038         dev_info->default_txconf = (struct rte_eth_txconf) {
3039                 .tx_thresh = {
3040                         .pthresh = IXGBE_DEFAULT_TX_PTHRESH,
3041                         .hthresh = IXGBE_DEFAULT_TX_HTHRESH,
3042                         .wthresh = IXGBE_DEFAULT_TX_WTHRESH,
3043                 },
3044                 .tx_free_thresh = IXGBE_DEFAULT_TX_FREE_THRESH,
3045                 .tx_rs_thresh = IXGBE_DEFAULT_TX_RSBIT_THRESH,
3046                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
3047                                 ETH_TXQ_FLAGS_NOOFFLOADS,
3048         };
3049
3050         dev_info->rx_desc_lim = rx_desc_lim;
3051         dev_info->tx_desc_lim = tx_desc_lim;
3052
3053         dev_info->hash_key_size = IXGBE_HKEY_MAX_INDEX * sizeof(uint32_t);
3054         dev_info->reta_size = ixgbe_reta_size_get(hw->mac.type);
3055         dev_info->flow_type_rss_offloads = IXGBE_RSS_OFFLOAD_ALL;
3056
3057         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G;
3058         if (hw->mac.type == ixgbe_mac_X540 ||
3059             hw->mac.type == ixgbe_mac_X540_vf ||
3060             hw->mac.type == ixgbe_mac_X550 ||
3061             hw->mac.type == ixgbe_mac_X550_vf) {
3062                 dev_info->speed_capa |= ETH_LINK_SPEED_100M;
3063         }
3064 }
3065
3066 static const uint32_t *
3067 ixgbe_dev_supported_ptypes_get(struct rte_eth_dev *dev)
3068 {
3069         static const uint32_t ptypes[] = {
3070                 /* For non-vec functions,
3071                  * refers to ixgbe_rxd_pkt_info_to_pkt_type();
3072                  * for vec functions,
3073                  * refers to _recv_raw_pkts_vec().
3074                  */
3075                 RTE_PTYPE_L2_ETHER,
3076                 RTE_PTYPE_L3_IPV4,
3077                 RTE_PTYPE_L3_IPV4_EXT,
3078                 RTE_PTYPE_L3_IPV6,
3079                 RTE_PTYPE_L3_IPV6_EXT,
3080                 RTE_PTYPE_L4_SCTP,
3081                 RTE_PTYPE_L4_TCP,
3082                 RTE_PTYPE_L4_UDP,
3083                 RTE_PTYPE_TUNNEL_IP,
3084                 RTE_PTYPE_INNER_L3_IPV6,
3085                 RTE_PTYPE_INNER_L3_IPV6_EXT,
3086                 RTE_PTYPE_INNER_L4_TCP,
3087                 RTE_PTYPE_INNER_L4_UDP,
3088                 RTE_PTYPE_UNKNOWN
3089         };
3090
3091         if (dev->rx_pkt_burst == ixgbe_recv_pkts ||
3092             dev->rx_pkt_burst == ixgbe_recv_pkts_lro_single_alloc ||
3093             dev->rx_pkt_burst == ixgbe_recv_pkts_lro_bulk_alloc ||
3094             dev->rx_pkt_burst == ixgbe_recv_pkts_bulk_alloc)
3095                 return ptypes;
3096         return NULL;
3097 }
3098
3099 static void
3100 ixgbevf_dev_info_get(struct rte_eth_dev *dev,
3101                      struct rte_eth_dev_info *dev_info)
3102 {
3103         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3104
3105         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
3106         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
3107         dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL reg */
3108         dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS reg */
3109         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
3110         dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC;
3111         dev_info->max_vfs = dev->pci_dev->max_vfs;
3112         if (hw->mac.type == ixgbe_mac_82598EB)
3113                 dev_info->max_vmdq_pools = ETH_16_POOLS;
3114         else
3115                 dev_info->max_vmdq_pools = ETH_64_POOLS;
3116         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
3117                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
3118                                 DEV_RX_OFFLOAD_UDP_CKSUM  |
3119                                 DEV_RX_OFFLOAD_TCP_CKSUM;
3120         dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
3121                                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
3122                                 DEV_TX_OFFLOAD_UDP_CKSUM   |
3123                                 DEV_TX_OFFLOAD_TCP_CKSUM   |
3124                                 DEV_TX_OFFLOAD_SCTP_CKSUM  |
3125                                 DEV_TX_OFFLOAD_TCP_TSO;
3126
3127         dev_info->default_rxconf = (struct rte_eth_rxconf) {
3128                 .rx_thresh = {
3129                         .pthresh = IXGBE_DEFAULT_RX_PTHRESH,
3130                         .hthresh = IXGBE_DEFAULT_RX_HTHRESH,
3131                         .wthresh = IXGBE_DEFAULT_RX_WTHRESH,
3132                 },
3133                 .rx_free_thresh = IXGBE_DEFAULT_RX_FREE_THRESH,
3134                 .rx_drop_en = 0,
3135         };
3136
3137         dev_info->default_txconf = (struct rte_eth_txconf) {
3138                 .tx_thresh = {
3139                         .pthresh = IXGBE_DEFAULT_TX_PTHRESH,
3140                         .hthresh = IXGBE_DEFAULT_TX_HTHRESH,
3141                         .wthresh = IXGBE_DEFAULT_TX_WTHRESH,
3142                 },
3143                 .tx_free_thresh = IXGBE_DEFAULT_TX_FREE_THRESH,
3144                 .tx_rs_thresh = IXGBE_DEFAULT_TX_RSBIT_THRESH,
3145                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
3146                                 ETH_TXQ_FLAGS_NOOFFLOADS,
3147         };
3148
3149         dev_info->rx_desc_lim = rx_desc_lim;
3150         dev_info->tx_desc_lim = tx_desc_lim;
3151 }
3152
3153 /* return 0 means link status changed, -1 means not changed */
3154 static int
3155 ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
3156 {
3157         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3158         struct rte_eth_link link, old;
3159         ixgbe_link_speed link_speed = IXGBE_LINK_SPEED_UNKNOWN;
3160         int link_up;
3161         int diag;
3162
3163         link.link_status = ETH_LINK_DOWN;
3164         link.link_speed = 0;
3165         link.link_duplex = ETH_LINK_HALF_DUPLEX;
3166         memset(&old, 0, sizeof(old));
3167         rte_ixgbe_dev_atomic_read_link_status(dev, &old);
3168
3169         hw->mac.get_link_status = true;
3170
3171         /* check if it needs to wait to complete, if lsc interrupt is enabled */
3172         if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
3173                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 0);
3174         else
3175                 diag = ixgbe_check_link(hw, &link_speed, &link_up, 1);
3176
3177         if (diag != 0) {
3178                 link.link_speed = ETH_SPEED_NUM_100M;
3179                 link.link_duplex = ETH_LINK_FULL_DUPLEX;
3180                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
3181                 if (link.link_status == old.link_status)
3182                         return -1;
3183                 return 0;
3184         }
3185
3186         if (link_up == 0) {
3187                 rte_ixgbe_dev_atomic_write_link_status(dev, &link);
3188                 if (link.link_status == old.link_status)
3189                         return -1;
3190                 return 0;
3191         }
3192         link.link_status = ETH_LINK_UP;
3193         link.link_duplex = ETH_LINK_FULL_DUPLEX;
3194
3195         switch (link_speed) {
3196         default:
3197         case IXGBE_LINK_SPEED_UNKNOWN:
3198                 link.link_duplex = ETH_LINK_FULL_DUPLEX;
3199                 link.link_speed = ETH_SPEED_NUM_100M;
3200                 break;
3201
3202         case IXGBE_LINK_SPEED_100_FULL:
3203                 link.link_speed = ETH_SPEED_NUM_100M;
3204                 break;
3205
3206         case IXGBE_LINK_SPEED_1GB_FULL:
3207                 link.link_speed = ETH_SPEED_NUM_1G;
3208                 break;
3209
3210         case IXGBE_LINK_SPEED_10GB_FULL:
3211                 link.link_speed = ETH_SPEED_NUM_10G;
3212                 break;
3213         }
3214         rte_ixgbe_dev_atomic_write_link_status(dev, &link);
3215
3216         if (link.link_status == old.link_status)
3217                 return -1;
3218
3219         return 0;
3220 }
3221
3222 static void
3223 ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev)
3224 {
3225         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3226         uint32_t fctrl;
3227
3228         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
3229         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
3230         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
3231 }
3232
3233 static void
3234 ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev)
3235 {
3236         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3237         uint32_t fctrl;
3238
3239         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
3240         fctrl &= (~IXGBE_FCTRL_UPE);
3241         if (dev->data->all_multicast == 1)
3242                 fctrl |= IXGBE_FCTRL_MPE;
3243         else
3244                 fctrl &= (~IXGBE_FCTRL_MPE);
3245         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
3246 }
3247
3248 static void
3249 ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev)
3250 {
3251         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3252         uint32_t fctrl;
3253
3254         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
3255         fctrl |= IXGBE_FCTRL_MPE;
3256         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
3257 }
3258
3259 static void
3260 ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev)
3261 {
3262         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3263         uint32_t fctrl;
3264
3265         if (dev->data->promiscuous == 1)
3266                 return; /* must remain in all_multicast mode */
3267
3268         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
3269         fctrl &= (~IXGBE_FCTRL_MPE);
3270         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
3271 }
3272
3273 /**
3274  * It clears the interrupt causes and enables the interrupt.
3275  * It will be called once only during nic initialized.
3276  *
3277  * @param dev
3278  *  Pointer to struct rte_eth_dev.
3279  *
3280  * @return
3281  *  - On success, zero.
3282  *  - On failure, a negative value.
3283  */
3284 static int
3285 ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev)
3286 {
3287         struct ixgbe_interrupt *intr =
3288                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
3289
3290         ixgbe_dev_link_status_print(dev);
3291         intr->mask |= IXGBE_EICR_LSC;
3292
3293         return 0;
3294 }
3295
3296 /**
3297  * It clears the interrupt causes and enables the interrupt.
3298  * It will be called once only during nic initialized.
3299  *
3300  * @param dev
3301  *  Pointer to struct rte_eth_dev.
3302  *
3303  * @return
3304  *  - On success, zero.
3305  *  - On failure, a negative value.
3306  */
3307 static int
3308 ixgbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
3309 {
3310         struct ixgbe_interrupt *intr =
3311                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
3312
3313         intr->mask |= IXGBE_EICR_RTX_QUEUE;
3314
3315         return 0;
3316 }
3317
3318 /*
3319  * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update.
3320  *
3321  * @param dev
3322  *  Pointer to struct rte_eth_dev.
3323  *
3324  * @return
3325  *  - On success, zero.
3326  *  - On failure, a negative value.
3327  */
3328 static int
3329 ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
3330 {
3331         uint32_t eicr;
3332         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3333         struct ixgbe_interrupt *intr =
3334                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
3335
3336         /* clear all cause mask */
3337         ixgbe_disable_intr(hw);
3338
3339         /* read-on-clear nic registers here */
3340         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
3341         PMD_DRV_LOG(DEBUG, "eicr %x", eicr);
3342
3343         intr->flags = 0;
3344
3345         /* set flag for async link update */
3346         if (eicr & IXGBE_EICR_LSC)
3347                 intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
3348
3349         if (eicr & IXGBE_EICR_MAILBOX)
3350                 intr->flags |= IXGBE_FLAG_MAILBOX;
3351
3352         if (hw->mac.type ==  ixgbe_mac_X550EM_x &&
3353             hw->phy.type == ixgbe_phy_x550em_ext_t &&
3354             (eicr & IXGBE_EICR_GPI_SDP0_X550EM_x))
3355                 intr->flags |= IXGBE_FLAG_PHY_INTERRUPT;
3356
3357         return 0;
3358 }
3359
3360 /**
3361  * It gets and then prints the link status.
3362  *
3363  * @param dev
3364  *  Pointer to struct rte_eth_dev.
3365  *
3366  * @return
3367  *  - On success, zero.
3368  *  - On failure, a negative value.
3369  */
3370 static void
3371 ixgbe_dev_link_status_print(struct rte_eth_dev *dev)
3372 {
3373         struct rte_eth_link link;
3374
3375         memset(&link, 0, sizeof(link));
3376         rte_ixgbe_dev_atomic_read_link_status(dev, &link);
3377         if (link.link_status) {
3378                 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
3379                                         (int)(dev->data->port_id),
3380                                         (unsigned)link.link_speed,
3381                         link.link_duplex == ETH_LINK_FULL_DUPLEX ?
3382                                         "full-duplex" : "half-duplex");
3383         } else {
3384                 PMD_INIT_LOG(INFO, " Port %d: Link Down",
3385                                 (int)(dev->data->port_id));
3386         }
3387         PMD_INIT_LOG(DEBUG, "PCI Address: %04d:%02d:%02d:%d",
3388                                 dev->pci_dev->addr.domain,
3389                                 dev->pci_dev->addr.bus,
3390                                 dev->pci_dev->addr.devid,
3391                                 dev->pci_dev->addr.function);
3392 }
3393
3394 /*
3395  * It executes link_update after knowing an interrupt occurred.
3396  *
3397  * @param dev
3398  *  Pointer to struct rte_eth_dev.
3399  *
3400  * @return
3401  *  - On success, zero.
3402  *  - On failure, a negative value.
3403  */
3404 static int
3405 ixgbe_dev_interrupt_action(struct rte_eth_dev *dev)
3406 {
3407         struct ixgbe_interrupt *intr =
3408                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
3409         int64_t timeout;
3410         struct rte_eth_link link;
3411         int intr_enable_delay = false;
3412         struct ixgbe_hw *hw =
3413                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3414
3415         PMD_DRV_LOG(DEBUG, "intr action type %d", intr->flags);
3416
3417         if (intr->flags & IXGBE_FLAG_MAILBOX) {
3418                 ixgbe_pf_mbx_process(dev);
3419                 intr->flags &= ~IXGBE_FLAG_MAILBOX;
3420         }
3421
3422         if (intr->flags & IXGBE_FLAG_PHY_INTERRUPT) {
3423                 ixgbe_handle_lasi(hw);
3424                 intr->flags &= ~IXGBE_FLAG_PHY_INTERRUPT;
3425         }
3426
3427         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
3428                 /* get the link status before link update, for predicting later */
3429                 memset(&link, 0, sizeof(link));
3430                 rte_ixgbe_dev_atomic_read_link_status(dev, &link);
3431
3432                 ixgbe_dev_link_update(dev, 0);
3433
3434                 /* likely to up */
3435                 if (!link.link_status)
3436                         /* handle it 1 sec later, wait it being stable */
3437                         timeout = IXGBE_LINK_UP_CHECK_TIMEOUT;
3438                 /* likely to down */
3439                 else
3440                         /* handle it 4 sec later, wait it being stable */
3441                         timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT;
3442
3443                 ixgbe_dev_link_status_print(dev);
3444
3445                 intr_enable_delay = true;
3446         }
3447
3448         if (intr_enable_delay) {
3449                 if (rte_eal_alarm_set(timeout * 1000,
3450                                       ixgbe_dev_interrupt_delayed_handler, (void *)dev) < 0)
3451                         PMD_DRV_LOG(ERR, "Error setting alarm");
3452         } else {
3453                 PMD_DRV_LOG(DEBUG, "enable intr immediately");
3454                 ixgbe_enable_intr(dev);
3455                 rte_intr_enable(&(dev->pci_dev->intr_handle));
3456         }
3457
3458
3459         return 0;
3460 }
3461
3462 /**
3463  * Interrupt handler which shall be registered for alarm callback for delayed
3464  * handling specific interrupt to wait for the stable nic state. As the
3465  * NIC interrupt state is not stable for ixgbe after link is just down,
3466  * it needs to wait 4 seconds to get the stable status.
3467  *
3468  * @param handle
3469  *  Pointer to interrupt handle.
3470  * @param param
3471  *  The address of parameter (struct rte_eth_dev *) regsitered before.
3472  *
3473  * @return
3474  *  void
3475  */
3476 static void
3477 ixgbe_dev_interrupt_delayed_handler(void *param)
3478 {
3479         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
3480         struct ixgbe_interrupt *intr =
3481                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
3482         struct ixgbe_hw *hw =
3483                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3484         uint32_t eicr;
3485
3486         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
3487         if (eicr & IXGBE_EICR_MAILBOX)
3488                 ixgbe_pf_mbx_process(dev);
3489
3490         if (intr->flags & IXGBE_FLAG_PHY_INTERRUPT) {
3491                 ixgbe_handle_lasi(hw);
3492                 intr->flags &= ~IXGBE_FLAG_PHY_INTERRUPT;
3493         }
3494
3495         if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
3496                 ixgbe_dev_link_update(dev, 0);
3497                 intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
3498                 ixgbe_dev_link_status_print(dev);
3499                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
3500         }
3501
3502         PMD_DRV_LOG(DEBUG, "enable intr in delayed handler S[%08x]", eicr);
3503         ixgbe_enable_intr(dev);
3504         rte_intr_enable(&(dev->pci_dev->intr_handle));
3505 }
3506
3507 /**
3508  * Interrupt handler triggered by NIC  for handling
3509  * specific interrupt.
3510  *
3511  * @param handle
3512  *  Pointer to interrupt handle.
3513  * @param param
3514  *  The address of parameter (struct rte_eth_dev *) regsitered before.
3515  *
3516  * @return
3517  *  void
3518  */
3519 static void
3520 ixgbe_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
3521                             void *param)
3522 {
3523         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
3524
3525         ixgbe_dev_interrupt_get_status(dev);
3526         ixgbe_dev_interrupt_action(dev);
3527 }
3528
3529 static int
3530 ixgbe_dev_led_on(struct rte_eth_dev *dev)
3531 {
3532         struct ixgbe_hw *hw;
3533
3534         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3535         return ixgbe_led_on(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP;
3536 }
3537
3538 static int
3539 ixgbe_dev_led_off(struct rte_eth_dev *dev)
3540 {
3541         struct ixgbe_hw *hw;
3542
3543         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3544         return ixgbe_led_off(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP;
3545 }
3546
3547 static int
3548 ixgbe_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
3549 {
3550         struct ixgbe_hw *hw;
3551         uint32_t mflcn_reg;
3552         uint32_t fccfg_reg;
3553         int rx_pause;
3554         int tx_pause;
3555
3556         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3557
3558         fc_conf->pause_time = hw->fc.pause_time;
3559         fc_conf->high_water = hw->fc.high_water[0];
3560         fc_conf->low_water = hw->fc.low_water[0];
3561         fc_conf->send_xon = hw->fc.send_xon;
3562         fc_conf->autoneg = !hw->fc.disable_fc_autoneg;
3563
3564         /*
3565          * Return rx_pause status according to actual setting of
3566          * MFLCN register.
3567          */
3568         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
3569         if (mflcn_reg & (IXGBE_MFLCN_RPFCE | IXGBE_MFLCN_RFCE))
3570                 rx_pause = 1;
3571         else
3572                 rx_pause = 0;
3573
3574         /*
3575          * Return tx_pause status according to actual setting of
3576          * FCCFG register.
3577          */
3578         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
3579         if (fccfg_reg & (IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY))
3580                 tx_pause = 1;
3581         else
3582                 tx_pause = 0;
3583
3584         if (rx_pause && tx_pause)
3585                 fc_conf->mode = RTE_FC_FULL;
3586         else if (rx_pause)
3587                 fc_conf->mode = RTE_FC_RX_PAUSE;
3588         else if (tx_pause)
3589                 fc_conf->mode = RTE_FC_TX_PAUSE;
3590         else
3591                 fc_conf->mode = RTE_FC_NONE;
3592
3593         return 0;
3594 }
3595
3596 static int
3597 ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
3598 {
3599         struct ixgbe_hw *hw;
3600         int err;
3601         uint32_t rx_buf_size;
3602         uint32_t max_high_water;
3603         uint32_t mflcn;
3604         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
3605                 ixgbe_fc_none,
3606                 ixgbe_fc_rx_pause,
3607                 ixgbe_fc_tx_pause,
3608                 ixgbe_fc_full
3609         };
3610
3611         PMD_INIT_FUNC_TRACE();
3612
3613         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3614         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0));
3615         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
3616
3617         /*
3618          * At least reserve one Ethernet frame for watermark
3619          * high_water/low_water in kilo bytes for ixgbe
3620          */
3621         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
3622         if ((fc_conf->high_water > max_high_water) ||
3623                 (fc_conf->high_water < fc_conf->low_water)) {
3624                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB");
3625                 PMD_INIT_LOG(ERR, "High_water must <= 0x%x", max_high_water);
3626                 return -EINVAL;
3627         }
3628
3629         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[fc_conf->mode];
3630         hw->fc.pause_time     = fc_conf->pause_time;
3631         hw->fc.high_water[0]  = fc_conf->high_water;
3632         hw->fc.low_water[0]   = fc_conf->low_water;
3633         hw->fc.send_xon       = fc_conf->send_xon;
3634         hw->fc.disable_fc_autoneg = !fc_conf->autoneg;
3635
3636         err = ixgbe_fc_enable(hw);
3637
3638         /* Not negotiated is not an error case */
3639         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) {
3640
3641                 /* check if we want to forward MAC frames - driver doesn't have native
3642                  * capability to do that, so we'll write the registers ourselves */
3643
3644                 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN);
3645
3646                 /* set or clear MFLCN.PMCF bit depending on configuration */
3647                 if (fc_conf->mac_ctrl_frame_fwd != 0)
3648                         mflcn |= IXGBE_MFLCN_PMCF;
3649                 else
3650                         mflcn &= ~IXGBE_MFLCN_PMCF;
3651
3652                 IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn);
3653                 IXGBE_WRITE_FLUSH(hw);
3654
3655                 return 0;
3656         }
3657
3658         PMD_INIT_LOG(ERR, "ixgbe_fc_enable = 0x%x", err);
3659         return -EIO;
3660 }
3661
3662 /**
3663  *  ixgbe_pfc_enable_generic - Enable flow control
3664  *  @hw: pointer to hardware structure
3665  *  @tc_num: traffic class number
3666  *  Enable flow control according to the current settings.
3667  */
3668 static int
3669 ixgbe_dcb_pfc_enable_generic(struct ixgbe_hw *hw, uint8_t tc_num)
3670 {
3671         int ret_val = 0;
3672         uint32_t mflcn_reg, fccfg_reg;
3673         uint32_t reg;
3674         uint32_t fcrtl, fcrth;
3675         uint8_t i;
3676         uint8_t nb_rx_en;
3677
3678         /* Validate the water mark configuration */
3679         if (!hw->fc.pause_time) {
3680                 ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
3681                 goto out;
3682         }
3683
3684         /* Low water mark of zero causes XOFF floods */
3685         if (hw->fc.current_mode & ixgbe_fc_tx_pause) {
3686                  /* High/Low water can not be 0 */
3687                 if ((!hw->fc.high_water[tc_num]) || (!hw->fc.low_water[tc_num])) {
3688                         PMD_INIT_LOG(ERR, "Invalid water mark configuration");
3689                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
3690                         goto out;
3691                 }
3692
3693                 if (hw->fc.low_water[tc_num] >= hw->fc.high_water[tc_num]) {
3694                         PMD_INIT_LOG(ERR, "Invalid water mark configuration");
3695                         ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
3696                         goto out;
3697                 }
3698         }
3699         /* Negotiate the fc mode to use */
3700         ixgbe_fc_autoneg(hw);
3701
3702         /* Disable any previous flow control settings */
3703         mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
3704         mflcn_reg &= ~(IXGBE_MFLCN_RPFCE_SHIFT | IXGBE_MFLCN_RFCE|IXGBE_MFLCN_RPFCE);
3705
3706         fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
3707         fccfg_reg &= ~(IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY);
3708
3709         switch (hw->fc.current_mode) {
3710         case ixgbe_fc_none:
3711                 /*
3712                  * If the count of enabled RX Priority Flow control >1,
3713                  * and the TX pause can not be disabled
3714                  */
3715                 nb_rx_en = 0;
3716                 for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
3717                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
3718                         if (reg & IXGBE_FCRTH_FCEN)
3719                                 nb_rx_en++;
3720                 }
3721                 if (nb_rx_en > 1)
3722                         fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
3723                 break;
3724         case ixgbe_fc_rx_pause:
3725                 /*
3726                  * Rx Flow control is enabled and Tx Flow control is
3727                  * disabled by software override. Since there really
3728                  * isn't a way to advertise that we are capable of RX
3729                  * Pause ONLY, we will advertise that we support both
3730                  * symmetric and asymmetric Rx PAUSE.  Later, we will
3731                  * disable the adapter's ability to send PAUSE frames.
3732                  */
3733                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
3734                 /*
3735                  * If the count of enabled RX Priority Flow control >1,
3736                  * and the TX pause can not be disabled
3737                  */
3738                 nb_rx_en = 0;
3739                 for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
3740                         reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
3741                         if (reg & IXGBE_FCRTH_FCEN)
3742                                 nb_rx_en++;
3743                 }
3744                 if (nb_rx_en > 1)
3745                         fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
3746                 break;
3747         case ixgbe_fc_tx_pause:
3748                 /*
3749                  * Tx Flow control is enabled, and Rx Flow control is
3750                  * disabled by software override.
3751                  */
3752                 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
3753                 break;
3754         case ixgbe_fc_full:
3755                 /* Flow control (both Rx and Tx) is enabled by SW override. */
3756                 mflcn_reg |= IXGBE_MFLCN_RPFCE;
3757                 fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
3758                 break;
3759         default:
3760                 PMD_DRV_LOG(DEBUG, "Flow control param set incorrectly");
3761                 ret_val = IXGBE_ERR_CONFIG;
3762                 goto out;
3763         }
3764
3765         /* Set 802.3x based flow control settings. */
3766         mflcn_reg |= IXGBE_MFLCN_DPF;
3767         IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg);
3768         IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg);
3769
3770         /* Set up and enable Rx high/low water mark thresholds, enable XON. */
3771         if ((hw->fc.current_mode & ixgbe_fc_tx_pause) &&
3772                 hw->fc.high_water[tc_num]) {
3773                 fcrtl = (hw->fc.low_water[tc_num] << 10) | IXGBE_FCRTL_XONE;
3774                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), fcrtl);
3775                 fcrth = (hw->fc.high_water[tc_num] << 10) | IXGBE_FCRTH_FCEN;
3776         } else {
3777                 IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), 0);
3778                 /*
3779                  * In order to prevent Tx hangs when the internal Tx
3780                  * switch is enabled we must set the high water mark
3781                  * to the maximum FCRTH value.  This allows the Tx
3782                  * switch to function even under heavy Rx workloads.
3783                  */
3784                 fcrth = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)) - 32;
3785         }
3786         IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(tc_num), fcrth);
3787
3788         /* Configure pause time (2 TCs per register) */
3789         reg = hw->fc.pause_time * 0x00010001;
3790         for (i = 0; i < (IXGBE_DCB_MAX_TRAFFIC_CLASS / 2); i++)
3791                 IXGBE_WRITE_REG(hw, IXGBE_FCTTV(i), reg);
3792
3793         /* Configure flow control refresh threshold value */
3794         IXGBE_WRITE_REG(hw, IXGBE_FCRTV, hw->fc.pause_time / 2);
3795
3796 out:
3797         return ret_val;
3798 }
3799
3800 static int
3801 ixgbe_dcb_pfc_enable(struct rte_eth_dev *dev, uint8_t tc_num)
3802 {
3803         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3804         int32_t ret_val = IXGBE_NOT_IMPLEMENTED;
3805
3806         if (hw->mac.type != ixgbe_mac_82598EB) {
3807                 ret_val = ixgbe_dcb_pfc_enable_generic(hw, tc_num);
3808         }
3809         return ret_val;
3810 }
3811
3812 static int
3813 ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf)
3814 {
3815         int err;
3816         uint32_t rx_buf_size;
3817         uint32_t max_high_water;
3818         uint8_t tc_num;
3819         uint8_t  map[IXGBE_DCB_MAX_USER_PRIORITY] = { 0 };
3820         struct ixgbe_hw *hw =
3821                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3822         struct ixgbe_dcb_config *dcb_config =
3823                 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
3824
3825         enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
3826                 ixgbe_fc_none,
3827                 ixgbe_fc_rx_pause,
3828                 ixgbe_fc_tx_pause,
3829                 ixgbe_fc_full
3830         };
3831
3832         PMD_INIT_FUNC_TRACE();
3833
3834         ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
3835         tc_num = map[pfc_conf->priority];
3836         rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num));
3837         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
3838         /*
3839          * At least reserve one Ethernet frame for watermark
3840          * high_water/low_water in kilo bytes for ixgbe
3841          */
3842         max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
3843         if ((pfc_conf->fc.high_water > max_high_water) ||
3844             (pfc_conf->fc.high_water <= pfc_conf->fc.low_water)) {
3845                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB");
3846                 PMD_INIT_LOG(ERR, "High_water must <= 0x%x", max_high_water);
3847                 return -EINVAL;
3848         }
3849
3850         hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[pfc_conf->fc.mode];
3851         hw->fc.pause_time = pfc_conf->fc.pause_time;
3852         hw->fc.send_xon = pfc_conf->fc.send_xon;
3853         hw->fc.low_water[tc_num] =  pfc_conf->fc.low_water;
3854         hw->fc.high_water[tc_num] = pfc_conf->fc.high_water;
3855
3856         err = ixgbe_dcb_pfc_enable(dev, tc_num);
3857
3858         /* Not negotiated is not an error case */
3859         if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED))
3860                 return 0;
3861
3862         PMD_INIT_LOG(ERR, "ixgbe_dcb_pfc_enable = 0x%x", err);
3863         return -EIO;
3864 }
3865
3866 static int
3867 ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
3868                           struct rte_eth_rss_reta_entry64 *reta_conf,
3869                           uint16_t reta_size)
3870 {
3871         uint16_t i, sp_reta_size;
3872         uint8_t j, mask;
3873         uint32_t reta, r;
3874         uint16_t idx, shift;
3875         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3876         uint32_t reta_reg;
3877
3878         PMD_INIT_FUNC_TRACE();
3879
3880         if (!ixgbe_rss_update_sp(hw->mac.type)) {
3881                 PMD_DRV_LOG(ERR, "RSS reta update is not supported on this "
3882                         "NIC.");
3883                 return -ENOTSUP;
3884         }
3885
3886         sp_reta_size = ixgbe_reta_size_get(hw->mac.type);
3887         if (reta_size != sp_reta_size) {
3888                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
3889                         "(%d) doesn't match the number hardware can supported "
3890                         "(%d)\n", reta_size, sp_reta_size);
3891                 return -EINVAL;
3892         }
3893
3894         for (i = 0; i < reta_size; i += IXGBE_4_BIT_WIDTH) {
3895                 idx = i / RTE_RETA_GROUP_SIZE;
3896                 shift = i % RTE_RETA_GROUP_SIZE;
3897                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
3898                                                 IXGBE_4_BIT_MASK);
3899                 if (!mask)
3900                         continue;
3901                 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i);
3902                 if (mask == IXGBE_4_BIT_MASK)
3903                         r = 0;
3904                 else
3905                         r = IXGBE_READ_REG(hw, reta_reg);
3906                 for (j = 0, reta = 0; j < IXGBE_4_BIT_WIDTH; j++) {
3907                         if (mask & (0x1 << j))
3908                                 reta |= reta_conf[idx].reta[shift + j] <<
3909                                                         (CHAR_BIT * j);
3910                         else
3911                                 reta |= r & (IXGBE_8_BIT_MASK <<
3912                                                 (CHAR_BIT * j));
3913                 }
3914                 IXGBE_WRITE_REG(hw, reta_reg, reta);
3915         }
3916
3917         return 0;
3918 }
3919
3920 static int
3921 ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
3922                          struct rte_eth_rss_reta_entry64 *reta_conf,
3923                          uint16_t reta_size)
3924 {
3925         uint16_t i, sp_reta_size;
3926         uint8_t j, mask;
3927         uint32_t reta;
3928         uint16_t idx, shift;
3929         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3930         uint32_t reta_reg;
3931
3932         PMD_INIT_FUNC_TRACE();
3933         sp_reta_size = ixgbe_reta_size_get(hw->mac.type);
3934         if (reta_size != sp_reta_size) {
3935                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
3936                         "(%d) doesn't match the number hardware can supported "
3937                         "(%d)\n", reta_size, sp_reta_size);
3938                 return -EINVAL;
3939         }
3940
3941         for (i = 0; i < reta_size; i += IXGBE_4_BIT_WIDTH) {
3942                 idx = i / RTE_RETA_GROUP_SIZE;
3943                 shift = i % RTE_RETA_GROUP_SIZE;
3944                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
3945                                                 IXGBE_4_BIT_MASK);
3946                 if (!mask)
3947                         continue;
3948
3949                 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i);
3950                 reta = IXGBE_READ_REG(hw, reta_reg);
3951                 for (j = 0; j < IXGBE_4_BIT_WIDTH; j++) {
3952                         if (mask & (0x1 << j))
3953                                 reta_conf[idx].reta[shift + j] =
3954                                         ((reta >> (CHAR_BIT * j)) &
3955                                                 IXGBE_8_BIT_MASK);
3956                 }
3957         }
3958
3959         return 0;
3960 }
3961
3962 static void
3963 ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
3964                                 uint32_t index, uint32_t pool)
3965 {
3966         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3967         uint32_t enable_addr = 1;
3968
3969         ixgbe_set_rar(hw, index, mac_addr->addr_bytes, pool, enable_addr);
3970 }
3971
3972 static void
3973 ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
3974 {
3975         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3976
3977         ixgbe_clear_rar(hw, index);
3978 }
3979
3980 static void
3981 ixgbe_set_default_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
3982 {
3983         ixgbe_remove_rar(dev, 0);
3984
3985         ixgbe_add_rar(dev, addr, 0, 0);
3986 }
3987
3988 static int
3989 ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
3990 {
3991         uint32_t hlreg0;
3992         uint32_t maxfrs;
3993         struct ixgbe_hw *hw;
3994         struct rte_eth_dev_info dev_info;
3995         uint32_t frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
3996
3997         ixgbe_dev_info_get(dev, &dev_info);
3998
3999         /* check that mtu is within the allowed range */
4000         if ((mtu < ETHER_MIN_MTU) || (frame_size > dev_info.max_rx_pktlen))
4001                 return -EINVAL;
4002
4003         /* refuse mtu that requires the support of scattered packets when this
4004          * feature has not been enabled before.
4005          */
4006         if (!dev->data->scattered_rx &&
4007             (frame_size + 2 * IXGBE_VLAN_TAG_SIZE >
4008              dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM))
4009                 return -EINVAL;
4010
4011         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4012         hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4013
4014         /* switch to jumbo mode if needed */
4015         if (frame_size > ETHER_MAX_LEN) {
4016                 dev->data->dev_conf.rxmode.jumbo_frame = 1;
4017                 hlreg0 |= IXGBE_HLREG0_JUMBOEN;
4018         } else {
4019                 dev->data->dev_conf.rxmode.jumbo_frame = 0;
4020                 hlreg0 &= ~IXGBE_HLREG0_JUMBOEN;
4021         }
4022         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
4023
4024         /* update max frame size */
4025         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
4026
4027         maxfrs = IXGBE_READ_REG(hw, IXGBE_MAXFRS);
4028         maxfrs &= 0x0000FFFF;
4029         maxfrs |= (dev->data->dev_conf.rxmode.max_rx_pkt_len << 16);
4030         IXGBE_WRITE_REG(hw, IXGBE_MAXFRS, maxfrs);
4031
4032         return 0;
4033 }
4034
4035 /*
4036  * Virtual Function operations
4037  */
4038 static void
4039 ixgbevf_intr_disable(struct ixgbe_hw *hw)
4040 {
4041         PMD_INIT_FUNC_TRACE();
4042
4043         /* Clear interrupt mask to stop from interrupts being generated */
4044         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, IXGBE_VF_IRQ_CLEAR_MASK);
4045
4046         IXGBE_WRITE_FLUSH(hw);
4047 }
4048
4049 static void
4050 ixgbevf_intr_enable(struct ixgbe_hw *hw)
4051 {
4052         PMD_INIT_FUNC_TRACE();
4053
4054         /* VF enable interrupt autoclean */
4055         IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_VF_IRQ_ENABLE_MASK);
4056         IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, IXGBE_VF_IRQ_ENABLE_MASK);
4057         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_VF_IRQ_ENABLE_MASK);
4058
4059         IXGBE_WRITE_FLUSH(hw);
4060 }
4061
4062 static int
4063 ixgbevf_dev_configure(struct rte_eth_dev *dev)
4064 {
4065         struct rte_eth_conf *conf = &dev->data->dev_conf;
4066         struct ixgbe_adapter *adapter =
4067                         (struct ixgbe_adapter *)dev->data->dev_private;
4068
4069         PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d",
4070                      dev->data->port_id);
4071
4072         /*
4073          * VF has no ability to enable/disable HW CRC
4074          * Keep the persistent behavior the same as Host PF
4075          */
4076 #ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC
4077         if (!conf->rxmode.hw_strip_crc) {
4078                 PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip");
4079                 conf->rxmode.hw_strip_crc = 1;
4080         }
4081 #else
4082         if (conf->rxmode.hw_strip_crc) {
4083                 PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip");
4084                 conf->rxmode.hw_strip_crc = 0;
4085         }
4086 #endif
4087
4088         /*
4089          * Initialize to TRUE. If any of Rx queues doesn't meet the bulk
4090          * allocation or vector Rx preconditions we will reset it.
4091          */
4092         adapter->rx_bulk_alloc_allowed = true;
4093         adapter->rx_vec_allowed = true;
4094
4095         return 0;
4096 }
4097
4098 static int
4099 ixgbevf_dev_start(struct rte_eth_dev *dev)
4100 {
4101         struct ixgbe_hw *hw =
4102                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4103         uint32_t intr_vector = 0;
4104         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
4105
4106         int err, mask = 0;
4107
4108         PMD_INIT_FUNC_TRACE();
4109
4110         hw->mac.ops.reset_hw(hw);
4111         hw->mac.get_link_status = true;
4112
4113         /* negotiate mailbox API version to use with the PF. */
4114         ixgbevf_negotiate_api(hw);
4115
4116         ixgbevf_dev_tx_init(dev);
4117
4118         /* This can fail when allocating mbufs for descriptor rings */
4119         err = ixgbevf_dev_rx_init(dev);
4120         if (err) {
4121                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware (%d)", err);
4122                 ixgbe_dev_clear_queues(dev);
4123                 return err;
4124         }
4125
4126         /* Set vfta */
4127         ixgbevf_set_vfta_all(dev, 1);
4128
4129         /* Set HW strip */
4130         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK |
4131                 ETH_VLAN_EXTEND_MASK;
4132         ixgbevf_vlan_offload_set(dev, mask);
4133
4134         ixgbevf_dev_rxtx_start(dev);
4135
4136         /* check and configure queue intr-vector mapping */
4137         if (dev->data->dev_conf.intr_conf.rxq != 0) {
4138                 intr_vector = dev->data->nb_rx_queues;
4139                 if (rte_intr_efd_enable(intr_handle, intr_vector))
4140                         return -1;
4141         }
4142
4143         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
4144                 intr_handle->intr_vec =
4145                         rte_zmalloc("intr_vec",
4146                                     dev->data->nb_rx_queues * sizeof(int), 0);
4147                 if (intr_handle->intr_vec == NULL) {
4148                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
4149                                      " intr_vec\n", dev->data->nb_rx_queues);
4150                         return -ENOMEM;
4151                 }
4152         }
4153         ixgbevf_configure_msix(dev);
4154
4155         rte_intr_enable(intr_handle);
4156
4157         /* Re-enable interrupt for VF */
4158         ixgbevf_intr_enable(hw);
4159
4160         return 0;
4161 }
4162
4163 static void
4164 ixgbevf_dev_stop(struct rte_eth_dev *dev)
4165 {
4166         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4167         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
4168
4169         PMD_INIT_FUNC_TRACE();
4170
4171         ixgbevf_intr_disable(hw);
4172
4173         hw->adapter_stopped = 1;
4174         ixgbe_stop_adapter(hw);
4175
4176         /*
4177           * Clear what we set, but we still keep shadow_vfta to
4178           * restore after device starts
4179           */
4180         ixgbevf_set_vfta_all(dev, 0);
4181
4182         /* Clear stored conf */
4183         dev->data->scattered_rx = 0;
4184
4185         ixgbe_dev_clear_queues(dev);
4186
4187         /* Clean datapath event and queue/vec mapping */
4188         rte_intr_efd_disable(intr_handle);
4189         if (intr_handle->intr_vec != NULL) {
4190                 rte_free(intr_handle->intr_vec);
4191                 intr_handle->intr_vec = NULL;
4192         }
4193 }
4194
4195 static void
4196 ixgbevf_dev_close(struct rte_eth_dev *dev)
4197 {
4198         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4199
4200         PMD_INIT_FUNC_TRACE();
4201
4202         ixgbe_reset_hw(hw);
4203
4204         ixgbevf_dev_stop(dev);
4205
4206         ixgbe_dev_free_queues(dev);
4207
4208         /**
4209          * Remove the VF MAC address ro ensure
4210          * that the VF traffic goes to the PF
4211          * after stop, close and detach of the VF
4212          **/
4213         ixgbevf_remove_mac_addr(dev, 0);
4214 }
4215
4216 static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on)
4217 {
4218         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4219         struct ixgbe_vfta *shadow_vfta =
4220                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
4221         int i = 0, j = 0, vfta = 0, mask = 1;
4222
4223         for (i = 0; i < IXGBE_VFTA_SIZE; i++) {
4224                 vfta = shadow_vfta->vfta[i];
4225                 if (vfta) {
4226                         mask = 1;
4227                         for (j = 0; j < 32; j++) {
4228                                 if (vfta & mask)
4229                                         ixgbe_set_vfta(hw, (i<<5)+j, 0, on);
4230                                 mask <<= 1;
4231                         }
4232                 }
4233         }
4234
4235 }
4236
4237 static int
4238 ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
4239 {
4240         struct ixgbe_hw *hw =
4241                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4242         struct ixgbe_vfta *shadow_vfta =
4243                 IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
4244         uint32_t vid_idx = 0;
4245         uint32_t vid_bit = 0;
4246         int ret = 0;
4247
4248         PMD_INIT_FUNC_TRACE();
4249
4250         /* vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf */
4251         ret = ixgbe_set_vfta(hw, vlan_id, 0, !!on);
4252         if (ret) {
4253                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
4254                 return ret;
4255         }
4256         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
4257         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
4258
4259         /* Save what we set and retore it after device reset */
4260         if (on)
4261                 shadow_vfta->vfta[vid_idx] |= vid_bit;
4262         else
4263                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
4264
4265         return 0;
4266 }
4267
4268 static void
4269 ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
4270 {
4271         struct ixgbe_hw *hw =
4272                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4273         uint32_t ctrl;
4274
4275         PMD_INIT_FUNC_TRACE();
4276
4277         if (queue >= hw->mac.max_rx_queues)
4278                 return;
4279
4280         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
4281         if (on)
4282                 ctrl |= IXGBE_RXDCTL_VME;
4283         else
4284                 ctrl &= ~IXGBE_RXDCTL_VME;
4285         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
4286
4287         ixgbe_vlan_hw_strip_bitmap_set(dev, queue, on);
4288 }
4289
4290 static void
4291 ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
4292 {
4293         struct ixgbe_hw *hw =
4294                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4295         uint16_t i;
4296         int on = 0;
4297
4298         /* VF function only support hw strip feature, others are not support */
4299         if (mask & ETH_VLAN_STRIP_MASK) {
4300                 on = !!(dev->data->dev_conf.rxmode.hw_vlan_strip);
4301
4302                 for (i = 0; i < hw->mac.max_rx_queues; i++)
4303                         ixgbevf_vlan_strip_queue_set(dev, i, on);
4304         }
4305 }
4306
4307 static int
4308 ixgbe_vmdq_mode_check(struct ixgbe_hw *hw)
4309 {
4310         uint32_t reg_val;
4311
4312         /* we only need to do this if VMDq is enabled */
4313         reg_val = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
4314         if (!(reg_val & IXGBE_VT_CTL_VT_ENABLE)) {
4315                 PMD_INIT_LOG(ERR, "VMDq must be enabled for this setting");
4316                 return -1;
4317         }
4318
4319         return 0;
4320 }
4321
4322 static uint32_t
4323 ixgbe_uta_vector(struct ixgbe_hw *hw, struct ether_addr *uc_addr)
4324 {
4325         uint32_t vector = 0;
4326
4327         switch (hw->mac.mc_filter_type) {
4328         case 0:   /* use bits [47:36] of the address */
4329                 vector = ((uc_addr->addr_bytes[4] >> 4) |
4330                         (((uint16_t)uc_addr->addr_bytes[5]) << 4));
4331                 break;
4332         case 1:   /* use bits [46:35] of the address */
4333                 vector = ((uc_addr->addr_bytes[4] >> 3) |
4334                         (((uint16_t)uc_addr->addr_bytes[5]) << 5));
4335                 break;
4336         case 2:   /* use bits [45:34] of the address */
4337                 vector = ((uc_addr->addr_bytes[4] >> 2) |
4338                         (((uint16_t)uc_addr->addr_bytes[5]) << 6));
4339                 break;
4340         case 3:   /* use bits [43:32] of the address */
4341                 vector = ((uc_addr->addr_bytes[4]) |
4342                         (((uint16_t)uc_addr->addr_bytes[5]) << 8));
4343                 break;
4344         default:  /* Invalid mc_filter_type */
4345                 break;
4346         }
4347
4348         /* vector can only be 12-bits or boundary will be exceeded */
4349         vector &= 0xFFF;
4350         return vector;
4351 }
4352
4353 static int
4354 ixgbe_uc_hash_table_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
4355                         uint8_t on)
4356 {
4357         uint32_t vector;
4358         uint32_t uta_idx;
4359         uint32_t reg_val;
4360         uint32_t uta_shift;
4361         uint32_t rc;
4362         const uint32_t ixgbe_uta_idx_mask = 0x7F;
4363         const uint32_t ixgbe_uta_bit_shift = 5;
4364         const uint32_t ixgbe_uta_bit_mask = (0x1 << ixgbe_uta_bit_shift) - 1;
4365         const uint32_t bit1 = 0x1;
4366
4367         struct ixgbe_hw *hw =
4368                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4369         struct ixgbe_uta_info *uta_info =
4370                 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private);
4371
4372         /* The UTA table only exists on 82599 hardware and newer */
4373         if (hw->mac.type < ixgbe_mac_82599EB)
4374                 return -ENOTSUP;
4375
4376         vector = ixgbe_uta_vector(hw, mac_addr);
4377         uta_idx = (vector >> ixgbe_uta_bit_shift) & ixgbe_uta_idx_mask;
4378         uta_shift = vector & ixgbe_uta_bit_mask;
4379
4380         rc = ((uta_info->uta_shadow[uta_idx] >> uta_shift & bit1) != 0);
4381         if (rc == on)
4382                 return 0;
4383
4384         reg_val = IXGBE_READ_REG(hw, IXGBE_UTA(uta_idx));
4385         if (on) {
4386                 uta_info->uta_in_use++;
4387                 reg_val |= (bit1 << uta_shift);
4388                 uta_info->uta_shadow[uta_idx] |= (bit1 << uta_shift);
4389         } else {
4390                 uta_info->uta_in_use--;
4391                 reg_val &= ~(bit1 << uta_shift);
4392                 uta_info->uta_shadow[uta_idx] &= ~(bit1 << uta_shift);
4393         }
4394
4395         IXGBE_WRITE_REG(hw, IXGBE_UTA(uta_idx), reg_val);
4396
4397         if (uta_info->uta_in_use > 0)
4398                 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL,
4399                                 IXGBE_MCSTCTRL_MFE | hw->mac.mc_filter_type);
4400         else
4401                 IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL, hw->mac.mc_filter_type);
4402
4403         return 0;
4404 }
4405
4406 static int
4407 ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on)
4408 {
4409         int i;
4410         struct ixgbe_hw *hw =
4411                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4412         struct ixgbe_uta_info *uta_info =
4413                 IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private);
4414
4415         /* The UTA table only exists on 82599 hardware and newer */
4416         if (hw->mac.type < ixgbe_mac_82599EB)
4417                 return -ENOTSUP;
4418
4419         if (on) {
4420                 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
4421                         uta_info->uta_shadow[i] = ~0;
4422                         IXGBE_WRITE_REG(hw, IXGBE_UTA(i), ~0);
4423                 }
4424         } else {
4425                 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
4426                         uta_info->uta_shadow[i] = 0;
4427                         IXGBE_WRITE_REG(hw, IXGBE_UTA(i), 0);
4428                 }
4429         }
4430         return 0;
4431
4432 }
4433
4434 uint32_t
4435 ixgbe_convert_vm_rx_mask_to_val(uint16_t rx_mask, uint32_t orig_val)
4436 {
4437         uint32_t new_val = orig_val;
4438
4439         if (rx_mask & ETH_VMDQ_ACCEPT_UNTAG)
4440                 new_val |= IXGBE_VMOLR_AUPE;
4441         if (rx_mask & ETH_VMDQ_ACCEPT_HASH_MC)
4442                 new_val |= IXGBE_VMOLR_ROMPE;
4443         if (rx_mask & ETH_VMDQ_ACCEPT_HASH_UC)
4444                 new_val |= IXGBE_VMOLR_ROPE;
4445         if (rx_mask & ETH_VMDQ_ACCEPT_BROADCAST)
4446                 new_val |= IXGBE_VMOLR_BAM;
4447         if (rx_mask & ETH_VMDQ_ACCEPT_MULTICAST)
4448                 new_val |= IXGBE_VMOLR_MPE;
4449
4450         return new_val;
4451 }
4452
4453 static int
4454 ixgbe_set_pool_rx_mode(struct rte_eth_dev *dev, uint16_t pool,
4455                                uint16_t rx_mask, uint8_t on)
4456 {
4457         int val = 0;
4458
4459         struct ixgbe_hw *hw =
4460                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4461         uint32_t vmolr = IXGBE_READ_REG(hw, IXGBE_VMOLR(pool));
4462
4463         if (hw->mac.type == ixgbe_mac_82598EB) {
4464                 PMD_INIT_LOG(ERR, "setting VF receive mode set should be done"
4465                              " on 82599 hardware and newer");
4466                 return -ENOTSUP;
4467         }
4468         if (ixgbe_vmdq_mode_check(hw) < 0)
4469                 return -ENOTSUP;
4470
4471         val = ixgbe_convert_vm_rx_mask_to_val(rx_mask, val);
4472
4473         if (on)
4474                 vmolr |= val;
4475         else
4476                 vmolr &= ~val;
4477
4478         IXGBE_WRITE_REG(hw, IXGBE_VMOLR(pool), vmolr);
4479
4480         return 0;
4481 }
4482
4483 static int
4484 ixgbe_set_pool_rx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on)
4485 {
4486         uint32_t reg, addr;
4487         uint32_t val;
4488         const uint8_t bit1 = 0x1;
4489
4490         struct ixgbe_hw *hw =
4491                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4492
4493         if (ixgbe_vmdq_mode_check(hw) < 0)
4494                 return -ENOTSUP;
4495
4496         if (pool >= ETH_64_POOLS)
4497                 return -EINVAL;
4498
4499         /* for pool >= 32, set bit in PFVFRE[1], otherwise PFVFRE[0] */
4500         if (pool >= 32) {
4501                 addr = IXGBE_VFRE(1);
4502                 val = bit1 << (pool - 32);
4503         } else {
4504                 addr = IXGBE_VFRE(0);
4505                 val = bit1 << pool;
4506         }
4507
4508         reg = IXGBE_READ_REG(hw, addr);
4509
4510         if (on)
4511                 reg |= val;
4512         else
4513                 reg &= ~val;
4514
4515         IXGBE_WRITE_REG(hw, addr, reg);
4516
4517         return 0;
4518 }
4519
4520 static int
4521 ixgbe_set_pool_tx(struct rte_eth_dev *dev, uint16_t pool, uint8_t on)
4522 {
4523         uint32_t reg, addr;
4524         uint32_t val;
4525         const uint8_t bit1 = 0x1;
4526
4527         struct ixgbe_hw *hw =
4528                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4529
4530         if (ixgbe_vmdq_mode_check(hw) < 0)
4531                 return -ENOTSUP;
4532
4533         if (pool >= ETH_64_POOLS)
4534                 return -EINVAL;
4535
4536         /* for pool >= 32, set bit in PFVFTE[1], otherwise PFVFTE[0] */
4537         if (pool >= 32) {
4538                 addr = IXGBE_VFTE(1);
4539                 val = bit1 << (pool - 32);
4540         } else {
4541                 addr = IXGBE_VFTE(0);
4542                 val = bit1 << pool;
4543         }
4544
4545         reg = IXGBE_READ_REG(hw, addr);
4546
4547         if (on)
4548                 reg |= val;
4549         else
4550                 reg &= ~val;
4551
4552         IXGBE_WRITE_REG(hw, addr, reg);
4553
4554         return 0;
4555 }
4556
4557 static int
4558 ixgbe_set_pool_vlan_filter(struct rte_eth_dev *dev, uint16_t vlan,
4559                         uint64_t pool_mask, uint8_t vlan_on)
4560 {
4561         int ret = 0;
4562         uint16_t pool_idx;
4563         struct ixgbe_hw *hw =
4564                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4565
4566         if (ixgbe_vmdq_mode_check(hw) < 0)
4567                 return -ENOTSUP;
4568         for (pool_idx = 0; pool_idx < ETH_64_POOLS; pool_idx++) {
4569                 if (pool_mask & ((uint64_t)(1ULL << pool_idx))) {
4570                         ret = hw->mac.ops.set_vfta(hw, vlan, pool_idx, vlan_on);
4571                         if (ret < 0)
4572                                 return ret;
4573                 }
4574         }
4575
4576         return ret;
4577 }
4578
4579 #define IXGBE_MRCTL_VPME  0x01 /* Virtual Pool Mirroring. */
4580 #define IXGBE_MRCTL_UPME  0x02 /* Uplink Port Mirroring. */
4581 #define IXGBE_MRCTL_DPME  0x04 /* Downlink Port Mirroring. */
4582 #define IXGBE_MRCTL_VLME  0x08 /* VLAN Mirroring. */
4583 #define IXGBE_INVALID_MIRROR_TYPE(mirror_type) \
4584         ((mirror_type) & ~(uint8_t)(ETH_MIRROR_VIRTUAL_POOL_UP | \
4585         ETH_MIRROR_UPLINK_PORT | ETH_MIRROR_DOWNLINK_PORT | ETH_MIRROR_VLAN))
4586
4587 static int
4588 ixgbe_mirror_rule_set(struct rte_eth_dev *dev,
4589                         struct rte_eth_mirror_conf *mirror_conf,
4590                         uint8_t rule_id, uint8_t on)
4591 {
4592         uint32_t mr_ctl, vlvf;
4593         uint32_t mp_lsb = 0;
4594         uint32_t mv_msb = 0;
4595         uint32_t mv_lsb = 0;
4596         uint32_t mp_msb = 0;
4597         uint8_t i = 0;
4598         int reg_index = 0;
4599         uint64_t vlan_mask = 0;
4600
4601         const uint8_t pool_mask_offset = 32;
4602         const uint8_t vlan_mask_offset = 32;
4603         const uint8_t dst_pool_offset = 8;
4604         const uint8_t rule_mr_offset  = 4;
4605         const uint8_t mirror_rule_mask = 0x0F;
4606
4607         struct ixgbe_mirror_info *mr_info =
4608                         (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private));
4609         struct ixgbe_hw *hw =
4610                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4611         uint8_t mirror_type = 0;
4612
4613         if (ixgbe_vmdq_mode_check(hw) < 0)
4614                 return -ENOTSUP;
4615
4616         if (rule_id >= IXGBE_MAX_MIRROR_RULES)
4617                 return -EINVAL;
4618
4619         if (IXGBE_INVALID_MIRROR_TYPE(mirror_conf->rule_type)) {
4620                 PMD_DRV_LOG(ERR, "unsupported mirror type 0x%x.",
4621                         mirror_conf->rule_type);
4622                 return -EINVAL;
4623         }
4624
4625         if (mirror_conf->rule_type & ETH_MIRROR_VLAN) {
4626                 mirror_type |= IXGBE_MRCTL_VLME;
4627                 /* Check if vlan id is valid and find conresponding VLAN ID index in VLVF */
4628                 for (i = 0; i < IXGBE_VLVF_ENTRIES; i++) {
4629                         if (mirror_conf->vlan.vlan_mask & (1ULL << i)) {
4630                                 /* search vlan id related pool vlan filter index */
4631                                 reg_index = ixgbe_find_vlvf_slot(hw,
4632                                                 mirror_conf->vlan.vlan_id[i]);
4633                                 if (reg_index < 0)
4634                                         return -EINVAL;
4635                                 vlvf = IXGBE_READ_REG(hw, IXGBE_VLVF(reg_index));
4636                                 if ((vlvf & IXGBE_VLVF_VIEN) &&
4637                                     ((vlvf & IXGBE_VLVF_VLANID_MASK) ==
4638                                       mirror_conf->vlan.vlan_id[i]))
4639                                         vlan_mask |= (1ULL << reg_index);
4640                                 else
4641                                         return -EINVAL;
4642                         }
4643                 }
4644
4645                 if (on) {
4646                         mv_lsb = vlan_mask & 0xFFFFFFFF;
4647                         mv_msb = vlan_mask >> vlan_mask_offset;
4648
4649                         mr_info->mr_conf[rule_id].vlan.vlan_mask =
4650                                                 mirror_conf->vlan.vlan_mask;
4651                         for (i = 0; i < ETH_VMDQ_MAX_VLAN_FILTERS; i++) {
4652                                 if (mirror_conf->vlan.vlan_mask & (1ULL << i))
4653                                         mr_info->mr_conf[rule_id].vlan.vlan_id[i] =
4654                                                 mirror_conf->vlan.vlan_id[i];
4655                         }
4656                 } else {
4657                         mv_lsb = 0;
4658                         mv_msb = 0;
4659                         mr_info->mr_conf[rule_id].vlan.vlan_mask = 0;
4660                         for (i = 0; i < ETH_VMDQ_MAX_VLAN_FILTERS; i++)
4661                                 mr_info->mr_conf[rule_id].vlan.vlan_id[i] = 0;
4662                 }
4663         }
4664
4665         /*
4666          * if enable pool mirror, write related pool mask register,if disable
4667          * pool mirror, clear PFMRVM register
4668          */
4669         if (mirror_conf->rule_type & ETH_MIRROR_VIRTUAL_POOL_UP) {
4670                 mirror_type |= IXGBE_MRCTL_VPME;
4671                 if (on) {
4672                         mp_lsb = mirror_conf->pool_mask & 0xFFFFFFFF;
4673                         mp_msb = mirror_conf->pool_mask >> pool_mask_offset;
4674                         mr_info->mr_conf[rule_id].pool_mask =
4675                                         mirror_conf->pool_mask;
4676
4677                 } else {
4678                         mp_lsb = 0;
4679                         mp_msb = 0;
4680                         mr_info->mr_conf[rule_id].pool_mask = 0;
4681                 }
4682         }
4683         if (mirror_conf->rule_type & ETH_MIRROR_UPLINK_PORT)
4684                 mirror_type |= IXGBE_MRCTL_UPME;
4685         if (mirror_conf->rule_type & ETH_MIRROR_DOWNLINK_PORT)
4686                 mirror_type |= IXGBE_MRCTL_DPME;
4687
4688         /* read  mirror control register and recalculate it */
4689         mr_ctl = IXGBE_READ_REG(hw, IXGBE_MRCTL(rule_id));
4690
4691         if (on) {
4692                 mr_ctl |= mirror_type;
4693                 mr_ctl &= mirror_rule_mask;
4694                 mr_ctl |= mirror_conf->dst_pool << dst_pool_offset;
4695         } else
4696                 mr_ctl &= ~(mirror_conf->rule_type & mirror_rule_mask);
4697
4698         mr_info->mr_conf[rule_id].rule_type = mirror_conf->rule_type;
4699         mr_info->mr_conf[rule_id].dst_pool = mirror_conf->dst_pool;
4700
4701         /* write mirrror control  register */
4702         IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl);
4703
4704         /* write pool mirrror control  register */
4705         if (mirror_conf->rule_type == ETH_MIRROR_VIRTUAL_POOL_UP) {
4706                 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), mp_lsb);
4707                 IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset),
4708                                 mp_msb);
4709         }
4710         /* write VLAN mirrror control  register */
4711         if (mirror_conf->rule_type == ETH_MIRROR_VLAN) {
4712                 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), mv_lsb);
4713                 IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset),
4714                                 mv_msb);
4715         }
4716
4717         return 0;
4718 }
4719
4720 static int
4721 ixgbe_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t rule_id)
4722 {
4723         int mr_ctl = 0;
4724         uint32_t lsb_val = 0;
4725         uint32_t msb_val = 0;
4726         const uint8_t rule_mr_offset = 4;
4727
4728         struct ixgbe_hw *hw =
4729                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4730         struct ixgbe_mirror_info *mr_info =
4731                 (IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private));
4732
4733         if (ixgbe_vmdq_mode_check(hw) < 0)
4734                 return -ENOTSUP;
4735
4736         memset(&mr_info->mr_conf[rule_id], 0,
4737                 sizeof(struct rte_eth_mirror_conf));
4738
4739         /* clear PFVMCTL register */
4740         IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl);
4741
4742         /* clear pool mask register */
4743         IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), lsb_val);
4744         IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset), msb_val);
4745
4746         /* clear vlan mask register */
4747         IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), lsb_val);
4748         IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset), msb_val);
4749
4750         return 0;
4751 }
4752
4753 static int
4754 ixgbevf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
4755 {
4756         uint32_t mask;
4757         struct ixgbe_hw *hw =
4758                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4759
4760         mask = IXGBE_READ_REG(hw, IXGBE_VTEIMS);
4761         mask |= (1 << IXGBE_MISC_VEC_ID);
4762         RTE_SET_USED(queue_id);
4763         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
4764
4765         rte_intr_enable(&dev->pci_dev->intr_handle);
4766
4767         return 0;
4768 }
4769
4770 static int
4771 ixgbevf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
4772 {
4773         uint32_t mask;
4774         struct ixgbe_hw *hw =
4775                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4776
4777         mask = IXGBE_READ_REG(hw, IXGBE_VTEIMS);
4778         mask &= ~(1 << IXGBE_MISC_VEC_ID);
4779         RTE_SET_USED(queue_id);
4780         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
4781
4782         return 0;
4783 }
4784
4785 static int
4786 ixgbe_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
4787 {
4788         uint32_t mask;
4789         struct ixgbe_hw *hw =
4790                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4791         struct ixgbe_interrupt *intr =
4792                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
4793
4794         if (queue_id < 16) {
4795                 ixgbe_disable_intr(hw);
4796                 intr->mask |= (1 << queue_id);
4797                 ixgbe_enable_intr(dev);
4798         } else if (queue_id < 32) {
4799                 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(0));
4800                 mask &= (1 << queue_id);
4801                 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
4802         } else if (queue_id < 64) {
4803                 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(1));
4804                 mask &= (1 << (queue_id - 32));
4805                 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
4806         }
4807         rte_intr_enable(&dev->pci_dev->intr_handle);
4808
4809         return 0;
4810 }
4811
4812 static int
4813 ixgbe_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
4814 {
4815         uint32_t mask;
4816         struct ixgbe_hw *hw =
4817                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4818         struct ixgbe_interrupt *intr =
4819                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
4820
4821         if (queue_id < 16) {
4822                 ixgbe_disable_intr(hw);
4823                 intr->mask &= ~(1 << queue_id);
4824                 ixgbe_enable_intr(dev);
4825         } else if (queue_id < 32) {
4826                 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(0));
4827                 mask &= ~(1 << queue_id);
4828                 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
4829         } else if (queue_id < 64) {
4830                 mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(1));
4831                 mask &= ~(1 << (queue_id - 32));
4832                 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
4833         }
4834
4835         return 0;
4836 }
4837
4838 static void
4839 ixgbevf_set_ivar_map(struct ixgbe_hw *hw, int8_t direction,
4840                      uint8_t queue, uint8_t msix_vector)
4841 {
4842         uint32_t tmp, idx;
4843
4844         if (direction == -1) {
4845                 /* other causes */
4846                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4847                 tmp = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
4848                 tmp &= ~0xFF;
4849                 tmp |= msix_vector;
4850                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, tmp);
4851         } else {
4852                 /* rx or tx cause */
4853                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4854                 idx = ((16 * (queue & 1)) + (8 * direction));
4855                 tmp = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1));
4856                 tmp &= ~(0xFF << idx);
4857                 tmp |= (msix_vector << idx);
4858                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), tmp);
4859         }
4860 }
4861
4862 /**
4863  * set the IVAR registers, mapping interrupt causes to vectors
4864  * @param hw
4865  *  pointer to ixgbe_hw struct
4866  * @direction
4867  *  0 for Rx, 1 for Tx, -1 for other causes
4868  * @queue
4869  *  queue to map the corresponding interrupt to
4870  * @msix_vector
4871  *  the vector to map to the corresponding queue
4872  */
4873 static void
4874 ixgbe_set_ivar_map(struct ixgbe_hw *hw, int8_t direction,
4875                    uint8_t queue, uint8_t msix_vector)
4876 {
4877         uint32_t tmp, idx;
4878
4879         msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4880         if (hw->mac.type == ixgbe_mac_82598EB) {
4881                 if (direction == -1)
4882                         direction = 0;
4883                 idx = (((direction * 64) + queue) >> 2) & 0x1F;
4884                 tmp = IXGBE_READ_REG(hw, IXGBE_IVAR(idx));
4885                 tmp &= ~(0xFF << (8 * (queue & 0x3)));
4886                 tmp |= (msix_vector << (8 * (queue & 0x3)));
4887                 IXGBE_WRITE_REG(hw, IXGBE_IVAR(idx), tmp);
4888         } else if ((hw->mac.type == ixgbe_mac_82599EB) ||
4889                         (hw->mac.type == ixgbe_mac_X540)) {
4890                 if (direction == -1) {
4891                         /* other causes */
4892                         idx = ((queue & 1) * 8);
4893                         tmp = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4894                         tmp &= ~(0xFF << idx);
4895                         tmp |= (msix_vector << idx);
4896                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, tmp);
4897                 } else {
4898                         /* rx or tx causes */
4899                         idx = ((16 * (queue & 1)) + (8 * direction));
4900                         tmp = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1));
4901                         tmp &= ~(0xFF << idx);
4902                         tmp |= (msix_vector << idx);
4903                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), tmp);
4904                 }
4905         }
4906 }
4907
4908 static void
4909 ixgbevf_configure_msix(struct rte_eth_dev *dev)
4910 {
4911         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
4912         struct ixgbe_hw *hw =
4913                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4914         uint32_t q_idx;
4915         uint32_t vector_idx = IXGBE_MISC_VEC_ID;
4916
4917         /* Configure VF other cause ivar */
4918         ixgbevf_set_ivar_map(hw, -1, 1, vector_idx);
4919
4920         /* won't configure msix register if no mapping is done
4921          * between intr vector and event fd.
4922          */
4923         if (!rte_intr_dp_is_en(intr_handle))
4924                 return;
4925
4926         /* Configure all RX queues of VF */
4927         for (q_idx = 0; q_idx < dev->data->nb_rx_queues; q_idx++) {
4928                 /* Force all queue use vector 0,
4929                  * as IXGBE_VF_MAXMSIVECOTR = 1
4930                  */
4931                 ixgbevf_set_ivar_map(hw, 0, q_idx, vector_idx);
4932                 intr_handle->intr_vec[q_idx] = vector_idx;
4933         }
4934 }
4935
4936 /**
4937  * Sets up the hardware to properly generate MSI-X interrupts
4938  * @hw
4939  *  board private structure
4940  */
4941 static void
4942 ixgbe_configure_msix(struct rte_eth_dev *dev)
4943 {
4944         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
4945         struct ixgbe_hw *hw =
4946                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4947         uint32_t queue_id, base = IXGBE_MISC_VEC_ID;
4948         uint32_t vec = IXGBE_MISC_VEC_ID;
4949         uint32_t mask;
4950         uint32_t gpie;
4951
4952         /* won't configure msix register if no mapping is done
4953          * between intr vector and event fd
4954          */
4955         if (!rte_intr_dp_is_en(intr_handle))
4956                 return;
4957
4958         if (rte_intr_allow_others(intr_handle))
4959                 vec = base = IXGBE_RX_VEC_START;
4960
4961         /* setup GPIE for MSI-x mode */
4962         gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
4963         gpie |= IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT |
4964                 IXGBE_GPIE_OCD | IXGBE_GPIE_EIAME;
4965         /* auto clearing and auto setting corresponding bits in EIMS
4966          * when MSI-X interrupt is triggered
4967          */
4968         if (hw->mac.type == ixgbe_mac_82598EB) {
4969                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
4970         } else {
4971                 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
4972                 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
4973         }
4974         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
4975
4976         /* Populate the IVAR table and set the ITR values to the
4977          * corresponding register.
4978          */
4979         for (queue_id = 0; queue_id < dev->data->nb_rx_queues;
4980              queue_id++) {
4981                 /* by default, 1:1 mapping */
4982                 ixgbe_set_ivar_map(hw, 0, queue_id, vec);
4983                 intr_handle->intr_vec[queue_id] = vec;
4984                 if (vec < base + intr_handle->nb_efd - 1)
4985                         vec++;
4986         }
4987
4988         switch (hw->mac.type) {
4989         case ixgbe_mac_82598EB:
4990                 ixgbe_set_ivar_map(hw, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX,
4991                                    IXGBE_MISC_VEC_ID);
4992                 break;
4993         case ixgbe_mac_82599EB:
4994         case ixgbe_mac_X540:
4995                 ixgbe_set_ivar_map(hw, -1, 1, IXGBE_MISC_VEC_ID);
4996                 break;
4997         default:
4998                 break;
4999         }
5000         IXGBE_WRITE_REG(hw, IXGBE_EITR(IXGBE_MISC_VEC_ID),
5001                         IXGBE_MIN_INTER_INTERRUPT_INTERVAL_DEFAULT & 0xFFF);
5002
5003         /* set up to autoclear timer, and the vectors */
5004         mask = IXGBE_EIMS_ENABLE_MASK;
5005         mask &= ~(IXGBE_EIMS_OTHER |
5006                   IXGBE_EIMS_MAILBOX |
5007                   IXGBE_EIMS_LSC);
5008
5009         IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
5010 }
5011
5012 static int ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev,
5013         uint16_t queue_idx, uint16_t tx_rate)
5014 {
5015         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5016         uint32_t rf_dec, rf_int;
5017         uint32_t bcnrc_val;
5018         uint16_t link_speed = dev->data->dev_link.link_speed;
5019
5020         if (queue_idx >= hw->mac.max_tx_queues)
5021                 return -EINVAL;
5022
5023         if (tx_rate != 0) {
5024                 /* Calculate the rate factor values to set */
5025                 rf_int = (uint32_t)link_speed / (uint32_t)tx_rate;
5026                 rf_dec = (uint32_t)link_speed % (uint32_t)tx_rate;
5027                 rf_dec = (rf_dec << IXGBE_RTTBCNRC_RF_INT_SHIFT) / tx_rate;
5028
5029                 bcnrc_val = IXGBE_RTTBCNRC_RS_ENA;
5030                 bcnrc_val |= ((rf_int << IXGBE_RTTBCNRC_RF_INT_SHIFT) &
5031                                 IXGBE_RTTBCNRC_RF_INT_MASK_M);
5032                 bcnrc_val |= (rf_dec & IXGBE_RTTBCNRC_RF_DEC_MASK);
5033         } else {
5034                 bcnrc_val = 0;
5035         }
5036
5037         /*
5038          * Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
5039          * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported, otherwise
5040          * set as 0x4.
5041          */
5042         if ((dev->data->dev_conf.rxmode.jumbo_frame == 1) &&
5043                 (dev->data->dev_conf.rxmode.max_rx_pkt_len >=
5044                                 IXGBE_MAX_JUMBO_FRAME_SIZE))
5045                 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRM,
5046                         IXGBE_MMW_SIZE_JUMBO_FRAME);
5047         else
5048                 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRM,
5049                         IXGBE_MMW_SIZE_DEFAULT);
5050
5051         /* Set RTTBCNRC of queue X */
5052         IXGBE_WRITE_REG(hw, IXGBE_RTTDQSEL, queue_idx);
5053         IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRC, bcnrc_val);
5054         IXGBE_WRITE_FLUSH(hw);
5055
5056         return 0;
5057 }
5058
5059 static int ixgbe_set_vf_rate_limit(struct rte_eth_dev *dev, uint16_t vf,
5060         uint16_t tx_rate, uint64_t q_msk)
5061 {
5062         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5063         struct ixgbe_vf_info *vfinfo =
5064                 *(IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private));
5065         uint8_t  nb_q_per_pool = RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool;
5066         uint32_t queue_stride =
5067                 IXGBE_MAX_RX_QUEUE_NUM / RTE_ETH_DEV_SRIOV(dev).active;
5068         uint32_t queue_idx = vf * queue_stride, idx = 0, vf_idx;
5069         uint32_t queue_end = queue_idx + nb_q_per_pool - 1;
5070         uint16_t total_rate = 0;
5071
5072         if (queue_end >= hw->mac.max_tx_queues)
5073                 return -EINVAL;
5074
5075         if (vfinfo != NULL) {
5076                 for (vf_idx = 0; vf_idx < dev->pci_dev->max_vfs; vf_idx++) {
5077                         if (vf_idx == vf)
5078                                 continue;
5079                         for (idx = 0; idx < RTE_DIM(vfinfo[vf_idx].tx_rate);
5080                                 idx++)
5081                                 total_rate += vfinfo[vf_idx].tx_rate[idx];
5082                 }
5083         } else
5084                 return -EINVAL;
5085
5086         /* Store tx_rate for this vf. */
5087         for (idx = 0; idx < nb_q_per_pool; idx++) {
5088                 if (((uint64_t)0x1 << idx) & q_msk) {
5089                         if (vfinfo[vf].tx_rate[idx] != tx_rate)
5090                                 vfinfo[vf].tx_rate[idx] = tx_rate;
5091                         total_rate += tx_rate;
5092                 }
5093         }
5094
5095         if (total_rate > dev->data->dev_link.link_speed) {
5096                 /*
5097                  * Reset stored TX rate of the VF if it causes exceed
5098                  * link speed.
5099                  */
5100                 memset(vfinfo[vf].tx_rate, 0, sizeof(vfinfo[vf].tx_rate));
5101                 return -EINVAL;
5102         }
5103
5104         /* Set RTTBCNRC of each queue/pool for vf X  */
5105         for (; queue_idx <= queue_end; queue_idx++) {
5106                 if (0x1 & q_msk)
5107                         ixgbe_set_queue_rate_limit(dev, queue_idx, tx_rate);
5108                 q_msk = q_msk >> 1;
5109         }
5110
5111         return 0;
5112 }
5113
5114 static void
5115 ixgbevf_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
5116                      __attribute__((unused)) uint32_t index,
5117                      __attribute__((unused)) uint32_t pool)
5118 {
5119         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5120         int diag;
5121
5122         /*
5123          * On a 82599 VF, adding again the same MAC addr is not an idempotent
5124          * operation. Trap this case to avoid exhausting the [very limited]
5125          * set of PF resources used to store VF MAC addresses.
5126          */
5127         if (memcmp(hw->mac.perm_addr, mac_addr, sizeof(struct ether_addr)) == 0)
5128                 return;
5129         diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes);
5130         if (diag == 0)
5131                 return;
5132         PMD_DRV_LOG(ERR, "Unable to add MAC address - diag=%d", diag);
5133 }
5134
5135 static void
5136 ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index)
5137 {
5138         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5139         struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr;
5140         struct ether_addr *mac_addr;
5141         uint32_t i;
5142         int diag;
5143
5144         /*
5145          * The IXGBE_VF_SET_MACVLAN command of the ixgbe-pf driver does
5146          * not support the deletion of a given MAC address.
5147          * Instead, it imposes to delete all MAC addresses, then to add again
5148          * all MAC addresses with the exception of the one to be deleted.
5149          */
5150         (void) ixgbevf_set_uc_addr_vf(hw, 0, NULL);
5151
5152         /*
5153          * Add again all MAC addresses, with the exception of the deleted one
5154          * and of the permanent MAC address.
5155          */
5156         for (i = 0, mac_addr = dev->data->mac_addrs;
5157              i < hw->mac.num_rar_entries; i++, mac_addr++) {
5158                 /* Skip the deleted MAC address */
5159                 if (i == index)
5160                         continue;
5161                 /* Skip NULL MAC addresses */
5162                 if (is_zero_ether_addr(mac_addr))
5163                         continue;
5164                 /* Skip the permanent MAC address */
5165                 if (memcmp(perm_addr, mac_addr, sizeof(struct ether_addr)) == 0)
5166                         continue;
5167                 diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes);
5168                 if (diag != 0)
5169                         PMD_DRV_LOG(ERR,
5170                                     "Adding again MAC address "
5171                                     "%02x:%02x:%02x:%02x:%02x:%02x failed "
5172                                     "diag=%d",
5173                                     mac_addr->addr_bytes[0],
5174                                     mac_addr->addr_bytes[1],
5175                                     mac_addr->addr_bytes[2],
5176                                     mac_addr->addr_bytes[3],
5177                                     mac_addr->addr_bytes[4],
5178                                     mac_addr->addr_bytes[5],
5179                                     diag);
5180         }
5181 }
5182
5183 static void
5184 ixgbevf_set_default_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
5185 {
5186         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5187
5188         hw->mac.ops.set_rar(hw, 0, (void *)addr, 0, 0);
5189 }
5190
5191 #define MAC_TYPE_FILTER_SUP(type)    do {\
5192         if ((type) != ixgbe_mac_82599EB && (type) != ixgbe_mac_X540 &&\
5193                 (type) != ixgbe_mac_X550 && (type) != ixgbe_mac_X550EM_x &&\
5194                 (type) != ixgbe_mac_X550EM_a)\
5195                 return -ENOTSUP;\
5196 } while (0)
5197
5198 static int
5199 ixgbe_syn_filter_set(struct rte_eth_dev *dev,
5200                         struct rte_eth_syn_filter *filter,
5201                         bool add)
5202 {
5203         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5204         uint32_t synqf;
5205
5206         if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM)
5207                 return -EINVAL;
5208
5209         synqf = IXGBE_READ_REG(hw, IXGBE_SYNQF);
5210
5211         if (add) {
5212                 if (synqf & IXGBE_SYN_FILTER_ENABLE)
5213                         return -EINVAL;
5214                 synqf = (uint32_t)(((filter->queue << IXGBE_SYN_FILTER_QUEUE_SHIFT) &
5215                         IXGBE_SYN_FILTER_QUEUE) | IXGBE_SYN_FILTER_ENABLE);
5216
5217                 if (filter->hig_pri)
5218                         synqf |= IXGBE_SYN_FILTER_SYNQFP;
5219                 else
5220                         synqf &= ~IXGBE_SYN_FILTER_SYNQFP;
5221         } else {
5222                 if (!(synqf & IXGBE_SYN_FILTER_ENABLE))
5223                         return -ENOENT;
5224                 synqf &= ~(IXGBE_SYN_FILTER_QUEUE | IXGBE_SYN_FILTER_ENABLE);
5225         }
5226         IXGBE_WRITE_REG(hw, IXGBE_SYNQF, synqf);
5227         IXGBE_WRITE_FLUSH(hw);
5228         return 0;
5229 }
5230
5231 static int
5232 ixgbe_syn_filter_get(struct rte_eth_dev *dev,
5233                         struct rte_eth_syn_filter *filter)
5234 {
5235         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5236         uint32_t synqf = IXGBE_READ_REG(hw, IXGBE_SYNQF);
5237
5238         if (synqf & IXGBE_SYN_FILTER_ENABLE) {
5239                 filter->hig_pri = (synqf & IXGBE_SYN_FILTER_SYNQFP) ? 1 : 0;
5240                 filter->queue = (uint16_t)((synqf & IXGBE_SYN_FILTER_QUEUE) >> 1);
5241                 return 0;
5242         }
5243         return -ENOENT;
5244 }
5245
5246 static int
5247 ixgbe_syn_filter_handle(struct rte_eth_dev *dev,
5248                         enum rte_filter_op filter_op,
5249                         void *arg)
5250 {
5251         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5252         int ret;
5253
5254         MAC_TYPE_FILTER_SUP(hw->mac.type);
5255
5256         if (filter_op == RTE_ETH_FILTER_NOP)
5257                 return 0;
5258
5259         if (arg == NULL) {
5260                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
5261                             filter_op);
5262                 return -EINVAL;
5263         }
5264
5265         switch (filter_op) {
5266         case RTE_ETH_FILTER_ADD:
5267                 ret = ixgbe_syn_filter_set(dev,
5268                                 (struct rte_eth_syn_filter *)arg,
5269                                 TRUE);
5270                 break;
5271         case RTE_ETH_FILTER_DELETE:
5272                 ret = ixgbe_syn_filter_set(dev,
5273                                 (struct rte_eth_syn_filter *)arg,
5274                                 FALSE);
5275                 break;
5276         case RTE_ETH_FILTER_GET:
5277                 ret = ixgbe_syn_filter_get(dev,
5278                                 (struct rte_eth_syn_filter *)arg);
5279                 break;
5280         default:
5281                 PMD_DRV_LOG(ERR, "unsupported operation %u\n", filter_op);
5282                 ret = -EINVAL;
5283                 break;
5284         }
5285
5286         return ret;
5287 }
5288
5289
5290 static inline enum ixgbe_5tuple_protocol
5291 convert_protocol_type(uint8_t protocol_value)
5292 {
5293         if (protocol_value == IPPROTO_TCP)
5294                 return IXGBE_FILTER_PROTOCOL_TCP;
5295         else if (protocol_value == IPPROTO_UDP)
5296                 return IXGBE_FILTER_PROTOCOL_UDP;
5297         else if (protocol_value == IPPROTO_SCTP)
5298                 return IXGBE_FILTER_PROTOCOL_SCTP;
5299         else
5300                 return IXGBE_FILTER_PROTOCOL_NONE;
5301 }
5302
5303 /*
5304  * add a 5tuple filter
5305  *
5306  * @param
5307  * dev: Pointer to struct rte_eth_dev.
5308  * index: the index the filter allocates.
5309  * filter: ponter to the filter that will be added.
5310  * rx_queue: the queue id the filter assigned to.
5311  *
5312  * @return
5313  *    - On success, zero.
5314  *    - On failure, a negative value.
5315  */
5316 static int
5317 ixgbe_add_5tuple_filter(struct rte_eth_dev *dev,
5318                         struct ixgbe_5tuple_filter *filter)
5319 {
5320         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5321         struct ixgbe_filter_info *filter_info =
5322                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
5323         int i, idx, shift;
5324         uint32_t ftqf, sdpqf;
5325         uint32_t l34timir = 0;
5326         uint8_t mask = 0xff;
5327
5328         /*
5329          * look for an unused 5tuple filter index,
5330          * and insert the filter to list.
5331          */
5332         for (i = 0; i < IXGBE_MAX_FTQF_FILTERS; i++) {
5333                 idx = i / (sizeof(uint32_t) * NBBY);
5334                 shift = i % (sizeof(uint32_t) * NBBY);
5335                 if (!(filter_info->fivetuple_mask[idx] & (1 << shift))) {
5336                         filter_info->fivetuple_mask[idx] |= 1 << shift;
5337                         filter->index = i;
5338                         TAILQ_INSERT_TAIL(&filter_info->fivetuple_list,
5339                                           filter,
5340                                           entries);
5341                         break;
5342                 }
5343         }
5344         if (i >= IXGBE_MAX_FTQF_FILTERS) {
5345                 PMD_DRV_LOG(ERR, "5tuple filters are full.");
5346                 return -ENOSYS;
5347         }
5348
5349         sdpqf = (uint32_t)(filter->filter_info.dst_port <<
5350                                 IXGBE_SDPQF_DSTPORT_SHIFT);
5351         sdpqf = sdpqf | (filter->filter_info.src_port & IXGBE_SDPQF_SRCPORT);
5352
5353         ftqf = (uint32_t)(filter->filter_info.proto &
5354                 IXGBE_FTQF_PROTOCOL_MASK);
5355         ftqf |= (uint32_t)((filter->filter_info.priority &
5356                 IXGBE_FTQF_PRIORITY_MASK) << IXGBE_FTQF_PRIORITY_SHIFT);
5357         if (filter->filter_info.src_ip_mask == 0) /* 0 means compare. */
5358                 mask &= IXGBE_FTQF_SOURCE_ADDR_MASK;
5359         if (filter->filter_info.dst_ip_mask == 0)
5360                 mask &= IXGBE_FTQF_DEST_ADDR_MASK;
5361         if (filter->filter_info.src_port_mask == 0)
5362                 mask &= IXGBE_FTQF_SOURCE_PORT_MASK;
5363         if (filter->filter_info.dst_port_mask == 0)
5364                 mask &= IXGBE_FTQF_DEST_PORT_MASK;
5365         if (filter->filter_info.proto_mask == 0)
5366                 mask &= IXGBE_FTQF_PROTOCOL_COMP_MASK;
5367         ftqf |= mask << IXGBE_FTQF_5TUPLE_MASK_SHIFT;
5368         ftqf |= IXGBE_FTQF_POOL_MASK_EN;
5369         ftqf |= IXGBE_FTQF_QUEUE_ENABLE;
5370
5371         IXGBE_WRITE_REG(hw, IXGBE_DAQF(i), filter->filter_info.dst_ip);
5372         IXGBE_WRITE_REG(hw, IXGBE_SAQF(i), filter->filter_info.src_ip);
5373         IXGBE_WRITE_REG(hw, IXGBE_SDPQF(i), sdpqf);
5374         IXGBE_WRITE_REG(hw, IXGBE_FTQF(i), ftqf);
5375
5376         l34timir |= IXGBE_L34T_IMIR_RESERVE;
5377         l34timir |= (uint32_t)(filter->queue <<
5378                                 IXGBE_L34T_IMIR_QUEUE_SHIFT);
5379         IXGBE_WRITE_REG(hw, IXGBE_L34T_IMIR(i), l34timir);
5380         return 0;
5381 }
5382
5383 /*
5384  * remove a 5tuple filter
5385  *
5386  * @param
5387  * dev: Pointer to struct rte_eth_dev.
5388  * filter: the pointer of the filter will be removed.
5389  */
5390 static void
5391 ixgbe_remove_5tuple_filter(struct rte_eth_dev *dev,
5392                         struct ixgbe_5tuple_filter *filter)
5393 {
5394         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5395         struct ixgbe_filter_info *filter_info =
5396                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
5397         uint16_t index = filter->index;
5398
5399         filter_info->fivetuple_mask[index / (sizeof(uint32_t) * NBBY)] &=
5400                                 ~(1 << (index % (sizeof(uint32_t) * NBBY)));
5401         TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries);
5402         rte_free(filter);
5403
5404         IXGBE_WRITE_REG(hw, IXGBE_DAQF(index), 0);
5405         IXGBE_WRITE_REG(hw, IXGBE_SAQF(index), 0);
5406         IXGBE_WRITE_REG(hw, IXGBE_SDPQF(index), 0);
5407         IXGBE_WRITE_REG(hw, IXGBE_FTQF(index), 0);
5408         IXGBE_WRITE_REG(hw, IXGBE_L34T_IMIR(index), 0);
5409 }
5410
5411 static int
5412 ixgbevf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
5413 {
5414         struct ixgbe_hw *hw;
5415         uint32_t max_frame = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
5416
5417         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5418
5419         if ((mtu < ETHER_MIN_MTU) || (max_frame > ETHER_MAX_JUMBO_FRAME_LEN))
5420                 return -EINVAL;
5421
5422         /* refuse mtu that requires the support of scattered packets when this
5423          * feature has not been enabled before.
5424          */
5425         if (!dev->data->scattered_rx &&
5426             (max_frame + 2 * IXGBE_VLAN_TAG_SIZE >
5427              dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM))
5428                 return -EINVAL;
5429
5430         /*
5431          * When supported by the underlying PF driver, use the IXGBE_VF_SET_MTU
5432          * request of the version 2.0 of the mailbox API.
5433          * For now, use the IXGBE_VF_SET_LPE request of the version 1.0
5434          * of the mailbox API.
5435          * This call to IXGBE_SET_LPE action won't work with ixgbe pf drivers
5436          * prior to 3.11.33 which contains the following change:
5437          * "ixgbe: Enable jumbo frames support w/ SR-IOV"
5438          */
5439         ixgbevf_rlpml_set_vf(hw, max_frame);
5440
5441         /* update max frame size */
5442         dev->data->dev_conf.rxmode.max_rx_pkt_len = max_frame;
5443         return 0;
5444 }
5445
5446 #define MAC_TYPE_FILTER_SUP_EXT(type)    do {\
5447         if ((type) != ixgbe_mac_82599EB && (type) != ixgbe_mac_X540)\
5448                 return -ENOTSUP;\
5449 } while (0)
5450
5451 static inline struct ixgbe_5tuple_filter *
5452 ixgbe_5tuple_filter_lookup(struct ixgbe_5tuple_filter_list *filter_list,
5453                         struct ixgbe_5tuple_filter_info *key)
5454 {
5455         struct ixgbe_5tuple_filter *it;
5456
5457         TAILQ_FOREACH(it, filter_list, entries) {
5458                 if (memcmp(key, &it->filter_info,
5459                         sizeof(struct ixgbe_5tuple_filter_info)) == 0) {
5460                         return it;
5461                 }
5462         }
5463         return NULL;
5464 }
5465
5466 /* translate elements in struct rte_eth_ntuple_filter to struct ixgbe_5tuple_filter_info*/
5467 static inline int
5468 ntuple_filter_to_5tuple(struct rte_eth_ntuple_filter *filter,
5469                         struct ixgbe_5tuple_filter_info *filter_info)
5470 {
5471         if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM ||
5472                 filter->priority > IXGBE_5TUPLE_MAX_PRI ||
5473                 filter->priority < IXGBE_5TUPLE_MIN_PRI)
5474                 return -EINVAL;
5475
5476         switch (filter->dst_ip_mask) {
5477         case UINT32_MAX:
5478                 filter_info->dst_ip_mask = 0;
5479                 filter_info->dst_ip = filter->dst_ip;
5480                 break;
5481         case 0:
5482                 filter_info->dst_ip_mask = 1;
5483                 break;
5484         default:
5485                 PMD_DRV_LOG(ERR, "invalid dst_ip mask.");
5486                 return -EINVAL;
5487         }
5488
5489         switch (filter->src_ip_mask) {
5490         case UINT32_MAX:
5491                 filter_info->src_ip_mask = 0;
5492                 filter_info->src_ip = filter->src_ip;
5493                 break;
5494         case 0:
5495                 filter_info->src_ip_mask = 1;
5496                 break;
5497         default:
5498                 PMD_DRV_LOG(ERR, "invalid src_ip mask.");
5499                 return -EINVAL;
5500         }
5501
5502         switch (filter->dst_port_mask) {
5503         case UINT16_MAX:
5504                 filter_info->dst_port_mask = 0;
5505                 filter_info->dst_port = filter->dst_port;
5506                 break;
5507         case 0:
5508                 filter_info->dst_port_mask = 1;
5509                 break;
5510         default:
5511                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
5512                 return -EINVAL;
5513         }
5514
5515         switch (filter->src_port_mask) {
5516         case UINT16_MAX:
5517                 filter_info->src_port_mask = 0;
5518                 filter_info->src_port = filter->src_port;
5519                 break;
5520         case 0:
5521                 filter_info->src_port_mask = 1;
5522                 break;
5523         default:
5524                 PMD_DRV_LOG(ERR, "invalid src_port mask.");
5525                 return -EINVAL;
5526         }
5527
5528         switch (filter->proto_mask) {
5529         case UINT8_MAX:
5530                 filter_info->proto_mask = 0;
5531                 filter_info->proto =
5532                         convert_protocol_type(filter->proto);
5533                 break;
5534         case 0:
5535                 filter_info->proto_mask = 1;
5536                 break;
5537         default:
5538                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
5539                 return -EINVAL;
5540         }
5541
5542         filter_info->priority = (uint8_t)filter->priority;
5543         return 0;
5544 }
5545
5546 /*
5547  * add or delete a ntuple filter
5548  *
5549  * @param
5550  * dev: Pointer to struct rte_eth_dev.
5551  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
5552  * add: if true, add filter, if false, remove filter
5553  *
5554  * @return
5555  *    - On success, zero.
5556  *    - On failure, a negative value.
5557  */
5558 static int
5559 ixgbe_add_del_ntuple_filter(struct rte_eth_dev *dev,
5560                         struct rte_eth_ntuple_filter *ntuple_filter,
5561                         bool add)
5562 {
5563         struct ixgbe_filter_info *filter_info =
5564                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
5565         struct ixgbe_5tuple_filter_info filter_5tuple;
5566         struct ixgbe_5tuple_filter *filter;
5567         int ret;
5568
5569         if (ntuple_filter->flags != RTE_5TUPLE_FLAGS) {
5570                 PMD_DRV_LOG(ERR, "only 5tuple is supported.");
5571                 return -EINVAL;
5572         }
5573
5574         memset(&filter_5tuple, 0, sizeof(struct ixgbe_5tuple_filter_info));
5575         ret = ntuple_filter_to_5tuple(ntuple_filter, &filter_5tuple);
5576         if (ret < 0)
5577                 return ret;
5578
5579         filter = ixgbe_5tuple_filter_lookup(&filter_info->fivetuple_list,
5580                                          &filter_5tuple);
5581         if (filter != NULL && add) {
5582                 PMD_DRV_LOG(ERR, "filter exists.");
5583                 return -EEXIST;
5584         }
5585         if (filter == NULL && !add) {
5586                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
5587                 return -ENOENT;
5588         }
5589
5590         if (add) {
5591                 filter = rte_zmalloc("ixgbe_5tuple_filter",
5592                                 sizeof(struct ixgbe_5tuple_filter), 0);
5593                 if (filter == NULL)
5594                         return -ENOMEM;
5595                 (void)rte_memcpy(&filter->filter_info,
5596                                  &filter_5tuple,
5597                                  sizeof(struct ixgbe_5tuple_filter_info));
5598                 filter->queue = ntuple_filter->queue;
5599                 ret = ixgbe_add_5tuple_filter(dev, filter);
5600                 if (ret < 0) {
5601                         rte_free(filter);
5602                         return ret;
5603                 }
5604         } else
5605                 ixgbe_remove_5tuple_filter(dev, filter);
5606
5607         return 0;
5608 }
5609
5610 /*
5611  * get a ntuple filter
5612  *
5613  * @param
5614  * dev: Pointer to struct rte_eth_dev.
5615  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
5616  *
5617  * @return
5618  *    - On success, zero.
5619  *    - On failure, a negative value.
5620  */
5621 static int
5622 ixgbe_get_ntuple_filter(struct rte_eth_dev *dev,
5623                         struct rte_eth_ntuple_filter *ntuple_filter)
5624 {
5625         struct ixgbe_filter_info *filter_info =
5626                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
5627         struct ixgbe_5tuple_filter_info filter_5tuple;
5628         struct ixgbe_5tuple_filter *filter;
5629         int ret;
5630
5631         if (ntuple_filter->flags != RTE_5TUPLE_FLAGS) {
5632                 PMD_DRV_LOG(ERR, "only 5tuple is supported.");
5633                 return -EINVAL;
5634         }
5635
5636         memset(&filter_5tuple, 0, sizeof(struct ixgbe_5tuple_filter_info));
5637         ret = ntuple_filter_to_5tuple(ntuple_filter, &filter_5tuple);
5638         if (ret < 0)
5639                 return ret;
5640
5641         filter = ixgbe_5tuple_filter_lookup(&filter_info->fivetuple_list,
5642                                          &filter_5tuple);
5643         if (filter == NULL) {
5644                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
5645                 return -ENOENT;
5646         }
5647         ntuple_filter->queue = filter->queue;
5648         return 0;
5649 }
5650
5651 /*
5652  * ixgbe_ntuple_filter_handle - Handle operations for ntuple filter.
5653  * @dev: pointer to rte_eth_dev structure
5654  * @filter_op:operation will be taken.
5655  * @arg: a pointer to specific structure corresponding to the filter_op
5656  *
5657  * @return
5658  *    - On success, zero.
5659  *    - On failure, a negative value.
5660  */
5661 static int
5662 ixgbe_ntuple_filter_handle(struct rte_eth_dev *dev,
5663                                 enum rte_filter_op filter_op,
5664                                 void *arg)
5665 {
5666         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5667         int ret;
5668
5669         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
5670
5671         if (filter_op == RTE_ETH_FILTER_NOP)
5672                 return 0;
5673
5674         if (arg == NULL) {
5675                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
5676                             filter_op);
5677                 return -EINVAL;
5678         }
5679
5680         switch (filter_op) {
5681         case RTE_ETH_FILTER_ADD:
5682                 ret = ixgbe_add_del_ntuple_filter(dev,
5683                         (struct rte_eth_ntuple_filter *)arg,
5684                         TRUE);
5685                 break;
5686         case RTE_ETH_FILTER_DELETE:
5687                 ret = ixgbe_add_del_ntuple_filter(dev,
5688                         (struct rte_eth_ntuple_filter *)arg,
5689                         FALSE);
5690                 break;
5691         case RTE_ETH_FILTER_GET:
5692                 ret = ixgbe_get_ntuple_filter(dev,
5693                         (struct rte_eth_ntuple_filter *)arg);
5694                 break;
5695         default:
5696                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
5697                 ret = -EINVAL;
5698                 break;
5699         }
5700         return ret;
5701 }
5702
5703 static inline int
5704 ixgbe_ethertype_filter_lookup(struct ixgbe_filter_info *filter_info,
5705                         uint16_t ethertype)
5706 {
5707         int i;
5708
5709         for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) {
5710                 if (filter_info->ethertype_filters[i] == ethertype &&
5711                     (filter_info->ethertype_mask & (1 << i)))
5712                         return i;
5713         }
5714         return -1;
5715 }
5716
5717 static inline int
5718 ixgbe_ethertype_filter_insert(struct ixgbe_filter_info *filter_info,
5719                         uint16_t ethertype)
5720 {
5721         int i;
5722
5723         for (i = 0; i < IXGBE_MAX_ETQF_FILTERS; i++) {
5724                 if (!(filter_info->ethertype_mask & (1 << i))) {
5725                         filter_info->ethertype_mask |= 1 << i;
5726                         filter_info->ethertype_filters[i] = ethertype;
5727                         return i;
5728                 }
5729         }
5730         return -1;
5731 }
5732
5733 static inline int
5734 ixgbe_ethertype_filter_remove(struct ixgbe_filter_info *filter_info,
5735                         uint8_t idx)
5736 {
5737         if (idx >= IXGBE_MAX_ETQF_FILTERS)
5738                 return -1;
5739         filter_info->ethertype_mask &= ~(1 << idx);
5740         filter_info->ethertype_filters[idx] = 0;
5741         return idx;
5742 }
5743
5744 static int
5745 ixgbe_add_del_ethertype_filter(struct rte_eth_dev *dev,
5746                         struct rte_eth_ethertype_filter *filter,
5747                         bool add)
5748 {
5749         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5750         struct ixgbe_filter_info *filter_info =
5751                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
5752         uint32_t etqf = 0;
5753         uint32_t etqs = 0;
5754         int ret;
5755
5756         if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM)
5757                 return -EINVAL;
5758
5759         if (filter->ether_type == ETHER_TYPE_IPv4 ||
5760                 filter->ether_type == ETHER_TYPE_IPv6) {
5761                 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in"
5762                         " ethertype filter.", filter->ether_type);
5763                 return -EINVAL;
5764         }
5765
5766         if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) {
5767                 PMD_DRV_LOG(ERR, "mac compare is unsupported.");
5768                 return -EINVAL;
5769         }
5770         if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) {
5771                 PMD_DRV_LOG(ERR, "drop option is unsupported.");
5772                 return -EINVAL;
5773         }
5774
5775         ret = ixgbe_ethertype_filter_lookup(filter_info, filter->ether_type);
5776         if (ret >= 0 && add) {
5777                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.",
5778                             filter->ether_type);
5779                 return -EEXIST;
5780         }
5781         if (ret < 0 && !add) {
5782                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
5783                             filter->ether_type);
5784                 return -ENOENT;
5785         }
5786
5787         if (add) {
5788                 ret = ixgbe_ethertype_filter_insert(filter_info,
5789                         filter->ether_type);
5790                 if (ret < 0) {
5791                         PMD_DRV_LOG(ERR, "ethertype filters are full.");
5792                         return -ENOSYS;
5793                 }
5794                 etqf = IXGBE_ETQF_FILTER_EN;
5795                 etqf |= (uint32_t)filter->ether_type;
5796                 etqs |= (uint32_t)((filter->queue <<
5797                                     IXGBE_ETQS_RX_QUEUE_SHIFT) &
5798                                     IXGBE_ETQS_RX_QUEUE);
5799                 etqs |= IXGBE_ETQS_QUEUE_EN;
5800         } else {
5801                 ret = ixgbe_ethertype_filter_remove(filter_info, (uint8_t)ret);
5802                 if (ret < 0)
5803                         return -ENOSYS;
5804         }
5805         IXGBE_WRITE_REG(hw, IXGBE_ETQF(ret), etqf);
5806         IXGBE_WRITE_REG(hw, IXGBE_ETQS(ret), etqs);
5807         IXGBE_WRITE_FLUSH(hw);
5808
5809         return 0;
5810 }
5811
5812 static int
5813 ixgbe_get_ethertype_filter(struct rte_eth_dev *dev,
5814                         struct rte_eth_ethertype_filter *filter)
5815 {
5816         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5817         struct ixgbe_filter_info *filter_info =
5818                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
5819         uint32_t etqf, etqs;
5820         int ret;
5821
5822         ret = ixgbe_ethertype_filter_lookup(filter_info, filter->ether_type);
5823         if (ret < 0) {
5824                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
5825                             filter->ether_type);
5826                 return -ENOENT;
5827         }
5828
5829         etqf = IXGBE_READ_REG(hw, IXGBE_ETQF(ret));
5830         if (etqf & IXGBE_ETQF_FILTER_EN) {
5831                 etqs = IXGBE_READ_REG(hw, IXGBE_ETQS(ret));
5832                 filter->ether_type = etqf & IXGBE_ETQF_ETHERTYPE;
5833                 filter->flags = 0;
5834                 filter->queue = (etqs & IXGBE_ETQS_RX_QUEUE) >>
5835                                IXGBE_ETQS_RX_QUEUE_SHIFT;
5836                 return 0;
5837         }
5838         return -ENOENT;
5839 }
5840
5841 /*
5842  * ixgbe_ethertype_filter_handle - Handle operations for ethertype filter.
5843  * @dev: pointer to rte_eth_dev structure
5844  * @filter_op:operation will be taken.
5845  * @arg: a pointer to specific structure corresponding to the filter_op
5846  */
5847 static int
5848 ixgbe_ethertype_filter_handle(struct rte_eth_dev *dev,
5849                                 enum rte_filter_op filter_op,
5850                                 void *arg)
5851 {
5852         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5853         int ret;
5854
5855         MAC_TYPE_FILTER_SUP(hw->mac.type);
5856
5857         if (filter_op == RTE_ETH_FILTER_NOP)
5858                 return 0;
5859
5860         if (arg == NULL) {
5861                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
5862                             filter_op);
5863                 return -EINVAL;
5864         }
5865
5866         switch (filter_op) {
5867         case RTE_ETH_FILTER_ADD:
5868                 ret = ixgbe_add_del_ethertype_filter(dev,
5869                         (struct rte_eth_ethertype_filter *)arg,
5870                         TRUE);
5871                 break;
5872         case RTE_ETH_FILTER_DELETE:
5873                 ret = ixgbe_add_del_ethertype_filter(dev,
5874                         (struct rte_eth_ethertype_filter *)arg,
5875                         FALSE);
5876                 break;
5877         case RTE_ETH_FILTER_GET:
5878                 ret = ixgbe_get_ethertype_filter(dev,
5879                         (struct rte_eth_ethertype_filter *)arg);
5880                 break;
5881         default:
5882                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
5883                 ret = -EINVAL;
5884                 break;
5885         }
5886         return ret;
5887 }
5888
5889 static int
5890 ixgbe_dev_filter_ctrl(struct rte_eth_dev *dev,
5891                      enum rte_filter_type filter_type,
5892                      enum rte_filter_op filter_op,
5893                      void *arg)
5894 {
5895         int ret = -EINVAL;
5896
5897         switch (filter_type) {
5898         case RTE_ETH_FILTER_NTUPLE:
5899                 ret = ixgbe_ntuple_filter_handle(dev, filter_op, arg);
5900                 break;
5901         case RTE_ETH_FILTER_ETHERTYPE:
5902                 ret = ixgbe_ethertype_filter_handle(dev, filter_op, arg);
5903                 break;
5904         case RTE_ETH_FILTER_SYN:
5905                 ret = ixgbe_syn_filter_handle(dev, filter_op, arg);
5906                 break;
5907         case RTE_ETH_FILTER_FDIR:
5908                 ret = ixgbe_fdir_ctrl_func(dev, filter_op, arg);
5909                 break;
5910         case RTE_ETH_FILTER_L2_TUNNEL:
5911                 ret = ixgbe_dev_l2_tunnel_filter_handle(dev, filter_op, arg);
5912                 break;
5913         default:
5914                 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported",
5915                                                         filter_type);
5916                 break;
5917         }
5918
5919         return ret;
5920 }
5921
5922 static u8 *
5923 ixgbe_dev_addr_list_itr(__attribute__((unused)) struct ixgbe_hw *hw,
5924                         u8 **mc_addr_ptr, u32 *vmdq)
5925 {
5926         u8 *mc_addr;
5927
5928         *vmdq = 0;
5929         mc_addr = *mc_addr_ptr;
5930         *mc_addr_ptr = (mc_addr + sizeof(struct ether_addr));
5931         return mc_addr;
5932 }
5933
5934 static int
5935 ixgbe_dev_set_mc_addr_list(struct rte_eth_dev *dev,
5936                           struct ether_addr *mc_addr_set,
5937                           uint32_t nb_mc_addr)
5938 {
5939         struct ixgbe_hw *hw;
5940         u8 *mc_addr_list;
5941
5942         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5943         mc_addr_list = (u8 *)mc_addr_set;
5944         return ixgbe_update_mc_addr_list(hw, mc_addr_list, nb_mc_addr,
5945                                          ixgbe_dev_addr_list_itr, TRUE);
5946 }
5947
5948 static uint64_t
5949 ixgbe_read_systime_cyclecounter(struct rte_eth_dev *dev)
5950 {
5951         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5952         uint64_t systime_cycles;
5953
5954         switch (hw->mac.type) {
5955         case ixgbe_mac_X550:
5956         case ixgbe_mac_X550EM_x:
5957         case ixgbe_mac_X550EM_a:
5958                 /* SYSTIMEL stores ns and SYSTIMEH stores seconds. */
5959                 systime_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_SYSTIML);
5960                 systime_cycles += (uint64_t)IXGBE_READ_REG(hw, IXGBE_SYSTIMH)
5961                                 * NSEC_PER_SEC;
5962                 break;
5963         default:
5964                 systime_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_SYSTIML);
5965                 systime_cycles |= (uint64_t)IXGBE_READ_REG(hw, IXGBE_SYSTIMH)
5966                                 << 32;
5967         }
5968
5969         return systime_cycles;
5970 }
5971
5972 static uint64_t
5973 ixgbe_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev)
5974 {
5975         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5976         uint64_t rx_tstamp_cycles;
5977
5978         switch (hw->mac.type) {
5979         case ixgbe_mac_X550:
5980         case ixgbe_mac_X550EM_x:
5981         case ixgbe_mac_X550EM_a:
5982                 /* RXSTMPL stores ns and RXSTMPH stores seconds. */
5983                 rx_tstamp_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_RXSTMPL);
5984                 rx_tstamp_cycles += (uint64_t)IXGBE_READ_REG(hw, IXGBE_RXSTMPH)
5985                                 * NSEC_PER_SEC;
5986                 break;
5987         default:
5988                 /* RXSTMPL stores ns and RXSTMPH stores seconds. */
5989                 rx_tstamp_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_RXSTMPL);
5990                 rx_tstamp_cycles |= (uint64_t)IXGBE_READ_REG(hw, IXGBE_RXSTMPH)
5991                                 << 32;
5992         }
5993
5994         return rx_tstamp_cycles;
5995 }
5996
5997 static uint64_t
5998 ixgbe_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev)
5999 {
6000         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6001         uint64_t tx_tstamp_cycles;
6002
6003         switch (hw->mac.type) {
6004         case ixgbe_mac_X550:
6005         case ixgbe_mac_X550EM_x:
6006         case ixgbe_mac_X550EM_a:
6007                 /* TXSTMPL stores ns and TXSTMPH stores seconds. */
6008                 tx_tstamp_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_TXSTMPL);
6009                 tx_tstamp_cycles += (uint64_t)IXGBE_READ_REG(hw, IXGBE_TXSTMPH)
6010                                 * NSEC_PER_SEC;
6011                 break;
6012         default:
6013                 /* TXSTMPL stores ns and TXSTMPH stores seconds. */
6014                 tx_tstamp_cycles = (uint64_t)IXGBE_READ_REG(hw, IXGBE_TXSTMPL);
6015                 tx_tstamp_cycles |= (uint64_t)IXGBE_READ_REG(hw, IXGBE_TXSTMPH)
6016                                 << 32;
6017         }
6018
6019         return tx_tstamp_cycles;
6020 }
6021
6022 static void
6023 ixgbe_start_timecounters(struct rte_eth_dev *dev)
6024 {
6025         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6026         struct ixgbe_adapter *adapter =
6027                 (struct ixgbe_adapter *)dev->data->dev_private;
6028         struct rte_eth_link link;
6029         uint32_t incval = 0;
6030         uint32_t shift = 0;
6031
6032         /* Get current link speed. */
6033         memset(&link, 0, sizeof(link));
6034         ixgbe_dev_link_update(dev, 1);
6035         rte_ixgbe_dev_atomic_read_link_status(dev, &link);
6036
6037         switch (link.link_speed) {
6038         case ETH_SPEED_NUM_100M:
6039                 incval = IXGBE_INCVAL_100;
6040                 shift = IXGBE_INCVAL_SHIFT_100;
6041                 break;
6042         case ETH_SPEED_NUM_1G:
6043                 incval = IXGBE_INCVAL_1GB;
6044                 shift = IXGBE_INCVAL_SHIFT_1GB;
6045                 break;
6046         case ETH_SPEED_NUM_10G:
6047         default:
6048                 incval = IXGBE_INCVAL_10GB;
6049                 shift = IXGBE_INCVAL_SHIFT_10GB;
6050                 break;
6051         }
6052
6053         switch (hw->mac.type) {
6054         case ixgbe_mac_X550:
6055         case ixgbe_mac_X550EM_x:
6056         case ixgbe_mac_X550EM_a:
6057                 /* Independent of link speed. */
6058                 incval = 1;
6059                 /* Cycles read will be interpreted as ns. */
6060                 shift = 0;
6061                 /* Fall-through */
6062         case ixgbe_mac_X540:
6063                 IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, incval);
6064                 break;
6065         case ixgbe_mac_82599EB:
6066                 incval >>= IXGBE_INCVAL_SHIFT_82599;
6067                 shift -= IXGBE_INCVAL_SHIFT_82599;
6068                 IXGBE_WRITE_REG(hw, IXGBE_TIMINCA,
6069                                 (1 << IXGBE_INCPER_SHIFT_82599) | incval);
6070                 break;
6071         default:
6072                 /* Not supported. */
6073                 return;
6074         }
6075
6076         memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter));
6077         memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter));
6078         memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter));
6079
6080         adapter->systime_tc.cc_mask = IXGBE_CYCLECOUNTER_MASK;
6081         adapter->systime_tc.cc_shift = shift;
6082         adapter->systime_tc.nsec_mask = (1ULL << shift) - 1;
6083
6084         adapter->rx_tstamp_tc.cc_mask = IXGBE_CYCLECOUNTER_MASK;
6085         adapter->rx_tstamp_tc.cc_shift = shift;
6086         adapter->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
6087
6088         adapter->tx_tstamp_tc.cc_mask = IXGBE_CYCLECOUNTER_MASK;
6089         adapter->tx_tstamp_tc.cc_shift = shift;
6090         adapter->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
6091 }
6092
6093 static int
6094 ixgbe_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta)
6095 {
6096         struct ixgbe_adapter *adapter =
6097                         (struct ixgbe_adapter *)dev->data->dev_private;
6098
6099         adapter->systime_tc.nsec += delta;
6100         adapter->rx_tstamp_tc.nsec += delta;
6101         adapter->tx_tstamp_tc.nsec += delta;
6102
6103         return 0;
6104 }
6105
6106 static int
6107 ixgbe_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts)
6108 {
6109         uint64_t ns;
6110         struct ixgbe_adapter *adapter =
6111                         (struct ixgbe_adapter *)dev->data->dev_private;
6112
6113         ns = rte_timespec_to_ns(ts);
6114         /* Set the timecounters to a new value. */
6115         adapter->systime_tc.nsec = ns;
6116         adapter->rx_tstamp_tc.nsec = ns;
6117         adapter->tx_tstamp_tc.nsec = ns;
6118
6119         return 0;
6120 }
6121
6122 static int
6123 ixgbe_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts)
6124 {
6125         uint64_t ns, systime_cycles;
6126         struct ixgbe_adapter *adapter =
6127                         (struct ixgbe_adapter *)dev->data->dev_private;
6128
6129         systime_cycles = ixgbe_read_systime_cyclecounter(dev);
6130         ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles);
6131         *ts = rte_ns_to_timespec(ns);
6132
6133         return 0;
6134 }
6135
6136 static int
6137 ixgbe_timesync_enable(struct rte_eth_dev *dev)
6138 {
6139         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6140         uint32_t tsync_ctl;
6141         uint32_t tsauxc;
6142
6143         /* Stop the timesync system time. */
6144         IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, 0x0);
6145         /* Reset the timesync system time value. */
6146         IXGBE_WRITE_REG(hw, IXGBE_SYSTIML, 0x0);
6147         IXGBE_WRITE_REG(hw, IXGBE_SYSTIMH, 0x0);
6148
6149         /* Enable system time for platforms where it isn't on by default. */
6150         tsauxc = IXGBE_READ_REG(hw, IXGBE_TSAUXC);
6151         tsauxc &= ~IXGBE_TSAUXC_DISABLE_SYSTIME;
6152         IXGBE_WRITE_REG(hw, IXGBE_TSAUXC, tsauxc);
6153
6154         ixgbe_start_timecounters(dev);
6155
6156         /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
6157         IXGBE_WRITE_REG(hw, IXGBE_ETQF(IXGBE_ETQF_FILTER_1588),
6158                         (ETHER_TYPE_1588 |
6159                          IXGBE_ETQF_FILTER_EN |
6160                          IXGBE_ETQF_1588));
6161
6162         /* Enable timestamping of received PTP packets. */
6163         tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL);
6164         tsync_ctl |= IXGBE_TSYNCRXCTL_ENABLED;
6165         IXGBE_WRITE_REG(hw, IXGBE_TSYNCRXCTL, tsync_ctl);
6166
6167         /* Enable timestamping of transmitted PTP packets. */
6168         tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL);
6169         tsync_ctl |= IXGBE_TSYNCTXCTL_ENABLED;
6170         IXGBE_WRITE_REG(hw, IXGBE_TSYNCTXCTL, tsync_ctl);
6171
6172         IXGBE_WRITE_FLUSH(hw);
6173
6174         return 0;
6175 }
6176
6177 static int
6178 ixgbe_timesync_disable(struct rte_eth_dev *dev)
6179 {
6180         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6181         uint32_t tsync_ctl;
6182
6183         /* Disable timestamping of transmitted PTP packets. */
6184         tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL);
6185         tsync_ctl &= ~IXGBE_TSYNCTXCTL_ENABLED;
6186         IXGBE_WRITE_REG(hw, IXGBE_TSYNCTXCTL, tsync_ctl);
6187
6188         /* Disable timestamping of received PTP packets. */
6189         tsync_ctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL);
6190         tsync_ctl &= ~IXGBE_TSYNCRXCTL_ENABLED;
6191         IXGBE_WRITE_REG(hw, IXGBE_TSYNCRXCTL, tsync_ctl);
6192
6193         /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
6194         IXGBE_WRITE_REG(hw, IXGBE_ETQF(IXGBE_ETQF_FILTER_1588), 0);
6195
6196         /* Stop incrementating the System Time registers. */
6197         IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, 0);
6198
6199         return 0;
6200 }
6201
6202 static int
6203 ixgbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
6204                                  struct timespec *timestamp,
6205                                  uint32_t flags __rte_unused)
6206 {
6207         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6208         struct ixgbe_adapter *adapter =
6209                 (struct ixgbe_adapter *)dev->data->dev_private;
6210         uint32_t tsync_rxctl;
6211         uint64_t rx_tstamp_cycles;
6212         uint64_t ns;
6213
6214         tsync_rxctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL);
6215         if ((tsync_rxctl & IXGBE_TSYNCRXCTL_VALID) == 0)
6216                 return -EINVAL;
6217
6218         rx_tstamp_cycles = ixgbe_read_rx_tstamp_cyclecounter(dev);
6219         ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles);
6220         *timestamp = rte_ns_to_timespec(ns);
6221
6222         return  0;
6223 }
6224
6225 static int
6226 ixgbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
6227                                  struct timespec *timestamp)
6228 {
6229         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6230         struct ixgbe_adapter *adapter =
6231                 (struct ixgbe_adapter *)dev->data->dev_private;
6232         uint32_t tsync_txctl;
6233         uint64_t tx_tstamp_cycles;
6234         uint64_t ns;
6235
6236         tsync_txctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL);
6237         if ((tsync_txctl & IXGBE_TSYNCTXCTL_VALID) == 0)
6238                 return -EINVAL;
6239
6240         tx_tstamp_cycles = ixgbe_read_tx_tstamp_cyclecounter(dev);
6241         ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles);
6242         *timestamp = rte_ns_to_timespec(ns);
6243
6244         return 0;
6245 }
6246
6247 static int
6248 ixgbe_get_reg_length(struct rte_eth_dev *dev)
6249 {
6250         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6251         int count = 0;
6252         int g_ind = 0;
6253         const struct reg_info *reg_group;
6254         const struct reg_info **reg_set = (hw->mac.type == ixgbe_mac_82598EB) ?
6255                                     ixgbe_regs_mac_82598EB : ixgbe_regs_others;
6256
6257         while ((reg_group = reg_set[g_ind++]))
6258                 count += ixgbe_regs_group_count(reg_group);
6259
6260         return count;
6261 }
6262
6263 static int
6264 ixgbevf_get_reg_length(struct rte_eth_dev *dev __rte_unused)
6265 {
6266         int count = 0;
6267         int g_ind = 0;
6268         const struct reg_info *reg_group;
6269
6270         while ((reg_group = ixgbevf_regs[g_ind++]))
6271                 count += ixgbe_regs_group_count(reg_group);
6272
6273         return count;
6274 }
6275
6276 static int
6277 ixgbe_get_regs(struct rte_eth_dev *dev,
6278               struct rte_dev_reg_info *regs)
6279 {
6280         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6281         uint32_t *data = regs->data;
6282         int g_ind = 0;
6283         int count = 0;
6284         const struct reg_info *reg_group;
6285         const struct reg_info **reg_set = (hw->mac.type == ixgbe_mac_82598EB) ?
6286                                     ixgbe_regs_mac_82598EB : ixgbe_regs_others;
6287
6288         /* Support only full register dump */
6289         if ((regs->length == 0) ||
6290             (regs->length == (uint32_t)ixgbe_get_reg_length(dev))) {
6291                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
6292                         hw->device_id;
6293                 while ((reg_group = reg_set[g_ind++]))
6294                         count += ixgbe_read_regs_group(dev, &data[count],
6295                                 reg_group);
6296                 return 0;
6297         }
6298
6299         return -ENOTSUP;
6300 }
6301
6302 static int
6303 ixgbevf_get_regs(struct rte_eth_dev *dev,
6304                 struct rte_dev_reg_info *regs)
6305 {
6306         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6307         uint32_t *data = regs->data;
6308         int g_ind = 0;
6309         int count = 0;
6310         const struct reg_info *reg_group;
6311
6312         /* Support only full register dump */
6313         if ((regs->length == 0) ||
6314             (regs->length == (uint32_t)ixgbevf_get_reg_length(dev))) {
6315                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
6316                         hw->device_id;
6317                 while ((reg_group = ixgbevf_regs[g_ind++]))
6318                         count += ixgbe_read_regs_group(dev, &data[count],
6319                                                       reg_group);
6320                 return 0;
6321         }
6322
6323         return -ENOTSUP;
6324 }
6325
6326 static int
6327 ixgbe_get_eeprom_length(struct rte_eth_dev *dev)
6328 {
6329         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6330
6331         /* Return unit is byte count */
6332         return hw->eeprom.word_size * 2;
6333 }
6334
6335 static int
6336 ixgbe_get_eeprom(struct rte_eth_dev *dev,
6337                 struct rte_dev_eeprom_info *in_eeprom)
6338 {
6339         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6340         struct ixgbe_eeprom_info *eeprom = &hw->eeprom;
6341         uint16_t *data = in_eeprom->data;
6342         int first, length;
6343
6344         first = in_eeprom->offset >> 1;
6345         length = in_eeprom->length >> 1;
6346         if ((first > hw->eeprom.word_size) ||
6347             ((first + length) > hw->eeprom.word_size))
6348                 return -EINVAL;
6349
6350         in_eeprom->magic = hw->vendor_id | (hw->device_id << 16);
6351
6352         return eeprom->ops.read_buffer(hw, first, length, data);
6353 }
6354
6355 static int
6356 ixgbe_set_eeprom(struct rte_eth_dev *dev,
6357                 struct rte_dev_eeprom_info *in_eeprom)
6358 {
6359         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6360         struct ixgbe_eeprom_info *eeprom = &hw->eeprom;
6361         uint16_t *data = in_eeprom->data;
6362         int first, length;
6363
6364         first = in_eeprom->offset >> 1;
6365         length = in_eeprom->length >> 1;
6366         if ((first > hw->eeprom.word_size) ||
6367             ((first + length) > hw->eeprom.word_size))
6368                 return -EINVAL;
6369
6370         in_eeprom->magic = hw->vendor_id | (hw->device_id << 16);
6371
6372         return eeprom->ops.write_buffer(hw,  first, length, data);
6373 }
6374
6375 uint16_t
6376 ixgbe_reta_size_get(enum ixgbe_mac_type mac_type) {
6377         switch (mac_type) {
6378         case ixgbe_mac_X550:
6379         case ixgbe_mac_X550EM_x:
6380         case ixgbe_mac_X550EM_a:
6381                 return ETH_RSS_RETA_SIZE_512;
6382         case ixgbe_mac_X550_vf:
6383         case ixgbe_mac_X550EM_x_vf:
6384         case ixgbe_mac_X550EM_a_vf:
6385                 return ETH_RSS_RETA_SIZE_64;
6386         default:
6387                 return ETH_RSS_RETA_SIZE_128;
6388         }
6389 }
6390
6391 uint32_t
6392 ixgbe_reta_reg_get(enum ixgbe_mac_type mac_type, uint16_t reta_idx) {
6393         switch (mac_type) {
6394         case ixgbe_mac_X550:
6395         case ixgbe_mac_X550EM_x:
6396         case ixgbe_mac_X550EM_a:
6397                 if (reta_idx < ETH_RSS_RETA_SIZE_128)
6398                         return IXGBE_RETA(reta_idx >> 2);
6399                 else
6400                         return IXGBE_ERETA((reta_idx - ETH_RSS_RETA_SIZE_128) >> 2);
6401         case ixgbe_mac_X550_vf:
6402         case ixgbe_mac_X550EM_x_vf:
6403         case ixgbe_mac_X550EM_a_vf:
6404                 return IXGBE_VFRETA(reta_idx >> 2);
6405         default:
6406                 return IXGBE_RETA(reta_idx >> 2);
6407         }
6408 }
6409
6410 uint32_t
6411 ixgbe_mrqc_reg_get(enum ixgbe_mac_type mac_type) {
6412         switch (mac_type) {
6413         case ixgbe_mac_X550_vf:
6414         case ixgbe_mac_X550EM_x_vf:
6415         case ixgbe_mac_X550EM_a_vf:
6416                 return IXGBE_VFMRQC;
6417         default:
6418                 return IXGBE_MRQC;
6419         }
6420 }
6421
6422 uint32_t
6423 ixgbe_rssrk_reg_get(enum ixgbe_mac_type mac_type, uint8_t i) {
6424         switch (mac_type) {
6425         case ixgbe_mac_X550_vf:
6426         case ixgbe_mac_X550EM_x_vf:
6427         case ixgbe_mac_X550EM_a_vf:
6428                 return IXGBE_VFRSSRK(i);
6429         default:
6430                 return IXGBE_RSSRK(i);
6431         }
6432 }
6433
6434 bool
6435 ixgbe_rss_update_sp(enum ixgbe_mac_type mac_type) {
6436         switch (mac_type) {
6437         case ixgbe_mac_82599_vf:
6438         case ixgbe_mac_X540_vf:
6439                 return 0;
6440         default:
6441                 return 1;
6442         }
6443 }
6444
6445 static int
6446 ixgbe_dev_get_dcb_info(struct rte_eth_dev *dev,
6447                         struct rte_eth_dcb_info *dcb_info)
6448 {
6449         struct ixgbe_dcb_config *dcb_config =
6450                         IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
6451         struct ixgbe_dcb_tc_config *tc;
6452         uint8_t i, j;
6453
6454         if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_DCB_FLAG)
6455                 dcb_info->nb_tcs = dcb_config->num_tcs.pg_tcs;
6456         else
6457                 dcb_info->nb_tcs = 1;
6458
6459         if (dcb_config->vt_mode) { /* vt is enabled*/
6460                 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
6461                                 &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf;
6462                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++)
6463                         dcb_info->prio_tc[i] = vmdq_rx_conf->dcb_tc[i];
6464                 for (i = 0; i < vmdq_rx_conf->nb_queue_pools; i++) {
6465                         for (j = 0; j < dcb_info->nb_tcs; j++) {
6466                                 dcb_info->tc_queue.tc_rxq[i][j].base =
6467                                                 i * dcb_info->nb_tcs + j;
6468                                 dcb_info->tc_queue.tc_rxq[i][j].nb_queue = 1;
6469                                 dcb_info->tc_queue.tc_txq[i][j].base =
6470                                                 i * dcb_info->nb_tcs + j;
6471                                 dcb_info->tc_queue.tc_txq[i][j].nb_queue = 1;
6472                         }
6473                 }
6474         } else { /* vt is disabled*/
6475                 struct rte_eth_dcb_rx_conf *rx_conf =
6476                                 &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf;
6477                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++)
6478                         dcb_info->prio_tc[i] = rx_conf->dcb_tc[i];
6479                 if (dcb_info->nb_tcs == ETH_4_TCS) {
6480                         for (i = 0; i < dcb_info->nb_tcs; i++) {
6481                                 dcb_info->tc_queue.tc_rxq[0][i].base = i * 32;
6482                                 dcb_info->tc_queue.tc_rxq[0][i].nb_queue = 16;
6483                         }
6484                         dcb_info->tc_queue.tc_txq[0][0].base = 0;
6485                         dcb_info->tc_queue.tc_txq[0][1].base = 64;
6486                         dcb_info->tc_queue.tc_txq[0][2].base = 96;
6487                         dcb_info->tc_queue.tc_txq[0][3].base = 112;
6488                         dcb_info->tc_queue.tc_txq[0][0].nb_queue = 64;
6489                         dcb_info->tc_queue.tc_txq[0][1].nb_queue = 32;
6490                         dcb_info->tc_queue.tc_txq[0][2].nb_queue = 16;
6491                         dcb_info->tc_queue.tc_txq[0][3].nb_queue = 16;
6492                 } else if (dcb_info->nb_tcs == ETH_8_TCS) {
6493                         for (i = 0; i < dcb_info->nb_tcs; i++) {
6494                                 dcb_info->tc_queue.tc_rxq[0][i].base = i * 16;
6495                                 dcb_info->tc_queue.tc_rxq[0][i].nb_queue = 16;
6496                         }
6497                         dcb_info->tc_queue.tc_txq[0][0].base = 0;
6498                         dcb_info->tc_queue.tc_txq[0][1].base = 32;
6499                         dcb_info->tc_queue.tc_txq[0][2].base = 64;
6500                         dcb_info->tc_queue.tc_txq[0][3].base = 80;
6501                         dcb_info->tc_queue.tc_txq[0][4].base = 96;
6502                         dcb_info->tc_queue.tc_txq[0][5].base = 104;
6503                         dcb_info->tc_queue.tc_txq[0][6].base = 112;
6504                         dcb_info->tc_queue.tc_txq[0][7].base = 120;
6505                         dcb_info->tc_queue.tc_txq[0][0].nb_queue = 32;
6506                         dcb_info->tc_queue.tc_txq[0][1].nb_queue = 32;
6507                         dcb_info->tc_queue.tc_txq[0][2].nb_queue = 16;
6508                         dcb_info->tc_queue.tc_txq[0][3].nb_queue = 16;
6509                         dcb_info->tc_queue.tc_txq[0][4].nb_queue = 8;
6510                         dcb_info->tc_queue.tc_txq[0][5].nb_queue = 8;
6511                         dcb_info->tc_queue.tc_txq[0][6].nb_queue = 8;
6512                         dcb_info->tc_queue.tc_txq[0][7].nb_queue = 8;
6513                 }
6514         }
6515         for (i = 0; i < dcb_info->nb_tcs; i++) {
6516                 tc = &dcb_config->tc_config[i];
6517                 dcb_info->tc_bws[i] = tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent;
6518         }
6519         return 0;
6520 }
6521
6522 /* Update e-tag ether type */
6523 static int
6524 ixgbe_update_e_tag_eth_type(struct ixgbe_hw *hw,
6525                             uint16_t ether_type)
6526 {
6527         uint32_t etag_etype;
6528
6529         if (hw->mac.type != ixgbe_mac_X550 &&
6530             hw->mac.type != ixgbe_mac_X550EM_x &&
6531             hw->mac.type != ixgbe_mac_X550EM_a) {
6532                 return -ENOTSUP;
6533         }
6534
6535         etag_etype = IXGBE_READ_REG(hw, IXGBE_ETAG_ETYPE);
6536         etag_etype &= ~IXGBE_ETAG_ETYPE_MASK;
6537         etag_etype |= ether_type;
6538         IXGBE_WRITE_REG(hw, IXGBE_ETAG_ETYPE, etag_etype);
6539         IXGBE_WRITE_FLUSH(hw);
6540
6541         return 0;
6542 }
6543
6544 /* Config l2 tunnel ether type */
6545 static int
6546 ixgbe_dev_l2_tunnel_eth_type_conf(struct rte_eth_dev *dev,
6547                                   struct rte_eth_l2_tunnel_conf *l2_tunnel)
6548 {
6549         int ret = 0;
6550         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6551
6552         if (l2_tunnel == NULL)
6553                 return -EINVAL;
6554
6555         switch (l2_tunnel->l2_tunnel_type) {
6556         case RTE_L2_TUNNEL_TYPE_E_TAG:
6557                 ret = ixgbe_update_e_tag_eth_type(hw, l2_tunnel->ether_type);
6558                 break;
6559         default:
6560                 PMD_DRV_LOG(ERR, "Invalid tunnel type");
6561                 ret = -EINVAL;
6562                 break;
6563         }
6564
6565         return ret;
6566 }
6567
6568 /* Enable e-tag tunnel */
6569 static int
6570 ixgbe_e_tag_enable(struct ixgbe_hw *hw)
6571 {
6572         uint32_t etag_etype;
6573
6574         if (hw->mac.type != ixgbe_mac_X550 &&
6575             hw->mac.type != ixgbe_mac_X550EM_x &&
6576             hw->mac.type != ixgbe_mac_X550EM_a) {
6577                 return -ENOTSUP;
6578         }
6579
6580         etag_etype = IXGBE_READ_REG(hw, IXGBE_ETAG_ETYPE);
6581         etag_etype |= IXGBE_ETAG_ETYPE_VALID;
6582         IXGBE_WRITE_REG(hw, IXGBE_ETAG_ETYPE, etag_etype);
6583         IXGBE_WRITE_FLUSH(hw);
6584
6585         return 0;
6586 }
6587
6588 /* Enable l2 tunnel */
6589 static int
6590 ixgbe_dev_l2_tunnel_enable(struct rte_eth_dev *dev,
6591                            enum rte_eth_tunnel_type l2_tunnel_type)
6592 {
6593         int ret = 0;
6594         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6595
6596         switch (l2_tunnel_type) {
6597         case RTE_L2_TUNNEL_TYPE_E_TAG:
6598                 ret = ixgbe_e_tag_enable(hw);
6599                 break;
6600         default:
6601                 PMD_DRV_LOG(ERR, "Invalid tunnel type");
6602                 ret = -EINVAL;
6603                 break;
6604         }
6605
6606         return ret;
6607 }
6608
6609 /* Disable e-tag tunnel */
6610 static int
6611 ixgbe_e_tag_disable(struct ixgbe_hw *hw)
6612 {
6613         uint32_t etag_etype;
6614
6615         if (hw->mac.type != ixgbe_mac_X550 &&
6616             hw->mac.type != ixgbe_mac_X550EM_x &&
6617             hw->mac.type != ixgbe_mac_X550EM_a) {
6618                 return -ENOTSUP;
6619         }
6620
6621         etag_etype = IXGBE_READ_REG(hw, IXGBE_ETAG_ETYPE);
6622         etag_etype &= ~IXGBE_ETAG_ETYPE_VALID;
6623         IXGBE_WRITE_REG(hw, IXGBE_ETAG_ETYPE, etag_etype);
6624         IXGBE_WRITE_FLUSH(hw);
6625
6626         return 0;
6627 }
6628
6629 /* Disable l2 tunnel */
6630 static int
6631 ixgbe_dev_l2_tunnel_disable(struct rte_eth_dev *dev,
6632                             enum rte_eth_tunnel_type l2_tunnel_type)
6633 {
6634         int ret = 0;
6635         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6636
6637         switch (l2_tunnel_type) {
6638         case RTE_L2_TUNNEL_TYPE_E_TAG:
6639                 ret = ixgbe_e_tag_disable(hw);
6640                 break;
6641         default:
6642                 PMD_DRV_LOG(ERR, "Invalid tunnel type");
6643                 ret = -EINVAL;
6644                 break;
6645         }
6646
6647         return ret;
6648 }
6649
6650 static int
6651 ixgbe_e_tag_filter_del(struct rte_eth_dev *dev,
6652                        struct rte_eth_l2_tunnel_conf *l2_tunnel)
6653 {
6654         int ret = 0;
6655         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6656         uint32_t i, rar_entries;
6657         uint32_t rar_low, rar_high;
6658
6659         if (hw->mac.type != ixgbe_mac_X550 &&
6660             hw->mac.type != ixgbe_mac_X550EM_x &&
6661             hw->mac.type != ixgbe_mac_X550EM_a) {
6662                 return -ENOTSUP;
6663         }
6664
6665         rar_entries = ixgbe_get_num_rx_addrs(hw);
6666
6667         for (i = 1; i < rar_entries; i++) {
6668                 rar_high = IXGBE_READ_REG(hw, IXGBE_RAH(i));
6669                 rar_low  = IXGBE_READ_REG(hw, IXGBE_RAL(i));
6670                 if ((rar_high & IXGBE_RAH_AV) &&
6671                     (rar_high & IXGBE_RAH_ADTYPE) &&
6672                     ((rar_low & IXGBE_RAL_ETAG_FILTER_MASK) ==
6673                      l2_tunnel->tunnel_id)) {
6674                         IXGBE_WRITE_REG(hw, IXGBE_RAL(i), 0);
6675                         IXGBE_WRITE_REG(hw, IXGBE_RAH(i), 0);
6676
6677                         ixgbe_clear_vmdq(hw, i, IXGBE_CLEAR_VMDQ_ALL);
6678
6679                         return ret;
6680                 }
6681         }
6682
6683         return ret;
6684 }
6685
6686 static int
6687 ixgbe_e_tag_filter_add(struct rte_eth_dev *dev,
6688                        struct rte_eth_l2_tunnel_conf *l2_tunnel)
6689 {
6690         int ret = 0;
6691         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6692         uint32_t i, rar_entries;
6693         uint32_t rar_low, rar_high;
6694
6695         if (hw->mac.type != ixgbe_mac_X550 &&
6696             hw->mac.type != ixgbe_mac_X550EM_x &&
6697             hw->mac.type != ixgbe_mac_X550EM_a) {
6698                 return -ENOTSUP;
6699         }
6700
6701         /* One entry for one tunnel. Try to remove potential existing entry. */
6702         ixgbe_e_tag_filter_del(dev, l2_tunnel);
6703
6704         rar_entries = ixgbe_get_num_rx_addrs(hw);
6705
6706         for (i = 1; i < rar_entries; i++) {
6707                 rar_high = IXGBE_READ_REG(hw, IXGBE_RAH(i));
6708                 if (rar_high & IXGBE_RAH_AV) {
6709                         continue;
6710                 } else {
6711                         ixgbe_set_vmdq(hw, i, l2_tunnel->pool);
6712                         rar_high = IXGBE_RAH_AV | IXGBE_RAH_ADTYPE;
6713                         rar_low = l2_tunnel->tunnel_id;
6714
6715                         IXGBE_WRITE_REG(hw, IXGBE_RAL(i), rar_low);
6716                         IXGBE_WRITE_REG(hw, IXGBE_RAH(i), rar_high);
6717
6718                         return ret;
6719                 }
6720         }
6721
6722         PMD_INIT_LOG(NOTICE, "The table of E-tag forwarding rule is full."
6723                      " Please remove a rule before adding a new one.");
6724         return -EINVAL;
6725 }
6726
6727 /* Add l2 tunnel filter */
6728 static int
6729 ixgbe_dev_l2_tunnel_filter_add(struct rte_eth_dev *dev,
6730                                struct rte_eth_l2_tunnel_conf *l2_tunnel)
6731 {
6732         int ret = 0;
6733
6734         switch (l2_tunnel->l2_tunnel_type) {
6735         case RTE_L2_TUNNEL_TYPE_E_TAG:
6736                 ret = ixgbe_e_tag_filter_add(dev, l2_tunnel);
6737                 break;
6738         default:
6739                 PMD_DRV_LOG(ERR, "Invalid tunnel type");
6740                 ret = -EINVAL;
6741                 break;
6742         }
6743
6744         return ret;
6745 }
6746
6747 /* Delete l2 tunnel filter */
6748 static int
6749 ixgbe_dev_l2_tunnel_filter_del(struct rte_eth_dev *dev,
6750                                struct rte_eth_l2_tunnel_conf *l2_tunnel)
6751 {
6752         int ret = 0;
6753
6754         switch (l2_tunnel->l2_tunnel_type) {
6755         case RTE_L2_TUNNEL_TYPE_E_TAG:
6756                 ret = ixgbe_e_tag_filter_del(dev, l2_tunnel);
6757                 break;
6758         default:
6759                 PMD_DRV_LOG(ERR, "Invalid tunnel type");
6760                 ret = -EINVAL;
6761                 break;
6762         }
6763
6764         return ret;
6765 }
6766
6767 /**
6768  * ixgbe_dev_l2_tunnel_filter_handle - Handle operations for l2 tunnel filter.
6769  * @dev: pointer to rte_eth_dev structure
6770  * @filter_op:operation will be taken.
6771  * @arg: a pointer to specific structure corresponding to the filter_op
6772  */
6773 static int
6774 ixgbe_dev_l2_tunnel_filter_handle(struct rte_eth_dev *dev,
6775                                   enum rte_filter_op filter_op,
6776                                   void *arg)
6777 {
6778         int ret = 0;
6779
6780         if (filter_op == RTE_ETH_FILTER_NOP)
6781                 return 0;
6782
6783         if (arg == NULL) {
6784                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
6785                             filter_op);
6786                 return -EINVAL;
6787         }
6788
6789         switch (filter_op) {
6790         case RTE_ETH_FILTER_ADD:
6791                 ret = ixgbe_dev_l2_tunnel_filter_add
6792                         (dev,
6793                          (struct rte_eth_l2_tunnel_conf *)arg);
6794                 break;
6795         case RTE_ETH_FILTER_DELETE:
6796                 ret = ixgbe_dev_l2_tunnel_filter_del
6797                         (dev,
6798                          (struct rte_eth_l2_tunnel_conf *)arg);
6799                 break;
6800         default:
6801                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
6802                 ret = -EINVAL;
6803                 break;
6804         }
6805         return ret;
6806 }
6807
6808 static int
6809 ixgbe_e_tag_forwarding_en_dis(struct rte_eth_dev *dev, bool en)
6810 {
6811         int ret = 0;
6812         uint32_t ctrl;
6813         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6814
6815         if (hw->mac.type != ixgbe_mac_X550 &&
6816             hw->mac.type != ixgbe_mac_X550EM_x &&
6817             hw->mac.type != ixgbe_mac_X550EM_a) {
6818                 return -ENOTSUP;
6819         }
6820
6821         ctrl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
6822         ctrl &= ~IXGBE_VT_CTL_POOLING_MODE_MASK;
6823         if (en)
6824                 ctrl |= IXGBE_VT_CTL_POOLING_MODE_ETAG;
6825         IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, ctrl);
6826
6827         return ret;
6828 }
6829
6830 /* Enable l2 tunnel forwarding */
6831 static int
6832 ixgbe_dev_l2_tunnel_forwarding_enable
6833         (struct rte_eth_dev *dev,
6834          enum rte_eth_tunnel_type l2_tunnel_type)
6835 {
6836         int ret = 0;
6837
6838         switch (l2_tunnel_type) {
6839         case RTE_L2_TUNNEL_TYPE_E_TAG:
6840                 ret = ixgbe_e_tag_forwarding_en_dis(dev, 1);
6841                 break;
6842         default:
6843                 PMD_DRV_LOG(ERR, "Invalid tunnel type");
6844                 ret = -EINVAL;
6845                 break;
6846         }
6847
6848         return ret;
6849 }
6850
6851 /* Disable l2 tunnel forwarding */
6852 static int
6853 ixgbe_dev_l2_tunnel_forwarding_disable
6854         (struct rte_eth_dev *dev,
6855          enum rte_eth_tunnel_type l2_tunnel_type)
6856 {
6857         int ret = 0;
6858
6859         switch (l2_tunnel_type) {
6860         case RTE_L2_TUNNEL_TYPE_E_TAG:
6861                 ret = ixgbe_e_tag_forwarding_en_dis(dev, 0);
6862                 break;
6863         default:
6864                 PMD_DRV_LOG(ERR, "Invalid tunnel type");
6865                 ret = -EINVAL;
6866                 break;
6867         }
6868
6869         return ret;
6870 }
6871
6872 static int
6873 ixgbe_e_tag_insertion_en_dis(struct rte_eth_dev *dev,
6874                              struct rte_eth_l2_tunnel_conf *l2_tunnel,
6875                              bool en)
6876 {
6877         int ret = 0;
6878         uint32_t vmtir, vmvir;
6879         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6880
6881         if (l2_tunnel->vf_id >= dev->pci_dev->max_vfs) {
6882                 PMD_DRV_LOG(ERR,
6883                             "VF id %u should be less than %u",
6884                             l2_tunnel->vf_id,
6885                             dev->pci_dev->max_vfs);
6886                 return -EINVAL;
6887         }
6888
6889         if (hw->mac.type != ixgbe_mac_X550 &&
6890             hw->mac.type != ixgbe_mac_X550EM_x &&
6891             hw->mac.type != ixgbe_mac_X550EM_a) {
6892                 return -ENOTSUP;
6893         }
6894
6895         if (en)
6896                 vmtir = l2_tunnel->tunnel_id;
6897         else
6898                 vmtir = 0;
6899
6900         IXGBE_WRITE_REG(hw, IXGBE_VMTIR(l2_tunnel->vf_id), vmtir);
6901
6902         vmvir = IXGBE_READ_REG(hw, IXGBE_VMVIR(l2_tunnel->vf_id));
6903         vmvir &= ~IXGBE_VMVIR_TAGA_MASK;
6904         if (en)
6905                 vmvir |= IXGBE_VMVIR_TAGA_ETAG_INSERT;
6906         IXGBE_WRITE_REG(hw, IXGBE_VMVIR(l2_tunnel->vf_id), vmvir);
6907
6908         return ret;
6909 }
6910
6911 /* Enable l2 tunnel tag insertion */
6912 static int
6913 ixgbe_dev_l2_tunnel_insertion_enable(struct rte_eth_dev *dev,
6914                                      struct rte_eth_l2_tunnel_conf *l2_tunnel)
6915 {
6916         int ret = 0;
6917
6918         switch (l2_tunnel->l2_tunnel_type) {
6919         case RTE_L2_TUNNEL_TYPE_E_TAG:
6920                 ret = ixgbe_e_tag_insertion_en_dis(dev, l2_tunnel, 1);
6921                 break;
6922         default:
6923                 PMD_DRV_LOG(ERR, "Invalid tunnel type");
6924                 ret = -EINVAL;
6925                 break;
6926         }
6927
6928         return ret;
6929 }
6930
6931 /* Disable l2 tunnel tag insertion */
6932 static int
6933 ixgbe_dev_l2_tunnel_insertion_disable
6934         (struct rte_eth_dev *dev,
6935          struct rte_eth_l2_tunnel_conf *l2_tunnel)
6936 {
6937         int ret = 0;
6938
6939         switch (l2_tunnel->l2_tunnel_type) {
6940         case RTE_L2_TUNNEL_TYPE_E_TAG:
6941                 ret = ixgbe_e_tag_insertion_en_dis(dev, l2_tunnel, 0);
6942                 break;
6943         default:
6944                 PMD_DRV_LOG(ERR, "Invalid tunnel type");
6945                 ret = -EINVAL;
6946                 break;
6947         }
6948
6949         return ret;
6950 }
6951
6952 static int
6953 ixgbe_e_tag_stripping_en_dis(struct rte_eth_dev *dev,
6954                              bool en)
6955 {
6956         int ret = 0;
6957         uint32_t qde;
6958         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
6959
6960         if (hw->mac.type != ixgbe_mac_X550 &&
6961             hw->mac.type != ixgbe_mac_X550EM_x &&
6962             hw->mac.type != ixgbe_mac_X550EM_a) {
6963                 return -ENOTSUP;
6964         }
6965
6966         qde = IXGBE_READ_REG(hw, IXGBE_QDE);
6967         if (en)
6968                 qde |= IXGBE_QDE_STRIP_TAG;
6969         else
6970                 qde &= ~IXGBE_QDE_STRIP_TAG;
6971         qde &= ~IXGBE_QDE_READ;
6972         qde |= IXGBE_QDE_WRITE;
6973         IXGBE_WRITE_REG(hw, IXGBE_QDE, qde);
6974
6975         return ret;
6976 }
6977
6978 /* Enable l2 tunnel tag stripping */
6979 static int
6980 ixgbe_dev_l2_tunnel_stripping_enable
6981         (struct rte_eth_dev *dev,
6982          enum rte_eth_tunnel_type l2_tunnel_type)
6983 {
6984         int ret = 0;
6985
6986         switch (l2_tunnel_type) {
6987         case RTE_L2_TUNNEL_TYPE_E_TAG:
6988                 ret = ixgbe_e_tag_stripping_en_dis(dev, 1);
6989                 break;
6990         default:
6991                 PMD_DRV_LOG(ERR, "Invalid tunnel type");
6992                 ret = -EINVAL;
6993                 break;
6994         }
6995
6996         return ret;
6997 }
6998
6999 /* Disable l2 tunnel tag stripping */
7000 static int
7001 ixgbe_dev_l2_tunnel_stripping_disable
7002         (struct rte_eth_dev *dev,
7003          enum rte_eth_tunnel_type l2_tunnel_type)
7004 {
7005         int ret = 0;
7006
7007         switch (l2_tunnel_type) {
7008         case RTE_L2_TUNNEL_TYPE_E_TAG:
7009                 ret = ixgbe_e_tag_stripping_en_dis(dev, 0);
7010                 break;
7011         default:
7012                 PMD_DRV_LOG(ERR, "Invalid tunnel type");
7013                 ret = -EINVAL;
7014                 break;
7015         }
7016
7017         return ret;
7018 }
7019
7020 /* Enable/disable l2 tunnel offload functions */
7021 static int
7022 ixgbe_dev_l2_tunnel_offload_set
7023         (struct rte_eth_dev *dev,
7024          struct rte_eth_l2_tunnel_conf *l2_tunnel,
7025          uint32_t mask,
7026          uint8_t en)
7027 {
7028         int ret = 0;
7029
7030         if (l2_tunnel == NULL)
7031                 return -EINVAL;
7032
7033         ret = -EINVAL;
7034         if (mask & ETH_L2_TUNNEL_ENABLE_MASK) {
7035                 if (en)
7036                         ret = ixgbe_dev_l2_tunnel_enable(
7037                                 dev,
7038                                 l2_tunnel->l2_tunnel_type);
7039                 else
7040                         ret = ixgbe_dev_l2_tunnel_disable(
7041                                 dev,
7042                                 l2_tunnel->l2_tunnel_type);
7043         }
7044
7045         if (mask & ETH_L2_TUNNEL_INSERTION_MASK) {
7046                 if (en)
7047                         ret = ixgbe_dev_l2_tunnel_insertion_enable(
7048                                 dev,
7049                                 l2_tunnel);
7050                 else
7051                         ret = ixgbe_dev_l2_tunnel_insertion_disable(
7052                                 dev,
7053                                 l2_tunnel);
7054         }
7055
7056         if (mask & ETH_L2_TUNNEL_STRIPPING_MASK) {
7057                 if (en)
7058                         ret = ixgbe_dev_l2_tunnel_stripping_enable(
7059                                 dev,
7060                                 l2_tunnel->l2_tunnel_type);
7061                 else
7062                         ret = ixgbe_dev_l2_tunnel_stripping_disable(
7063                                 dev,
7064                                 l2_tunnel->l2_tunnel_type);
7065         }
7066
7067         if (mask & ETH_L2_TUNNEL_FORWARDING_MASK) {
7068                 if (en)
7069                         ret = ixgbe_dev_l2_tunnel_forwarding_enable(
7070                                 dev,
7071                                 l2_tunnel->l2_tunnel_type);
7072                 else
7073                         ret = ixgbe_dev_l2_tunnel_forwarding_disable(
7074                                 dev,
7075                                 l2_tunnel->l2_tunnel_type);
7076         }
7077
7078         return ret;
7079 }
7080
7081 static int
7082 ixgbe_update_vxlan_port(struct ixgbe_hw *hw,
7083                         uint16_t port)
7084 {
7085         IXGBE_WRITE_REG(hw, IXGBE_VXLANCTRL, port);
7086         IXGBE_WRITE_FLUSH(hw);
7087
7088         return 0;
7089 }
7090
7091 /* There's only one register for VxLAN UDP port.
7092  * So, we cannot add several ports. Will update it.
7093  */
7094 static int
7095 ixgbe_add_vxlan_port(struct ixgbe_hw *hw,
7096                      uint16_t port)
7097 {
7098         if (port == 0) {
7099                 PMD_DRV_LOG(ERR, "Add VxLAN port 0 is not allowed.");
7100                 return -EINVAL;
7101         }
7102
7103         return ixgbe_update_vxlan_port(hw, port);
7104 }
7105
7106 /* We cannot delete the VxLAN port. For there's a register for VxLAN
7107  * UDP port, it must have a value.
7108  * So, will reset it to the original value 0.
7109  */
7110 static int
7111 ixgbe_del_vxlan_port(struct ixgbe_hw *hw,
7112                      uint16_t port)
7113 {
7114         uint16_t cur_port;
7115
7116         cur_port = (uint16_t)IXGBE_READ_REG(hw, IXGBE_VXLANCTRL);
7117
7118         if (cur_port != port) {
7119                 PMD_DRV_LOG(ERR, "Port %u does not exist.", port);
7120                 return -EINVAL;
7121         }
7122
7123         return ixgbe_update_vxlan_port(hw, 0);
7124 }
7125
7126 /* Add UDP tunneling port */
7127 static int
7128 ixgbe_dev_udp_tunnel_port_add(struct rte_eth_dev *dev,
7129                               struct rte_eth_udp_tunnel *udp_tunnel)
7130 {
7131         int ret = 0;
7132         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
7133
7134         if (hw->mac.type != ixgbe_mac_X550 &&
7135             hw->mac.type != ixgbe_mac_X550EM_x &&
7136             hw->mac.type != ixgbe_mac_X550EM_a) {
7137                 return -ENOTSUP;
7138         }
7139
7140         if (udp_tunnel == NULL)
7141                 return -EINVAL;
7142
7143         switch (udp_tunnel->prot_type) {
7144         case RTE_TUNNEL_TYPE_VXLAN:
7145                 ret = ixgbe_add_vxlan_port(hw, udp_tunnel->udp_port);
7146                 break;
7147
7148         case RTE_TUNNEL_TYPE_GENEVE:
7149         case RTE_TUNNEL_TYPE_TEREDO:
7150                 PMD_DRV_LOG(ERR, "Tunnel type is not supported now.");
7151                 ret = -EINVAL;
7152                 break;
7153
7154         default:
7155                 PMD_DRV_LOG(ERR, "Invalid tunnel type");
7156                 ret = -EINVAL;
7157                 break;
7158         }
7159
7160         return ret;
7161 }
7162
7163 /* Remove UDP tunneling port */
7164 static int
7165 ixgbe_dev_udp_tunnel_port_del(struct rte_eth_dev *dev,
7166                               struct rte_eth_udp_tunnel *udp_tunnel)
7167 {
7168         int ret = 0;
7169         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
7170
7171         if (hw->mac.type != ixgbe_mac_X550 &&
7172             hw->mac.type != ixgbe_mac_X550EM_x &&
7173             hw->mac.type != ixgbe_mac_X550EM_a) {
7174                 return -ENOTSUP;
7175         }
7176
7177         if (udp_tunnel == NULL)
7178                 return -EINVAL;
7179
7180         switch (udp_tunnel->prot_type) {
7181         case RTE_TUNNEL_TYPE_VXLAN:
7182                 ret = ixgbe_del_vxlan_port(hw, udp_tunnel->udp_port);
7183                 break;
7184         case RTE_TUNNEL_TYPE_GENEVE:
7185         case RTE_TUNNEL_TYPE_TEREDO:
7186                 PMD_DRV_LOG(ERR, "Tunnel type is not supported now.");
7187                 ret = -EINVAL;
7188                 break;
7189         default:
7190                 PMD_DRV_LOG(ERR, "Invalid tunnel type");
7191                 ret = -EINVAL;
7192                 break;
7193         }
7194
7195         return ret;
7196 }
7197
7198 /* ixgbevf_update_xcast_mode - Update Multicast mode
7199  * @hw: pointer to the HW structure
7200  * @netdev: pointer to net device structure
7201  * @xcast_mode: new multicast mode
7202  *
7203  * Updates the Multicast Mode of VF.
7204  */
7205 static int ixgbevf_update_xcast_mode(struct ixgbe_hw *hw,
7206                                      int xcast_mode)
7207 {
7208         struct ixgbe_mbx_info *mbx = &hw->mbx;
7209         u32 msgbuf[2];
7210         s32 err;
7211
7212         switch (hw->api_version) {
7213         case ixgbe_mbox_api_12:
7214                 break;
7215         default:
7216                 return -EOPNOTSUPP;
7217         }
7218
7219         msgbuf[0] = IXGBE_VF_UPDATE_XCAST_MODE;
7220         msgbuf[1] = xcast_mode;
7221
7222         err = mbx->ops.write_posted(hw, msgbuf, 2, 0);
7223         if (err)
7224                 return err;
7225
7226         err = mbx->ops.read_posted(hw, msgbuf, 2, 0);
7227         if (err)
7228                 return err;
7229
7230         msgbuf[0] &= ~IXGBE_VT_MSGTYPE_CTS;
7231         if (msgbuf[0] == (IXGBE_VF_UPDATE_XCAST_MODE | IXGBE_VT_MSGTYPE_NACK))
7232                 return -EPERM;
7233
7234         return 0;
7235 }
7236
7237 static void
7238 ixgbevf_dev_allmulticast_enable(struct rte_eth_dev *dev)
7239 {
7240         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
7241
7242         ixgbevf_update_xcast_mode(hw, IXGBEVF_XCAST_MODE_ALLMULTI);
7243 }
7244
7245 static void
7246 ixgbevf_dev_allmulticast_disable(struct rte_eth_dev *dev)
7247 {
7248         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
7249
7250         ixgbevf_update_xcast_mode(hw, IXGBEVF_XCAST_MODE_NONE);
7251 }
7252
7253 static void ixgbevf_mbx_process(struct rte_eth_dev *dev)
7254 {
7255         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
7256         u32 in_msg = 0;
7257
7258         if (ixgbe_read_mbx(hw, &in_msg, 1, 0))
7259                 return;
7260
7261         /* PF reset VF event */
7262         if (in_msg == IXGBE_PF_CONTROL_MSG)
7263                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET);
7264 }
7265
7266 static int
7267 ixgbevf_dev_interrupt_get_status(struct rte_eth_dev *dev)
7268 {
7269         uint32_t eicr;
7270         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
7271         struct ixgbe_interrupt *intr =
7272                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
7273         ixgbevf_intr_disable(hw);
7274
7275         /* read-on-clear nic registers here */
7276         eicr = IXGBE_READ_REG(hw, IXGBE_VTEICR);
7277         intr->flags = 0;
7278
7279         /* only one misc vector supported - mailbox */
7280         eicr &= IXGBE_VTEICR_MASK;
7281         if (eicr == IXGBE_MISC_VEC_ID)
7282                 intr->flags |= IXGBE_FLAG_MAILBOX;
7283
7284         return 0;
7285 }
7286
7287 static int
7288 ixgbevf_dev_interrupt_action(struct rte_eth_dev *dev)
7289 {
7290         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
7291         struct ixgbe_interrupt *intr =
7292                 IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
7293
7294         if (intr->flags & IXGBE_FLAG_MAILBOX) {
7295                 ixgbevf_mbx_process(dev);
7296                 intr->flags &= ~IXGBE_FLAG_MAILBOX;
7297         }
7298
7299         ixgbevf_intr_enable(hw);
7300
7301         return 0;
7302 }
7303
7304 static void
7305 ixgbevf_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
7306                               void *param)
7307 {
7308         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
7309
7310         ixgbevf_dev_interrupt_get_status(dev);
7311         ixgbevf_dev_interrupt_action(dev);
7312 }
7313
7314 static struct rte_driver rte_ixgbe_driver = {
7315         .type = PMD_PDEV,
7316         .init = rte_ixgbe_pmd_init,
7317 };
7318
7319 static struct rte_driver rte_ixgbevf_driver = {
7320         .type = PMD_PDEV,
7321         .init = rte_ixgbevf_pmd_init,
7322 };
7323
7324 PMD_REGISTER_DRIVER(rte_ixgbe_driver);
7325 PMD_REGISTER_DRIVER(rte_ixgbevf_driver);