lib/librte_pmd_e1000/igb_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <stdarg.h>
  39
  40 #include <rte_common.h>
  41 #include <rte_interrupts.h>
  42 #include <rte_byteorder.h>
  43 #include <rte_log.h>
  44 #include <rte_debug.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_memory.h>
  49 #include <rte_memzone.h>
  50 #include <rte_tailq.h>
  51 #include <rte_eal.h>
  52 #include <rte_atomic.h>
  53 #include <rte_malloc.h>
  54 #include <rte_dev.h>
  55
  56 #include "e1000_logs.h"
  57 #include "e1000/e1000_api.h"
  58 #include "e1000_ethdev.h"
  59
  60 /*
  61  * Default values for port configuration
  62  */
  63 #define IGB_DEFAULT_RX_FREE_THRESH  32
  64 #define IGB_DEFAULT_RX_PTHRESH      8
  65 #define IGB_DEFAULT_RX_HTHRESH      8
  66 #define IGB_DEFAULT_RX_WTHRESH      0
  67
  68 #define IGB_DEFAULT_TX_PTHRESH      32
  69 #define IGB_DEFAULT_TX_HTHRESH      0
  70 #define IGB_DEFAULT_TX_WTHRESH      0
  71
  72 /* Bit shift and mask */
  73 #define IGB_4_BIT_WIDTH  (CHAR_BIT / 2)
  74 #define IGB_4_BIT_MASK   RTE_LEN2MASK(IGB_4_BIT_WIDTH, uint8_t)
  75 #define IGB_8_BIT_WIDTH  CHAR_BIT
  76 #define IGB_8_BIT_MASK   UINT8_MAX
  77
  78 static int  eth_igb_configure(struct rte_eth_dev *dev);
  79 static int  eth_igb_start(struct rte_eth_dev *dev);
  80 static void eth_igb_stop(struct rte_eth_dev *dev);
  81 static void eth_igb_close(struct rte_eth_dev *dev);
  82 static void eth_igb_promiscuous_enable(struct rte_eth_dev *dev);
  83 static void eth_igb_promiscuous_disable(struct rte_eth_dev *dev);
  84 static void eth_igb_allmulticast_enable(struct rte_eth_dev *dev);
  85 static void eth_igb_allmulticast_disable(struct rte_eth_dev *dev);
  86 static int  eth_igb_link_update(struct rte_eth_dev *dev,
  87                                 int wait_to_complete);
  88 static void eth_igb_stats_get(struct rte_eth_dev *dev,
  89                                 struct rte_eth_stats *rte_stats);
  90 static void eth_igb_stats_reset(struct rte_eth_dev *dev);
  91 static void eth_igb_infos_get(struct rte_eth_dev *dev,
  92                               struct rte_eth_dev_info *dev_info);
  93 static void eth_igbvf_infos_get(struct rte_eth_dev *dev,
  94                                 struct rte_eth_dev_info *dev_info);
  95 static int  eth_igb_flow_ctrl_get(struct rte_eth_dev *dev,
  96                                 struct rte_eth_fc_conf *fc_conf);
  97 static int  eth_igb_flow_ctrl_set(struct rte_eth_dev *dev,
  98                                 struct rte_eth_fc_conf *fc_conf);
  99 static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev);
 100 static int eth_igb_interrupt_get_status(struct rte_eth_dev *dev);
 101 static int eth_igb_interrupt_action(struct rte_eth_dev *dev);
 102 static void eth_igb_interrupt_handler(struct rte_intr_handle *handle,
 103                                                         void *param);
 104 static int  igb_hardware_init(struct e1000_hw *hw);
 105 static void igb_hw_control_acquire(struct e1000_hw *hw);
 106 static void igb_hw_control_release(struct e1000_hw *hw);
 107 static void igb_init_manageability(struct e1000_hw *hw);
 108 static void igb_release_manageability(struct e1000_hw *hw);
 109
 110 static int  eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 111
 112 static int eth_igb_vlan_filter_set(struct rte_eth_dev *dev,
 113                 uint16_t vlan_id, int on);
 114 static void eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 115 static void eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 116
 117 static void igb_vlan_hw_filter_enable(struct rte_eth_dev *dev);
 118 static void igb_vlan_hw_filter_disable(struct rte_eth_dev *dev);
 119 static void igb_vlan_hw_strip_enable(struct rte_eth_dev *dev);
 120 static void igb_vlan_hw_strip_disable(struct rte_eth_dev *dev);
 121 static void igb_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 122 static void igb_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 123
 124 static int eth_igb_led_on(struct rte_eth_dev *dev);
 125 static int eth_igb_led_off(struct rte_eth_dev *dev);
 126
 127 static void igb_intr_disable(struct e1000_hw *hw);
 128 static int  igb_get_rx_buffer_size(struct e1000_hw *hw);
 129 static void eth_igb_rar_set(struct rte_eth_dev *dev,
 130                 struct ether_addr *mac_addr,
 131                 uint32_t index, uint32_t pool);
 132 static void eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index);
 133
 134 static void igbvf_intr_disable(struct e1000_hw *hw);
 135 static int igbvf_dev_configure(struct rte_eth_dev *dev);
 136 static int igbvf_dev_start(struct rte_eth_dev *dev);
 137 static void igbvf_dev_stop(struct rte_eth_dev *dev);
 138 static void igbvf_dev_close(struct rte_eth_dev *dev);
 139 static int eth_igbvf_link_update(struct e1000_hw *hw);
 140 static void eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats);
 141 static void eth_igbvf_stats_reset(struct rte_eth_dev *dev);
 142 static int igbvf_vlan_filter_set(struct rte_eth_dev *dev,
 143                 uint16_t vlan_id, int on);
 144 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on);
 145 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 146 static int eth_igb_rss_reta_update(struct rte_eth_dev *dev,
 147                                    struct rte_eth_rss_reta_entry64 *reta_conf,
 148                                    uint16_t reta_size);
 149 static int eth_igb_rss_reta_query(struct rte_eth_dev *dev,
 150                                   struct rte_eth_rss_reta_entry64 *reta_conf,
 151                                   uint16_t reta_size);
 152
 153 static int eth_igb_syn_filter_set(struct rte_eth_dev *dev,
 154                         struct rte_eth_syn_filter *filter,
 155                         bool add);
 156 static int eth_igb_syn_filter_get(struct rte_eth_dev *dev,
 157                         struct rte_eth_syn_filter *filter);
 158 static int eth_igb_syn_filter_handle(struct rte_eth_dev *dev,
 159                         enum rte_filter_op filter_op,
 160                         void *arg);
 161 static int igb_add_2tuple_filter(struct rte_eth_dev *dev,
 162                         struct rte_eth_ntuple_filter *ntuple_filter);
 163 static int igb_remove_2tuple_filter(struct rte_eth_dev *dev,
 164                         struct rte_eth_ntuple_filter *ntuple_filter);
 165 static int eth_igb_add_del_flex_filter(struct rte_eth_dev *dev,
 166                         struct rte_eth_flex_filter *filter,
 167                         bool add);
 168 static int eth_igb_get_flex_filter(struct rte_eth_dev *dev,
 169                         struct rte_eth_flex_filter *filter);
 170 static int eth_igb_flex_filter_handle(struct rte_eth_dev *dev,
 171                         enum rte_filter_op filter_op,
 172                         void *arg);
 173 static int igb_add_5tuple_filter_82576(struct rte_eth_dev *dev,
 174                         struct rte_eth_ntuple_filter *ntuple_filter);
 175 static int igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev,
 176                         struct rte_eth_ntuple_filter *ntuple_filter);
 177 static int igb_add_del_ntuple_filter(struct rte_eth_dev *dev,
 178                         struct rte_eth_ntuple_filter *filter,
 179                         bool add);
 180 static int igb_get_ntuple_filter(struct rte_eth_dev *dev,
 181                         struct rte_eth_ntuple_filter *filter);
 182 static int igb_ntuple_filter_handle(struct rte_eth_dev *dev,
 183                                 enum rte_filter_op filter_op,
 184                                 void *arg);
 185 static int igb_add_del_ethertype_filter(struct rte_eth_dev *dev,
 186                         struct rte_eth_ethertype_filter *filter,
 187                         bool add);
 188 static int igb_ethertype_filter_handle(struct rte_eth_dev *dev,
 189                                 enum rte_filter_op filter_op,
 190                                 void *arg);
 191 static int igb_get_ethertype_filter(struct rte_eth_dev *dev,
 192                         struct rte_eth_ethertype_filter *filter);
 193 static int eth_igb_filter_ctrl(struct rte_eth_dev *dev,
 194                      enum rte_filter_type filter_type,
 195                      enum rte_filter_op filter_op,
 196                      void *arg);
 197
 198 /*
 199  * Define VF Stats MACRO for Non "cleared on read" register
 200  */
 201 #define UPDATE_VF_STAT(reg, last, cur)            \
 202 {                                                 \
 203         u32 latest = E1000_READ_REG(hw, reg);     \
 204         cur += latest - last;                     \
 205         last = latest;                            \
 206 }
 207
 208
 209 #define IGB_FC_PAUSE_TIME 0x0680
 210 #define IGB_LINK_UPDATE_CHECK_TIMEOUT  90  /* 9s */
 211 #define IGB_LINK_UPDATE_CHECK_INTERVAL 100 /* ms */
 212
 213 #define IGBVF_PMD_NAME "rte_igbvf_pmd"     /* PMD name */
 214
 215 static enum e1000_fc_mode igb_fc_setting = e1000_fc_full;
 216
 217 /*
 218  * The set of PCI devices this driver supports
 219  */
 220 static struct rte_pci_id pci_id_igb_map[] = {
 221
 222 #define RTE_PCI_DEV_ID_DECL_IGB(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 223 #include "rte_pci_dev_ids.h"
 224
 225 {.device_id = 0},
 226 };
 227
 228 /*
 229  * The set of PCI devices this driver supports (for 82576&I350 VF)
 230  */
 231 static struct rte_pci_id pci_id_igbvf_map[] = {
 232
 233 #define RTE_PCI_DEV_ID_DECL_IGBVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 234 #include "rte_pci_dev_ids.h"
 235
 236 {.device_id = 0},
 237 };
 238
 239 static struct eth_dev_ops eth_igb_ops = {
 240         .dev_configure        = eth_igb_configure,
 241         .dev_start            = eth_igb_start,
 242         .dev_stop             = eth_igb_stop,
 243         .dev_close            = eth_igb_close,
 244         .promiscuous_enable   = eth_igb_promiscuous_enable,
 245         .promiscuous_disable  = eth_igb_promiscuous_disable,
 246         .allmulticast_enable  = eth_igb_allmulticast_enable,
 247         .allmulticast_disable = eth_igb_allmulticast_disable,
 248         .link_update          = eth_igb_link_update,
 249         .stats_get            = eth_igb_stats_get,
 250         .stats_reset          = eth_igb_stats_reset,
 251         .dev_infos_get        = eth_igb_infos_get,
 252         .mtu_set              = eth_igb_mtu_set,
 253         .vlan_filter_set      = eth_igb_vlan_filter_set,
 254         .vlan_tpid_set        = eth_igb_vlan_tpid_set,
 255         .vlan_offload_set     = eth_igb_vlan_offload_set,
 256         .rx_queue_setup       = eth_igb_rx_queue_setup,
 257         .rx_queue_release     = eth_igb_rx_queue_release,
 258         .rx_queue_count       = eth_igb_rx_queue_count,
 259         .rx_descriptor_done   = eth_igb_rx_descriptor_done,
 260         .tx_queue_setup       = eth_igb_tx_queue_setup,
 261         .tx_queue_release     = eth_igb_tx_queue_release,
 262         .dev_led_on           = eth_igb_led_on,
 263         .dev_led_off          = eth_igb_led_off,
 264         .flow_ctrl_get        = eth_igb_flow_ctrl_get,
 265         .flow_ctrl_set        = eth_igb_flow_ctrl_set,
 266         .mac_addr_add         = eth_igb_rar_set,
 267         .mac_addr_remove      = eth_igb_rar_clear,
 268         .reta_update          = eth_igb_rss_reta_update,
 269         .reta_query           = eth_igb_rss_reta_query,
 270         .rss_hash_update      = eth_igb_rss_hash_update,
 271         .rss_hash_conf_get    = eth_igb_rss_hash_conf_get,
 272         .filter_ctrl          = eth_igb_filter_ctrl,
 273 };
 274
 275 /*
 276  * dev_ops for virtual function, bare necessities for basic vf
 277  * operation have been implemented
 278  */
 279 static struct eth_dev_ops igbvf_eth_dev_ops = {
 280         .dev_configure        = igbvf_dev_configure,
 281         .dev_start            = igbvf_dev_start,
 282         .dev_stop             = igbvf_dev_stop,
 283         .dev_close            = igbvf_dev_close,
 284         .link_update          = eth_igb_link_update,
 285         .stats_get            = eth_igbvf_stats_get,
 286         .stats_reset          = eth_igbvf_stats_reset,
 287         .vlan_filter_set      = igbvf_vlan_filter_set,
 288         .dev_infos_get        = eth_igbvf_infos_get,
 289         .rx_queue_setup       = eth_igb_rx_queue_setup,
 290         .rx_queue_release     = eth_igb_rx_queue_release,
 291         .tx_queue_setup       = eth_igb_tx_queue_setup,
 292         .tx_queue_release     = eth_igb_tx_queue_release,
 293 };
 294
 295 /**
 296  * Atomically reads the link status information from global
 297  * structure rte_eth_dev.
 298  *
 299  * @param dev
 300  *   - Pointer to the structure rte_eth_dev to read from.
 301  *   - Pointer to the buffer to be saved with the link status.
 302  *
 303  * @return
 304  *   - On success, zero.
 305  *   - On failure, negative value.
 306  */
 307 static inline int
 308 rte_igb_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 309                                 struct rte_eth_link *link)
 310 {
 311         struct rte_eth_link *dst = link;
 312         struct rte_eth_link *src = &(dev->data->dev_link);
 313
 314         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 315                                         *(uint64_t *)src) == 0)
 316                 return -1;
 317
 318         return 0;
 319 }
 320
 321 /**
 322  * Atomically writes the link status information into global
 323  * structure rte_eth_dev.
 324  *
 325  * @param dev
 326  *   - Pointer to the structure rte_eth_dev to read from.
 327  *   - Pointer to the buffer to be saved with the link status.
 328  *
 329  * @return
 330  *   - On success, zero.
 331  *   - On failure, negative value.
 332  */
 333 static inline int
 334 rte_igb_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 335                                 struct rte_eth_link *link)
 336 {
 337         struct rte_eth_link *dst = &(dev->data->dev_link);
 338         struct rte_eth_link *src = link;
 339
 340         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 341                                         *(uint64_t *)src) == 0)
 342                 return -1;
 343
 344         return 0;
 345 }
 346
 347 static inline void
 348 igb_intr_enable(struct rte_eth_dev *dev)
 349 {
 350         struct e1000_interrupt *intr =
 351                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 352         struct e1000_hw *hw =
 353                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 354
 355         E1000_WRITE_REG(hw, E1000_IMS, intr->mask);
 356         E1000_WRITE_FLUSH(hw);
 357 }
 358
 359 static void
 360 igb_intr_disable(struct e1000_hw *hw)
 361 {
 362         E1000_WRITE_REG(hw, E1000_IMC, ~0);
 363         E1000_WRITE_FLUSH(hw);
 364 }
 365
 366 static inline int32_t
 367 igb_pf_reset_hw(struct e1000_hw *hw)
 368 {
 369         uint32_t ctrl_ext;
 370         int32_t status;
 371
 372         status = e1000_reset_hw(hw);
 373
 374         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 375         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 376         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 377         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 378         E1000_WRITE_FLUSH(hw);
 379
 380         return status;
 381 }
 382
 383 static void
 384 igb_identify_hardware(struct rte_eth_dev *dev)
 385 {
 386         struct e1000_hw *hw =
 387                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 388
 389         hw->vendor_id = dev->pci_dev->id.vendor_id;
 390         hw->device_id = dev->pci_dev->id.device_id;
 391         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
 392         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
 393
 394         e1000_set_mac_type(hw);
 395
 396         /* need to check if it is a vf device below */
 397 }
 398
 399 static int
 400 igb_reset_swfw_lock(struct e1000_hw *hw)
 401 {
 402         int ret_val;
 403
 404         /*
 405          * Do mac ops initialization manually here, since we will need
 406          * some function pointers set by this call.
 407          */
 408         ret_val = e1000_init_mac_params(hw);
 409         if (ret_val)
 410                 return ret_val;
 411
 412         /*
 413          * SMBI lock should not fail in this early stage. If this is the case,
 414          * it is due to an improper exit of the application.
 415          * So force the release of the faulty lock.
 416          */
 417         if (e1000_get_hw_semaphore_generic(hw) < 0) {
 418                 PMD_DRV_LOG(DEBUG, "SMBI lock released");
 419         }
 420         e1000_put_hw_semaphore_generic(hw);
 421
 422         if (hw->mac.ops.acquire_swfw_sync != NULL) {
 423                 uint16_t mask;
 424
 425                 /*
 426                  * Phy lock should not fail in this early stage. If this is the case,
 427                  * it is due to an improper exit of the application.
 428                  * So force the release of the faulty lock.
 429                  */
 430                 mask = E1000_SWFW_PHY0_SM << hw->bus.func;
 431                 if (hw->bus.func > E1000_FUNC_1)
 432                         mask <<= 2;
 433                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 434                         PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released",
 435                                     hw->bus.func);
 436                 }
 437                 hw->mac.ops.release_swfw_sync(hw, mask);
 438
 439                 /*
 440                  * This one is more tricky since it is common to all ports; but
 441                  * swfw_sync retries last long enough (1s) to be almost sure that if
 442                  * lock can not be taken it is due to an improper lock of the
 443                  * semaphore.
 444                  */
 445                 mask = E1000_SWFW_EEP_SM;
 446                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 447                         PMD_DRV_LOG(DEBUG, "SWFW common locks released");
 448                 }
 449                 hw->mac.ops.release_swfw_sync(hw, mask);
 450         }
 451
 452         return E1000_SUCCESS;
 453 }
 454
 455 static int
 456 eth_igb_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 457                    struct rte_eth_dev *eth_dev)
 458 {
 459         int error = 0;
 460         struct rte_pci_device *pci_dev;
 461         struct e1000_hw *hw =
 462                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 463         struct e1000_vfta * shadow_vfta =
 464                         E1000_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 465         struct e1000_filter_info *filter_info =
 466                 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private);
 467         uint32_t ctrl_ext;
 468
 469         pci_dev = eth_dev->pci_dev;
 470         eth_dev->dev_ops = &eth_igb_ops;
 471         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 472         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 473
 474         /* for secondary processes, we don't initialise any further as primary
 475          * has already done this work. Only check we don't need a different
 476          * RX function */
 477         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 478                 if (eth_dev->data->scattered_rx)
 479                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 480                 return 0;
 481         }
 482
 483         hw->hw_addr= (void *)pci_dev->mem_resource[0].addr;
 484
 485         igb_identify_hardware(eth_dev);
 486         if (e1000_setup_init_funcs(hw, FALSE) != E1000_SUCCESS) {
 487                 error = -EIO;
 488                 goto err_late;
 489         }
 490
 491         e1000_get_bus_info(hw);
 492
 493         /* Reset any pending lock */
 494         if (igb_reset_swfw_lock(hw) != E1000_SUCCESS) {
 495                 error = -EIO;
 496                 goto err_late;
 497         }
 498
 499         /* Finish initialization */
 500         if (e1000_setup_init_funcs(hw, TRUE) != E1000_SUCCESS) {
 501                 error = -EIO;
 502                 goto err_late;
 503         }
 504
 505         hw->mac.autoneg = 1;
 506         hw->phy.autoneg_wait_to_complete = 0;
 507         hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
 508
 509         /* Copper options */
 510         if (hw->phy.media_type == e1000_media_type_copper) {
 511                 hw->phy.mdix = 0; /* AUTO_ALL_MODES */
 512                 hw->phy.disable_polarity_correction = 0;
 513                 hw->phy.ms_type = e1000_ms_hw_default;
 514         }
 515
 516         /*
 517          * Start from a known state, this is important in reading the nvm
 518          * and mac from that.
 519          */
 520         igb_pf_reset_hw(hw);
 521
 522         /* Make sure we have a good EEPROM before we read from it */
 523         if (e1000_validate_nvm_checksum(hw) < 0) {
 524                 /*
 525                  * Some PCI-E parts fail the first check due to
 526                  * the link being in sleep state, call it again,
 527                  * if it fails a second time its a real issue.
 528                  */
 529                 if (e1000_validate_nvm_checksum(hw) < 0) {
 530                         PMD_INIT_LOG(ERR, "EEPROM checksum invalid");
 531                         error = -EIO;
 532                         goto err_late;
 533                 }
 534         }
 535
 536         /* Read the permanent MAC address out of the EEPROM */
 537         if (e1000_read_mac_addr(hw) != 0) {
 538                 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address");
 539                 error = -EIO;
 540                 goto err_late;
 541         }
 542
 543         /* Allocate memory for storing MAC addresses */
 544         eth_dev->data->mac_addrs = rte_zmalloc("e1000",
 545                 ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0);
 546         if (eth_dev->data->mac_addrs == NULL) {
 547                 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to "
 548                                                 "store MAC addresses",
 549                                 ETHER_ADDR_LEN * hw->mac.rar_entry_count);
 550                 error = -ENOMEM;
 551                 goto err_late;
 552         }
 553
 554         /* Copy the permanent MAC address */
 555         ether_addr_copy((struct ether_addr *)hw->mac.addr, &eth_dev->data->mac_addrs[0]);
 556
 557         /* initialize the vfta */
 558         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 559
 560         /* Now initialize the hardware */
 561         if (igb_hardware_init(hw) != 0) {
 562                 PMD_INIT_LOG(ERR, "Hardware initialization failed");
 563                 rte_free(eth_dev->data->mac_addrs);
 564                 eth_dev->data->mac_addrs = NULL;
 565                 error = -ENODEV;
 566                 goto err_late;
 567         }
 568         hw->mac.get_link_status = 1;
 569
 570         /* Indicate SOL/IDER usage */
 571         if (e1000_check_reset_block(hw) < 0) {
 572                 PMD_INIT_LOG(ERR, "PHY reset is blocked due to"
 573                                         "SOL/IDER session");
 574         }
 575
 576         /* initialize PF if max_vfs not zero */
 577         igb_pf_host_init(eth_dev);
 578
 579         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 580         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 581         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 582         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 583         E1000_WRITE_FLUSH(hw);
 584
 585         PMD_INIT_LOG(INFO, "port_id %d vendorID=0x%x deviceID=0x%x",
 586                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 587                      pci_dev->id.device_id);
 588
 589         rte_intr_callback_register(&(pci_dev->intr_handle),
 590                 eth_igb_interrupt_handler, (void *)eth_dev);
 591
 592         /* enable uio intr after callback register */
 593         rte_intr_enable(&(pci_dev->intr_handle));
 594
 595         /* enable support intr */
 596         igb_intr_enable(eth_dev);
 597
 598         TAILQ_INIT(&filter_info->flex_list);
 599         filter_info->flex_mask = 0;
 600         TAILQ_INIT(&filter_info->twotuple_list);
 601         filter_info->twotuple_mask = 0;
 602         TAILQ_INIT(&filter_info->fivetuple_list);
 603         filter_info->fivetuple_mask = 0;
 604
 605         return 0;
 606
 607 err_late:
 608         igb_hw_control_release(hw);
 609
 610         return (error);
 611 }
 612
 613 /*
 614  * Virtual Function device init
 615  */
 616 static int
 617 eth_igbvf_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 618                 struct rte_eth_dev *eth_dev)
 619 {
 620         struct rte_pci_device *pci_dev;
 621         struct e1000_hw *hw =
 622                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 623         int diag;
 624
 625         PMD_INIT_FUNC_TRACE();
 626
 627         eth_dev->dev_ops = &igbvf_eth_dev_ops;
 628         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 629         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 630
 631         /* for secondary processes, we don't initialise any further as primary
 632          * has already done this work. Only check we don't need a different
 633          * RX function */
 634         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 635                 if (eth_dev->data->scattered_rx)
 636                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 637                 return 0;
 638         }
 639
 640         pci_dev = eth_dev->pci_dev;
 641
 642         hw->device_id = pci_dev->id.device_id;
 643         hw->vendor_id = pci_dev->id.vendor_id;
 644         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 645
 646         /* Initialize the shared code (base driver) */
 647         diag = e1000_setup_init_funcs(hw, TRUE);
 648         if (diag != 0) {
 649                 PMD_INIT_LOG(ERR, "Shared code init failed for igbvf: %d",
 650                         diag);
 651                 return -EIO;
 652         }
 653
 654         /* init_mailbox_params */
 655         hw->mbx.ops.init_params(hw);
 656
 657         /* Disable the interrupts for VF */
 658         igbvf_intr_disable(hw);
 659
 660         diag = hw->mac.ops.reset_hw(hw);
 661
 662         /* Allocate memory for storing MAC addresses */
 663         eth_dev->data->mac_addrs = rte_zmalloc("igbvf", ETHER_ADDR_LEN *
 664                 hw->mac.rar_entry_count, 0);
 665         if (eth_dev->data->mac_addrs == NULL) {
 666                 PMD_INIT_LOG(ERR,
 667                         "Failed to allocate %d bytes needed to store MAC "
 668                         "addresses",
 669                         ETHER_ADDR_LEN * hw->mac.rar_entry_count);
 670                 return -ENOMEM;
 671         }
 672
 673         /* Copy the permanent MAC address */
 674         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 675                         &eth_dev->data->mac_addrs[0]);
 676
 677         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x "
 678                      "mac.type=%s",
 679                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 680                      pci_dev->id.device_id, "igb_mac_82576_vf");
 681
 682         return 0;
 683 }
 684
 685 static struct eth_driver rte_igb_pmd = {
 686         {
 687                 .name = "rte_igb_pmd",
 688                 .id_table = pci_id_igb_map,
 689                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
 690         },
 691         .eth_dev_init = eth_igb_dev_init,
 692         .dev_private_size = sizeof(struct e1000_adapter),
 693 };
 694
 695 /*
 696  * virtual function driver struct
 697  */
 698 static struct eth_driver rte_igbvf_pmd = {
 699         {
 700                 .name = "rte_igbvf_pmd",
 701                 .id_table = pci_id_igbvf_map,
 702                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
 703         },
 704         .eth_dev_init = eth_igbvf_dev_init,
 705         .dev_private_size = sizeof(struct e1000_adapter),
 706 };
 707
 708 static int
 709 rte_igb_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
 710 {
 711         rte_eth_driver_register(&rte_igb_pmd);
 712         return 0;
 713 }
 714
 715 static void
 716 igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
 717 {
 718         struct e1000_hw *hw =
 719                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 720         /* RCTL: enable VLAN filter since VMDq always use VLAN filter */
 721         uint32_t rctl = E1000_READ_REG(hw, E1000_RCTL);
 722         rctl |= E1000_RCTL_VFE;
 723         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
 724 }
 725
 726 /*
 727  * VF Driver initialization routine.
 728  * Invoked one at EAL init time.
 729  * Register itself as the [Virtual Poll Mode] Driver of PCI IGB devices.
 730  */
 731 static int
 732 rte_igbvf_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
 733 {
 734         PMD_INIT_FUNC_TRACE();
 735
 736         rte_eth_driver_register(&rte_igbvf_pmd);
 737         return (0);
 738 }
 739
 740 static int
 741 eth_igb_configure(struct rte_eth_dev *dev)
 742 {
 743         struct e1000_interrupt *intr =
 744                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 745
 746         PMD_INIT_FUNC_TRACE();
 747         intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
 748         PMD_INIT_FUNC_TRACE();
 749
 750         return (0);
 751 }
 752
 753 static int
 754 eth_igb_start(struct rte_eth_dev *dev)
 755 {
 756         struct e1000_hw *hw =
 757                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 758         int ret, i, mask;
 759         uint32_t ctrl_ext;
 760
 761         PMD_INIT_FUNC_TRACE();
 762
 763         /* Power up the phy. Needed to make the link go Up */
 764         e1000_power_up_phy(hw);
 765
 766         /*
 767          * Packet Buffer Allocation (PBA)
 768          * Writing PBA sets the receive portion of the buffer
 769          * the remainder is used for the transmit buffer.
 770          */
 771         if (hw->mac.type == e1000_82575) {
 772                 uint32_t pba;
 773
 774                 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
 775                 E1000_WRITE_REG(hw, E1000_PBA, pba);
 776         }
 777
 778         /* Put the address into the Receive Address Array */
 779         e1000_rar_set(hw, hw->mac.addr, 0);
 780
 781         /* Initialize the hardware */
 782         if (igb_hardware_init(hw)) {
 783                 PMD_INIT_LOG(ERR, "Unable to initialize the hardware");
 784                 return (-EIO);
 785         }
 786
 787         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
 788
 789         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 790         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 791         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 792         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 793         E1000_WRITE_FLUSH(hw);
 794
 795         /* configure PF module if SRIOV enabled */
 796         igb_pf_host_configure(dev);
 797
 798         /* Configure for OS presence */
 799         igb_init_manageability(hw);
 800
 801         eth_igb_tx_init(dev);
 802
 803         /* This can fail when allocating mbufs for descriptor rings */
 804         ret = eth_igb_rx_init(dev);
 805         if (ret) {
 806                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
 807                 igb_dev_clear_queues(dev);
 808                 return ret;
 809         }
 810
 811         e1000_clear_hw_cntrs_base_generic(hw);
 812
 813         /*
 814          * VLAN Offload Settings
 815          */
 816         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
 817                         ETH_VLAN_EXTEND_MASK;
 818         eth_igb_vlan_offload_set(dev, mask);
 819
 820         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
 821                 /* Enable VLAN filter since VMDq always use VLAN filter */
 822                 igb_vmdq_vlan_hw_filter_enable(dev);
 823         }
 824
 825         /*
 826          * Configure the Interrupt Moderation register (EITR) with the maximum
 827          * possible value (0xFFFF) to minimize "System Partial Write" issued by
 828          * spurious [DMA] memory updates of RX and TX ring descriptors.
 829          *
 830          * With a EITR granularity of 2 microseconds in the 82576, only 7/8
 831          * spurious memory updates per second should be expected.
 832          * ((65535 * 2) / 1000.1000 ~= 0.131 second).
 833          *
 834          * Because interrupts are not used at all, the MSI-X is not activated
 835          * and interrupt moderation is controlled by EITR[0].
 836          *
 837          * Note that having [almost] disabled memory updates of RX and TX ring
 838          * descriptors through the Interrupt Moderation mechanism, memory
 839          * updates of ring descriptors are now moderated by the configurable
 840          * value of Write-Back Threshold registers.
 841          */
 842         if ((hw->mac.type == e1000_82576) || (hw->mac.type == e1000_82580) ||
 843                 (hw->mac.type == e1000_i350) || (hw->mac.type == e1000_i210) ||
 844                 (hw->mac.type == e1000_i211)) {
 845                 uint32_t ivar;
 846
 847                 /* Enable all RX & TX queues in the IVAR registers */
 848                 ivar = (uint32_t) ((E1000_IVAR_VALID << 16) | E1000_IVAR_VALID);
 849                 for (i = 0; i < 8; i++)
 850                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, i, ivar);
 851
 852                 /* Configure EITR with the maximum possible value (0xFFFF) */
 853                 E1000_WRITE_REG(hw, E1000_EITR(0), 0xFFFF);
 854         }
 855
 856         /* Setup link speed and duplex */
 857         switch (dev->data->dev_conf.link_speed) {
 858         case ETH_LINK_SPEED_AUTONEG:
 859                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
 860                         hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
 861                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
 862                         hw->phy.autoneg_advertised = E1000_ALL_HALF_DUPLEX;
 863                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
 864                         hw->phy.autoneg_advertised = E1000_ALL_FULL_DUPLEX;
 865                 else
 866                         goto error_invalid_config;
 867                 break;
 868         case ETH_LINK_SPEED_10:
 869                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
 870                         hw->phy.autoneg_advertised = E1000_ALL_10_SPEED;
 871                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
 872                         hw->phy.autoneg_advertised = ADVERTISE_10_HALF;
 873                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
 874                         hw->phy.autoneg_advertised = ADVERTISE_10_FULL;
 875                 else
 876                         goto error_invalid_config;
 877                 break;
 878         case ETH_LINK_SPEED_100:
 879                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
 880                         hw->phy.autoneg_advertised = E1000_ALL_100_SPEED;
 881                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
 882                         hw->phy.autoneg_advertised = ADVERTISE_100_HALF;
 883                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
 884                         hw->phy.autoneg_advertised = ADVERTISE_100_FULL;
 885                 else
 886                         goto error_invalid_config;
 887                 break;
 888         case ETH_LINK_SPEED_1000:
 889                 if ((dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX) ||
 890                                 (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX))
 891                         hw->phy.autoneg_advertised = ADVERTISE_1000_FULL;
 892                 else
 893                         goto error_invalid_config;
 894                 break;
 895         case ETH_LINK_SPEED_10000:
 896         default:
 897                 goto error_invalid_config;
 898         }
 899         e1000_setup_link(hw);
 900
 901         /* check if lsc interrupt feature is enabled */
 902         if (dev->data->dev_conf.intr_conf.lsc != 0)
 903                 ret = eth_igb_lsc_interrupt_setup(dev);
 904
 905         /* resume enabled intr since hw reset */
 906         igb_intr_enable(dev);
 907
 908         PMD_INIT_LOG(DEBUG, "<<");
 909
 910         return (0);
 911
 912 error_invalid_config:
 913         PMD_INIT_LOG(ERR, "Invalid link_speed/link_duplex (%u/%u) for port %u",
 914                      dev->data->dev_conf.link_speed,
 915                      dev->data->dev_conf.link_duplex, dev->data->port_id);
 916         igb_dev_clear_queues(dev);
 917         return (-EINVAL);
 918 }
 919
 920 /*********************************************************************
 921  *
 922  *  This routine disables all traffic on the adapter by issuing a
 923  *  global reset on the MAC.
 924  *
 925  **********************************************************************/
 926 static void
 927 eth_igb_stop(struct rte_eth_dev *dev)
 928 {
 929         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 930         struct e1000_filter_info *filter_info =
 931                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
 932         struct rte_eth_link link;
 933         struct e1000_flex_filter *p_flex;
 934         struct e1000_5tuple_filter *p_5tuple, *p_5tuple_next;
 935         struct e1000_2tuple_filter *p_2tuple, *p_2tuple_next;
 936
 937         igb_intr_disable(hw);
 938         igb_pf_reset_hw(hw);
 939         E1000_WRITE_REG(hw, E1000_WUC, 0);
 940
 941         /* Set bit for Go Link disconnect */
 942         if (hw->mac.type >= e1000_82580) {
 943                 uint32_t phpm_reg;
 944
 945                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
 946                 phpm_reg |= E1000_82580_PM_GO_LINKD;
 947                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
 948         }
 949
 950         /* Power down the phy. Needed to make the link go Down */
 951         e1000_power_down_phy(hw);
 952
 953         igb_dev_clear_queues(dev);
 954
 955         /* clear the recorded link status */
 956         memset(&link, 0, sizeof(link));
 957         rte_igb_dev_atomic_write_link_status(dev, &link);
 958
 959         /* Remove all flex filters of the device */
 960         while ((p_flex = TAILQ_FIRST(&filter_info->flex_list))) {
 961                 TAILQ_REMOVE(&filter_info->flex_list, p_flex, entries);
 962                 rte_free(p_flex);
 963         }
 964         filter_info->flex_mask = 0;
 965
 966         /* Remove all ntuple filters of the device */
 967         for (p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list);
 968              p_5tuple != NULL; p_5tuple = p_5tuple_next) {
 969                 p_5tuple_next = TAILQ_NEXT(p_5tuple, entries);
 970                 TAILQ_REMOVE(&filter_info->fivetuple_list,
 971                              p_5tuple, entries);
 972                 rte_free(p_5tuple);
 973         }
 974         filter_info->fivetuple_mask = 0;
 975         for (p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list);
 976              p_2tuple != NULL; p_2tuple = p_2tuple_next) {
 977                 p_2tuple_next = TAILQ_NEXT(p_2tuple, entries);
 978                 TAILQ_REMOVE(&filter_info->twotuple_list,
 979                              p_2tuple, entries);
 980                 rte_free(p_2tuple);
 981         }
 982         filter_info->twotuple_mask = 0;
 983 }
 984
 985 static void
 986 eth_igb_close(struct rte_eth_dev *dev)
 987 {
 988         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 989         struct rte_eth_link link;
 990
 991         eth_igb_stop(dev);
 992         e1000_phy_hw_reset(hw);
 993         igb_release_manageability(hw);
 994         igb_hw_control_release(hw);
 995
 996         /* Clear bit for Go Link disconnect */
 997         if (hw->mac.type >= e1000_82580) {
 998                 uint32_t phpm_reg;
 999
1000                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
1001                 phpm_reg &= ~E1000_82580_PM_GO_LINKD;
1002                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
1003         }
1004
1005         igb_dev_clear_queues(dev);
1006
1007         memset(&link, 0, sizeof(link));
1008         rte_igb_dev_atomic_write_link_status(dev, &link);
1009 }
1010
1011 static int
1012 igb_get_rx_buffer_size(struct e1000_hw *hw)
1013 {
1014         uint32_t rx_buf_size;
1015         if (hw->mac.type == e1000_82576) {
1016                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xffff) << 10;
1017         } else if (hw->mac.type == e1000_82580 || hw->mac.type == e1000_i350) {
1018                 /* PBS needs to be translated according to a lookup table */
1019                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xf);
1020                 rx_buf_size = (uint32_t) e1000_rxpbs_adjust_82580(rx_buf_size);
1021                 rx_buf_size = (rx_buf_size << 10);
1022         } else if (hw->mac.type == e1000_i210 || hw->mac.type == e1000_i211) {
1023                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0x3f) << 10;
1024         } else {
1025                 rx_buf_size = (E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10;
1026         }
1027
1028         return rx_buf_size;
1029 }
1030
1031 /*********************************************************************
1032  *
1033  *  Initialize the hardware
1034  *
1035  **********************************************************************/
1036 static int
1037 igb_hardware_init(struct e1000_hw *hw)
1038 {
1039         uint32_t rx_buf_size;
1040         int diag;
1041
1042         /* Let the firmware know the OS is in control */
1043         igb_hw_control_acquire(hw);
1044
1045         /*
1046          * These parameters control the automatic generation (Tx) and
1047          * response (Rx) to Ethernet PAUSE frames.
1048          * - High water mark should allow for at least two standard size (1518)
1049          *   frames to be received after sending an XOFF.
1050          * - Low water mark works best when it is very near the high water mark.
1051          *   This allows the receiver to restart by sending XON when it has
1052          *   drained a bit. Here we use an arbitrary value of 1500 which will
1053          *   restart after one full frame is pulled from the buffer. There
1054          *   could be several smaller frames in the buffer and if so they will
1055          *   not trigger the XON until their total number reduces the buffer
1056          *   by 1500.
1057          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
1058          */
1059         rx_buf_size = igb_get_rx_buffer_size(hw);
1060
1061         hw->fc.high_water = rx_buf_size - (ETHER_MAX_LEN * 2);
1062         hw->fc.low_water = hw->fc.high_water - 1500;
1063         hw->fc.pause_time = IGB_FC_PAUSE_TIME;
1064         hw->fc.send_xon = 1;
1065
1066         /* Set Flow control, use the tunable location if sane */
1067         if ((igb_fc_setting != e1000_fc_none) && (igb_fc_setting < 4))
1068                 hw->fc.requested_mode = igb_fc_setting;
1069         else
1070                 hw->fc.requested_mode = e1000_fc_none;
1071
1072         /* Issue a global reset */
1073         igb_pf_reset_hw(hw);
1074         E1000_WRITE_REG(hw, E1000_WUC, 0);
1075
1076         diag = e1000_init_hw(hw);
1077         if (diag < 0)
1078                 return (diag);
1079
1080         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
1081         e1000_get_phy_info(hw);
1082         e1000_check_for_link(hw);
1083
1084         return (0);
1085 }
1086
1087 /* This function is based on igb_update_stats_counters() in igb/if_igb.c */
1088 static void
1089 eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1090 {
1091         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1092         struct e1000_hw_stats *stats =
1093                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1094         int pause_frames;
1095
1096         if(hw->phy.media_type == e1000_media_type_copper ||
1097             (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) {
1098                 stats->symerrs +=
1099                     E1000_READ_REG(hw,E1000_SYMERRS);
1100                 stats->sec += E1000_READ_REG(hw, E1000_SEC);
1101         }
1102
1103         stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
1104         stats->mpc += E1000_READ_REG(hw, E1000_MPC);
1105         stats->scc += E1000_READ_REG(hw, E1000_SCC);
1106         stats->ecol += E1000_READ_REG(hw, E1000_ECOL);
1107
1108         stats->mcc += E1000_READ_REG(hw, E1000_MCC);
1109         stats->latecol += E1000_READ_REG(hw, E1000_LATECOL);
1110         stats->colc += E1000_READ_REG(hw, E1000_COLC);
1111         stats->dc += E1000_READ_REG(hw, E1000_DC);
1112         stats->rlec += E1000_READ_REG(hw, E1000_RLEC);
1113         stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
1114         stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC);
1115         /*
1116         ** For watchdog management we need to know if we have been
1117         ** paused during the last interval, so capture that here.
1118         */
1119         pause_frames = E1000_READ_REG(hw, E1000_XOFFRXC);
1120         stats->xoffrxc += pause_frames;
1121         stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
1122         stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC);
1123         stats->prc64 += E1000_READ_REG(hw, E1000_PRC64);
1124         stats->prc127 += E1000_READ_REG(hw, E1000_PRC127);
1125         stats->prc255 += E1000_READ_REG(hw, E1000_PRC255);
1126         stats->prc511 += E1000_READ_REG(hw, E1000_PRC511);
1127         stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
1128         stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
1129         stats->gprc += E1000_READ_REG(hw, E1000_GPRC);
1130         stats->bprc += E1000_READ_REG(hw, E1000_BPRC);
1131         stats->mprc += E1000_READ_REG(hw, E1000_MPRC);
1132         stats->gptc += E1000_READ_REG(hw, E1000_GPTC);
1133
1134         /* For the 64-bit byte counters the low dword must be read first. */
1135         /* Both registers clear on the read of the high dword */
1136
1137         stats->gorc += E1000_READ_REG(hw, E1000_GORCL);
1138         stats->gorc += ((uint64_t)E1000_READ_REG(hw, E1000_GORCH) << 32);
1139         stats->gotc += E1000_READ_REG(hw, E1000_GOTCL);
1140         stats->gotc += ((uint64_t)E1000_READ_REG(hw, E1000_GOTCH) << 32);
1141
1142         stats->rnbc += E1000_READ_REG(hw, E1000_RNBC);
1143         stats->ruc += E1000_READ_REG(hw, E1000_RUC);
1144         stats->rfc += E1000_READ_REG(hw, E1000_RFC);
1145         stats->roc += E1000_READ_REG(hw, E1000_ROC);
1146         stats->rjc += E1000_READ_REG(hw, E1000_RJC);
1147
1148         stats->tor += E1000_READ_REG(hw, E1000_TORH);
1149         stats->tot += E1000_READ_REG(hw, E1000_TOTH);
1150
1151         stats->tpr += E1000_READ_REG(hw, E1000_TPR);
1152         stats->tpt += E1000_READ_REG(hw, E1000_TPT);
1153         stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64);
1154         stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127);
1155         stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255);
1156         stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511);
1157         stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
1158         stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
1159         stats->mptc += E1000_READ_REG(hw, E1000_MPTC);
1160         stats->bptc += E1000_READ_REG(hw, E1000_BPTC);
1161
1162         /* Interrupt Counts */
1163
1164         stats->iac += E1000_READ_REG(hw, E1000_IAC);
1165         stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
1166         stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
1167         stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
1168         stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
1169         stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
1170         stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
1171         stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
1172         stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
1173
1174         /* Host to Card Statistics */
1175
1176         stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC);
1177         stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC);
1178         stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC);
1179         stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC);
1180         stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC);
1181         stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC);
1182         stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC);
1183         stats->hgorc += E1000_READ_REG(hw, E1000_HGORCL);
1184         stats->hgorc += ((uint64_t)E1000_READ_REG(hw, E1000_HGORCH) << 32);
1185         stats->hgotc += E1000_READ_REG(hw, E1000_HGOTCL);
1186         stats->hgotc += ((uint64_t)E1000_READ_REG(hw, E1000_HGOTCH) << 32);
1187         stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS);
1188         stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC);
1189         stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC);
1190
1191         stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
1192         stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
1193         stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS);
1194         stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR);
1195         stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC);
1196         stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
1197
1198         if (rte_stats == NULL)
1199                 return;
1200
1201         /* Rx Errors */
1202         rte_stats->ibadcrc = stats->crcerrs;
1203         rte_stats->ibadlen = stats->rlec + stats->ruc + stats->roc;
1204         rte_stats->imissed = stats->mpc;
1205         rte_stats->ierrors = rte_stats->ibadcrc +
1206                              rte_stats->ibadlen +
1207                              rte_stats->imissed +
1208                              stats->rxerrc + stats->algnerrc + stats->cexterr;
1209
1210         /* Tx Errors */
1211         rte_stats->oerrors = stats->ecol + stats->latecol;
1212
1213         /* XON/XOFF pause frames */
1214         rte_stats->tx_pause_xon  = stats->xontxc;
1215         rte_stats->rx_pause_xon  = stats->xonrxc;
1216         rte_stats->tx_pause_xoff = stats->xofftxc;
1217         rte_stats->rx_pause_xoff = stats->xoffrxc;
1218
1219         rte_stats->ipackets = stats->gprc;
1220         rte_stats->opackets = stats->gptc;
1221         rte_stats->ibytes   = stats->gorc;
1222         rte_stats->obytes   = stats->gotc;
1223 }
1224
1225 static void
1226 eth_igb_stats_reset(struct rte_eth_dev *dev)
1227 {
1228         struct e1000_hw_stats *hw_stats =
1229                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1230
1231         /* HW registers are cleared on read */
1232         eth_igb_stats_get(dev, NULL);
1233
1234         /* Reset software totals */
1235         memset(hw_stats, 0, sizeof(*hw_stats));
1236 }
1237
1238 static void
1239 eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1240 {
1241         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1242         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*)
1243                           E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1244
1245         /* Good Rx packets, include VF loopback */
1246         UPDATE_VF_STAT(E1000_VFGPRC,
1247             hw_stats->last_gprc, hw_stats->gprc);
1248
1249         /* Good Rx octets, include VF loopback */
1250         UPDATE_VF_STAT(E1000_VFGORC,
1251             hw_stats->last_gorc, hw_stats->gorc);
1252
1253         /* Good Tx packets, include VF loopback */
1254         UPDATE_VF_STAT(E1000_VFGPTC,
1255             hw_stats->last_gptc, hw_stats->gptc);
1256
1257         /* Good Tx octets, include VF loopback */
1258         UPDATE_VF_STAT(E1000_VFGOTC,
1259             hw_stats->last_gotc, hw_stats->gotc);
1260
1261         /* Rx Multicst packets */
1262         UPDATE_VF_STAT(E1000_VFMPRC,
1263             hw_stats->last_mprc, hw_stats->mprc);
1264
1265         /* Good Rx loopback packets */
1266         UPDATE_VF_STAT(E1000_VFGPRLBC,
1267             hw_stats->last_gprlbc, hw_stats->gprlbc);
1268
1269         /* Good Rx loopback octets */
1270         UPDATE_VF_STAT(E1000_VFGORLBC,
1271             hw_stats->last_gorlbc, hw_stats->gorlbc);
1272
1273         /* Good Tx loopback packets */
1274         UPDATE_VF_STAT(E1000_VFGPTLBC,
1275             hw_stats->last_gptlbc, hw_stats->gptlbc);
1276
1277         /* Good Tx loopback octets */
1278         UPDATE_VF_STAT(E1000_VFGOTLBC,
1279             hw_stats->last_gotlbc, hw_stats->gotlbc);
1280
1281         if (rte_stats == NULL)
1282                 return;
1283
1284         rte_stats->ipackets = hw_stats->gprc;
1285         rte_stats->ibytes = hw_stats->gorc;
1286         rte_stats->opackets = hw_stats->gptc;
1287         rte_stats->obytes = hw_stats->gotc;
1288         rte_stats->imcasts = hw_stats->mprc;
1289         rte_stats->ilbpackets = hw_stats->gprlbc;
1290         rte_stats->ilbbytes = hw_stats->gorlbc;
1291         rte_stats->olbpackets = hw_stats->gptlbc;
1292         rte_stats->olbbytes = hw_stats->gotlbc;
1293
1294 }
1295
1296 static void
1297 eth_igbvf_stats_reset(struct rte_eth_dev *dev)
1298 {
1299         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*)
1300                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1301
1302         /* Sync HW register to the last stats */
1303         eth_igbvf_stats_get(dev, NULL);
1304
1305         /* reset HW current stats*/
1306         memset(&hw_stats->gprc, 0, sizeof(*hw_stats) -
1307                offsetof(struct e1000_vf_stats, gprc));
1308
1309 }
1310
1311 static void
1312 eth_igb_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1313 {
1314         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1315
1316         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1317         dev_info->max_rx_pktlen  = 0x3FFF; /* See RLPML register. */
1318         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1319         dev_info->rx_offload_capa =
1320                 DEV_RX_OFFLOAD_VLAN_STRIP |
1321                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1322                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1323                 DEV_RX_OFFLOAD_TCP_CKSUM;
1324         dev_info->tx_offload_capa =
1325                 DEV_TX_OFFLOAD_VLAN_INSERT |
1326                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1327                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1328                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1329                 DEV_TX_OFFLOAD_SCTP_CKSUM;
1330
1331         switch (hw->mac.type) {
1332         case e1000_82575:
1333                 dev_info->max_rx_queues = 4;
1334                 dev_info->max_tx_queues = 4;
1335                 dev_info->max_vmdq_pools = 0;
1336                 break;
1337
1338         case e1000_82576:
1339                 dev_info->max_rx_queues = 16;
1340                 dev_info->max_tx_queues = 16;
1341                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1342                 dev_info->vmdq_queue_num = 16;
1343                 break;
1344
1345         case e1000_82580:
1346                 dev_info->max_rx_queues = 8;
1347                 dev_info->max_tx_queues = 8;
1348                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1349                 dev_info->vmdq_queue_num = 8;
1350                 break;
1351
1352         case e1000_i350:
1353                 dev_info->max_rx_queues = 8;
1354                 dev_info->max_tx_queues = 8;
1355                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1356                 dev_info->vmdq_queue_num = 8;
1357                 break;
1358
1359         case e1000_i354:
1360                 dev_info->max_rx_queues = 8;
1361                 dev_info->max_tx_queues = 8;
1362                 break;
1363
1364         case e1000_i210:
1365                 dev_info->max_rx_queues = 4;
1366                 dev_info->max_tx_queues = 4;
1367                 dev_info->max_vmdq_pools = 0;
1368                 break;
1369
1370         case e1000_i211:
1371                 dev_info->max_rx_queues = 2;
1372                 dev_info->max_tx_queues = 2;
1373                 dev_info->max_vmdq_pools = 0;
1374                 break;
1375
1376         default:
1377                 /* Should not happen */
1378                 break;
1379         }
1380         dev_info->reta_size = ETH_RSS_RETA_SIZE_128;
1381
1382         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1383                 .rx_thresh = {
1384                         .pthresh = IGB_DEFAULT_RX_PTHRESH,
1385                         .hthresh = IGB_DEFAULT_RX_HTHRESH,
1386                         .wthresh = IGB_DEFAULT_RX_WTHRESH,
1387                 },
1388                 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH,
1389                 .rx_drop_en = 0,
1390         };
1391
1392         dev_info->default_txconf = (struct rte_eth_txconf) {
1393                 .tx_thresh = {
1394                         .pthresh = IGB_DEFAULT_TX_PTHRESH,
1395                         .hthresh = IGB_DEFAULT_TX_HTHRESH,
1396                         .wthresh = IGB_DEFAULT_TX_WTHRESH,
1397                 },
1398                 .txq_flags = 0,
1399         };
1400 }
1401
1402 static void
1403 eth_igbvf_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1404 {
1405         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1406
1407         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1408         dev_info->max_rx_pktlen  = 0x3FFF; /* See RLPML register. */
1409         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1410         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
1411                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1412                                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1413                                 DEV_RX_OFFLOAD_TCP_CKSUM;
1414         dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
1415                                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1416                                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1417                                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1418                                 DEV_TX_OFFLOAD_SCTP_CKSUM;
1419         switch (hw->mac.type) {
1420         case e1000_vfadapt:
1421                 dev_info->max_rx_queues = 2;
1422                 dev_info->max_tx_queues = 2;
1423                 break;
1424         case e1000_vfadapt_i350:
1425                 dev_info->max_rx_queues = 1;
1426                 dev_info->max_tx_queues = 1;
1427                 break;
1428         default:
1429                 /* Should not happen */
1430                 break;
1431         }
1432
1433         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1434                 .rx_thresh = {
1435                         .pthresh = IGB_DEFAULT_RX_PTHRESH,
1436                         .hthresh = IGB_DEFAULT_RX_HTHRESH,
1437                         .wthresh = IGB_DEFAULT_RX_WTHRESH,
1438                 },
1439                 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH,
1440                 .rx_drop_en = 0,
1441         };
1442
1443         dev_info->default_txconf = (struct rte_eth_txconf) {
1444                 .tx_thresh = {
1445                         .pthresh = IGB_DEFAULT_TX_PTHRESH,
1446                         .hthresh = IGB_DEFAULT_TX_HTHRESH,
1447                         .wthresh = IGB_DEFAULT_TX_WTHRESH,
1448                 },
1449                 .txq_flags = 0,
1450         };
1451 }
1452
1453 /* return 0 means link status changed, -1 means not changed */
1454 static int
1455 eth_igb_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1456 {
1457         struct e1000_hw *hw =
1458                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1459         struct rte_eth_link link, old;
1460         int link_check, count;
1461
1462         link_check = 0;
1463         hw->mac.get_link_status = 1;
1464
1465         /* possible wait-to-complete in up to 9 seconds */
1466         for (count = 0; count < IGB_LINK_UPDATE_CHECK_TIMEOUT; count ++) {
1467                 /* Read the real link status */
1468                 switch (hw->phy.media_type) {
1469                 case e1000_media_type_copper:
1470                         /* Do the work to read phy */
1471                         e1000_check_for_link(hw);
1472                         link_check = !hw->mac.get_link_status;
1473                         break;
1474
1475                 case e1000_media_type_fiber:
1476                         e1000_check_for_link(hw);
1477                         link_check = (E1000_READ_REG(hw, E1000_STATUS) &
1478                                       E1000_STATUS_LU);
1479                         break;
1480
1481                 case e1000_media_type_internal_serdes:
1482                         e1000_check_for_link(hw);
1483                         link_check = hw->mac.serdes_has_link;
1484                         break;
1485
1486                 /* VF device is type_unknown */
1487                 case e1000_media_type_unknown:
1488                         eth_igbvf_link_update(hw);
1489                         link_check = !hw->mac.get_link_status;
1490                         break;
1491
1492                 default:
1493                         break;
1494                 }
1495                 if (link_check || wait_to_complete == 0)
1496                         break;
1497                 rte_delay_ms(IGB_LINK_UPDATE_CHECK_INTERVAL);
1498         }
1499         memset(&link, 0, sizeof(link));
1500         rte_igb_dev_atomic_read_link_status(dev, &link);
1501         old = link;
1502
1503         /* Now we check if a transition has happened */
1504         if (link_check) {
1505                 hw->mac.ops.get_link_up_info(hw, &link.link_speed,
1506                                           &link.link_duplex);
1507                 link.link_status = 1;
1508         } else if (!link_check) {
1509                 link.link_speed = 0;
1510                 link.link_duplex = 0;
1511                 link.link_status = 0;
1512         }
1513         rte_igb_dev_atomic_write_link_status(dev, &link);
1514
1515         /* not changed */
1516         if (old.link_status == link.link_status)
1517                 return -1;
1518
1519         /* changed */
1520         return 0;
1521 }
1522
1523 /*
1524  * igb_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit.
1525  * For ASF and Pass Through versions of f/w this means
1526  * that the driver is loaded.
1527  */
1528 static void
1529 igb_hw_control_acquire(struct e1000_hw *hw)
1530 {
1531         uint32_t ctrl_ext;
1532
1533         /* Let firmware know the driver has taken over */
1534         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1535         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1536 }
1537
1538 /*
1539  * igb_hw_control_release resets CTRL_EXT:DRV_LOAD bit.
1540  * For ASF and Pass Through versions of f/w this means that the
1541  * driver is no longer loaded.
1542  */
1543 static void
1544 igb_hw_control_release(struct e1000_hw *hw)
1545 {
1546         uint32_t ctrl_ext;
1547
1548         /* Let firmware taken over control of h/w */
1549         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1550         E1000_WRITE_REG(hw, E1000_CTRL_EXT,
1551                         ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1552 }
1553
1554 /*
1555  * Bit of a misnomer, what this really means is
1556  * to enable OS management of the system... aka
1557  * to disable special hardware management features.
1558  */
1559 static void
1560 igb_init_manageability(struct e1000_hw *hw)
1561 {
1562         if (e1000_enable_mng_pass_thru(hw)) {
1563                 uint32_t manc2h = E1000_READ_REG(hw, E1000_MANC2H);
1564                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
1565
1566                 /* disable hardware interception of ARP */
1567                 manc &= ~(E1000_MANC_ARP_EN);
1568
1569                 /* enable receiving management packets to the host */
1570                 manc |= E1000_MANC_EN_MNG2HOST;
1571                 manc2h |= 1 << 5;  /* Mng Port 623 */
1572                 manc2h |= 1 << 6;  /* Mng Port 664 */
1573                 E1000_WRITE_REG(hw, E1000_MANC2H, manc2h);
1574                 E1000_WRITE_REG(hw, E1000_MANC, manc);
1575         }
1576 }
1577
1578 static void
1579 igb_release_manageability(struct e1000_hw *hw)
1580 {
1581         if (e1000_enable_mng_pass_thru(hw)) {
1582                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
1583
1584                 manc |= E1000_MANC_ARP_EN;
1585                 manc &= ~E1000_MANC_EN_MNG2HOST;
1586
1587                 E1000_WRITE_REG(hw, E1000_MANC, manc);
1588         }
1589 }
1590
1591 static void
1592 eth_igb_promiscuous_enable(struct rte_eth_dev *dev)
1593 {
1594         struct e1000_hw *hw =
1595                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1596         uint32_t rctl;
1597
1598         rctl = E1000_READ_REG(hw, E1000_RCTL);
1599         rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1600         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1601 }
1602
1603 static void
1604 eth_igb_promiscuous_disable(struct rte_eth_dev *dev)
1605 {
1606         struct e1000_hw *hw =
1607                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1608         uint32_t rctl;
1609
1610         rctl = E1000_READ_REG(hw, E1000_RCTL);
1611         rctl &= (~E1000_RCTL_UPE);
1612         if (dev->data->all_multicast == 1)
1613                 rctl |= E1000_RCTL_MPE;
1614         else
1615                 rctl &= (~E1000_RCTL_MPE);
1616         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1617 }
1618
1619 static void
1620 eth_igb_allmulticast_enable(struct rte_eth_dev *dev)
1621 {
1622         struct e1000_hw *hw =
1623                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1624         uint32_t rctl;
1625
1626         rctl = E1000_READ_REG(hw, E1000_RCTL);
1627         rctl |= E1000_RCTL_MPE;
1628         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1629 }
1630
1631 static void
1632 eth_igb_allmulticast_disable(struct rte_eth_dev *dev)
1633 {
1634         struct e1000_hw *hw =
1635                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1636         uint32_t rctl;
1637
1638         if (dev->data->promiscuous == 1)
1639                 return; /* must remain in all_multicast mode */
1640         rctl = E1000_READ_REG(hw, E1000_RCTL);
1641         rctl &= (~E1000_RCTL_MPE);
1642         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1643 }
1644
1645 static int
1646 eth_igb_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1647 {
1648         struct e1000_hw *hw =
1649                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1650         struct e1000_vfta * shadow_vfta =
1651                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1652         uint32_t vfta;
1653         uint32_t vid_idx;
1654         uint32_t vid_bit;
1655
1656         vid_idx = (uint32_t) ((vlan_id >> E1000_VFTA_ENTRY_SHIFT) &
1657                               E1000_VFTA_ENTRY_MASK);
1658         vid_bit = (uint32_t) (1 << (vlan_id & E1000_VFTA_ENTRY_BIT_SHIFT_MASK));
1659         vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, vid_idx);
1660         if (on)
1661                 vfta |= vid_bit;
1662         else
1663                 vfta &= ~vid_bit;
1664         E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, vid_idx, vfta);
1665
1666         /* update local VFTA copy */
1667         shadow_vfta->vfta[vid_idx] = vfta;
1668
1669         return 0;
1670 }
1671
1672 static void
1673 eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
1674 {
1675         struct e1000_hw *hw =
1676                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1677         uint32_t reg = ETHER_TYPE_VLAN ;
1678
1679         reg |= (tpid << 16);
1680         E1000_WRITE_REG(hw, E1000_VET, reg);
1681 }
1682
1683 static void
1684 igb_vlan_hw_filter_disable(struct rte_eth_dev *dev)
1685 {
1686         struct e1000_hw *hw =
1687                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1688         uint32_t reg;
1689
1690         /* Filter Table Disable */
1691         reg = E1000_READ_REG(hw, E1000_RCTL);
1692         reg &= ~E1000_RCTL_CFIEN;
1693         reg &= ~E1000_RCTL_VFE;
1694         E1000_WRITE_REG(hw, E1000_RCTL, reg);
1695 }
1696
1697 static void
1698 igb_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1699 {
1700         struct e1000_hw *hw =
1701                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1702         struct e1000_vfta * shadow_vfta =
1703                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1704         uint32_t reg;
1705         int i;
1706
1707         /* Filter Table Enable, CFI not used for packet acceptance */
1708         reg = E1000_READ_REG(hw, E1000_RCTL);
1709         reg &= ~E1000_RCTL_CFIEN;
1710         reg |= E1000_RCTL_VFE;
1711         E1000_WRITE_REG(hw, E1000_RCTL, reg);
1712
1713         /* restore VFTA table */
1714         for (i = 0; i < IGB_VFTA_SIZE; i++)
1715                 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, i, shadow_vfta->vfta[i]);
1716 }
1717
1718 static void
1719 igb_vlan_hw_strip_disable(struct rte_eth_dev *dev)
1720 {
1721         struct e1000_hw *hw =
1722                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1723         uint32_t reg;
1724
1725         /* VLAN Mode Disable */
1726         reg = E1000_READ_REG(hw, E1000_CTRL);
1727         reg &= ~E1000_CTRL_VME;
1728         E1000_WRITE_REG(hw, E1000_CTRL, reg);
1729 }
1730
1731 static void
1732 igb_vlan_hw_strip_enable(struct rte_eth_dev *dev)
1733 {
1734         struct e1000_hw *hw =
1735                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1736         uint32_t reg;
1737
1738         /* VLAN Mode Enable */
1739         reg = E1000_READ_REG(hw, E1000_CTRL);
1740         reg |= E1000_CTRL_VME;
1741         E1000_WRITE_REG(hw, E1000_CTRL, reg);
1742 }
1743
1744 static void
1745 igb_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1746 {
1747         struct e1000_hw *hw =
1748                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1749         uint32_t reg;
1750
1751         /* CTRL_EXT: Extended VLAN */
1752         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
1753         reg &= ~E1000_CTRL_EXT_EXTEND_VLAN;
1754         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
1755
1756         /* Update maximum packet length */
1757         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
1758                 E1000_WRITE_REG(hw, E1000_RLPML,
1759                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
1760                                                 VLAN_TAG_SIZE);
1761 }
1762
1763 static void
1764 igb_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1765 {
1766         struct e1000_hw *hw =
1767                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1768         uint32_t reg;
1769
1770         /* CTRL_EXT: Extended VLAN */
1771         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
1772         reg |= E1000_CTRL_EXT_EXTEND_VLAN;
1773         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
1774
1775         /* Update maximum packet length */
1776         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
1777                 E1000_WRITE_REG(hw, E1000_RLPML,
1778                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
1779                                                 2 * VLAN_TAG_SIZE);
1780 }
1781
1782 static void
1783 eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1784 {
1785         if(mask & ETH_VLAN_STRIP_MASK){
1786                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1787                         igb_vlan_hw_strip_enable(dev);
1788                 else
1789                         igb_vlan_hw_strip_disable(dev);
1790         }
1791
1792         if(mask & ETH_VLAN_FILTER_MASK){
1793                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1794                         igb_vlan_hw_filter_enable(dev);
1795                 else
1796                         igb_vlan_hw_filter_disable(dev);
1797         }
1798
1799         if(mask & ETH_VLAN_EXTEND_MASK){
1800                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1801                         igb_vlan_hw_extend_enable(dev);
1802                 else
1803                         igb_vlan_hw_extend_disable(dev);
1804         }
1805 }
1806
1807
1808 /**
1809  * It enables the interrupt mask and then enable the interrupt.
1810  *
1811  * @param dev
1812  *  Pointer to struct rte_eth_dev.
1813  *
1814  * @return
1815  *  - On success, zero.
1816  *  - On failure, a negative value.
1817  */
1818 static int
1819 eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev)
1820 {
1821         struct e1000_interrupt *intr =
1822                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1823
1824         intr->mask |= E1000_ICR_LSC;
1825
1826         return 0;
1827 }
1828
1829 /*
1830  * It reads ICR and gets interrupt causes, check it and set a bit flag
1831  * to update link status.
1832  *
1833  * @param dev
1834  *  Pointer to struct rte_eth_dev.
1835  *
1836  * @return
1837  *  - On success, zero.
1838  *  - On failure, a negative value.
1839  */
1840 static int
1841 eth_igb_interrupt_get_status(struct rte_eth_dev *dev)
1842 {
1843         uint32_t icr;
1844         struct e1000_hw *hw =
1845                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1846         struct e1000_interrupt *intr =
1847                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1848
1849         igb_intr_disable(hw);
1850
1851         /* read-on-clear nic registers here */
1852         icr = E1000_READ_REG(hw, E1000_ICR);
1853
1854         intr->flags = 0;
1855         if (icr & E1000_ICR_LSC) {
1856                 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
1857         }
1858
1859         if (icr & E1000_ICR_VMMB)
1860                 intr->flags |= E1000_FLAG_MAILBOX;
1861
1862         return 0;
1863 }
1864
1865 /*
1866  * It executes link_update after knowing an interrupt is prsent.
1867  *
1868  * @param dev
1869  *  Pointer to struct rte_eth_dev.
1870  *
1871  * @return
1872  *  - On success, zero.
1873  *  - On failure, a negative value.
1874  */
1875 static int
1876 eth_igb_interrupt_action(struct rte_eth_dev *dev)
1877 {
1878         struct e1000_hw *hw =
1879                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1880         struct e1000_interrupt *intr =
1881                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1882         uint32_t tctl, rctl;
1883         struct rte_eth_link link;
1884         int ret;
1885
1886         if (intr->flags & E1000_FLAG_MAILBOX) {
1887                 igb_pf_mbx_process(dev);
1888                 intr->flags &= ~E1000_FLAG_MAILBOX;
1889         }
1890
1891         igb_intr_enable(dev);
1892         rte_intr_enable(&(dev->pci_dev->intr_handle));
1893
1894         if (intr->flags & E1000_FLAG_NEED_LINK_UPDATE) {
1895                 intr->flags &= ~E1000_FLAG_NEED_LINK_UPDATE;
1896
1897                 /* set get_link_status to check register later */
1898                 hw->mac.get_link_status = 1;
1899                 ret = eth_igb_link_update(dev, 0);
1900
1901                 /* check if link has changed */
1902                 if (ret < 0)
1903                         return 0;
1904
1905                 memset(&link, 0, sizeof(link));
1906                 rte_igb_dev_atomic_read_link_status(dev, &link);
1907                 if (link.link_status) {
1908                         PMD_INIT_LOG(INFO,
1909                                      " Port %d: Link Up - speed %u Mbps - %s",
1910                                      dev->data->port_id,
1911                                      (unsigned)link.link_speed,
1912                                      link.link_duplex == ETH_LINK_FULL_DUPLEX ?
1913                                      "full-duplex" : "half-duplex");
1914                 } else {
1915                         PMD_INIT_LOG(INFO, " Port %d: Link Down",
1916                                      dev->data->port_id);
1917                 }
1918                 PMD_INIT_LOG(INFO, "PCI Address: %04d:%02d:%02d:%d",
1919                              dev->pci_dev->addr.domain,
1920                              dev->pci_dev->addr.bus,
1921                              dev->pci_dev->addr.devid,
1922                              dev->pci_dev->addr.function);
1923                 tctl = E1000_READ_REG(hw, E1000_TCTL);
1924                 rctl = E1000_READ_REG(hw, E1000_RCTL);
1925                 if (link.link_status) {
1926                         /* enable Tx/Rx */
1927                         tctl |= E1000_TCTL_EN;
1928                         rctl |= E1000_RCTL_EN;
1929                 } else {
1930                         /* disable Tx/Rx */
1931                         tctl &= ~E1000_TCTL_EN;
1932                         rctl &= ~E1000_RCTL_EN;
1933                 }
1934                 E1000_WRITE_REG(hw, E1000_TCTL, tctl);
1935                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1936                 E1000_WRITE_FLUSH(hw);
1937                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
1938         }
1939
1940         return 0;
1941 }
1942
1943 /**
1944  * Interrupt handler which shall be registered at first.
1945  *
1946  * @param handle
1947  *  Pointer to interrupt handle.
1948  * @param param
1949  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1950  *
1951  * @return
1952  *  void
1953  */
1954 static void
1955 eth_igb_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
1956                                                         void *param)
1957 {
1958         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1959
1960         eth_igb_interrupt_get_status(dev);
1961         eth_igb_interrupt_action(dev);
1962 }
1963
1964 static int
1965 eth_igb_led_on(struct rte_eth_dev *dev)
1966 {
1967         struct e1000_hw *hw;
1968
1969         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1970         return (e1000_led_on(hw) == E1000_SUCCESS ? 0 : -ENOTSUP);
1971 }
1972
1973 static int
1974 eth_igb_led_off(struct rte_eth_dev *dev)
1975 {
1976         struct e1000_hw *hw;
1977
1978         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1979         return (e1000_led_off(hw) == E1000_SUCCESS ? 0 : -ENOTSUP);
1980 }
1981
1982 static int
1983 eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1984 {
1985         struct e1000_hw *hw;
1986         uint32_t ctrl;
1987         int tx_pause;
1988         int rx_pause;
1989
1990         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1991         fc_conf->pause_time = hw->fc.pause_time;
1992         fc_conf->high_water = hw->fc.high_water;
1993         fc_conf->low_water = hw->fc.low_water;
1994         fc_conf->send_xon = hw->fc.send_xon;
1995         fc_conf->autoneg = hw->mac.autoneg;
1996
1997         /*
1998          * Return rx_pause and tx_pause status according to actual setting of
1999          * the TFCE and RFCE bits in the CTRL register.
2000          */
2001         ctrl = E1000_READ_REG(hw, E1000_CTRL);
2002         if (ctrl & E1000_CTRL_TFCE)
2003                 tx_pause = 1;
2004         else
2005                 tx_pause = 0;
2006
2007         if (ctrl & E1000_CTRL_RFCE)
2008                 rx_pause = 1;
2009         else
2010                 rx_pause = 0;
2011
2012         if (rx_pause && tx_pause)
2013                 fc_conf->mode = RTE_FC_FULL;
2014         else if (rx_pause)
2015                 fc_conf->mode = RTE_FC_RX_PAUSE;
2016         else if (tx_pause)
2017                 fc_conf->mode = RTE_FC_TX_PAUSE;
2018         else
2019                 fc_conf->mode = RTE_FC_NONE;
2020
2021         return 0;
2022 }
2023
2024 static int
2025 eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2026 {
2027         struct e1000_hw *hw;
2028         int err;
2029         enum e1000_fc_mode rte_fcmode_2_e1000_fcmode[] = {
2030                 e1000_fc_none,
2031                 e1000_fc_rx_pause,
2032                 e1000_fc_tx_pause,
2033                 e1000_fc_full
2034         };
2035         uint32_t rx_buf_size;
2036         uint32_t max_high_water;
2037         uint32_t rctl;
2038
2039         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2040         if (fc_conf->autoneg != hw->mac.autoneg)
2041                 return -ENOTSUP;
2042         rx_buf_size = igb_get_rx_buffer_size(hw);
2043         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
2044
2045         /* At least reserve one Ethernet frame for watermark */
2046         max_high_water = rx_buf_size - ETHER_MAX_LEN;
2047         if ((fc_conf->high_water > max_high_water) ||
2048             (fc_conf->high_water < fc_conf->low_water)) {
2049                 PMD_INIT_LOG(ERR, "e1000 incorrect high/low water value");
2050                 PMD_INIT_LOG(ERR, "high water must <=  0x%x", max_high_water);
2051                 return (-EINVAL);
2052         }
2053
2054         hw->fc.requested_mode = rte_fcmode_2_e1000_fcmode[fc_conf->mode];
2055         hw->fc.pause_time     = fc_conf->pause_time;
2056         hw->fc.high_water     = fc_conf->high_water;
2057         hw->fc.low_water      = fc_conf->low_water;
2058         hw->fc.send_xon       = fc_conf->send_xon;
2059
2060         err = e1000_setup_link_generic(hw);
2061         if (err == E1000_SUCCESS) {
2062
2063                 /* check if we want to forward MAC frames - driver doesn't have native
2064                  * capability to do that, so we'll write the registers ourselves */
2065
2066                 rctl = E1000_READ_REG(hw, E1000_RCTL);
2067
2068                 /* set or clear MFLCN.PMCF bit depending on configuration */
2069                 if (fc_conf->mac_ctrl_frame_fwd != 0)
2070                         rctl |= E1000_RCTL_PMCF;
2071                 else
2072                         rctl &= ~E1000_RCTL_PMCF;
2073
2074                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2075                 E1000_WRITE_FLUSH(hw);
2076
2077                 return 0;
2078         }
2079
2080         PMD_INIT_LOG(ERR, "e1000_setup_link_generic = 0x%x", err);
2081         return (-EIO);
2082 }
2083
2084 #define E1000_RAH_POOLSEL_SHIFT      (18)
2085 static void
2086 eth_igb_rar_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
2087                 uint32_t index, __rte_unused uint32_t pool)
2088 {
2089         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2090         uint32_t rah;
2091
2092         e1000_rar_set(hw, mac_addr->addr_bytes, index);
2093         rah = E1000_READ_REG(hw, E1000_RAH(index));
2094         rah |= (0x1 << (E1000_RAH_POOLSEL_SHIFT + pool));
2095         E1000_WRITE_REG(hw, E1000_RAH(index), rah);
2096 }
2097
2098 static void
2099 eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index)
2100 {
2101         uint8_t addr[ETHER_ADDR_LEN];
2102         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2103
2104         memset(addr, 0, sizeof(addr));
2105
2106         e1000_rar_set(hw, addr, index);
2107 }
2108
2109 /*
2110  * Virtual Function operations
2111  */
2112 static void
2113 igbvf_intr_disable(struct e1000_hw *hw)
2114 {
2115         PMD_INIT_FUNC_TRACE();
2116
2117         /* Clear interrupt mask to stop from interrupts being generated */
2118         E1000_WRITE_REG(hw, E1000_EIMC, 0xFFFF);
2119
2120         E1000_WRITE_FLUSH(hw);
2121 }
2122
2123 static void
2124 igbvf_stop_adapter(struct rte_eth_dev *dev)
2125 {
2126         u32 reg_val;
2127         u16 i;
2128         struct rte_eth_dev_info dev_info;
2129         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2130
2131         memset(&dev_info, 0, sizeof(dev_info));
2132         eth_igbvf_infos_get(dev, &dev_info);
2133
2134         /* Clear interrupt mask to stop from interrupts being generated */
2135         igbvf_intr_disable(hw);
2136
2137         /* Clear any pending interrupts, flush previous writes */
2138         E1000_READ_REG(hw, E1000_EICR);
2139
2140         /* Disable the transmit unit.  Each queue must be disabled. */
2141         for (i = 0; i < dev_info.max_tx_queues; i++)
2142                 E1000_WRITE_REG(hw, E1000_TXDCTL(i), E1000_TXDCTL_SWFLSH);
2143
2144         /* Disable the receive unit by stopping each queue */
2145         for (i = 0; i < dev_info.max_rx_queues; i++) {
2146                 reg_val = E1000_READ_REG(hw, E1000_RXDCTL(i));
2147                 reg_val &= ~E1000_RXDCTL_QUEUE_ENABLE;
2148                 E1000_WRITE_REG(hw, E1000_RXDCTL(i), reg_val);
2149                 while (E1000_READ_REG(hw, E1000_RXDCTL(i)) & E1000_RXDCTL_QUEUE_ENABLE)
2150                         ;
2151         }
2152
2153         /* flush all queues disables */
2154         E1000_WRITE_FLUSH(hw);
2155         msec_delay(2);
2156 }
2157
2158 static int eth_igbvf_link_update(struct e1000_hw *hw)
2159 {
2160         struct e1000_mbx_info *mbx = &hw->mbx;
2161         struct e1000_mac_info *mac = &hw->mac;
2162         int ret_val = E1000_SUCCESS;
2163
2164         PMD_INIT_LOG(DEBUG, "e1000_check_for_link_vf");
2165
2166         /*
2167          * We only want to run this if there has been a rst asserted.
2168          * in this case that could mean a link change, device reset,
2169          * or a virtual function reset
2170          */
2171
2172         /* If we were hit with a reset or timeout drop the link */
2173         if (!e1000_check_for_rst(hw, 0) || !mbx->timeout)
2174                 mac->get_link_status = TRUE;
2175
2176         if (!mac->get_link_status)
2177                 goto out;
2178
2179         /* if link status is down no point in checking to see if pf is up */
2180         if (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU))
2181                 goto out;
2182
2183         /* if we passed all the tests above then the link is up and we no
2184          * longer need to check for link */
2185         mac->get_link_status = FALSE;
2186
2187 out:
2188         return ret_val;
2189 }
2190
2191
2192 static int
2193 igbvf_dev_configure(struct rte_eth_dev *dev)
2194 {
2195         struct rte_eth_conf* conf = &dev->data->dev_conf;
2196
2197         PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d",
2198                      dev->data->port_id);
2199
2200         /*
2201          * VF has no ability to enable/disable HW CRC
2202          * Keep the persistent behavior the same as Host PF
2203          */
2204 #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC
2205         if (!conf->rxmode.hw_strip_crc) {
2206                 PMD_INIT_LOG(INFO, "VF can't disable HW CRC Strip");
2207                 conf->rxmode.hw_strip_crc = 1;
2208         }
2209 #else
2210         if (conf->rxmode.hw_strip_crc) {
2211                 PMD_INIT_LOG(INFO, "VF can't enable HW CRC Strip");
2212                 conf->rxmode.hw_strip_crc = 0;
2213         }
2214 #endif
2215
2216         return 0;
2217 }
2218
2219 static int
2220 igbvf_dev_start(struct rte_eth_dev *dev)
2221 {
2222         struct e1000_hw *hw =
2223                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2224         int ret;
2225
2226         PMD_INIT_FUNC_TRACE();
2227
2228         hw->mac.ops.reset_hw(hw);
2229
2230         /* Set all vfta */
2231         igbvf_set_vfta_all(dev,1);
2232
2233         eth_igbvf_tx_init(dev);
2234
2235         /* This can fail when allocating mbufs for descriptor rings */
2236         ret = eth_igbvf_rx_init(dev);
2237         if (ret) {
2238                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
2239                 igb_dev_clear_queues(dev);
2240                 return ret;
2241         }
2242
2243         return 0;
2244 }
2245
2246 static void
2247 igbvf_dev_stop(struct rte_eth_dev *dev)
2248 {
2249         PMD_INIT_FUNC_TRACE();
2250
2251         igbvf_stop_adapter(dev);
2252
2253         /*
2254           * Clear what we set, but we still keep shadow_vfta to
2255           * restore after device starts
2256           */
2257         igbvf_set_vfta_all(dev,0);
2258
2259         igb_dev_clear_queues(dev);
2260 }
2261
2262 static void
2263 igbvf_dev_close(struct rte_eth_dev *dev)
2264 {
2265         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2266
2267         PMD_INIT_FUNC_TRACE();
2268
2269         e1000_reset_hw(hw);
2270
2271         igbvf_dev_stop(dev);
2272 }
2273
2274 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on)
2275 {
2276         struct e1000_mbx_info *mbx = &hw->mbx;
2277         uint32_t msgbuf[2];
2278
2279         /* After set vlan, vlan strip will also be enabled in igb driver*/
2280         msgbuf[0] = E1000_VF_SET_VLAN;
2281         msgbuf[1] = vid;
2282         /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */
2283         if (on)
2284                 msgbuf[0] |= E1000_VF_SET_VLAN_ADD;
2285
2286         return (mbx->ops.write_posted(hw, msgbuf, 2, 0));
2287 }
2288
2289 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on)
2290 {
2291         struct e1000_hw *hw =
2292                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2293         struct e1000_vfta * shadow_vfta =
2294                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2295         int i = 0, j = 0, vfta = 0, mask = 1;
2296
2297         for (i = 0; i < IGB_VFTA_SIZE; i++){
2298                 vfta = shadow_vfta->vfta[i];
2299                 if(vfta){
2300                         mask = 1;
2301                         for (j = 0; j < 32; j++){
2302                                 if(vfta & mask)
2303                                         igbvf_set_vfta(hw,
2304                                                 (uint16_t)((i<<5)+j), on);
2305                                 mask<<=1;
2306                         }
2307                 }
2308         }
2309
2310 }
2311
2312 static int
2313 igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2314 {
2315         struct e1000_hw *hw =
2316                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2317         struct e1000_vfta * shadow_vfta =
2318                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2319         uint32_t vid_idx = 0;
2320         uint32_t vid_bit = 0;
2321         int ret = 0;
2322
2323         PMD_INIT_FUNC_TRACE();
2324
2325         /*vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf*/
2326         ret = igbvf_set_vfta(hw, vlan_id, !!on);
2327         if(ret){
2328                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
2329                 return ret;
2330         }
2331         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
2332         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
2333
2334         /*Save what we set and retore it after device reset*/
2335         if (on)
2336                 shadow_vfta->vfta[vid_idx] |= vid_bit;
2337         else
2338                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
2339
2340         return 0;
2341 }
2342
2343 static int
2344 eth_igb_rss_reta_update(struct rte_eth_dev *dev,
2345                         struct rte_eth_rss_reta_entry64 *reta_conf,
2346                         uint16_t reta_size)
2347 {
2348         uint8_t i, j, mask;
2349         uint32_t reta, r;
2350         uint16_t idx, shift;
2351         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2352
2353         if (reta_size != ETH_RSS_RETA_SIZE_128) {
2354                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2355                         "(%d) doesn't match the number hardware can supported "
2356                         "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
2357                 return -EINVAL;
2358         }
2359
2360         for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) {
2361                 idx = i / RTE_RETA_GROUP_SIZE;
2362                 shift = i % RTE_RETA_GROUP_SIZE;
2363                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2364                                                 IGB_4_BIT_MASK);
2365                 if (!mask)
2366                         continue;
2367                 if (mask == IGB_4_BIT_MASK)
2368                         r = 0;
2369                 else
2370                         r = E1000_READ_REG(hw, E1000_RETA(i >> 2));
2371                 for (j = 0, reta = 0; j < IGB_4_BIT_WIDTH; j++) {
2372                         if (mask & (0x1 << j))
2373                                 reta |= reta_conf[idx].reta[shift + j] <<
2374                                                         (CHAR_BIT * j);
2375                         else
2376                                 reta |= r & (IGB_8_BIT_MASK << (CHAR_BIT * j));
2377                 }
2378                 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta);
2379         }
2380
2381         return 0;
2382 }
2383
2384 static int
2385 eth_igb_rss_reta_query(struct rte_eth_dev *dev,
2386                        struct rte_eth_rss_reta_entry64 *reta_conf,
2387                        uint16_t reta_size)
2388 {
2389         uint8_t i, j, mask;
2390         uint32_t reta;
2391         uint16_t idx, shift;
2392         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2393
2394         if (reta_size != ETH_RSS_RETA_SIZE_128) {
2395                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2396                         "(%d) doesn't match the number hardware can supported "
2397                         "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
2398                 return -EINVAL;
2399         }
2400
2401         for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) {
2402                 idx = i / RTE_RETA_GROUP_SIZE;
2403                 shift = i % RTE_RETA_GROUP_SIZE;
2404                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2405                                                 IGB_4_BIT_MASK);
2406                 if (!mask)
2407                         continue;
2408                 reta = E1000_READ_REG(hw, E1000_RETA(i >> 2));
2409                 for (j = 0; j < IGB_4_BIT_WIDTH; j++) {
2410                         if (mask & (0x1 << j))
2411                                 reta_conf[idx].reta[shift + j] =
2412                                         ((reta >> (CHAR_BIT * j)) &
2413                                                 IGB_8_BIT_MASK);
2414                 }
2415         }
2416
2417         return 0;
2418 }
2419
2420 #define MAC_TYPE_FILTER_SUP(type)    do {\
2421         if ((type) != e1000_82580 && (type) != e1000_i350 &&\
2422                 (type) != e1000_82576)\
2423                 return -ENOTSUP;\
2424 } while (0)
2425
2426 static int
2427 eth_igb_syn_filter_set(struct rte_eth_dev *dev,
2428                         struct rte_eth_syn_filter *filter,
2429                         bool add)
2430 {
2431         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2432         uint32_t synqf, rfctl;
2433
2434         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM)
2435                 return -EINVAL;
2436
2437         synqf = E1000_READ_REG(hw, E1000_SYNQF(0));
2438
2439         if (add) {
2440                 if (synqf & E1000_SYN_FILTER_ENABLE)
2441                         return -EINVAL;
2442
2443                 synqf = (uint32_t)(((filter->queue << E1000_SYN_FILTER_QUEUE_SHIFT) &
2444                         E1000_SYN_FILTER_QUEUE) | E1000_SYN_FILTER_ENABLE);
2445
2446                 rfctl = E1000_READ_REG(hw, E1000_RFCTL);
2447                 if (filter->hig_pri)
2448                         rfctl |= E1000_RFCTL_SYNQFP;
2449                 else
2450                         rfctl &= ~E1000_RFCTL_SYNQFP;
2451
2452                 E1000_WRITE_REG(hw, E1000_RFCTL, rfctl);
2453         } else {
2454                 if (!(synqf & E1000_SYN_FILTER_ENABLE))
2455                         return -ENOENT;
2456                 synqf = 0;
2457         }
2458
2459         E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf);
2460         E1000_WRITE_FLUSH(hw);
2461         return 0;
2462 }
2463
2464 static int
2465 eth_igb_syn_filter_get(struct rte_eth_dev *dev,
2466                         struct rte_eth_syn_filter *filter)
2467 {
2468         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2469         uint32_t synqf, rfctl;
2470
2471         synqf = E1000_READ_REG(hw, E1000_SYNQF(0));
2472         if (synqf & E1000_SYN_FILTER_ENABLE) {
2473                 rfctl = E1000_READ_REG(hw, E1000_RFCTL);
2474                 filter->hig_pri = (rfctl & E1000_RFCTL_SYNQFP) ? 1 : 0;
2475                 filter->queue = (uint8_t)((synqf & E1000_SYN_FILTER_QUEUE) >>
2476                                 E1000_SYN_FILTER_QUEUE_SHIFT);
2477                 return 0;
2478         }
2479
2480         return -ENOENT;
2481 }
2482
2483 static int
2484 eth_igb_syn_filter_handle(struct rte_eth_dev *dev,
2485                         enum rte_filter_op filter_op,
2486                         void *arg)
2487 {
2488         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2489         int ret;
2490
2491         MAC_TYPE_FILTER_SUP(hw->mac.type);
2492
2493         if (filter_op == RTE_ETH_FILTER_NOP)
2494                 return 0;
2495
2496         if (arg == NULL) {
2497                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
2498                             filter_op);
2499                 return -EINVAL;
2500         }
2501
2502         switch (filter_op) {
2503         case RTE_ETH_FILTER_ADD:
2504                 ret = eth_igb_syn_filter_set(dev,
2505                                 (struct rte_eth_syn_filter *)arg,
2506                                 TRUE);
2507                 break;
2508         case RTE_ETH_FILTER_DELETE:
2509                 ret = eth_igb_syn_filter_set(dev,
2510                                 (struct rte_eth_syn_filter *)arg,
2511                                 FALSE);
2512                 break;
2513         case RTE_ETH_FILTER_GET:
2514                 ret = eth_igb_syn_filter_get(dev,
2515                                 (struct rte_eth_syn_filter *)arg);
2516                 break;
2517         default:
2518                 PMD_DRV_LOG(ERR, "unsupported operation %u\n", filter_op);
2519                 ret = -EINVAL;
2520                 break;
2521         }
2522
2523         return ret;
2524 }
2525
2526 #define MAC_TYPE_FILTER_SUP_EXT(type)    do {\
2527         if ((type) != e1000_82580 && (type) != e1000_i350)\
2528                 return -ENOSYS; \
2529 } while (0)
2530
2531 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_2tuple_filter_info*/
2532 static inline int
2533 ntuple_filter_to_2tuple(struct rte_eth_ntuple_filter *filter,
2534                         struct e1000_2tuple_filter_info *filter_info)
2535 {
2536         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM)
2537                 return -EINVAL;
2538         if (filter->priority > E1000_2TUPLE_MAX_PRI)
2539                 return -EINVAL;  /* filter index is out of range. */
2540         if (filter->tcp_flags > TCP_FLAG_ALL)
2541                 return -EINVAL;  /* flags is invalid. */
2542
2543         switch (filter->dst_port_mask) {
2544         case UINT16_MAX:
2545                 filter_info->dst_port_mask = 0;
2546                 filter_info->dst_port = filter->dst_port;
2547                 break;
2548         case 0:
2549                 filter_info->dst_port_mask = 1;
2550                 break;
2551         default:
2552                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
2553                 return -EINVAL;
2554         }
2555
2556         switch (filter->proto_mask) {
2557         case UINT8_MAX:
2558                 filter_info->proto_mask = 0;
2559                 filter_info->proto = filter->proto;
2560                 break;
2561         case 0:
2562                 filter_info->proto_mask = 1;
2563                 break;
2564         default:
2565                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
2566                 return -EINVAL;
2567         }
2568
2569         filter_info->priority = (uint8_t)filter->priority;
2570         if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG)
2571                 filter_info->tcp_flags = filter->tcp_flags;
2572         else
2573                 filter_info->tcp_flags = 0;
2574
2575         return 0;
2576 }
2577
2578 static inline struct e1000_2tuple_filter *
2579 igb_2tuple_filter_lookup(struct e1000_2tuple_filter_list *filter_list,
2580                         struct e1000_2tuple_filter_info *key)
2581 {
2582         struct e1000_2tuple_filter *it;
2583
2584         TAILQ_FOREACH(it, filter_list, entries) {
2585                 if (memcmp(key, &it->filter_info,
2586                         sizeof(struct e1000_2tuple_filter_info)) == 0) {
2587                         return it;
2588                 }
2589         }
2590         return NULL;
2591 }
2592
2593 /*
2594  * igb_add_2tuple_filter - add a 2tuple filter
2595  *
2596  * @param
2597  * dev: Pointer to struct rte_eth_dev.
2598  * ntuple_filter: ponter to the filter that will be added.
2599  *
2600  * @return
2601  *    - On success, zero.
2602  *    - On failure, a negative value.
2603  */
2604 static int
2605 igb_add_2tuple_filter(struct rte_eth_dev *dev,
2606                         struct rte_eth_ntuple_filter *ntuple_filter)
2607 {
2608         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2609         struct e1000_filter_info *filter_info =
2610                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
2611         struct e1000_2tuple_filter *filter;
2612         uint32_t ttqf = E1000_TTQF_DISABLE_MASK;
2613         uint32_t imir, imir_ext = E1000_IMIREXT_SIZE_BP;
2614         int i, ret;
2615
2616         filter = rte_zmalloc("e1000_2tuple_filter",
2617                         sizeof(struct e1000_2tuple_filter), 0);
2618         if (filter == NULL)
2619                 return -ENOMEM;
2620
2621         ret = ntuple_filter_to_2tuple(ntuple_filter,
2622                                       &filter->filter_info);
2623         if (ret < 0) {
2624                 rte_free(filter);
2625                 return ret;
2626         }
2627         if (igb_2tuple_filter_lookup(&filter_info->twotuple_list,
2628                                          &filter->filter_info) != NULL) {
2629                 PMD_DRV_LOG(ERR, "filter exists.");
2630                 rte_free(filter);
2631                 return -EEXIST;
2632         }
2633         filter->queue = ntuple_filter->queue;
2634
2635         /*
2636          * look for an unused 2tuple filter index,
2637          * and insert the filter to list.
2638          */
2639         for (i = 0; i < E1000_MAX_TTQF_FILTERS; i++) {
2640                 if (!(filter_info->twotuple_mask & (1 << i))) {
2641                         filter_info->twotuple_mask |= 1 << i;
2642                         filter->index = i;
2643                         TAILQ_INSERT_TAIL(&filter_info->twotuple_list,
2644                                           filter,
2645                                           entries);
2646                         break;
2647                 }
2648         }
2649         if (i >= E1000_MAX_TTQF_FILTERS) {
2650                 PMD_DRV_LOG(ERR, "2tuple filters are full.");
2651                 rte_free(filter);
2652                 return -ENOSYS;
2653         }
2654
2655         imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT);
2656         if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */
2657                 imir |= E1000_IMIR_PORT_BP;
2658         else
2659                 imir &= ~E1000_IMIR_PORT_BP;
2660
2661         imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT;
2662
2663         ttqf |= E1000_TTQF_QUEUE_ENABLE;
2664         ttqf |= (uint32_t)(filter->queue << E1000_TTQF_QUEUE_SHIFT);
2665         ttqf |= (uint32_t)(filter->filter_info.proto & E1000_TTQF_PROTOCOL_MASK);
2666         if (filter->filter_info.proto_mask == 0)
2667                 ttqf &= ~E1000_TTQF_MASK_ENABLE;
2668
2669         /* tcp flags bits setting. */
2670         if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) {
2671                 if (filter->filter_info.tcp_flags & TCP_URG_FLAG)
2672                         imir_ext |= E1000_IMIREXT_CTRL_URG;
2673                 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG)
2674                         imir_ext |= E1000_IMIREXT_CTRL_ACK;
2675                 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG)
2676                         imir_ext |= E1000_IMIREXT_CTRL_PSH;
2677                 if (filter->filter_info.tcp_flags & TCP_RST_FLAG)
2678                         imir_ext |= E1000_IMIREXT_CTRL_RST;
2679                 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG)
2680                         imir_ext |= E1000_IMIREXT_CTRL_SYN;
2681                 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG)
2682                         imir_ext |= E1000_IMIREXT_CTRL_FIN;
2683         } else
2684                 imir_ext |= E1000_IMIREXT_CTRL_BP;
2685         E1000_WRITE_REG(hw, E1000_IMIR(i), imir);
2686         E1000_WRITE_REG(hw, E1000_TTQF(i), ttqf);
2687         E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext);
2688         return 0;
2689 }
2690
2691 /*
2692  * igb_remove_2tuple_filter - remove a 2tuple filter
2693  *
2694  * @param
2695  * dev: Pointer to struct rte_eth_dev.
2696  * ntuple_filter: ponter to the filter that will be removed.
2697  *
2698  * @return
2699  *    - On success, zero.
2700  *    - On failure, a negative value.
2701  */
2702 static int
2703 igb_remove_2tuple_filter(struct rte_eth_dev *dev,
2704                         struct rte_eth_ntuple_filter *ntuple_filter)
2705 {
2706         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2707         struct e1000_filter_info *filter_info =
2708                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
2709         struct e1000_2tuple_filter_info filter_2tuple;
2710         struct e1000_2tuple_filter *filter;
2711         int ret;
2712
2713         memset(&filter_2tuple, 0, sizeof(struct e1000_2tuple_filter_info));
2714         ret = ntuple_filter_to_2tuple(ntuple_filter,
2715                                       &filter_2tuple);
2716         if (ret < 0)
2717                 return ret;
2718
2719         filter = igb_2tuple_filter_lookup(&filter_info->twotuple_list,
2720                                          &filter_2tuple);
2721         if (filter == NULL) {
2722                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
2723                 return -ENOENT;
2724         }
2725
2726         filter_info->twotuple_mask &= ~(1 << filter->index);
2727         TAILQ_REMOVE(&filter_info->twotuple_list, filter, entries);
2728         rte_free(filter);
2729
2730         E1000_WRITE_REG(hw, E1000_TTQF(filter->index), E1000_TTQF_DISABLE_MASK);
2731         E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0);
2732         E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0);
2733         return 0;
2734 }
2735
2736 static inline struct e1000_flex_filter *
2737 eth_igb_flex_filter_lookup(struct e1000_flex_filter_list *filter_list,
2738                         struct e1000_flex_filter_info *key)
2739 {
2740         struct e1000_flex_filter *it;
2741
2742         TAILQ_FOREACH(it, filter_list, entries) {
2743                 if (memcmp(key, &it->filter_info,
2744                         sizeof(struct e1000_flex_filter_info)) == 0)
2745                         return it;
2746         }
2747
2748         return NULL;
2749 }
2750
2751 static int
2752 eth_igb_add_del_flex_filter(struct rte_eth_dev *dev,
2753                         struct rte_eth_flex_filter *filter,
2754                         bool add)
2755 {
2756         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2757         struct e1000_filter_info *filter_info =
2758                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
2759         struct e1000_flex_filter *flex_filter, *it;
2760         uint32_t wufc, queueing, mask;
2761         uint32_t reg_off;
2762         uint8_t shift, i, j = 0;
2763
2764         flex_filter = rte_zmalloc("e1000_flex_filter",
2765                         sizeof(struct e1000_flex_filter), 0);
2766         if (flex_filter == NULL)
2767                 return -ENOMEM;
2768
2769         flex_filter->filter_info.len = filter->len;
2770         flex_filter->filter_info.priority = filter->priority;
2771         memcpy(flex_filter->filter_info.dwords, filter->bytes, filter->len);
2772         for (i = 0; i < RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT; i++) {
2773                 mask = 0;
2774                 /* reverse bits in flex filter's mask*/
2775                 for (shift = 0; shift < CHAR_BIT; shift++) {
2776                         if (filter->mask[i] & (0x01 << shift))
2777                                 mask |= (0x80 >> shift);
2778                 }
2779                 flex_filter->filter_info.mask[i] = mask;
2780         }
2781
2782         wufc = E1000_READ_REG(hw, E1000_WUFC);
2783         if (flex_filter->index < E1000_MAX_FHFT)
2784                 reg_off = E1000_FHFT(flex_filter->index);
2785         else
2786                 reg_off = E1000_FHFT_EXT(flex_filter->index - E1000_MAX_FHFT);
2787
2788         if (add) {
2789                 if (eth_igb_flex_filter_lookup(&filter_info->flex_list,
2790                                 &flex_filter->filter_info) != NULL) {
2791                         PMD_DRV_LOG(ERR, "filter exists.");
2792                         rte_free(flex_filter);
2793                         return -EEXIST;
2794                 }
2795                 flex_filter->queue = filter->queue;
2796                 /*
2797                  * look for an unused flex filter index
2798                  * and insert the filter into the list.
2799                  */
2800                 for (i = 0; i < E1000_MAX_FLEX_FILTERS; i++) {
2801                         if (!(filter_info->flex_mask & (1 << i))) {
2802                                 filter_info->flex_mask |= 1 << i;
2803                                 flex_filter->index = i;
2804                                 TAILQ_INSERT_TAIL(&filter_info->flex_list,
2805                                         flex_filter,
2806                                         entries);
2807                                 break;
2808                         }
2809                 }
2810                 if (i >= E1000_MAX_FLEX_FILTERS) {
2811                         PMD_DRV_LOG(ERR, "flex filters are full.");
2812                         rte_free(flex_filter);
2813                         return -ENOSYS;
2814                 }
2815
2816                 E1000_WRITE_REG(hw, E1000_WUFC, wufc | E1000_WUFC_FLEX_HQ |
2817                                 (E1000_WUFC_FLX0 << flex_filter->index));
2818                 queueing = filter->len |
2819                         (filter->queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) |
2820                         (filter->priority << E1000_FHFT_QUEUEING_PRIO_SHIFT);
2821                 E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET,
2822                                 queueing);
2823                 for (i = 0; i < E1000_FLEX_FILTERS_MASK_SIZE; i++) {
2824                         E1000_WRITE_REG(hw, reg_off,
2825                                         flex_filter->filter_info.dwords[j]);
2826                         reg_off += sizeof(uint32_t);
2827                         E1000_WRITE_REG(hw, reg_off,
2828                                         flex_filter->filter_info.dwords[++j]);
2829                         reg_off += sizeof(uint32_t);
2830                         E1000_WRITE_REG(hw, reg_off,
2831                                 (uint32_t)flex_filter->filter_info.mask[i]);
2832                         reg_off += sizeof(uint32_t) * 2;
2833                         ++j;
2834                 }
2835         } else {
2836                 it = eth_igb_flex_filter_lookup(&filter_info->flex_list,
2837                                 &flex_filter->filter_info);
2838                 if (it == NULL) {
2839                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
2840                         rte_free(flex_filter);
2841                         return -ENOENT;
2842                 }
2843
2844                 for (i = 0; i < E1000_FHFT_SIZE_IN_DWD; i++)
2845                         E1000_WRITE_REG(hw, reg_off + i * sizeof(uint32_t), 0);
2846                 E1000_WRITE_REG(hw, E1000_WUFC, wufc &
2847                         (~(E1000_WUFC_FLX0 << it->index)));
2848
2849                 filter_info->flex_mask &= ~(1 << it->index);
2850                 TAILQ_REMOVE(&filter_info->flex_list, it, entries);
2851                 rte_free(it);
2852                 rte_free(flex_filter);
2853         }
2854
2855         return 0;
2856 }
2857
2858 static int
2859 eth_igb_get_flex_filter(struct rte_eth_dev *dev,
2860                         struct rte_eth_flex_filter *filter)
2861 {
2862         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2863         struct e1000_filter_info *filter_info =
2864                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
2865         struct e1000_flex_filter flex_filter, *it;
2866         uint32_t wufc, queueing, wufc_en = 0;
2867
2868         memset(&flex_filter, 0, sizeof(struct e1000_flex_filter));
2869         flex_filter.filter_info.len = filter->len;
2870         flex_filter.filter_info.priority = filter->priority;
2871         memcpy(flex_filter.filter_info.dwords, filter->bytes, filter->len);
2872         memcpy(flex_filter.filter_info.mask, filter->mask,
2873                         RTE_ALIGN(filter->len, sizeof(char)) / sizeof(char));
2874
2875         it = eth_igb_flex_filter_lookup(&filter_info->flex_list,
2876                                 &flex_filter.filter_info);
2877         if (it == NULL) {
2878                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
2879                 return -ENOENT;
2880         }
2881
2882         wufc = E1000_READ_REG(hw, E1000_WUFC);
2883         wufc_en = E1000_WUFC_FLEX_HQ | (E1000_WUFC_FLX0 << it->index);
2884
2885         if ((wufc & wufc_en) == wufc_en) {
2886                 uint32_t reg_off = 0;
2887                 if (it->index < E1000_MAX_FHFT)
2888                         reg_off = E1000_FHFT(it->index);
2889                 else
2890                         reg_off = E1000_FHFT_EXT(it->index - E1000_MAX_FHFT);
2891
2892                 queueing = E1000_READ_REG(hw,
2893                                 reg_off + E1000_FHFT_QUEUEING_OFFSET);
2894                 filter->len = queueing & E1000_FHFT_QUEUEING_LEN;
2895                 filter->priority = (queueing & E1000_FHFT_QUEUEING_PRIO) >>
2896                         E1000_FHFT_QUEUEING_PRIO_SHIFT;
2897                 filter->queue = (queueing & E1000_FHFT_QUEUEING_QUEUE) >>
2898                         E1000_FHFT_QUEUEING_QUEUE_SHIFT;
2899                 return 0;
2900         }
2901         return -ENOENT;
2902 }
2903
2904 static int
2905 eth_igb_flex_filter_handle(struct rte_eth_dev *dev,
2906                         enum rte_filter_op filter_op,
2907                         void *arg)
2908 {
2909         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2910         struct rte_eth_flex_filter *filter;
2911         int ret = 0;
2912
2913         MAC_TYPE_FILTER_SUP(hw->mac.type);
2914
2915         if (filter_op == RTE_ETH_FILTER_NOP)
2916                 return ret;
2917
2918         if (arg == NULL) {
2919                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
2920                             filter_op);
2921                 return -EINVAL;
2922         }
2923
2924         filter = (struct rte_eth_flex_filter *)arg;
2925         if (filter->len == 0 || filter->len > E1000_MAX_FLEX_FILTER_LEN
2926             || filter->len % sizeof(uint64_t) != 0) {
2927                 PMD_DRV_LOG(ERR, "filter's length is out of range");
2928                 return -EINVAL;
2929         }
2930         if (filter->priority > E1000_MAX_FLEX_FILTER_PRI) {
2931                 PMD_DRV_LOG(ERR, "filter's priority is out of range");
2932                 return -EINVAL;
2933         }
2934
2935         switch (filter_op) {
2936         case RTE_ETH_FILTER_ADD:
2937                 ret = eth_igb_add_del_flex_filter(dev, filter, TRUE);
2938                 break;
2939         case RTE_ETH_FILTER_DELETE:
2940                 ret = eth_igb_add_del_flex_filter(dev, filter, FALSE);
2941                 break;
2942         case RTE_ETH_FILTER_GET:
2943                 ret = eth_igb_get_flex_filter(dev, filter);
2944                 break;
2945         default:
2946                 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op);
2947                 ret = -EINVAL;
2948                 break;
2949         }
2950
2951         return ret;
2952 }
2953
2954 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_5tuple_filter_info*/
2955 static inline int
2956 ntuple_filter_to_5tuple_82576(struct rte_eth_ntuple_filter *filter,
2957                         struct e1000_5tuple_filter_info *filter_info)
2958 {
2959         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576)
2960                 return -EINVAL;
2961         if (filter->priority > E1000_2TUPLE_MAX_PRI)
2962                 return -EINVAL;  /* filter index is out of range. */
2963         if (filter->tcp_flags > TCP_FLAG_ALL)
2964                 return -EINVAL;  /* flags is invalid. */
2965
2966         switch (filter->dst_ip_mask) {
2967         case UINT32_MAX:
2968                 filter_info->dst_ip_mask = 0;
2969                 filter_info->dst_ip = filter->dst_ip;
2970                 break;
2971         case 0:
2972                 filter_info->dst_ip_mask = 1;
2973                 break;
2974         default:
2975                 PMD_DRV_LOG(ERR, "invalid dst_ip mask.");
2976                 return -EINVAL;
2977         }
2978
2979         switch (filter->src_ip_mask) {
2980         case UINT32_MAX:
2981                 filter_info->src_ip_mask = 0;
2982                 filter_info->src_ip = filter->src_ip;
2983                 break;
2984         case 0:
2985                 filter_info->src_ip_mask = 1;
2986                 break;
2987         default:
2988                 PMD_DRV_LOG(ERR, "invalid src_ip mask.");
2989                 return -EINVAL;
2990         }
2991
2992         switch (filter->dst_port_mask) {
2993         case UINT16_MAX:
2994                 filter_info->dst_port_mask = 0;
2995                 filter_info->dst_port = filter->dst_port;
2996                 break;
2997         case 0:
2998                 filter_info->dst_port_mask = 1;
2999                 break;
3000         default:
3001                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
3002                 return -EINVAL;
3003         }
3004
3005         switch (filter->src_port_mask) {
3006         case UINT16_MAX:
3007                 filter_info->src_port_mask = 0;
3008                 filter_info->src_port = filter->src_port;
3009                 break;
3010         case 0:
3011                 filter_info->src_port_mask = 1;
3012                 break;
3013         default:
3014                 PMD_DRV_LOG(ERR, "invalid src_port mask.");
3015                 return -EINVAL;
3016         }
3017
3018         switch (filter->proto_mask) {
3019         case UINT8_MAX:
3020                 filter_info->proto_mask = 0;
3021                 filter_info->proto = filter->proto;
3022                 break;
3023         case 0:
3024                 filter_info->proto_mask = 1;
3025                 break;
3026         default:
3027                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
3028                 return -EINVAL;
3029         }
3030
3031         filter_info->priority = (uint8_t)filter->priority;
3032         if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG)
3033                 filter_info->tcp_flags = filter->tcp_flags;
3034         else
3035                 filter_info->tcp_flags = 0;
3036
3037         return 0;
3038 }
3039
3040 static inline struct e1000_5tuple_filter *
3041 igb_5tuple_filter_lookup_82576(struct e1000_5tuple_filter_list *filter_list,
3042                         struct e1000_5tuple_filter_info *key)
3043 {
3044         struct e1000_5tuple_filter *it;
3045
3046         TAILQ_FOREACH(it, filter_list, entries) {
3047                 if (memcmp(key, &it->filter_info,
3048                         sizeof(struct e1000_5tuple_filter_info)) == 0) {
3049                         return it;
3050                 }
3051         }
3052         return NULL;
3053 }
3054
3055 /*
3056  * igb_add_5tuple_filter_82576 - add a 5tuple filter
3057  *
3058  * @param
3059  * dev: Pointer to struct rte_eth_dev.
3060  * ntuple_filter: ponter to the filter that will be added.
3061  *
3062  * @return
3063  *    - On success, zero.
3064  *    - On failure, a negative value.
3065  */
3066 static int
3067 igb_add_5tuple_filter_82576(struct rte_eth_dev *dev,
3068                         struct rte_eth_ntuple_filter *ntuple_filter)
3069 {
3070         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3071         struct e1000_filter_info *filter_info =
3072                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3073         struct e1000_5tuple_filter *filter;
3074         uint32_t ftqf = E1000_FTQF_VF_BP | E1000_FTQF_MASK;
3075         uint32_t spqf, imir, imir_ext = E1000_IMIREXT_SIZE_BP;
3076         uint8_t i;
3077         int ret;
3078
3079         filter = rte_zmalloc("e1000_5tuple_filter",
3080                         sizeof(struct e1000_5tuple_filter), 0);
3081         if (filter == NULL)
3082                 return -ENOMEM;
3083
3084         ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
3085                                             &filter->filter_info);
3086         if (ret < 0) {
3087                 rte_free(filter);
3088                 return ret;
3089         }
3090
3091         if (igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list,
3092                                          &filter->filter_info) != NULL) {
3093                 PMD_DRV_LOG(ERR, "filter exists.");
3094                 rte_free(filter);
3095                 return -EEXIST;
3096         }
3097         filter->queue = ntuple_filter->queue;
3098
3099         /*
3100          * look for an unused 5tuple filter index,
3101          * and insert the filter to list.
3102          */
3103         for (i = 0; i < E1000_MAX_FTQF_FILTERS; i++) {
3104                 if (!(filter_info->fivetuple_mask & (1 << i))) {
3105                         filter_info->fivetuple_mask |= 1 << i;
3106                         filter->index = i;
3107                         TAILQ_INSERT_TAIL(&filter_info->fivetuple_list,
3108                                           filter,
3109                                           entries);
3110                         break;
3111                 }
3112         }
3113         if (i >= E1000_MAX_FTQF_FILTERS) {
3114                 PMD_DRV_LOG(ERR, "5tuple filters are full.");
3115                 rte_free(filter);
3116                 return -ENOSYS;
3117         }
3118
3119         ftqf |= filter->filter_info.proto & E1000_FTQF_PROTOCOL_MASK;
3120         if (filter->filter_info.src_ip_mask == 0) /* 0b means compare. */
3121                 ftqf &= ~E1000_FTQF_MASK_SOURCE_ADDR_BP;
3122         if (filter->filter_info.dst_ip_mask == 0)
3123                 ftqf &= ~E1000_FTQF_MASK_DEST_ADDR_BP;
3124         if (filter->filter_info.src_port_mask == 0)
3125                 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
3126         if (filter->filter_info.proto_mask == 0)
3127                 ftqf &= ~E1000_FTQF_MASK_PROTO_BP;
3128         ftqf |= (filter->queue << E1000_FTQF_QUEUE_SHIFT) &
3129                 E1000_FTQF_QUEUE_MASK;
3130         ftqf |= E1000_FTQF_QUEUE_ENABLE;
3131         E1000_WRITE_REG(hw, E1000_FTQF(i), ftqf);
3132         E1000_WRITE_REG(hw, E1000_DAQF(i), filter->filter_info.dst_ip);
3133         E1000_WRITE_REG(hw, E1000_SAQF(i), filter->filter_info.src_ip);
3134
3135         spqf = filter->filter_info.src_port & E1000_SPQF_SRCPORT;
3136         E1000_WRITE_REG(hw, E1000_SPQF(i), spqf);
3137
3138         imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT);
3139         if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */
3140                 imir |= E1000_IMIR_PORT_BP;
3141         else
3142                 imir &= ~E1000_IMIR_PORT_BP;
3143         imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT;
3144
3145         /* tcp flags bits setting. */
3146         if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) {
3147                 if (filter->filter_info.tcp_flags & TCP_URG_FLAG)
3148                         imir_ext |= E1000_IMIREXT_CTRL_URG;
3149                 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG)
3150                         imir_ext |= E1000_IMIREXT_CTRL_ACK;
3151                 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG)
3152                         imir_ext |= E1000_IMIREXT_CTRL_PSH;
3153                 if (filter->filter_info.tcp_flags & TCP_RST_FLAG)
3154                         imir_ext |= E1000_IMIREXT_CTRL_RST;
3155                 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG)
3156                         imir_ext |= E1000_IMIREXT_CTRL_SYN;
3157                 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG)
3158                         imir_ext |= E1000_IMIREXT_CTRL_FIN;
3159         } else
3160                 imir_ext |= E1000_IMIREXT_CTRL_BP;
3161         E1000_WRITE_REG(hw, E1000_IMIR(i), imir);
3162         E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext);
3163         return 0;
3164 }
3165
3166 /*
3167  * igb_remove_5tuple_filter_82576 - remove a 5tuple filter
3168  *
3169  * @param
3170  * dev: Pointer to struct rte_eth_dev.
3171  * ntuple_filter: ponter to the filter that will be removed.
3172  *
3173  * @return
3174  *    - On success, zero.
3175  *    - On failure, a negative value.
3176  */
3177 static int
3178 igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev,
3179                                 struct rte_eth_ntuple_filter *ntuple_filter)
3180 {
3181         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3182         struct e1000_filter_info *filter_info =
3183                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3184         struct e1000_5tuple_filter_info filter_5tuple;
3185         struct e1000_5tuple_filter *filter;
3186         int ret;
3187
3188         memset(&filter_5tuple, 0, sizeof(struct e1000_5tuple_filter_info));
3189         ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
3190                                             &filter_5tuple);
3191         if (ret < 0)
3192                 return ret;
3193
3194         filter = igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list,
3195                                          &filter_5tuple);
3196         if (filter == NULL) {
3197                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
3198                 return -ENOENT;
3199         }
3200
3201         filter_info->fivetuple_mask &= ~(1 << filter->index);
3202         TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries);
3203         rte_free(filter);
3204
3205         E1000_WRITE_REG(hw, E1000_FTQF(filter->index),
3206                         E1000_FTQF_VF_BP | E1000_FTQF_MASK);
3207         E1000_WRITE_REG(hw, E1000_DAQF(filter->index), 0);
3208         E1000_WRITE_REG(hw, E1000_SAQF(filter->index), 0);
3209         E1000_WRITE_REG(hw, E1000_SPQF(filter->index), 0);
3210         E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0);
3211         E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0);
3212         return 0;
3213 }
3214
3215 static int
3216 eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
3217 {
3218         uint32_t rctl;
3219         struct e1000_hw *hw;
3220         struct rte_eth_dev_info dev_info;
3221         uint32_t frame_size = mtu + (ETHER_HDR_LEN + ETHER_CRC_LEN +
3222                                      VLAN_TAG_SIZE);
3223
3224         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3225
3226 #ifdef RTE_LIBRTE_82571_SUPPORT
3227         /* XXX: not bigger than max_rx_pktlen */
3228         if (hw->mac.type == e1000_82571)
3229                 return -ENOTSUP;
3230 #endif
3231         eth_igb_infos_get(dev, &dev_info);
3232
3233         /* check that mtu is within the allowed range */
3234         if ((mtu < ETHER_MIN_MTU) ||
3235             (frame_size > dev_info.max_rx_pktlen))
3236                 return -EINVAL;
3237
3238         /* refuse mtu that requires the support of scattered packets when this
3239          * feature has not been enabled before. */
3240         if (!dev->data->scattered_rx &&
3241             frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)
3242                 return -EINVAL;
3243
3244         rctl = E1000_READ_REG(hw, E1000_RCTL);
3245
3246         /* switch to jumbo mode if needed */
3247         if (frame_size > ETHER_MAX_LEN) {
3248                 dev->data->dev_conf.rxmode.jumbo_frame = 1;
3249                 rctl |= E1000_RCTL_LPE;
3250         } else {
3251                 dev->data->dev_conf.rxmode.jumbo_frame = 0;
3252                 rctl &= ~E1000_RCTL_LPE;
3253         }
3254         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
3255
3256         /* update max frame size */
3257         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
3258
3259         E1000_WRITE_REG(hw, E1000_RLPML,
3260                         dev->data->dev_conf.rxmode.max_rx_pkt_len);
3261
3262         return 0;
3263 }
3264
3265 /*
3266  * igb_add_del_ntuple_filter - add or delete a ntuple filter
3267  *
3268  * @param
3269  * dev: Pointer to struct rte_eth_dev.
3270  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
3271  * add: if true, add filter, if false, remove filter
3272  *
3273  * @return
3274  *    - On success, zero.
3275  *    - On failure, a negative value.
3276  */
3277 static int
3278 igb_add_del_ntuple_filter(struct rte_eth_dev *dev,
3279                         struct rte_eth_ntuple_filter *ntuple_filter,
3280                         bool add)
3281 {
3282         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3283         int ret;
3284
3285         switch (ntuple_filter->flags) {
3286         case RTE_5TUPLE_FLAGS:
3287         case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3288                 if (hw->mac.type != e1000_82576)
3289                         return -ENOTSUP;
3290                 if (add)
3291                         ret = igb_add_5tuple_filter_82576(dev,
3292                                                           ntuple_filter);
3293                 else
3294                         ret = igb_remove_5tuple_filter_82576(dev,
3295                                                              ntuple_filter);
3296                 break;
3297         case RTE_2TUPLE_FLAGS:
3298         case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3299                 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350)
3300                         return -ENOTSUP;
3301                 if (add)
3302                         ret = igb_add_2tuple_filter(dev, ntuple_filter);
3303                 else
3304                         ret = igb_remove_2tuple_filter(dev, ntuple_filter);
3305                 break;
3306         default:
3307                 ret = -EINVAL;
3308                 break;
3309         }
3310
3311         return ret;
3312 }
3313
3314 /*
3315  * igb_get_ntuple_filter - get a ntuple filter
3316  *
3317  * @param
3318  * dev: Pointer to struct rte_eth_dev.
3319  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
3320  *
3321  * @return
3322  *    - On success, zero.
3323  *    - On failure, a negative value.
3324  */
3325 static int
3326 igb_get_ntuple_filter(struct rte_eth_dev *dev,
3327                         struct rte_eth_ntuple_filter *ntuple_filter)
3328 {
3329         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3330         struct e1000_filter_info *filter_info =
3331                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3332         struct e1000_5tuple_filter_info filter_5tuple;
3333         struct e1000_2tuple_filter_info filter_2tuple;
3334         struct e1000_5tuple_filter *p_5tuple_filter;
3335         struct e1000_2tuple_filter *p_2tuple_filter;
3336         int ret;
3337
3338         switch (ntuple_filter->flags) {
3339         case RTE_5TUPLE_FLAGS:
3340         case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3341                 if (hw->mac.type != e1000_82576)
3342                         return -ENOTSUP;
3343                 memset(&filter_5tuple,
3344                         0,
3345                         sizeof(struct e1000_5tuple_filter_info));
3346                 ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
3347                                                     &filter_5tuple);
3348                 if (ret < 0)
3349                         return ret;
3350                 p_5tuple_filter = igb_5tuple_filter_lookup_82576(
3351                                         &filter_info->fivetuple_list,
3352                                         &filter_5tuple);
3353                 if (p_5tuple_filter == NULL) {
3354                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
3355                         return -ENOENT;
3356                 }
3357                 ntuple_filter->queue = p_5tuple_filter->queue;
3358                 break;
3359         case RTE_2TUPLE_FLAGS:
3360         case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3361                 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350)
3362                         return -ENOTSUP;
3363                 memset(&filter_2tuple,
3364                         0,
3365                         sizeof(struct e1000_2tuple_filter_info));
3366                 ret = ntuple_filter_to_2tuple(ntuple_filter, &filter_2tuple);
3367                 if (ret < 0)
3368                         return ret;
3369                 p_2tuple_filter = igb_2tuple_filter_lookup(
3370                                         &filter_info->twotuple_list,
3371                                         &filter_2tuple);
3372                 if (p_2tuple_filter == NULL) {
3373                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
3374                         return -ENOENT;
3375                 }
3376                 ntuple_filter->queue = p_2tuple_filter->queue;
3377                 break;
3378         default:
3379                 ret = -EINVAL;
3380                 break;
3381         }
3382
3383         return 0;
3384 }
3385
3386 /*
3387  * igb_ntuple_filter_handle - Handle operations for ntuple filter.
3388  * @dev: pointer to rte_eth_dev structure
3389  * @filter_op:operation will be taken.
3390  * @arg: a pointer to specific structure corresponding to the filter_op
3391  */
3392 static int
3393 igb_ntuple_filter_handle(struct rte_eth_dev *dev,
3394                                 enum rte_filter_op filter_op,
3395                                 void *arg)
3396 {
3397         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3398         int ret;
3399
3400         MAC_TYPE_FILTER_SUP(hw->mac.type);
3401
3402         if (filter_op == RTE_ETH_FILTER_NOP)
3403                 return 0;
3404
3405         if (arg == NULL) {
3406                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
3407                             filter_op);
3408                 return -EINVAL;
3409         }
3410
3411         switch (filter_op) {
3412         case RTE_ETH_FILTER_ADD:
3413                 ret = igb_add_del_ntuple_filter(dev,
3414                         (struct rte_eth_ntuple_filter *)arg,
3415                         TRUE);
3416                 break;
3417         case RTE_ETH_FILTER_DELETE:
3418                 ret = igb_add_del_ntuple_filter(dev,
3419                         (struct rte_eth_ntuple_filter *)arg,
3420                         FALSE);
3421                 break;
3422         case RTE_ETH_FILTER_GET:
3423                 ret = igb_get_ntuple_filter(dev,
3424                         (struct rte_eth_ntuple_filter *)arg);
3425                 break;
3426         default:
3427                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
3428                 ret = -EINVAL;
3429                 break;
3430         }
3431         return ret;
3432 }
3433
3434 static inline int
3435 igb_ethertype_filter_lookup(struct e1000_filter_info *filter_info,
3436                         uint16_t ethertype)
3437 {
3438         int i;
3439
3440         for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) {
3441                 if (filter_info->ethertype_filters[i] == ethertype &&
3442                     (filter_info->ethertype_mask & (1 << i)))
3443                         return i;
3444         }
3445         return -1;
3446 }
3447
3448 static inline int
3449 igb_ethertype_filter_insert(struct e1000_filter_info *filter_info,
3450                         uint16_t ethertype)
3451 {
3452         int i;
3453
3454         for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) {
3455                 if (!(filter_info->ethertype_mask & (1 << i))) {
3456                         filter_info->ethertype_mask |= 1 << i;
3457                         filter_info->ethertype_filters[i] = ethertype;
3458                         return i;
3459                 }
3460         }
3461         return -1;
3462 }
3463
3464 static inline int
3465 igb_ethertype_filter_remove(struct e1000_filter_info *filter_info,
3466                         uint8_t idx)
3467 {
3468         if (idx >= E1000_MAX_ETQF_FILTERS)
3469                 return -1;
3470         filter_info->ethertype_mask &= ~(1 << idx);
3471         filter_info->ethertype_filters[idx] = 0;
3472         return idx;
3473 }
3474
3475
3476 static int
3477 igb_add_del_ethertype_filter(struct rte_eth_dev *dev,
3478                         struct rte_eth_ethertype_filter *filter,
3479                         bool add)
3480 {
3481         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3482         struct e1000_filter_info *filter_info =
3483                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3484         uint32_t etqf = 0;
3485         int ret;
3486
3487         if (filter->ether_type == ETHER_TYPE_IPv4 ||
3488                 filter->ether_type == ETHER_TYPE_IPv6) {
3489                 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in"
3490                         " ethertype filter.", filter->ether_type);
3491                 return -EINVAL;
3492         }
3493
3494         if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) {
3495                 PMD_DRV_LOG(ERR, "mac compare is unsupported.");
3496                 return -EINVAL;
3497         }
3498         if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) {
3499                 PMD_DRV_LOG(ERR, "drop option is unsupported.");
3500                 return -EINVAL;
3501         }
3502
3503         ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type);
3504         if (ret >= 0 && add) {
3505                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.",
3506                             filter->ether_type);
3507                 return -EEXIST;
3508         }
3509         if (ret < 0 && !add) {
3510                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
3511                             filter->ether_type);
3512                 return -ENOENT;
3513         }
3514
3515         if (add) {
3516                 ret = igb_ethertype_filter_insert(filter_info,
3517                         filter->ether_type);
3518                 if (ret < 0) {
3519                         PMD_DRV_LOG(ERR, "ethertype filters are full.");
3520                         return -ENOSYS;
3521                 }
3522
3523                 etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE;
3524                 etqf |= (uint32_t)(filter->ether_type & E1000_ETQF_ETHERTYPE);
3525                 etqf |= filter->queue << E1000_ETQF_QUEUE_SHIFT;
3526         } else {
3527                 ret = igb_ethertype_filter_remove(filter_info, (uint8_t)ret);
3528                 if (ret < 0)
3529                         return -ENOSYS;
3530         }
3531         E1000_WRITE_REG(hw, E1000_ETQF(ret), etqf);
3532         E1000_WRITE_FLUSH(hw);
3533
3534         return 0;
3535 }
3536
3537 static int
3538 igb_get_ethertype_filter(struct rte_eth_dev *dev,
3539                         struct rte_eth_ethertype_filter *filter)
3540 {
3541         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3542         struct e1000_filter_info *filter_info =
3543                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3544         uint32_t etqf;
3545         int ret;
3546
3547         ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type);
3548         if (ret < 0) {
3549                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
3550                             filter->ether_type);
3551                 return -ENOENT;
3552         }
3553
3554         etqf = E1000_READ_REG(hw, E1000_ETQF(ret));
3555         if (etqf & E1000_ETQF_FILTER_ENABLE) {
3556                 filter->ether_type = etqf & E1000_ETQF_ETHERTYPE;
3557                 filter->flags = 0;
3558                 filter->queue = (etqf & E1000_ETQF_QUEUE) >>
3559                                 E1000_ETQF_QUEUE_SHIFT;
3560                 return 0;
3561         }
3562
3563         return -ENOENT;
3564 }
3565
3566 /*
3567  * igb_ethertype_filter_handle - Handle operations for ethertype filter.
3568  * @dev: pointer to rte_eth_dev structure
3569  * @filter_op:operation will be taken.
3570  * @arg: a pointer to specific structure corresponding to the filter_op
3571  */
3572 static int
3573 igb_ethertype_filter_handle(struct rte_eth_dev *dev,
3574                                 enum rte_filter_op filter_op,
3575                                 void *arg)
3576 {
3577         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3578         int ret;
3579
3580         MAC_TYPE_FILTER_SUP(hw->mac.type);
3581
3582         if (filter_op == RTE_ETH_FILTER_NOP)
3583                 return 0;
3584
3585         if (arg == NULL) {
3586                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
3587                             filter_op);
3588                 return -EINVAL;
3589         }
3590
3591         switch (filter_op) {
3592         case RTE_ETH_FILTER_ADD:
3593                 ret = igb_add_del_ethertype_filter(dev,
3594                         (struct rte_eth_ethertype_filter *)arg,
3595                         TRUE);
3596                 break;
3597         case RTE_ETH_FILTER_DELETE:
3598                 ret = igb_add_del_ethertype_filter(dev,
3599                         (struct rte_eth_ethertype_filter *)arg,
3600                         FALSE);
3601                 break;
3602         case RTE_ETH_FILTER_GET:
3603                 ret = igb_get_ethertype_filter(dev,
3604                         (struct rte_eth_ethertype_filter *)arg);
3605                 break;
3606         default:
3607                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
3608                 ret = -EINVAL;
3609                 break;
3610         }
3611         return ret;
3612 }
3613
3614 static int
3615 eth_igb_filter_ctrl(struct rte_eth_dev *dev,
3616                      enum rte_filter_type filter_type,
3617                      enum rte_filter_op filter_op,
3618                      void *arg)
3619 {
3620         int ret = -EINVAL;
3621
3622         switch (filter_type) {
3623         case RTE_ETH_FILTER_NTUPLE:
3624                 ret = igb_ntuple_filter_handle(dev, filter_op, arg);
3625                 break;
3626         case RTE_ETH_FILTER_ETHERTYPE:
3627                 ret = igb_ethertype_filter_handle(dev, filter_op, arg);
3628                 break;
3629         case RTE_ETH_FILTER_SYN:
3630                 ret = eth_igb_syn_filter_handle(dev, filter_op, arg);
3631                 break;
3632         case RTE_ETH_FILTER_FLEXIBLE:
3633                 ret = eth_igb_flex_filter_handle(dev, filter_op, arg);
3634                 break;
3635         default:
3636                 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported",
3637                                                         filter_type);
3638                 break;
3639         }
3640
3641         return ret;
3642 }
3643
3644 static struct rte_driver pmd_igb_drv = {
3645         .type = PMD_PDEV,
3646         .init = rte_igb_pmd_init,
3647 };
3648
3649 static struct rte_driver pmd_igbvf_drv = {
3650         .type = PMD_PDEV,
3651         .init = rte_igbvf_pmd_init,
3652 };
3653
3654 PMD_REGISTER_DRIVER(pmd_igb_drv);
3655 PMD_REGISTER_DRIVER(pmd_igbvf_drv);