drivers/net/txgbe/txgbe_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2020
   3  */
   4
   5 #include <stdio.h>
   6 #include <errno.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <rte_common.h>
  10 #include <rte_ethdev_pci.h>
  11
  12 #include <rte_interrupts.h>
  13 #include <rte_log.h>
  14 #include <rte_debug.h>
  15 #include <rte_pci.h>
  16 #include <rte_memory.h>
  17 #include <rte_eal.h>
  18 #include <rte_alarm.h>
  19
  20 #include "txgbe_logs.h"
  21 #include "base/txgbe.h"
  22 #include "txgbe_ethdev.h"
  23 #include "txgbe_rxtx.h"
  24
  25 static int  txgbe_dev_set_link_up(struct rte_eth_dev *dev);
  26 static int  txgbe_dev_set_link_down(struct rte_eth_dev *dev);
  27 static int txgbe_dev_close(struct rte_eth_dev *dev);
  28 static int txgbe_dev_link_update(struct rte_eth_dev *dev,
  29                                 int wait_to_complete);
  30
  31 static void txgbe_dev_link_status_print(struct rte_eth_dev *dev);
  32 static int txgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on);
  33 static int txgbe_dev_macsec_interrupt_setup(struct rte_eth_dev *dev);
  34 static int txgbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
  35 static int txgbe_dev_interrupt_get_status(struct rte_eth_dev *dev);
  36 static int txgbe_dev_interrupt_action(struct rte_eth_dev *dev,
  37                                       struct rte_intr_handle *handle);
  38 static void txgbe_dev_interrupt_handler(void *param);
  39 static void txgbe_dev_interrupt_delayed_handler(void *param);
  40 static void txgbe_configure_msix(struct rte_eth_dev *dev);
  41
  42 /*
  43  * The set of PCI devices this driver supports
  44  */
  45 static const struct rte_pci_id pci_id_txgbe_map[] = {
  46         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, TXGBE_DEV_ID_RAPTOR_SFP) },
  47         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, TXGBE_DEV_ID_WX1820_SFP) },
  48         { .vendor_id = 0, /* sentinel */ },
  49 };
  50
  51 static const struct rte_eth_desc_lim rx_desc_lim = {
  52         .nb_max = TXGBE_RING_DESC_MAX,
  53         .nb_min = TXGBE_RING_DESC_MIN,
  54         .nb_align = TXGBE_RXD_ALIGN,
  55 };
  56
  57 static const struct rte_eth_desc_lim tx_desc_lim = {
  58         .nb_max = TXGBE_RING_DESC_MAX,
  59         .nb_min = TXGBE_RING_DESC_MIN,
  60         .nb_align = TXGBE_TXD_ALIGN,
  61         .nb_seg_max = TXGBE_TX_MAX_SEG,
  62         .nb_mtu_seg_max = TXGBE_TX_MAX_SEG,
  63 };
  64
  65 static const struct eth_dev_ops txgbe_eth_dev_ops;
  66
  67 static inline int
  68 txgbe_is_sfp(struct txgbe_hw *hw)
  69 {
  70         switch (hw->phy.type) {
  71         case txgbe_phy_sfp_avago:
  72         case txgbe_phy_sfp_ftl:
  73         case txgbe_phy_sfp_intel:
  74         case txgbe_phy_sfp_unknown:
  75         case txgbe_phy_sfp_tyco_passive:
  76         case txgbe_phy_sfp_unknown_passive:
  77                 return 1;
  78         default:
  79                 return 0;
  80         }
  81 }
  82
  83 static inline void
  84 txgbe_enable_intr(struct rte_eth_dev *dev)
  85 {
  86         struct txgbe_interrupt *intr = TXGBE_DEV_INTR(dev);
  87         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
  88
  89         wr32(hw, TXGBE_IENMISC, intr->mask_misc);
  90         wr32(hw, TXGBE_IMC(0), TXGBE_IMC_MASK);
  91         wr32(hw, TXGBE_IMC(1), TXGBE_IMC_MASK);
  92         txgbe_flush(hw);
  93 }
  94
  95 static void
  96 txgbe_disable_intr(struct txgbe_hw *hw)
  97 {
  98         PMD_INIT_FUNC_TRACE();
  99
 100         wr32(hw, TXGBE_IENMISC, ~BIT_MASK32);
 101         wr32(hw, TXGBE_IMS(0), TXGBE_IMC_MASK);
 102         wr32(hw, TXGBE_IMS(1), TXGBE_IMC_MASK);
 103         txgbe_flush(hw);
 104 }
 105
 106 static int
 107 eth_txgbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused)
 108 {
 109         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
 110         struct txgbe_hw *hw = TXGBE_DEV_HW(eth_dev);
 111         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 112         const struct rte_memzone *mz;
 113         uint16_t csum;
 114         int err;
 115
 116         PMD_INIT_FUNC_TRACE();
 117
 118         eth_dev->dev_ops = &txgbe_eth_dev_ops;
 119
 120         rte_eth_copy_pci_info(eth_dev, pci_dev);
 121
 122         /* Vendor and Device ID need to be set before init of shared code */
 123         hw->device_id = pci_dev->id.device_id;
 124         hw->vendor_id = pci_dev->id.vendor_id;
 125         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 126         hw->allow_unsupported_sfp = 1;
 127
 128         /* Reserve memory for interrupt status block */
 129         mz = rte_eth_dma_zone_reserve(eth_dev, "txgbe_driver", -1,
 130                 16, TXGBE_ALIGN, SOCKET_ID_ANY);
 131         if (mz == NULL)
 132                 return -ENOMEM;
 133
 134         hw->isb_dma = TMZ_PADDR(mz);
 135         hw->isb_mem = TMZ_VADDR(mz);
 136
 137         /* Initialize the shared code (base driver) */
 138         err = txgbe_init_shared_code(hw);
 139         if (err != 0) {
 140                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", err);
 141                 return -EIO;
 142         }
 143
 144         err = hw->rom.init_params(hw);
 145         if (err != 0) {
 146                 PMD_INIT_LOG(ERR, "The EEPROM init failed: %d", err);
 147                 return -EIO;
 148         }
 149
 150         /* Make sure we have a good EEPROM before we read from it */
 151         err = hw->rom.validate_checksum(hw, &csum);
 152         if (err != 0) {
 153                 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", err);
 154                 return -EIO;
 155         }
 156
 157         err = hw->mac.init_hw(hw);
 158
 159         /*
 160          * Devices with copper phys will fail to initialise if txgbe_init_hw()
 161          * is called too soon after the kernel driver unbinding/binding occurs.
 162          * The failure occurs in txgbe_identify_phy() for all devices,
 163          * but for non-copper devies, txgbe_identify_sfp_module() is
 164          * also called. See txgbe_identify_phy(). The reason for the
 165          * failure is not known, and only occuts when virtualisation features
 166          * are disabled in the bios. A delay of 200ms  was found to be enough by
 167          * trial-and-error, and is doubled to be safe.
 168          */
 169         if (err && hw->phy.media_type == txgbe_media_type_copper) {
 170                 rte_delay_ms(200);
 171                 err = hw->mac.init_hw(hw);
 172         }
 173
 174         if (err == TXGBE_ERR_SFP_NOT_PRESENT)
 175                 err = 0;
 176
 177         if (err == TXGBE_ERR_EEPROM_VERSION) {
 178                 PMD_INIT_LOG(ERR, "This device is a pre-production adapter/"
 179                              "LOM.  Please be aware there may be issues associated "
 180                              "with your hardware.");
 181                 PMD_INIT_LOG(ERR, "If you are experiencing problems "
 182                              "please contact your hardware representative "
 183                              "who provided you with this hardware.");
 184         } else if (err == TXGBE_ERR_SFP_NOT_SUPPORTED) {
 185                 PMD_INIT_LOG(ERR, "Unsupported SFP+ Module");
 186         }
 187         if (err) {
 188                 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", err);
 189                 return -EIO;
 190         }
 191
 192         /* disable interrupt */
 193         txgbe_disable_intr(hw);
 194
 195         /* Allocate memory for storing MAC addresses */
 196         eth_dev->data->mac_addrs = rte_zmalloc("txgbe", RTE_ETHER_ADDR_LEN *
 197                                                hw->mac.num_rar_entries, 0);
 198         if (eth_dev->data->mac_addrs == NULL) {
 199                 PMD_INIT_LOG(ERR,
 200                              "Failed to allocate %u bytes needed to store "
 201                              "MAC addresses",
 202                              RTE_ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 203                 return -ENOMEM;
 204         }
 205
 206         /* Copy the permanent MAC address */
 207         rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.perm_addr,
 208                         &eth_dev->data->mac_addrs[0]);
 209
 210         /* Allocate memory for storing hash filter MAC addresses */
 211         eth_dev->data->hash_mac_addrs = rte_zmalloc("txgbe",
 212                         RTE_ETHER_ADDR_LEN * TXGBE_VMDQ_NUM_UC_MAC, 0);
 213         if (eth_dev->data->hash_mac_addrs == NULL) {
 214                 PMD_INIT_LOG(ERR,
 215                              "Failed to allocate %d bytes needed to store MAC addresses",
 216                              RTE_ETHER_ADDR_LEN * TXGBE_VMDQ_NUM_UC_MAC);
 217                 return -ENOMEM;
 218         }
 219
 220         if (txgbe_is_sfp(hw) && hw->phy.sfp_type != txgbe_sfp_type_not_present)
 221                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d, SFP+: %d",
 222                              (int)hw->mac.type, (int)hw->phy.type,
 223                              (int)hw->phy.sfp_type);
 224         else
 225                 PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d",
 226                              (int)hw->mac.type, (int)hw->phy.type);
 227
 228         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
 229                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 230                      pci_dev->id.device_id);
 231
 232         rte_intr_callback_register(intr_handle,
 233                                    txgbe_dev_interrupt_handler, eth_dev);
 234
 235         /* enable uio/vfio intr/eventfd mapping */
 236         rte_intr_enable(intr_handle);
 237
 238         /* enable support intr */
 239         txgbe_enable_intr(eth_dev);
 240
 241         return 0;
 242 }
 243
 244 static int
 245 eth_txgbe_dev_uninit(struct rte_eth_dev *eth_dev)
 246 {
 247         PMD_INIT_FUNC_TRACE();
 248
 249         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 250                 return 0;
 251
 252         txgbe_dev_close(eth_dev);
 253
 254         return 0;
 255 }
 256
 257 static int
 258 eth_txgbe_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 259                 struct rte_pci_device *pci_dev)
 260 {
 261         struct rte_eth_dev *pf_ethdev;
 262         struct rte_eth_devargs eth_da;
 263         int retval;
 264
 265         if (pci_dev->device.devargs) {
 266                 retval = rte_eth_devargs_parse(pci_dev->device.devargs->args,
 267                                 &eth_da);
 268                 if (retval)
 269                         return retval;
 270         } else {
 271                 memset(&eth_da, 0, sizeof(eth_da));
 272         }
 273
 274         retval = rte_eth_dev_create(&pci_dev->device, pci_dev->device.name,
 275                         sizeof(struct txgbe_adapter),
 276                         eth_dev_pci_specific_init, pci_dev,
 277                         eth_txgbe_dev_init, NULL);
 278
 279         if (retval || eth_da.nb_representor_ports < 1)
 280                 return retval;
 281
 282         pf_ethdev = rte_eth_dev_allocated(pci_dev->device.name);
 283         if (pf_ethdev == NULL)
 284                 return -ENODEV;
 285
 286         return 0;
 287 }
 288
 289 static int eth_txgbe_pci_remove(struct rte_pci_device *pci_dev)
 290 {
 291         struct rte_eth_dev *ethdev;
 292
 293         ethdev = rte_eth_dev_allocated(pci_dev->device.name);
 294         if (!ethdev)
 295                 return -ENODEV;
 296
 297         return rte_eth_dev_destroy(ethdev, eth_txgbe_dev_uninit);
 298 }
 299
 300 static struct rte_pci_driver rte_txgbe_pmd = {
 301         .id_table = pci_id_txgbe_map,
 302         .drv_flags = RTE_PCI_DRV_NEED_MAPPING |
 303                      RTE_PCI_DRV_INTR_LSC,
 304         .probe = eth_txgbe_pci_probe,
 305         .remove = eth_txgbe_pci_remove,
 306 };
 307
 308 static int
 309 txgbe_check_vf_rss_rxq_num(struct rte_eth_dev *dev, uint16_t nb_rx_q)
 310 {
 311         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 312
 313         switch (nb_rx_q) {
 314         case 1:
 315         case 2:
 316                 RTE_ETH_DEV_SRIOV(dev).active = ETH_64_POOLS;
 317                 break;
 318         case 4:
 319                 RTE_ETH_DEV_SRIOV(dev).active = ETH_32_POOLS;
 320                 break;
 321         default:
 322                 return -EINVAL;
 323         }
 324
 325         RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool =
 326                 TXGBE_MAX_RX_QUEUE_NUM / RTE_ETH_DEV_SRIOV(dev).active;
 327         RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx =
 328                 pci_dev->max_vfs * RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool;
 329         return 0;
 330 }
 331
 332 static int
 333 txgbe_check_mq_mode(struct rte_eth_dev *dev)
 334 {
 335         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 336         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 337         uint16_t nb_tx_q = dev->data->nb_tx_queues;
 338
 339         if (RTE_ETH_DEV_SRIOV(dev).active != 0) {
 340                 /* check multi-queue mode */
 341                 switch (dev_conf->rxmode.mq_mode) {
 342                 case ETH_MQ_RX_VMDQ_DCB:
 343                         PMD_INIT_LOG(INFO, "ETH_MQ_RX_VMDQ_DCB mode supported in SRIOV");
 344                         break;
 345                 case ETH_MQ_RX_VMDQ_DCB_RSS:
 346                         /* DCB/RSS VMDQ in SRIOV mode, not implement yet */
 347                         PMD_INIT_LOG(ERR, "SRIOV active,"
 348                                         " unsupported mq_mode rx %d.",
 349                                         dev_conf->rxmode.mq_mode);
 350                         return -EINVAL;
 351                 case ETH_MQ_RX_RSS:
 352                 case ETH_MQ_RX_VMDQ_RSS:
 353                         dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_RSS;
 354                         if (nb_rx_q <= RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool)
 355                                 if (txgbe_check_vf_rss_rxq_num(dev, nb_rx_q)) {
 356                                         PMD_INIT_LOG(ERR, "SRIOV is active,"
 357                                                 " invalid queue number"
 358                                                 " for VMDQ RSS, allowed"
 359                                                 " value are 1, 2 or 4.");
 360                                         return -EINVAL;
 361                                 }
 362                         break;
 363                 case ETH_MQ_RX_VMDQ_ONLY:
 364                 case ETH_MQ_RX_NONE:
 365                         /* if nothing mq mode configure, use default scheme */
 366                         dev->data->dev_conf.rxmode.mq_mode =
 367                                 ETH_MQ_RX_VMDQ_ONLY;
 368                         break;
 369                 default: /* ETH_MQ_RX_DCB, ETH_MQ_RX_DCB_RSS or ETH_MQ_TX_DCB*/
 370                         /* SRIOV only works in VMDq enable mode */
 371                         PMD_INIT_LOG(ERR, "SRIOV is active,"
 372                                         " wrong mq_mode rx %d.",
 373                                         dev_conf->rxmode.mq_mode);
 374                         return -EINVAL;
 375                 }
 376
 377                 switch (dev_conf->txmode.mq_mode) {
 378                 case ETH_MQ_TX_VMDQ_DCB:
 379                         PMD_INIT_LOG(INFO, "ETH_MQ_TX_VMDQ_DCB mode supported in SRIOV");
 380                         dev->data->dev_conf.txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
 381                         break;
 382                 default: /* ETH_MQ_TX_VMDQ_ONLY or ETH_MQ_TX_NONE */
 383                         dev->data->dev_conf.txmode.mq_mode =
 384                                 ETH_MQ_TX_VMDQ_ONLY;
 385                         break;
 386                 }
 387
 388                 /* check valid queue number */
 389                 if ((nb_rx_q > RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool) ||
 390                     (nb_tx_q > RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool)) {
 391                         PMD_INIT_LOG(ERR, "SRIOV is active,"
 392                                         " nb_rx_q=%d nb_tx_q=%d queue number"
 393                                         " must be less than or equal to %d.",
 394                                         nb_rx_q, nb_tx_q,
 395                                         RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool);
 396                         return -EINVAL;
 397                 }
 398         } else {
 399                 if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_VMDQ_DCB_RSS) {
 400                         PMD_INIT_LOG(ERR, "VMDQ+DCB+RSS mq_mode is"
 401                                           " not supported.");
 402                         return -EINVAL;
 403                 }
 404                 /* check configuration for vmdb+dcb mode */
 405                 if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_VMDQ_DCB) {
 406                         const struct rte_eth_vmdq_dcb_conf *conf;
 407
 408                         if (nb_rx_q != TXGBE_VMDQ_DCB_NB_QUEUES) {
 409                                 PMD_INIT_LOG(ERR, "VMDQ+DCB, nb_rx_q != %d.",
 410                                                 TXGBE_VMDQ_DCB_NB_QUEUES);
 411                                 return -EINVAL;
 412                         }
 413                         conf = &dev_conf->rx_adv_conf.vmdq_dcb_conf;
 414                         if (!(conf->nb_queue_pools == ETH_16_POOLS ||
 415                                conf->nb_queue_pools == ETH_32_POOLS)) {
 416                                 PMD_INIT_LOG(ERR, "VMDQ+DCB selected,"
 417                                                 " nb_queue_pools must be %d or %d.",
 418                                                 ETH_16_POOLS, ETH_32_POOLS);
 419                                 return -EINVAL;
 420                         }
 421                 }
 422                 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_VMDQ_DCB) {
 423                         const struct rte_eth_vmdq_dcb_tx_conf *conf;
 424
 425                         if (nb_tx_q != TXGBE_VMDQ_DCB_NB_QUEUES) {
 426                                 PMD_INIT_LOG(ERR, "VMDQ+DCB, nb_tx_q != %d",
 427                                                  TXGBE_VMDQ_DCB_NB_QUEUES);
 428                                 return -EINVAL;
 429                         }
 430                         conf = &dev_conf->tx_adv_conf.vmdq_dcb_tx_conf;
 431                         if (!(conf->nb_queue_pools == ETH_16_POOLS ||
 432                                conf->nb_queue_pools == ETH_32_POOLS)) {
 433                                 PMD_INIT_LOG(ERR, "VMDQ+DCB selected,"
 434                                                 " nb_queue_pools != %d and"
 435                                                 " nb_queue_pools != %d.",
 436                                                 ETH_16_POOLS, ETH_32_POOLS);
 437                                 return -EINVAL;
 438                         }
 439                 }
 440
 441                 /* For DCB mode check our configuration before we go further */
 442                 if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_DCB) {
 443                         const struct rte_eth_dcb_rx_conf *conf;
 444
 445                         conf = &dev_conf->rx_adv_conf.dcb_rx_conf;
 446                         if (!(conf->nb_tcs == ETH_4_TCS ||
 447                                conf->nb_tcs == ETH_8_TCS)) {
 448                                 PMD_INIT_LOG(ERR, "DCB selected, nb_tcs != %d"
 449                                                 " and nb_tcs != %d.",
 450                                                 ETH_4_TCS, ETH_8_TCS);
 451                                 return -EINVAL;
 452                         }
 453                 }
 454
 455                 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_DCB) {
 456                         const struct rte_eth_dcb_tx_conf *conf;
 457
 458                         conf = &dev_conf->tx_adv_conf.dcb_tx_conf;
 459                         if (!(conf->nb_tcs == ETH_4_TCS ||
 460                                conf->nb_tcs == ETH_8_TCS)) {
 461                                 PMD_INIT_LOG(ERR, "DCB selected, nb_tcs != %d"
 462                                                 " and nb_tcs != %d.",
 463                                                 ETH_4_TCS, ETH_8_TCS);
 464                                 return -EINVAL;
 465                         }
 466                 }
 467         }
 468         return 0;
 469 }
 470
 471 static int
 472 txgbe_dev_configure(struct rte_eth_dev *dev)
 473 {
 474         struct txgbe_interrupt *intr = TXGBE_DEV_INTR(dev);
 475         struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev);
 476         int ret;
 477
 478         PMD_INIT_FUNC_TRACE();
 479
 480         if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)
 481                 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH;
 482
 483         /* multiple queue mode checking */
 484         ret  = txgbe_check_mq_mode(dev);
 485         if (ret != 0) {
 486                 PMD_DRV_LOG(ERR, "txgbe_check_mq_mode fails with %d.",
 487                             ret);
 488                 return ret;
 489         }
 490
 491         /* set flag to update link status after init */
 492         intr->flags |= TXGBE_FLAG_NEED_LINK_UPDATE;
 493
 494         /*
 495          * Initialize to TRUE. If any of Rx queues doesn't meet the bulk
 496          * allocation Rx preconditions we will reset it.
 497          */
 498         adapter->rx_bulk_alloc_allowed = true;
 499
 500         return 0;
 501 }
 502
 503 static void
 504 txgbe_dev_phy_intr_setup(struct rte_eth_dev *dev)
 505 {
 506         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
 507         struct txgbe_interrupt *intr = TXGBE_DEV_INTR(dev);
 508         uint32_t gpie;
 509
 510         gpie = rd32(hw, TXGBE_GPIOINTEN);
 511         gpie |= TXGBE_GPIOBIT_6;
 512         wr32(hw, TXGBE_GPIOINTEN, gpie);
 513         intr->mask_misc |= TXGBE_ICRMISC_GPIO;
 514 }
 515
 516 /*
 517  * Set device link up: enable tx.
 518  */
 519 static int
 520 txgbe_dev_set_link_up(struct rte_eth_dev *dev)
 521 {
 522         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
 523
 524         if (hw->phy.media_type == txgbe_media_type_copper) {
 525                 /* Turn on the copper */
 526                 hw->phy.set_phy_power(hw, true);
 527         } else {
 528                 /* Turn on the laser */
 529                 hw->mac.enable_tx_laser(hw);
 530                 txgbe_dev_link_update(dev, 0);
 531         }
 532
 533         return 0;
 534 }
 535
 536 /*
 537  * Set device link down: disable tx.
 538  */
 539 static int
 540 txgbe_dev_set_link_down(struct rte_eth_dev *dev)
 541 {
 542         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
 543
 544         if (hw->phy.media_type == txgbe_media_type_copper) {
 545                 /* Turn off the copper */
 546                 hw->phy.set_phy_power(hw, false);
 547         } else {
 548                 /* Turn off the laser */
 549                 hw->mac.disable_tx_laser(hw);
 550                 txgbe_dev_link_update(dev, 0);
 551         }
 552
 553         return 0;
 554 }
 555
 556 /*
 557  * Reset and stop device.
 558  */
 559 static int
 560 txgbe_dev_close(struct rte_eth_dev *dev)
 561 {
 562         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 563         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 564         int retries = 0;
 565         int ret;
 566
 567         PMD_INIT_FUNC_TRACE();
 568
 569         txgbe_dev_free_queues(dev);
 570
 571         /* disable uio intr before callback unregister */
 572         rte_intr_disable(intr_handle);
 573
 574         do {
 575                 ret = rte_intr_callback_unregister(intr_handle,
 576                                 txgbe_dev_interrupt_handler, dev);
 577                 if (ret >= 0 || ret == -ENOENT) {
 578                         break;
 579                 } else if (ret != -EAGAIN) {
 580                         PMD_INIT_LOG(ERR,
 581                                 "intr callback unregister failed: %d",
 582                                 ret);
 583                 }
 584                 rte_delay_ms(100);
 585         } while (retries++ < (10 + TXGBE_LINK_UP_TIME));
 586
 587         /* cancel the delay handler before remove dev */
 588         rte_eal_alarm_cancel(txgbe_dev_interrupt_delayed_handler, dev);
 589
 590         rte_free(dev->data->mac_addrs);
 591         dev->data->mac_addrs = NULL;
 592
 593         rte_free(dev->data->hash_mac_addrs);
 594         dev->data->hash_mac_addrs = NULL;
 595
 596         return 0;
 597 }
 598
 599 static int
 600 txgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 601 {
 602         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 603         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
 604
 605         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
 606         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
 607         dev_info->min_rx_bufsize = 1024;
 608         dev_info->max_rx_pktlen = 15872;
 609         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
 610         dev_info->max_hash_mac_addrs = TXGBE_VMDQ_NUM_UC_MAC;
 611         dev_info->max_vfs = pci_dev->max_vfs;
 612         dev_info->max_vmdq_pools = ETH_64_POOLS;
 613         dev_info->vmdq_queue_num = dev_info->max_rx_queues;
 614         dev_info->rx_queue_offload_capa = txgbe_get_rx_queue_offloads(dev);
 615         dev_info->rx_offload_capa = (txgbe_get_rx_port_offloads(dev) |
 616                                      dev_info->rx_queue_offload_capa);
 617         dev_info->tx_queue_offload_capa = txgbe_get_tx_queue_offloads(dev);
 618         dev_info->tx_offload_capa = txgbe_get_tx_port_offloads(dev);
 619
 620         dev_info->default_rxconf = (struct rte_eth_rxconf) {
 621                 .rx_thresh = {
 622                         .pthresh = TXGBE_DEFAULT_RX_PTHRESH,
 623                         .hthresh = TXGBE_DEFAULT_RX_HTHRESH,
 624                         .wthresh = TXGBE_DEFAULT_RX_WTHRESH,
 625                 },
 626                 .rx_free_thresh = TXGBE_DEFAULT_RX_FREE_THRESH,
 627                 .rx_drop_en = 0,
 628                 .offloads = 0,
 629         };
 630
 631         dev_info->default_txconf = (struct rte_eth_txconf) {
 632                 .tx_thresh = {
 633                         .pthresh = TXGBE_DEFAULT_TX_PTHRESH,
 634                         .hthresh = TXGBE_DEFAULT_TX_HTHRESH,
 635                         .wthresh = TXGBE_DEFAULT_TX_WTHRESH,
 636                 },
 637                 .tx_free_thresh = TXGBE_DEFAULT_TX_FREE_THRESH,
 638                 .offloads = 0,
 639         };
 640
 641         dev_info->rx_desc_lim = rx_desc_lim;
 642         dev_info->tx_desc_lim = tx_desc_lim;
 643
 644         dev_info->hash_key_size = TXGBE_HKEY_MAX_INDEX * sizeof(uint32_t);
 645         dev_info->reta_size = ETH_RSS_RETA_SIZE_128;
 646         dev_info->flow_type_rss_offloads = TXGBE_RSS_OFFLOAD_ALL;
 647
 648         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G;
 649         dev_info->speed_capa |= ETH_LINK_SPEED_100M;
 650
 651         /* Driver-preferred Rx/Tx parameters */
 652         dev_info->default_rxportconf.burst_size = 32;
 653         dev_info->default_txportconf.burst_size = 32;
 654         dev_info->default_rxportconf.nb_queues = 1;
 655         dev_info->default_txportconf.nb_queues = 1;
 656         dev_info->default_rxportconf.ring_size = 256;
 657         dev_info->default_txportconf.ring_size = 256;
 658
 659         return 0;
 660 }
 661
 662 void
 663 txgbe_dev_setup_link_alarm_handler(void *param)
 664 {
 665         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
 666         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
 667         struct txgbe_interrupt *intr = TXGBE_DEV_INTR(dev);
 668         u32 speed;
 669         bool autoneg = false;
 670
 671         speed = hw->phy.autoneg_advertised;
 672         if (!speed)
 673                 hw->mac.get_link_capabilities(hw, &speed, &autoneg);
 674
 675         hw->mac.setup_link(hw, speed, true);
 676
 677         intr->flags &= ~TXGBE_FLAG_NEED_LINK_CONFIG;
 678 }
 679
 680 /* return 0 means link status changed, -1 means not changed */
 681 int
 682 txgbe_dev_link_update_share(struct rte_eth_dev *dev,
 683                             int wait_to_complete)
 684 {
 685         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
 686         struct rte_eth_link link;
 687         u32 link_speed = TXGBE_LINK_SPEED_UNKNOWN;
 688         struct txgbe_interrupt *intr = TXGBE_DEV_INTR(dev);
 689         bool link_up;
 690         int err;
 691         int wait = 1;
 692
 693         memset(&link, 0, sizeof(link));
 694         link.link_status = ETH_LINK_DOWN;
 695         link.link_speed = ETH_SPEED_NUM_NONE;
 696         link.link_duplex = ETH_LINK_HALF_DUPLEX;
 697         link.link_autoneg = ETH_LINK_AUTONEG;
 698
 699         hw->mac.get_link_status = true;
 700
 701         if (intr->flags & TXGBE_FLAG_NEED_LINK_CONFIG)
 702                 return rte_eth_linkstatus_set(dev, &link);
 703
 704         /* check if it needs to wait to complete, if lsc interrupt is enabled */
 705         if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
 706                 wait = 0;
 707
 708         err = hw->mac.check_link(hw, &link_speed, &link_up, wait);
 709
 710         if (err != 0) {
 711                 link.link_speed = ETH_SPEED_NUM_100M;
 712                 link.link_duplex = ETH_LINK_FULL_DUPLEX;
 713                 return rte_eth_linkstatus_set(dev, &link);
 714         }
 715
 716         if (link_up == 0) {
 717                 if (hw->phy.media_type == txgbe_media_type_fiber) {
 718                         intr->flags |= TXGBE_FLAG_NEED_LINK_CONFIG;
 719                         rte_eal_alarm_set(10,
 720                                 txgbe_dev_setup_link_alarm_handler, dev);
 721                 }
 722                 return rte_eth_linkstatus_set(dev, &link);
 723         }
 724
 725         intr->flags &= ~TXGBE_FLAG_NEED_LINK_CONFIG;
 726         link.link_status = ETH_LINK_UP;
 727         link.link_duplex = ETH_LINK_FULL_DUPLEX;
 728
 729         switch (link_speed) {
 730         default:
 731         case TXGBE_LINK_SPEED_UNKNOWN:
 732                 link.link_duplex = ETH_LINK_FULL_DUPLEX;
 733                 link.link_speed = ETH_SPEED_NUM_100M;
 734                 break;
 735
 736         case TXGBE_LINK_SPEED_100M_FULL:
 737                 link.link_speed = ETH_SPEED_NUM_100M;
 738                 break;
 739
 740         case TXGBE_LINK_SPEED_1GB_FULL:
 741                 link.link_speed = ETH_SPEED_NUM_1G;
 742                 break;
 743
 744         case TXGBE_LINK_SPEED_2_5GB_FULL:
 745                 link.link_speed = ETH_SPEED_NUM_2_5G;
 746                 break;
 747
 748         case TXGBE_LINK_SPEED_5GB_FULL:
 749                 link.link_speed = ETH_SPEED_NUM_5G;
 750                 break;
 751
 752         case TXGBE_LINK_SPEED_10GB_FULL:
 753                 link.link_speed = ETH_SPEED_NUM_10G;
 754                 break;
 755         }
 756
 757         return rte_eth_linkstatus_set(dev, &link);
 758 }
 759
 760 static int
 761 txgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
 762 {
 763         return txgbe_dev_link_update_share(dev, wait_to_complete);
 764 }
 765
 766 /**
 767  * It clears the interrupt causes and enables the interrupt.
 768  * It will be called once only during nic initialized.
 769  *
 770  * @param dev
 771  *  Pointer to struct rte_eth_dev.
 772  * @param on
 773  *  Enable or Disable.
 774  *
 775  * @return
 776  *  - On success, zero.
 777  *  - On failure, a negative value.
 778  */
 779 static int
 780 txgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on)
 781 {
 782         struct txgbe_interrupt *intr = TXGBE_DEV_INTR(dev);
 783
 784         txgbe_dev_link_status_print(dev);
 785         if (on)
 786                 intr->mask_misc |= TXGBE_ICRMISC_LSC;
 787         else
 788                 intr->mask_misc &= ~TXGBE_ICRMISC_LSC;
 789
 790         return 0;
 791 }
 792
 793 /**
 794  * It clears the interrupt causes and enables the interrupt.
 795  * It will be called once only during nic initialized.
 796  *
 797  * @param dev
 798  *  Pointer to struct rte_eth_dev.
 799  *
 800  * @return
 801  *  - On success, zero.
 802  *  - On failure, a negative value.
 803  */
 804 static int
 805 txgbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
 806 {
 807         struct txgbe_interrupt *intr = TXGBE_DEV_INTR(dev);
 808
 809         intr->mask[0] |= TXGBE_ICR_MASK;
 810         intr->mask[1] |= TXGBE_ICR_MASK;
 811
 812         return 0;
 813 }
 814
 815 /**
 816  * It clears the interrupt causes and enables the interrupt.
 817  * It will be called once only during nic initialized.
 818  *
 819  * @param dev
 820  *  Pointer to struct rte_eth_dev.
 821  *
 822  * @return
 823  *  - On success, zero.
 824  *  - On failure, a negative value.
 825  */
 826 static int
 827 txgbe_dev_macsec_interrupt_setup(struct rte_eth_dev *dev)
 828 {
 829         struct txgbe_interrupt *intr = TXGBE_DEV_INTR(dev);
 830
 831         intr->mask_misc |= TXGBE_ICRMISC_LNKSEC;
 832
 833         return 0;
 834 }
 835
 836 /*
 837  * It reads ICR and sets flag (TXGBE_ICRMISC_LSC) for the link_update.
 838  *
 839  * @param dev
 840  *  Pointer to struct rte_eth_dev.
 841  *
 842  * @return
 843  *  - On success, zero.
 844  *  - On failure, a negative value.
 845  */
 846 static int
 847 txgbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
 848 {
 849         uint32_t eicr;
 850         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
 851         struct txgbe_interrupt *intr = TXGBE_DEV_INTR(dev);
 852
 853         /* clear all cause mask */
 854         txgbe_disable_intr(hw);
 855
 856         /* read-on-clear nic registers here */
 857         eicr = ((u32 *)hw->isb_mem)[TXGBE_ISB_MISC];
 858         PMD_DRV_LOG(DEBUG, "eicr %x", eicr);
 859
 860         intr->flags = 0;
 861
 862         /* set flag for async link update */
 863         if (eicr & TXGBE_ICRMISC_LSC)
 864                 intr->flags |= TXGBE_FLAG_NEED_LINK_UPDATE;
 865
 866         if (eicr & TXGBE_ICRMISC_VFMBX)
 867                 intr->flags |= TXGBE_FLAG_MAILBOX;
 868
 869         if (eicr & TXGBE_ICRMISC_LNKSEC)
 870                 intr->flags |= TXGBE_FLAG_MACSEC;
 871
 872         if (eicr & TXGBE_ICRMISC_GPIO)
 873                 intr->flags |= TXGBE_FLAG_PHY_INTERRUPT;
 874
 875         return 0;
 876 }
 877
 878 /**
 879  * It gets and then prints the link status.
 880  *
 881  * @param dev
 882  *  Pointer to struct rte_eth_dev.
 883  *
 884  * @return
 885  *  - On success, zero.
 886  *  - On failure, a negative value.
 887  */
 888 static void
 889 txgbe_dev_link_status_print(struct rte_eth_dev *dev)
 890 {
 891         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 892         struct rte_eth_link link;
 893
 894         rte_eth_linkstatus_get(dev, &link);
 895
 896         if (link.link_status) {
 897                 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
 898                                         (int)(dev->data->port_id),
 899                                         (unsigned int)link.link_speed,
 900                         link.link_duplex == ETH_LINK_FULL_DUPLEX ?
 901                                         "full-duplex" : "half-duplex");
 902         } else {
 903                 PMD_INIT_LOG(INFO, " Port %d: Link Down",
 904                                 (int)(dev->data->port_id));
 905         }
 906         PMD_INIT_LOG(DEBUG, "PCI Address: " PCI_PRI_FMT,
 907                                 pci_dev->addr.domain,
 908                                 pci_dev->addr.bus,
 909                                 pci_dev->addr.devid,
 910                                 pci_dev->addr.function);
 911 }
 912
 913 /*
 914  * It executes link_update after knowing an interrupt occurred.
 915  *
 916  * @param dev
 917  *  Pointer to struct rte_eth_dev.
 918  *
 919  * @return
 920  *  - On success, zero.
 921  *  - On failure, a negative value.
 922  */
 923 static int
 924 txgbe_dev_interrupt_action(struct rte_eth_dev *dev,
 925                            struct rte_intr_handle *intr_handle)
 926 {
 927         struct txgbe_interrupt *intr = TXGBE_DEV_INTR(dev);
 928         int64_t timeout;
 929         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
 930
 931         PMD_DRV_LOG(DEBUG, "intr action type %d", intr->flags);
 932
 933         if (intr->flags & TXGBE_FLAG_MAILBOX)
 934                 intr->flags &= ~TXGBE_FLAG_MAILBOX;
 935
 936         if (intr->flags & TXGBE_FLAG_PHY_INTERRUPT) {
 937                 hw->phy.handle_lasi(hw);
 938                 intr->flags &= ~TXGBE_FLAG_PHY_INTERRUPT;
 939         }
 940
 941         if (intr->flags & TXGBE_FLAG_NEED_LINK_UPDATE) {
 942                 struct rte_eth_link link;
 943
 944                 /*get the link status before link update, for predicting later*/
 945                 rte_eth_linkstatus_get(dev, &link);
 946
 947                 txgbe_dev_link_update(dev, 0);
 948
 949                 /* likely to up */
 950                 if (!link.link_status)
 951                         /* handle it 1 sec later, wait it being stable */
 952                         timeout = TXGBE_LINK_UP_CHECK_TIMEOUT;
 953                 /* likely to down */
 954                 else
 955                         /* handle it 4 sec later, wait it being stable */
 956                         timeout = TXGBE_LINK_DOWN_CHECK_TIMEOUT;
 957
 958                 txgbe_dev_link_status_print(dev);
 959                 if (rte_eal_alarm_set(timeout * 1000,
 960                                       txgbe_dev_interrupt_delayed_handler,
 961                                       (void *)dev) < 0) {
 962                         PMD_DRV_LOG(ERR, "Error setting alarm");
 963                 } else {
 964                         /* remember original mask */
 965                         intr->mask_misc_orig = intr->mask_misc;
 966                         /* only disable lsc interrupt */
 967                         intr->mask_misc &= ~TXGBE_ICRMISC_LSC;
 968                 }
 969         }
 970
 971         PMD_DRV_LOG(DEBUG, "enable intr immediately");
 972         txgbe_enable_intr(dev);
 973         rte_intr_enable(intr_handle);
 974
 975         return 0;
 976 }
 977
 978 /**
 979  * Interrupt handler which shall be registered for alarm callback for delayed
 980  * handling specific interrupt to wait for the stable nic state. As the
 981  * NIC interrupt state is not stable for txgbe after link is just down,
 982  * it needs to wait 4 seconds to get the stable status.
 983  *
 984  * @param handle
 985  *  Pointer to interrupt handle.
 986  * @param param
 987  *  The address of parameter (struct rte_eth_dev *) registered before.
 988  *
 989  * @return
 990  *  void
 991  */
 992 static void
 993 txgbe_dev_interrupt_delayed_handler(void *param)
 994 {
 995         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
 996         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 997         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 998         struct txgbe_interrupt *intr = TXGBE_DEV_INTR(dev);
 999         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
1000         uint32_t eicr;
1001
1002         txgbe_disable_intr(hw);
1003
1004         eicr = ((u32 *)hw->isb_mem)[TXGBE_ISB_MISC];
1005
1006         if (intr->flags & TXGBE_FLAG_PHY_INTERRUPT) {
1007                 hw->phy.handle_lasi(hw);
1008                 intr->flags &= ~TXGBE_FLAG_PHY_INTERRUPT;
1009         }
1010
1011         if (intr->flags & TXGBE_FLAG_NEED_LINK_UPDATE) {
1012                 txgbe_dev_link_update(dev, 0);
1013                 intr->flags &= ~TXGBE_FLAG_NEED_LINK_UPDATE;
1014                 txgbe_dev_link_status_print(dev);
1015                 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
1016                                               NULL);
1017         }
1018
1019         if (intr->flags & TXGBE_FLAG_MACSEC) {
1020                 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_MACSEC,
1021                                               NULL);
1022                 intr->flags &= ~TXGBE_FLAG_MACSEC;
1023         }
1024
1025         /* restore original mask */
1026         intr->mask_misc = intr->mask_misc_orig;
1027         intr->mask_misc_orig = 0;
1028
1029         PMD_DRV_LOG(DEBUG, "enable intr in delayed handler S[%08x]", eicr);
1030         txgbe_enable_intr(dev);
1031         rte_intr_enable(intr_handle);
1032 }
1033
1034 /**
1035  * Interrupt handler triggered by NIC  for handling
1036  * specific interrupt.
1037  *
1038  * @param handle
1039  *  Pointer to interrupt handle.
1040  * @param param
1041  *  The address of parameter (struct rte_eth_dev *) registered before.
1042  *
1043  * @return
1044  *  void
1045  */
1046 static void
1047 txgbe_dev_interrupt_handler(void *param)
1048 {
1049         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1050
1051         txgbe_dev_interrupt_get_status(dev);
1052         txgbe_dev_interrupt_action(dev, dev->intr_handle);
1053 }
1054
1055 static int
1056 txgbe_add_rar(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
1057                                 uint32_t index, uint32_t pool)
1058 {
1059         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
1060         uint32_t enable_addr = 1;
1061
1062         return txgbe_set_rar(hw, index, mac_addr->addr_bytes,
1063                              pool, enable_addr);
1064 }
1065
1066 static void
1067 txgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
1068 {
1069         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
1070
1071         txgbe_clear_rar(hw, index);
1072 }
1073
1074 static int
1075 txgbe_set_default_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *addr)
1076 {
1077         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1078
1079         txgbe_remove_rar(dev, 0);
1080         txgbe_add_rar(dev, addr, 0, pci_dev->max_vfs);
1081
1082         return 0;
1083 }
1084
1085 static uint32_t
1086 txgbe_uta_vector(struct txgbe_hw *hw, struct rte_ether_addr *uc_addr)
1087 {
1088         uint32_t vector = 0;
1089
1090         switch (hw->mac.mc_filter_type) {
1091         case 0:   /* use bits [47:36] of the address */
1092                 vector = ((uc_addr->addr_bytes[4] >> 4) |
1093                         (((uint16_t)uc_addr->addr_bytes[5]) << 4));
1094                 break;
1095         case 1:   /* use bits [46:35] of the address */
1096                 vector = ((uc_addr->addr_bytes[4] >> 3) |
1097                         (((uint16_t)uc_addr->addr_bytes[5]) << 5));
1098                 break;
1099         case 2:   /* use bits [45:34] of the address */
1100                 vector = ((uc_addr->addr_bytes[4] >> 2) |
1101                         (((uint16_t)uc_addr->addr_bytes[5]) << 6));
1102                 break;
1103         case 3:   /* use bits [43:32] of the address */
1104                 vector = ((uc_addr->addr_bytes[4]) |
1105                         (((uint16_t)uc_addr->addr_bytes[5]) << 8));
1106                 break;
1107         default:  /* Invalid mc_filter_type */
1108                 break;
1109         }
1110
1111         /* vector can only be 12-bits or boundary will be exceeded */
1112         vector &= 0xFFF;
1113         return vector;
1114 }
1115
1116 static int
1117 txgbe_uc_hash_table_set(struct rte_eth_dev *dev,
1118                         struct rte_ether_addr *mac_addr, uint8_t on)
1119 {
1120         uint32_t vector;
1121         uint32_t uta_idx;
1122         uint32_t reg_val;
1123         uint32_t uta_mask;
1124         uint32_t psrctl;
1125
1126         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
1127         struct txgbe_uta_info *uta_info = TXGBE_DEV_UTA_INFO(dev);
1128
1129         /* The UTA table only exists on pf hardware */
1130         if (hw->mac.type < txgbe_mac_raptor)
1131                 return -ENOTSUP;
1132
1133         vector = txgbe_uta_vector(hw, mac_addr);
1134         uta_idx = (vector >> 5) & 0x7F;
1135         uta_mask = 0x1UL << (vector & 0x1F);
1136
1137         if (!!on == !!(uta_info->uta_shadow[uta_idx] & uta_mask))
1138                 return 0;
1139
1140         reg_val = rd32(hw, TXGBE_UCADDRTBL(uta_idx));
1141         if (on) {
1142                 uta_info->uta_in_use++;
1143                 reg_val |= uta_mask;
1144                 uta_info->uta_shadow[uta_idx] |= uta_mask;
1145         } else {
1146                 uta_info->uta_in_use--;
1147                 reg_val &= ~uta_mask;
1148                 uta_info->uta_shadow[uta_idx] &= ~uta_mask;
1149         }
1150
1151         wr32(hw, TXGBE_UCADDRTBL(uta_idx), reg_val);
1152
1153         psrctl = rd32(hw, TXGBE_PSRCTL);
1154         if (uta_info->uta_in_use > 0)
1155                 psrctl |= TXGBE_PSRCTL_UCHFENA;
1156         else
1157                 psrctl &= ~TXGBE_PSRCTL_UCHFENA;
1158
1159         psrctl &= ~TXGBE_PSRCTL_ADHF12_MASK;
1160         psrctl |= TXGBE_PSRCTL_ADHF12(hw->mac.mc_filter_type);
1161         wr32(hw, TXGBE_PSRCTL, psrctl);
1162
1163         return 0;
1164 }
1165
1166 static int
1167 txgbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on)
1168 {
1169         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
1170         struct txgbe_uta_info *uta_info = TXGBE_DEV_UTA_INFO(dev);
1171         uint32_t psrctl;
1172         int i;
1173
1174         /* The UTA table only exists on pf hardware */
1175         if (hw->mac.type < txgbe_mac_raptor)
1176                 return -ENOTSUP;
1177
1178         if (on) {
1179                 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
1180                         uta_info->uta_shadow[i] = ~0;
1181                         wr32(hw, TXGBE_UCADDRTBL(i), ~0);
1182                 }
1183         } else {
1184                 for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
1185                         uta_info->uta_shadow[i] = 0;
1186                         wr32(hw, TXGBE_UCADDRTBL(i), 0);
1187                 }
1188         }
1189
1190         psrctl = rd32(hw, TXGBE_PSRCTL);
1191         if (on)
1192                 psrctl |= TXGBE_PSRCTL_UCHFENA;
1193         else
1194                 psrctl &= ~TXGBE_PSRCTL_UCHFENA;
1195
1196         psrctl &= ~TXGBE_PSRCTL_ADHF12_MASK;
1197         psrctl |= TXGBE_PSRCTL_ADHF12(hw->mac.mc_filter_type);
1198         wr32(hw, TXGBE_PSRCTL, psrctl);
1199
1200         return 0;
1201 }
1202
1203 /**
1204  * set the IVAR registers, mapping interrupt causes to vectors
1205  * @param hw
1206  *  pointer to txgbe_hw struct
1207  * @direction
1208  *  0 for Rx, 1 for Tx, -1 for other causes
1209  * @queue
1210  *  queue to map the corresponding interrupt to
1211  * @msix_vector
1212  *  the vector to map to the corresponding queue
1213  */
1214 void
1215 txgbe_set_ivar_map(struct txgbe_hw *hw, int8_t direction,
1216                    uint8_t queue, uint8_t msix_vector)
1217 {
1218         uint32_t tmp, idx;
1219
1220         if (direction == -1) {
1221                 /* other causes */
1222                 msix_vector |= TXGBE_IVARMISC_VLD;
1223                 idx = 0;
1224                 tmp = rd32(hw, TXGBE_IVARMISC);
1225                 tmp &= ~(0xFF << idx);
1226                 tmp |= (msix_vector << idx);
1227                 wr32(hw, TXGBE_IVARMISC, tmp);
1228         } else {
1229                 /* rx or tx causes */
1230                 /* Workround for ICR lost */
1231                 idx = ((16 * (queue & 1)) + (8 * direction));
1232                 tmp = rd32(hw, TXGBE_IVAR(queue >> 1));
1233                 tmp &= ~(0xFF << idx);
1234                 tmp |= (msix_vector << idx);
1235                 wr32(hw, TXGBE_IVAR(queue >> 1), tmp);
1236         }
1237 }
1238
1239 /**
1240  * Sets up the hardware to properly generate MSI-X interrupts
1241  * @hw
1242  *  board private structure
1243  */
1244 static void
1245 txgbe_configure_msix(struct rte_eth_dev *dev)
1246 {
1247         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1248         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
1249         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
1250         uint32_t queue_id, base = TXGBE_MISC_VEC_ID;
1251         uint32_t vec = TXGBE_MISC_VEC_ID;
1252         uint32_t gpie;
1253
1254         /* won't configure msix register if no mapping is done
1255          * between intr vector and event fd
1256          * but if misx has been enabled already, need to configure
1257          * auto clean, auto mask and throttling.
1258          */
1259         gpie = rd32(hw, TXGBE_GPIE);
1260         if (!rte_intr_dp_is_en(intr_handle) &&
1261             !(gpie & TXGBE_GPIE_MSIX))
1262                 return;
1263
1264         if (rte_intr_allow_others(intr_handle)) {
1265                 base = TXGBE_RX_VEC_START;
1266                 vec = base;
1267         }
1268
1269         /* setup GPIE for MSI-x mode */
1270         gpie = rd32(hw, TXGBE_GPIE);
1271         gpie |= TXGBE_GPIE_MSIX;
1272         wr32(hw, TXGBE_GPIE, gpie);
1273
1274         /* Populate the IVAR table and set the ITR values to the
1275          * corresponding register.
1276          */
1277         if (rte_intr_dp_is_en(intr_handle)) {
1278                 for (queue_id = 0; queue_id < dev->data->nb_rx_queues;
1279                         queue_id++) {
1280                         /* by default, 1:1 mapping */
1281                         txgbe_set_ivar_map(hw, 0, queue_id, vec);
1282                         intr_handle->intr_vec[queue_id] = vec;
1283                         if (vec < base + intr_handle->nb_efd - 1)
1284                                 vec++;
1285                 }
1286
1287                 txgbe_set_ivar_map(hw, -1, 1, TXGBE_MISC_VEC_ID);
1288         }
1289         wr32(hw, TXGBE_ITR(TXGBE_MISC_VEC_ID),
1290                         TXGBE_ITR_IVAL_10G(TXGBE_QUEUE_ITR_INTERVAL_DEFAULT)
1291                         | TXGBE_ITR_WRDSA);
1292 }
1293
1294 static u8 *
1295 txgbe_dev_addr_list_itr(__rte_unused struct txgbe_hw *hw,
1296                         u8 **mc_addr_ptr, u32 *vmdq)
1297 {
1298         u8 *mc_addr;
1299
1300         *vmdq = 0;
1301         mc_addr = *mc_addr_ptr;
1302         *mc_addr_ptr = (mc_addr + sizeof(struct rte_ether_addr));
1303         return mc_addr;
1304 }
1305
1306 int
1307 txgbe_dev_set_mc_addr_list(struct rte_eth_dev *dev,
1308                           struct rte_ether_addr *mc_addr_set,
1309                           uint32_t nb_mc_addr)
1310 {
1311         struct txgbe_hw *hw;
1312         u8 *mc_addr_list;
1313
1314         hw = TXGBE_DEV_HW(dev);
1315         mc_addr_list = (u8 *)mc_addr_set;
1316         return txgbe_update_mc_addr_list(hw, mc_addr_list, nb_mc_addr,
1317                                          txgbe_dev_addr_list_itr, TRUE);
1318 }
1319
1320 static const struct eth_dev_ops txgbe_eth_dev_ops = {
1321         .dev_configure              = txgbe_dev_configure,
1322         .dev_infos_get              = txgbe_dev_info_get,
1323         .dev_set_link_up            = txgbe_dev_set_link_up,
1324         .dev_set_link_down          = txgbe_dev_set_link_down,
1325         .rx_queue_start             = txgbe_dev_rx_queue_start,
1326         .rx_queue_stop              = txgbe_dev_rx_queue_stop,
1327         .tx_queue_start             = txgbe_dev_tx_queue_start,
1328         .tx_queue_stop              = txgbe_dev_tx_queue_stop,
1329         .rx_queue_setup             = txgbe_dev_rx_queue_setup,
1330         .rx_queue_release           = txgbe_dev_rx_queue_release,
1331         .tx_queue_setup             = txgbe_dev_tx_queue_setup,
1332         .tx_queue_release           = txgbe_dev_tx_queue_release,
1333         .mac_addr_add               = txgbe_add_rar,
1334         .mac_addr_remove            = txgbe_remove_rar,
1335         .mac_addr_set               = txgbe_set_default_mac_addr,
1336         .uc_hash_table_set          = txgbe_uc_hash_table_set,
1337         .uc_all_hash_table_set      = txgbe_uc_all_hash_table_set,
1338         .set_mc_addr_list           = txgbe_dev_set_mc_addr_list,
1339 };
1340
1341 RTE_PMD_REGISTER_PCI(net_txgbe, rte_txgbe_pmd);
1342 RTE_PMD_REGISTER_PCI_TABLE(net_txgbe, pci_id_txgbe_map);
1343 RTE_PMD_REGISTER_KMOD_DEP(net_txgbe, "* igb_uio | uio_pci_generic | vfio-pci");
1344
1345 RTE_LOG_REGISTER(txgbe_logtype_init, pmd.net.txgbe.init, NOTICE);
1346 RTE_LOG_REGISTER(txgbe_logtype_driver, pmd.net.txgbe.driver, NOTICE);
1347
1348 #ifdef RTE_LIBRTE_TXGBE_DEBUG_RX
1349         RTE_LOG_REGISTER(txgbe_logtype_rx, pmd.net.txgbe.rx, DEBUG);
1350 #endif
1351 #ifdef RTE_LIBRTE_TXGBE_DEBUG_TX
1352         RTE_LOG_REGISTER(txgbe_logtype_tx, pmd.net.txgbe.tx, DEBUG);
1353 #endif
1354
1355 #ifdef RTE_LIBRTE_TXGBE_DEBUG_TX_FREE
1356         RTE_LOG_REGISTER(txgbe_logtype_tx_free, pmd.net.txgbe.tx_free, DEBUG);
1357 #endif