drivers/net/atlantic/atl_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Aquantia Corporation
   3  */
   4
   5 #include <rte_ethdev_pci.h>
   6
   7 #include "atl_ethdev.h"
   8 #include "atl_common.h"
   9 #include "atl_hw_regs.h"
  10 #include "atl_logs.h"
  11 #include "hw_atl/hw_atl_llh.h"
  12 #include "hw_atl/hw_atl_b0.h"
  13 #include "hw_atl/hw_atl_b0_internal.h"
  14
  15 static int eth_atl_dev_init(struct rte_eth_dev *eth_dev);
  16 static int eth_atl_dev_uninit(struct rte_eth_dev *eth_dev);
  17
  18 static int  atl_dev_configure(struct rte_eth_dev *dev);
  19 static int  atl_dev_start(struct rte_eth_dev *dev);
  20 static void atl_dev_stop(struct rte_eth_dev *dev);
  21 static int  atl_dev_set_link_up(struct rte_eth_dev *dev);
  22 static int  atl_dev_set_link_down(struct rte_eth_dev *dev);
  23 static void atl_dev_close(struct rte_eth_dev *dev);
  24 static int  atl_dev_reset(struct rte_eth_dev *dev);
  25 static void atl_dev_promiscuous_enable(struct rte_eth_dev *dev);
  26 static void atl_dev_promiscuous_disable(struct rte_eth_dev *dev);
  27 static void atl_dev_allmulticast_enable(struct rte_eth_dev *dev);
  28 static void atl_dev_allmulticast_disable(struct rte_eth_dev *dev);
  29 static int  atl_dev_link_update(struct rte_eth_dev *dev, int wait);
  30
  31 static int atl_dev_xstats_get_names(struct rte_eth_dev *dev __rte_unused,
  32                                     struct rte_eth_xstat_name *xstats_names,
  33                                     unsigned int size);
  34
  35 static int atl_dev_stats_get(struct rte_eth_dev *dev,
  36                                 struct rte_eth_stats *stats);
  37
  38 static int atl_dev_xstats_get(struct rte_eth_dev *dev,
  39                               struct rte_eth_xstat *stats, unsigned int n);
  40
  41 static void atl_dev_stats_reset(struct rte_eth_dev *dev);
  42
  43 static int atl_fw_version_get(struct rte_eth_dev *dev, char *fw_version,
  44                               size_t fw_size);
  45
  46 static void atl_dev_info_get(struct rte_eth_dev *dev,
  47                                struct rte_eth_dev_info *dev_info);
  48
  49 static const uint32_t *atl_dev_supported_ptypes_get(struct rte_eth_dev *dev);
  50
  51 static int atl_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
  52
  53 /* VLAN stuff */
  54 static int atl_vlan_filter_set(struct rte_eth_dev *dev,
  55                 uint16_t vlan_id, int on);
  56
  57 static int atl_vlan_offload_set(struct rte_eth_dev *dev, int mask);
  58
  59 static void atl_vlan_strip_queue_set(struct rte_eth_dev *dev,
  60                                      uint16_t queue_id, int on);
  61
  62 static int atl_vlan_tpid_set(struct rte_eth_dev *dev,
  63                              enum rte_vlan_type vlan_type, uint16_t tpid);
  64
  65 /* Flow control */
  66 static int atl_flow_ctrl_get(struct rte_eth_dev *dev,
  67                                struct rte_eth_fc_conf *fc_conf);
  68 static int atl_flow_ctrl_set(struct rte_eth_dev *dev,
  69                                struct rte_eth_fc_conf *fc_conf);
  70
  71 static void atl_dev_link_status_print(struct rte_eth_dev *dev);
  72
  73 /* Interrupts */
  74 static int atl_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
  75 static int atl_dev_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on);
  76 static int atl_dev_interrupt_get_status(struct rte_eth_dev *dev);
  77 static int atl_dev_interrupt_action(struct rte_eth_dev *dev,
  78                                     struct rte_intr_handle *handle);
  79 static void atl_dev_interrupt_handler(void *param);
  80
  81
  82 static int atl_add_mac_addr(struct rte_eth_dev *dev,
  83                             struct ether_addr *mac_addr,
  84                             uint32_t index, uint32_t pool);
  85 static void atl_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index);
  86 static int atl_set_default_mac_addr(struct rte_eth_dev *dev,
  87                                            struct ether_addr *mac_addr);
  88
  89 static int atl_dev_set_mc_addr_list(struct rte_eth_dev *dev,
  90                                     struct ether_addr *mc_addr_set,
  91                                     uint32_t nb_mc_addr);
  92
  93 /* RSS */
  94 static int atl_reta_update(struct rte_eth_dev *dev,
  95                              struct rte_eth_rss_reta_entry64 *reta_conf,
  96                              uint16_t reta_size);
  97 static int atl_reta_query(struct rte_eth_dev *dev,
  98                             struct rte_eth_rss_reta_entry64 *reta_conf,
  99                             uint16_t reta_size);
 100 static int atl_rss_hash_update(struct rte_eth_dev *dev,
 101                                  struct rte_eth_rss_conf *rss_conf);
 102 static int atl_rss_hash_conf_get(struct rte_eth_dev *dev,
 103                                    struct rte_eth_rss_conf *rss_conf);
 104
 105
 106 static int eth_atl_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 107         struct rte_pci_device *pci_dev);
 108 static int eth_atl_pci_remove(struct rte_pci_device *pci_dev);
 109
 110 static void atl_dev_info_get(struct rte_eth_dev *dev,
 111                                 struct rte_eth_dev_info *dev_info);
 112
 113 int atl_logtype_init;
 114 int atl_logtype_driver;
 115
 116 /*
 117  * The set of PCI devices this driver supports
 118  */
 119 static const struct rte_pci_id pci_id_atl_map[] = {
 120         { RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_0001) },
 121         { RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_D100) },
 122         { RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_D107) },
 123         { RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_D108) },
 124         { RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_D109) },
 125
 126         { RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC100) },
 127         { RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC107) },
 128         { RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC108) },
 129         { RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC109) },
 130         { RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC111) },
 131         { RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC112) },
 132
 133         { RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC100S) },
 134         { RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC107S) },
 135         { RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC108S) },
 136         { RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC109S) },
 137         { RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC111S) },
 138         { RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC112S) },
 139
 140         { RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC111E) },
 141         { RTE_PCI_DEVICE(PCI_VENDOR_ID_AQUANTIA, AQ_DEVICE_ID_AQC112E) },
 142         { .vendor_id = 0, /* sentinel */ },
 143 };
 144
 145 static struct rte_pci_driver rte_atl_pmd = {
 146         .id_table = pci_id_atl_map,
 147         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
 148                      RTE_PCI_DRV_IOVA_AS_VA,
 149         .probe = eth_atl_pci_probe,
 150         .remove = eth_atl_pci_remove,
 151 };
 152
 153 #define ATL_RX_OFFLOADS (DEV_RX_OFFLOAD_VLAN_STRIP \
 154                         | DEV_RX_OFFLOAD_IPV4_CKSUM \
 155                         | DEV_RX_OFFLOAD_UDP_CKSUM \
 156                         | DEV_RX_OFFLOAD_TCP_CKSUM \
 157                         | DEV_RX_OFFLOAD_JUMBO_FRAME)
 158
 159 #define ATL_TX_OFFLOADS (DEV_TX_OFFLOAD_VLAN_INSERT \
 160                         | DEV_TX_OFFLOAD_IPV4_CKSUM \
 161                         | DEV_TX_OFFLOAD_UDP_CKSUM \
 162                         | DEV_TX_OFFLOAD_TCP_CKSUM \
 163                         | DEV_TX_OFFLOAD_TCP_TSO \
 164                         | DEV_TX_OFFLOAD_MULTI_SEGS)
 165
 166 static const struct rte_eth_desc_lim rx_desc_lim = {
 167         .nb_max = ATL_MAX_RING_DESC,
 168         .nb_min = ATL_MIN_RING_DESC,
 169         .nb_align = ATL_RXD_ALIGN,
 170 };
 171
 172 static const struct rte_eth_desc_lim tx_desc_lim = {
 173         .nb_max = ATL_MAX_RING_DESC,
 174         .nb_min = ATL_MIN_RING_DESC,
 175         .nb_align = ATL_TXD_ALIGN,
 176         .nb_seg_max = ATL_TX_MAX_SEG,
 177         .nb_mtu_seg_max = ATL_TX_MAX_SEG,
 178 };
 179
 180 #define ATL_XSTATS_FIELD(name) { \
 181         #name, \
 182         offsetof(struct aq_stats_s, name) \
 183 }
 184
 185 struct atl_xstats_tbl_s {
 186         const char *name;
 187         unsigned int offset;
 188 };
 189
 190 static struct atl_xstats_tbl_s atl_xstats_tbl[] = {
 191         ATL_XSTATS_FIELD(uprc),
 192         ATL_XSTATS_FIELD(mprc),
 193         ATL_XSTATS_FIELD(bprc),
 194         ATL_XSTATS_FIELD(erpt),
 195         ATL_XSTATS_FIELD(uptc),
 196         ATL_XSTATS_FIELD(mptc),
 197         ATL_XSTATS_FIELD(bptc),
 198         ATL_XSTATS_FIELD(erpr),
 199         ATL_XSTATS_FIELD(ubrc),
 200         ATL_XSTATS_FIELD(ubtc),
 201         ATL_XSTATS_FIELD(mbrc),
 202         ATL_XSTATS_FIELD(mbtc),
 203         ATL_XSTATS_FIELD(bbrc),
 204         ATL_XSTATS_FIELD(bbtc),
 205 };
 206
 207 static const struct eth_dev_ops atl_eth_dev_ops = {
 208         .dev_configure        = atl_dev_configure,
 209         .dev_start            = atl_dev_start,
 210         .dev_stop             = atl_dev_stop,
 211         .dev_set_link_up      = atl_dev_set_link_up,
 212         .dev_set_link_down    = atl_dev_set_link_down,
 213         .dev_close            = atl_dev_close,
 214         .dev_reset            = atl_dev_reset,
 215
 216         /* PROMISC */
 217         .promiscuous_enable   = atl_dev_promiscuous_enable,
 218         .promiscuous_disable  = atl_dev_promiscuous_disable,
 219         .allmulticast_enable  = atl_dev_allmulticast_enable,
 220         .allmulticast_disable = atl_dev_allmulticast_disable,
 221
 222         /* Link */
 223         .link_update          = atl_dev_link_update,
 224
 225         /* Stats */
 226         .stats_get            = atl_dev_stats_get,
 227         .xstats_get           = atl_dev_xstats_get,
 228         .xstats_get_names     = atl_dev_xstats_get_names,
 229         .stats_reset          = atl_dev_stats_reset,
 230         .xstats_reset         = atl_dev_stats_reset,
 231
 232         .fw_version_get       = atl_fw_version_get,
 233         .dev_infos_get        = atl_dev_info_get,
 234         .dev_supported_ptypes_get = atl_dev_supported_ptypes_get,
 235
 236         .mtu_set              = atl_dev_mtu_set,
 237
 238         /* VLAN */
 239         .vlan_filter_set      = atl_vlan_filter_set,
 240         .vlan_offload_set     = atl_vlan_offload_set,
 241         .vlan_tpid_set        = atl_vlan_tpid_set,
 242         .vlan_strip_queue_set = atl_vlan_strip_queue_set,
 243
 244         /* Queue Control */
 245         .rx_queue_start       = atl_rx_queue_start,
 246         .rx_queue_stop        = atl_rx_queue_stop,
 247         .rx_queue_setup       = atl_rx_queue_setup,
 248         .rx_queue_release     = atl_rx_queue_release,
 249
 250         .tx_queue_start       = atl_tx_queue_start,
 251         .tx_queue_stop        = atl_tx_queue_stop,
 252         .tx_queue_setup       = atl_tx_queue_setup,
 253         .tx_queue_release     = atl_tx_queue_release,
 254
 255         .rx_queue_intr_enable = atl_dev_rx_queue_intr_enable,
 256         .rx_queue_intr_disable = atl_dev_rx_queue_intr_disable,
 257
 258         .rx_queue_count       = atl_rx_queue_count,
 259         .rx_descriptor_status = atl_dev_rx_descriptor_status,
 260         .tx_descriptor_status = atl_dev_tx_descriptor_status,
 261
 262         /* Flow Control */
 263         .flow_ctrl_get        = atl_flow_ctrl_get,
 264         .flow_ctrl_set        = atl_flow_ctrl_set,
 265
 266         /* MAC */
 267         .mac_addr_add         = atl_add_mac_addr,
 268         .mac_addr_remove      = atl_remove_mac_addr,
 269         .mac_addr_set         = atl_set_default_mac_addr,
 270         .set_mc_addr_list     = atl_dev_set_mc_addr_list,
 271         .rxq_info_get         = atl_rxq_info_get,
 272         .txq_info_get         = atl_txq_info_get,
 273
 274         .reta_update          = atl_reta_update,
 275         .reta_query           = atl_reta_query,
 276         .rss_hash_update      = atl_rss_hash_update,
 277         .rss_hash_conf_get    = atl_rss_hash_conf_get,
 278 };
 279
 280 static inline int32_t
 281 atl_reset_hw(struct aq_hw_s *hw)
 282 {
 283         return hw_atl_b0_hw_reset(hw);
 284 }
 285
 286 static inline void
 287 atl_enable_intr(struct rte_eth_dev *dev)
 288 {
 289         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 290
 291         hw_atl_itr_irq_msk_setlsw_set(hw, 0xffffffff);
 292 }
 293
 294 static void
 295 atl_disable_intr(struct aq_hw_s *hw)
 296 {
 297         PMD_INIT_FUNC_TRACE();
 298         hw_atl_itr_irq_msk_clearlsw_set(hw, 0xffffffff);
 299 }
 300
 301 static int
 302 eth_atl_dev_init(struct rte_eth_dev *eth_dev)
 303 {
 304         struct atl_adapter *adapter =
 305                 (struct atl_adapter *)eth_dev->data->dev_private;
 306         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
 307         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 308         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 309         int err = 0;
 310
 311         PMD_INIT_FUNC_TRACE();
 312
 313         eth_dev->dev_ops = &atl_eth_dev_ops;
 314         eth_dev->rx_pkt_burst = &atl_recv_pkts;
 315         eth_dev->tx_pkt_burst = &atl_xmit_pkts;
 316         eth_dev->tx_pkt_prepare = &atl_prep_pkts;
 317
 318         /* For secondary processes, the primary process has done all the work */
 319         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 320                 return 0;
 321
 322         /* Vendor and Device ID need to be set before init of shared code */
 323         hw->device_id = pci_dev->id.device_id;
 324         hw->vendor_id = pci_dev->id.vendor_id;
 325         hw->mmio = (void *)pci_dev->mem_resource[0].addr;
 326
 327         /* Hardware configuration - hardcode */
 328         adapter->hw_cfg.is_lro = false;
 329         adapter->hw_cfg.wol = false;
 330         adapter->hw_cfg.is_rss = false;
 331         adapter->hw_cfg.num_rss_queues = HW_ATL_B0_RSS_MAX;
 332
 333         adapter->hw_cfg.link_speed_msk = AQ_NIC_RATE_10G |
 334                           AQ_NIC_RATE_5G |
 335                           AQ_NIC_RATE_2G5 |
 336                           AQ_NIC_RATE_1G |
 337                           AQ_NIC_RATE_100M;
 338
 339         adapter->hw_cfg.flow_control = (AQ_NIC_FC_RX | AQ_NIC_FC_TX);
 340         adapter->hw_cfg.aq_rss.indirection_table_size =
 341                 HW_ATL_B0_RSS_REDIRECTION_MAX;
 342
 343         hw->aq_nic_cfg = &adapter->hw_cfg;
 344
 345         /* disable interrupt */
 346         atl_disable_intr(hw);
 347
 348         /* Allocate memory for storing MAC addresses */
 349         eth_dev->data->mac_addrs = rte_zmalloc("atlantic", ETHER_ADDR_LEN, 0);
 350         if (eth_dev->data->mac_addrs == NULL) {
 351                 PMD_INIT_LOG(ERR, "MAC Malloc failed");
 352                 return -ENOMEM;
 353         }
 354
 355         err = hw_atl_utils_initfw(hw, &hw->aq_fw_ops);
 356         if (err)
 357                 return err;
 358
 359         /* Copy the permanent MAC address */
 360         if (hw->aq_fw_ops->get_mac_permanent(hw,
 361                         eth_dev->data->mac_addrs->addr_bytes) != 0)
 362                 return -EINVAL;
 363
 364         /* Reset the hw statistics */
 365         atl_dev_stats_reset(eth_dev);
 366
 367         rte_intr_callback_register(intr_handle,
 368                                    atl_dev_interrupt_handler, eth_dev);
 369
 370         /* enable uio/vfio intr/eventfd mapping */
 371         rte_intr_enable(intr_handle);
 372
 373         /* enable support intr */
 374         atl_enable_intr(eth_dev);
 375
 376         return err;
 377 }
 378
 379 static int
 380 eth_atl_dev_uninit(struct rte_eth_dev *eth_dev)
 381 {
 382         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
 383         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 384         struct aq_hw_s *hw;
 385
 386         PMD_INIT_FUNC_TRACE();
 387
 388         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 389                 return -EPERM;
 390
 391         hw = ATL_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 392
 393         if (hw->adapter_stopped == 0)
 394                 atl_dev_close(eth_dev);
 395
 396         eth_dev->dev_ops = NULL;
 397         eth_dev->rx_pkt_burst = NULL;
 398         eth_dev->tx_pkt_burst = NULL;
 399
 400         /* disable uio intr before callback unregister */
 401         rte_intr_disable(intr_handle);
 402         rte_intr_callback_unregister(intr_handle,
 403                                      atl_dev_interrupt_handler, eth_dev);
 404
 405         rte_free(eth_dev->data->mac_addrs);
 406         eth_dev->data->mac_addrs = NULL;
 407
 408         return 0;
 409 }
 410
 411 static int
 412 eth_atl_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 413         struct rte_pci_device *pci_dev)
 414 {
 415         return rte_eth_dev_pci_generic_probe(pci_dev,
 416                 sizeof(struct atl_adapter), eth_atl_dev_init);
 417 }
 418
 419 static int
 420 eth_atl_pci_remove(struct rte_pci_device *pci_dev)
 421 {
 422         return rte_eth_dev_pci_generic_remove(pci_dev, eth_atl_dev_uninit);
 423 }
 424
 425 static int
 426 atl_dev_configure(struct rte_eth_dev *dev)
 427 {
 428         struct atl_interrupt *intr =
 429                 ATL_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 430
 431         PMD_INIT_FUNC_TRACE();
 432
 433         /* set flag to update link status after init */
 434         intr->flags |= ATL_FLAG_NEED_LINK_UPDATE;
 435
 436         return 0;
 437 }
 438
 439 /*
 440  * Configure device link speed and setup link.
 441  * It returns 0 on success.
 442  */
 443 static int
 444 atl_dev_start(struct rte_eth_dev *dev)
 445 {
 446         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 447         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 448         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 449         uint32_t intr_vector = 0;
 450         uint32_t *link_speeds;
 451         uint32_t speed = 0;
 452         int status;
 453         int err;
 454
 455         PMD_INIT_FUNC_TRACE();
 456
 457         /* set adapter started */
 458         hw->adapter_stopped = 0;
 459
 460         if (dev->data->dev_conf.link_speeds & ETH_LINK_SPEED_FIXED) {
 461                 PMD_INIT_LOG(ERR,
 462                 "Invalid link_speeds for port %u, fix speed not supported",
 463                                 dev->data->port_id);
 464                 return -EINVAL;
 465         }
 466
 467         /* disable uio/vfio intr/eventfd mapping */
 468         rte_intr_disable(intr_handle);
 469
 470         /* reinitialize adapter
 471          * this calls reset and start
 472          */
 473         status = atl_reset_hw(hw);
 474         if (status != 0)
 475                 return -EIO;
 476
 477         err = hw_atl_b0_hw_init(hw, dev->data->mac_addrs->addr_bytes);
 478
 479         hw_atl_b0_hw_start(hw);
 480         /* check and configure queue intr-vector mapping */
 481         if ((rte_intr_cap_multiple(intr_handle) ||
 482             !RTE_ETH_DEV_SRIOV(dev).active) &&
 483             dev->data->dev_conf.intr_conf.rxq != 0) {
 484                 intr_vector = dev->data->nb_rx_queues;
 485                 if (intr_vector > ATL_MAX_INTR_QUEUE_NUM) {
 486                         PMD_INIT_LOG(ERR, "At most %d intr queues supported",
 487                                         ATL_MAX_INTR_QUEUE_NUM);
 488                         return -ENOTSUP;
 489                 }
 490                 if (rte_intr_efd_enable(intr_handle, intr_vector)) {
 491                         PMD_INIT_LOG(ERR, "rte_intr_efd_enable failed");
 492                         return -1;
 493                 }
 494         }
 495
 496         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
 497                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
 498                                     dev->data->nb_rx_queues * sizeof(int), 0);
 499                 if (intr_handle->intr_vec == NULL) {
 500                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
 501                                      " intr_vec", dev->data->nb_rx_queues);
 502                         return -ENOMEM;
 503                 }
 504         }
 505
 506         /* initialize transmission unit */
 507         atl_tx_init(dev);
 508
 509         /* This can fail when allocating mbufs for descriptor rings */
 510         err = atl_rx_init(dev);
 511         if (err) {
 512                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
 513                 goto error;
 514         }
 515
 516         PMD_INIT_LOG(DEBUG, "FW version: %u.%u.%u",
 517                 hw->fw_ver_actual >> 24,
 518                 (hw->fw_ver_actual >> 16) & 0xFF,
 519                 hw->fw_ver_actual & 0xFFFF);
 520         PMD_INIT_LOG(DEBUG, "Driver version: %s", ATL_PMD_DRIVER_VERSION);
 521
 522         err = atl_start_queues(dev);
 523         if (err < 0) {
 524                 PMD_INIT_LOG(ERR, "Unable to start rxtx queues");
 525                 goto error;
 526         }
 527
 528         err = hw->aq_fw_ops->update_link_status(hw);
 529
 530         if (err)
 531                 goto error;
 532
 533         dev->data->dev_link.link_status = hw->aq_link_status.mbps != 0;
 534
 535         link_speeds = &dev->data->dev_conf.link_speeds;
 536
 537         speed = 0x0;
 538
 539         if (*link_speeds == ETH_LINK_SPEED_AUTONEG) {
 540                 speed = hw->aq_nic_cfg->link_speed_msk;
 541         } else {
 542                 if (*link_speeds & ETH_LINK_SPEED_10G)
 543                         speed |= AQ_NIC_RATE_10G;
 544                 if (*link_speeds & ETH_LINK_SPEED_5G)
 545                         speed |= AQ_NIC_RATE_5G;
 546                 if (*link_speeds & ETH_LINK_SPEED_1G)
 547                         speed |= AQ_NIC_RATE_1G;
 548                 if (*link_speeds & ETH_LINK_SPEED_2_5G)
 549                         speed |=  AQ_NIC_RATE_2G5;
 550                 if (*link_speeds & ETH_LINK_SPEED_100M)
 551                         speed |= AQ_NIC_RATE_100M;
 552         }
 553
 554         err = hw->aq_fw_ops->set_link_speed(hw, speed);
 555         if (err)
 556                 goto error;
 557
 558         if (rte_intr_allow_others(intr_handle)) {
 559                 /* check if lsc interrupt is enabled */
 560                 if (dev->data->dev_conf.intr_conf.lsc != 0)
 561                         atl_dev_lsc_interrupt_setup(dev, true);
 562                 else
 563                         atl_dev_lsc_interrupt_setup(dev, false);
 564         } else {
 565                 rte_intr_callback_unregister(intr_handle,
 566                                              atl_dev_interrupt_handler, dev);
 567                 if (dev->data->dev_conf.intr_conf.lsc != 0)
 568                         PMD_INIT_LOG(INFO, "lsc won't enable because of"
 569                                      " no intr multiplex");
 570         }
 571
 572         /* check if rxq interrupt is enabled */
 573         if (dev->data->dev_conf.intr_conf.rxq != 0 &&
 574             rte_intr_dp_is_en(intr_handle))
 575                 atl_dev_rxq_interrupt_setup(dev);
 576
 577         /* enable uio/vfio intr/eventfd mapping */
 578         rte_intr_enable(intr_handle);
 579
 580         /* resume enabled intr since hw reset */
 581         atl_enable_intr(dev);
 582
 583         return 0;
 584
 585 error:
 586         atl_stop_queues(dev);
 587         return -EIO;
 588 }
 589
 590 /*
 591  * Stop device: disable rx and tx functions to allow for reconfiguring.
 592  */
 593 static void
 594 atl_dev_stop(struct rte_eth_dev *dev)
 595 {
 596         struct rte_eth_link link;
 597         struct aq_hw_s *hw =
 598                 ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 599         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 600         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 601
 602         PMD_INIT_FUNC_TRACE();
 603
 604         /* disable interrupts */
 605         atl_disable_intr(hw);
 606
 607         /* reset the NIC */
 608         atl_reset_hw(hw);
 609         hw->adapter_stopped = 1;
 610
 611         atl_stop_queues(dev);
 612
 613         /* Clear stored conf */
 614         dev->data->scattered_rx = 0;
 615         dev->data->lro = 0;
 616
 617         /* Clear recorded link status */
 618         memset(&link, 0, sizeof(link));
 619         rte_eth_linkstatus_set(dev, &link);
 620
 621         if (!rte_intr_allow_others(intr_handle))
 622                 /* resume to the default handler */
 623                 rte_intr_callback_register(intr_handle,
 624                                            atl_dev_interrupt_handler,
 625                                            (void *)dev);
 626
 627         /* Clean datapath event and queue/vec mapping */
 628         rte_intr_efd_disable(intr_handle);
 629         if (intr_handle->intr_vec != NULL) {
 630                 rte_free(intr_handle->intr_vec);
 631                 intr_handle->intr_vec = NULL;
 632         }
 633 }
 634
 635 /*
 636  * Set device link up: enable tx.
 637  */
 638 static int
 639 atl_dev_set_link_up(struct rte_eth_dev *dev)
 640 {
 641         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 642
 643         return hw->aq_fw_ops->set_link_speed(hw,
 644                         hw->aq_nic_cfg->link_speed_msk);
 645 }
 646
 647 /*
 648  * Set device link down: disable tx.
 649  */
 650 static int
 651 atl_dev_set_link_down(struct rte_eth_dev *dev)
 652 {
 653         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 654
 655         return hw->aq_fw_ops->set_link_speed(hw, 0);
 656 }
 657
 658 /*
 659  * Reset and stop device.
 660  */
 661 static void
 662 atl_dev_close(struct rte_eth_dev *dev)
 663 {
 664         PMD_INIT_FUNC_TRACE();
 665
 666         atl_dev_stop(dev);
 667
 668         atl_free_queues(dev);
 669 }
 670
 671 static int
 672 atl_dev_reset(struct rte_eth_dev *dev)
 673 {
 674         int ret;
 675
 676         ret = eth_atl_dev_uninit(dev);
 677         if (ret)
 678                 return ret;
 679
 680         ret = eth_atl_dev_init(dev);
 681
 682         return ret;
 683 }
 684
 685
 686 static int
 687 atl_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 688 {
 689         struct atl_adapter *adapter = ATL_DEV_TO_ADAPTER(dev);
 690         struct aq_hw_s *hw = &adapter->hw;
 691         struct atl_sw_stats *swstats = &adapter->sw_stats;
 692         unsigned int i;
 693
 694         hw->aq_fw_ops->update_stats(hw);
 695
 696         /* Fill out the rte_eth_stats statistics structure */
 697         stats->ipackets = hw->curr_stats.dma_pkt_rc;
 698         stats->ibytes = hw->curr_stats.dma_oct_rc;
 699         stats->imissed = hw->curr_stats.dpc;
 700         stats->ierrors = hw->curr_stats.erpt;
 701
 702         stats->opackets = hw->curr_stats.dma_pkt_tc;
 703         stats->obytes = hw->curr_stats.dma_oct_tc;
 704         stats->oerrors = 0;
 705
 706         stats->rx_nombuf = swstats->rx_nombuf;
 707
 708         for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
 709                 stats->q_ipackets[i] = swstats->q_ipackets[i];
 710                 stats->q_opackets[i] = swstats->q_opackets[i];
 711                 stats->q_ibytes[i] = swstats->q_ibytes[i];
 712                 stats->q_obytes[i] = swstats->q_obytes[i];
 713                 stats->q_errors[i] = swstats->q_errors[i];
 714         }
 715         return 0;
 716 }
 717
 718 static void
 719 atl_dev_stats_reset(struct rte_eth_dev *dev)
 720 {
 721         struct atl_adapter *adapter = ATL_DEV_TO_ADAPTER(dev);
 722         struct aq_hw_s *hw = &adapter->hw;
 723
 724         hw->aq_fw_ops->update_stats(hw);
 725
 726         /* Reset software totals */
 727         memset(&hw->curr_stats, 0, sizeof(hw->curr_stats));
 728
 729         memset(&adapter->sw_stats, 0, sizeof(adapter->sw_stats));
 730 }
 731
 732 static int
 733 atl_dev_xstats_get_names(struct rte_eth_dev *dev __rte_unused,
 734                          struct rte_eth_xstat_name *xstats_names,
 735                          unsigned int size)
 736 {
 737         unsigned int i;
 738
 739         if (!xstats_names)
 740                 return RTE_DIM(atl_xstats_tbl);
 741
 742         for (i = 0; i < size && i < RTE_DIM(atl_xstats_tbl); i++)
 743                 snprintf(xstats_names[i].name, RTE_ETH_XSTATS_NAME_SIZE, "%s",
 744                         atl_xstats_tbl[i].name);
 745
 746         return size;
 747 }
 748
 749 static int
 750 atl_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *stats,
 751                    unsigned int n)
 752 {
 753         struct atl_adapter *adapter = ATL_DEV_TO_ADAPTER(dev);
 754         struct aq_hw_s *hw = &adapter->hw;
 755         unsigned int i;
 756
 757         if (!stats)
 758                 return 0;
 759
 760         for (i = 0; i < n && i < RTE_DIM(atl_xstats_tbl); i++) {
 761                 stats[i].id = i;
 762                 stats[i].value = *(u64 *)((uint8_t *)&hw->curr_stats +
 763                                         atl_xstats_tbl[i].offset);
 764         }
 765
 766         return n;
 767 }
 768
 769 static int
 770 atl_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size)
 771 {
 772         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 773         uint32_t fw_ver = 0;
 774         unsigned int ret = 0;
 775
 776         ret = hw_atl_utils_get_fw_version(hw, &fw_ver);
 777         if (ret)
 778                 return -EIO;
 779
 780         ret = snprintf(fw_version, fw_size, "%u.%u.%u", fw_ver >> 24,
 781                        (fw_ver >> 16) & 0xFFU, fw_ver & 0xFFFFU);
 782
 783         ret += 1; /* add string null-terminator */
 784
 785         if (fw_size < ret)
 786                 return ret;
 787
 788         return 0;
 789 }
 790
 791 static void
 792 atl_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 793 {
 794         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 795
 796         dev_info->max_rx_queues = AQ_HW_MAX_RX_QUEUES;
 797         dev_info->max_tx_queues = AQ_HW_MAX_TX_QUEUES;
 798
 799         dev_info->min_rx_bufsize = 1024;
 800         dev_info->max_rx_pktlen = HW_ATL_B0_MTU_JUMBO;
 801         dev_info->max_mac_addrs = HW_ATL_B0_MAC_MAX;
 802         dev_info->max_vfs = pci_dev->max_vfs;
 803
 804         dev_info->max_hash_mac_addrs = 0;
 805         dev_info->max_vmdq_pools = 0;
 806         dev_info->vmdq_queue_num = 0;
 807
 808         dev_info->rx_offload_capa = ATL_RX_OFFLOADS;
 809
 810         dev_info->tx_offload_capa = ATL_TX_OFFLOADS;
 811
 812
 813         dev_info->default_rxconf = (struct rte_eth_rxconf) {
 814                 .rx_free_thresh = ATL_DEFAULT_RX_FREE_THRESH,
 815         };
 816
 817         dev_info->default_txconf = (struct rte_eth_txconf) {
 818                 .tx_free_thresh = ATL_DEFAULT_TX_FREE_THRESH,
 819         };
 820
 821         dev_info->rx_desc_lim = rx_desc_lim;
 822         dev_info->tx_desc_lim = tx_desc_lim;
 823
 824         dev_info->hash_key_size = HW_ATL_B0_RSS_HASHKEY_BITS / 8;
 825         dev_info->reta_size = HW_ATL_B0_RSS_REDIRECTION_MAX;
 826         dev_info->flow_type_rss_offloads = ATL_RSS_OFFLOAD_ALL;
 827
 828         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G;
 829         dev_info->speed_capa |= ETH_LINK_SPEED_100M;
 830         dev_info->speed_capa |= ETH_LINK_SPEED_2_5G;
 831         dev_info->speed_capa |= ETH_LINK_SPEED_5G;
 832 }
 833
 834 static const uint32_t *
 835 atl_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 836 {
 837         static const uint32_t ptypes[] = {
 838                 RTE_PTYPE_L2_ETHER,
 839                 RTE_PTYPE_L2_ETHER_ARP,
 840                 RTE_PTYPE_L2_ETHER_VLAN,
 841                 RTE_PTYPE_L3_IPV4,
 842                 RTE_PTYPE_L3_IPV6,
 843                 RTE_PTYPE_L4_TCP,
 844                 RTE_PTYPE_L4_UDP,
 845                 RTE_PTYPE_L4_SCTP,
 846                 RTE_PTYPE_L4_ICMP,
 847                 RTE_PTYPE_UNKNOWN
 848         };
 849
 850         if (dev->rx_pkt_burst == atl_recv_pkts)
 851                 return ptypes;
 852
 853         return NULL;
 854 }
 855
 856 /* return 0 means link status changed, -1 means not changed */
 857 static int
 858 atl_dev_link_update(struct rte_eth_dev *dev, int wait __rte_unused)
 859 {
 860         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 861         struct atl_interrupt *intr =
 862                 ATL_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 863         struct rte_eth_link link, old;
 864         int err = 0;
 865
 866         link.link_status = ETH_LINK_DOWN;
 867         link.link_speed = 0;
 868         link.link_duplex = ETH_LINK_FULL_DUPLEX;
 869         link.link_autoneg = hw->is_autoneg ? ETH_LINK_AUTONEG : ETH_LINK_FIXED;
 870         memset(&old, 0, sizeof(old));
 871
 872         /* load old link status */
 873         rte_eth_linkstatus_get(dev, &old);
 874
 875         /* read current link status */
 876         err = hw->aq_fw_ops->update_link_status(hw);
 877
 878         if (err)
 879                 return 0;
 880
 881         if (hw->aq_link_status.mbps == 0) {
 882                 /* write default (down) link status */
 883                 rte_eth_linkstatus_set(dev, &link);
 884                 if (link.link_status == old.link_status)
 885                         return -1;
 886                 return 0;
 887         }
 888
 889         intr->flags &= ~ATL_FLAG_NEED_LINK_CONFIG;
 890
 891         link.link_status = ETH_LINK_UP;
 892         link.link_duplex = ETH_LINK_FULL_DUPLEX;
 893         link.link_speed = hw->aq_link_status.mbps;
 894
 895         rte_eth_linkstatus_set(dev, &link);
 896
 897         if (link.link_status == old.link_status)
 898                 return -1;
 899
 900         return 0;
 901 }
 902
 903 static void
 904 atl_dev_promiscuous_enable(struct rte_eth_dev *dev)
 905 {
 906         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 907
 908         hw_atl_rpfl2promiscuous_mode_en_set(hw, true);
 909 }
 910
 911 static void
 912 atl_dev_promiscuous_disable(struct rte_eth_dev *dev)
 913 {
 914         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 915
 916         hw_atl_rpfl2promiscuous_mode_en_set(hw, false);
 917 }
 918
 919 static void
 920 atl_dev_allmulticast_enable(struct rte_eth_dev *dev)
 921 {
 922         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 923
 924         hw_atl_rpfl2_accept_all_mc_packets_set(hw, true);
 925 }
 926
 927 static void
 928 atl_dev_allmulticast_disable(struct rte_eth_dev *dev)
 929 {
 930         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 931
 932         if (dev->data->promiscuous == 1)
 933                 return; /* must remain in all_multicast mode */
 934
 935         hw_atl_rpfl2_accept_all_mc_packets_set(hw, false);
 936 }
 937
 938 /**
 939  * It clears the interrupt causes and enables the interrupt.
 940  * It will be called once only during nic initialized.
 941  *
 942  * @param dev
 943  *  Pointer to struct rte_eth_dev.
 944  * @param on
 945  *  Enable or Disable.
 946  *
 947  * @return
 948  *  - On success, zero.
 949  *  - On failure, a negative value.
 950  */
 951
 952 static int
 953 atl_dev_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on __rte_unused)
 954 {
 955         atl_dev_link_status_print(dev);
 956         return 0;
 957 }
 958
 959 static int
 960 atl_dev_rxq_interrupt_setup(struct rte_eth_dev *dev __rte_unused)
 961 {
 962         return 0;
 963 }
 964
 965
 966 static int
 967 atl_dev_interrupt_get_status(struct rte_eth_dev *dev)
 968 {
 969         struct atl_interrupt *intr =
 970                 ATL_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 971         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 972         u64 cause = 0;
 973
 974         hw_atl_b0_hw_irq_read(hw, &cause);
 975
 976         atl_disable_intr(hw);
 977         intr->flags = cause & BIT(ATL_IRQ_CAUSE_LINK) ?
 978                         ATL_FLAG_NEED_LINK_UPDATE : 0;
 979
 980         return 0;
 981 }
 982
 983 /**
 984  * It gets and then prints the link status.
 985  *
 986  * @param dev
 987  *  Pointer to struct rte_eth_dev.
 988  *
 989  * @return
 990  *  - On success, zero.
 991  *  - On failure, a negative value.
 992  */
 993 static void
 994 atl_dev_link_status_print(struct rte_eth_dev *dev)
 995 {
 996         struct rte_eth_link link;
 997
 998         memset(&link, 0, sizeof(link));
 999         rte_eth_linkstatus_get(dev, &link);
1000         if (link.link_status) {
1001                 PMD_DRV_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
1002                                         (int)(dev->data->port_id),
1003                                         (unsigned int)link.link_speed,
1004                         link.link_duplex == ETH_LINK_FULL_DUPLEX ?
1005                                         "full-duplex" : "half-duplex");
1006         } else {
1007                 PMD_DRV_LOG(INFO, " Port %d: Link Down",
1008                                 (int)(dev->data->port_id));
1009         }
1010
1011
1012 #ifdef DEBUG
1013 {
1014         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1015
1016         PMD_DRV_LOG(DEBUG, "PCI Address: " PCI_PRI_FMT,
1017                                 pci_dev->addr.domain,
1018                                 pci_dev->addr.bus,
1019                                 pci_dev->addr.devid,
1020                                 pci_dev->addr.function);
1021 }
1022 #endif
1023
1024         PMD_DRV_LOG(INFO, "Link speed:%d", link.link_speed);
1025 }
1026
1027 /*
1028  * It executes link_update after knowing an interrupt occurred.
1029  *
1030  * @param dev
1031  *  Pointer to struct rte_eth_dev.
1032  *
1033  * @return
1034  *  - On success, zero.
1035  *  - On failure, a negative value.
1036  */
1037 static int
1038 atl_dev_interrupt_action(struct rte_eth_dev *dev,
1039                            struct rte_intr_handle *intr_handle)
1040 {
1041         struct atl_interrupt *intr =
1042                 ATL_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1043
1044         if (intr->flags & ATL_FLAG_NEED_LINK_UPDATE) {
1045                 atl_dev_link_update(dev, 0);
1046                 intr->flags &= ~ATL_FLAG_NEED_LINK_UPDATE;
1047                 atl_dev_link_status_print(dev);
1048                 _rte_eth_dev_callback_process(dev,
1049                         RTE_ETH_EVENT_INTR_LSC, NULL);
1050         }
1051
1052         atl_enable_intr(dev);
1053         rte_intr_enable(intr_handle);
1054
1055         return 0;
1056 }
1057
1058 /**
1059  * Interrupt handler triggered by NIC  for handling
1060  * specific interrupt.
1061  *
1062  * @param handle
1063  *  Pointer to interrupt handle.
1064  * @param param
1065  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1066  *
1067  * @return
1068  *  void
1069  */
1070 static void
1071 atl_dev_interrupt_handler(void *param)
1072 {
1073         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1074
1075         atl_dev_interrupt_get_status(dev);
1076         atl_dev_interrupt_action(dev, dev->intr_handle);
1077 }
1078
1079
1080 static int
1081 atl_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1082 {
1083         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1084
1085         if (hw->aq_nic_cfg->flow_control == AQ_NIC_FC_OFF)
1086                 fc_conf->mode = RTE_FC_NONE;
1087         else if (hw->aq_nic_cfg->flow_control & (AQ_NIC_FC_RX | AQ_NIC_FC_TX))
1088                 fc_conf->mode = RTE_FC_FULL;
1089         else if (hw->aq_nic_cfg->flow_control & AQ_NIC_FC_RX)
1090                 fc_conf->mode = RTE_FC_RX_PAUSE;
1091         else if (hw->aq_nic_cfg->flow_control & AQ_NIC_FC_RX)
1092                 fc_conf->mode = RTE_FC_TX_PAUSE;
1093
1094         return 0;
1095 }
1096
1097 static int
1098 atl_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1099 {
1100         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1101         uint32_t old_flow_control = hw->aq_nic_cfg->flow_control;
1102
1103
1104         if (hw->aq_fw_ops->set_flow_control == NULL)
1105                 return -ENOTSUP;
1106
1107         if (fc_conf->mode == RTE_FC_NONE)
1108                 hw->aq_nic_cfg->flow_control = AQ_NIC_FC_OFF;
1109         else if (fc_conf->mode == RTE_FC_RX_PAUSE)
1110                 hw->aq_nic_cfg->flow_control = AQ_NIC_FC_RX;
1111         else if (fc_conf->mode == RTE_FC_TX_PAUSE)
1112                 hw->aq_nic_cfg->flow_control = AQ_NIC_FC_TX;
1113         else if (fc_conf->mode == RTE_FC_FULL)
1114                 hw->aq_nic_cfg->flow_control = (AQ_NIC_FC_RX | AQ_NIC_FC_TX);
1115
1116         if (old_flow_control != hw->aq_nic_cfg->flow_control)
1117                 return hw->aq_fw_ops->set_flow_control(hw);
1118
1119         return 0;
1120 }
1121
1122 static int
1123 atl_update_mac_addr(struct rte_eth_dev *dev, uint32_t index,
1124                     u8 *mac_addr, bool enable)
1125 {
1126         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1127         unsigned int h = 0U;
1128         unsigned int l = 0U;
1129         int err;
1130
1131         if (mac_addr) {
1132                 h = (mac_addr[0] << 8) | (mac_addr[1]);
1133                 l = (mac_addr[2] << 24) | (mac_addr[3] << 16) |
1134                         (mac_addr[4] << 8) | mac_addr[5];
1135         }
1136
1137         hw_atl_rpfl2_uc_flr_en_set(hw, 0U, index);
1138         hw_atl_rpfl2unicast_dest_addresslsw_set(hw, l, index);
1139         hw_atl_rpfl2unicast_dest_addressmsw_set(hw, h, index);
1140
1141         if (enable)
1142                 hw_atl_rpfl2_uc_flr_en_set(hw, 1U, index);
1143
1144         err = aq_hw_err_from_flags(hw);
1145
1146         return err;
1147 }
1148
1149 static int
1150 atl_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
1151                         uint32_t index __rte_unused, uint32_t pool __rte_unused)
1152 {
1153         if (is_zero_ether_addr(mac_addr)) {
1154                 PMD_DRV_LOG(ERR, "Invalid Ethernet Address");
1155                 return -EINVAL;
1156         }
1157
1158         return atl_update_mac_addr(dev, index, (u8 *)mac_addr, true);
1159 }
1160
1161 static void
1162 atl_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index)
1163 {
1164         atl_update_mac_addr(dev, index, NULL, false);
1165 }
1166
1167 static int
1168 atl_set_default_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
1169 {
1170         atl_remove_mac_addr(dev, 0);
1171         atl_add_mac_addr(dev, addr, 0, 0);
1172         return 0;
1173 }
1174
1175 static int
1176 atl_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
1177 {
1178         struct rte_eth_dev_info dev_info;
1179         uint32_t frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1180
1181         atl_dev_info_get(dev, &dev_info);
1182
1183         if ((mtu < ETHER_MIN_MTU) || (frame_size > dev_info.max_rx_pktlen))
1184                 return -EINVAL;
1185
1186         /* update max frame size */
1187         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
1188
1189         return 0;
1190 }
1191
1192 static int
1193 atl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1194 {
1195         struct aq_hw_cfg_s *cfg =
1196                 ATL_DEV_PRIVATE_TO_CFG(dev->data->dev_private);
1197         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1198         int err = 0;
1199         int i = 0;
1200
1201         PMD_INIT_FUNC_TRACE();
1202
1203         for (i = 0; i < HW_ATL_B0_MAX_VLAN_IDS; i++) {
1204                 if (cfg->vlan_filter[i] == vlan_id) {
1205                         if (!on) {
1206                                 /* Disable VLAN filter. */
1207                                 hw_atl_rpf_vlan_flr_en_set(hw, 0U, i);
1208
1209                                 /* Clear VLAN filter entry */
1210                                 cfg->vlan_filter[i] = 0;
1211                         }
1212                         break;
1213                 }
1214         }
1215
1216         /* VLAN_ID was not found. So, nothing to delete. */
1217         if (i == HW_ATL_B0_MAX_VLAN_IDS && !on)
1218                 goto exit;
1219
1220         /* VLAN_ID already exist, or already removed above. Nothing to do. */
1221         if (i != HW_ATL_B0_MAX_VLAN_IDS)
1222                 goto exit;
1223
1224         /* Try to found free VLAN filter to add new VLAN_ID */
1225         for (i = 0; i < HW_ATL_B0_MAX_VLAN_IDS; i++) {
1226                 if (cfg->vlan_filter[i] == 0)
1227                         break;
1228         }
1229
1230         if (i == HW_ATL_B0_MAX_VLAN_IDS) {
1231                 /* We have no free VLAN filter to add new VLAN_ID*/
1232                 err = -ENOMEM;
1233                 goto exit;
1234         }
1235
1236         cfg->vlan_filter[i] = vlan_id;
1237         hw_atl_rpf_vlan_flr_act_set(hw, 1U, i);
1238         hw_atl_rpf_vlan_id_flr_set(hw, vlan_id, i);
1239         hw_atl_rpf_vlan_flr_en_set(hw, 1U, i);
1240
1241 exit:
1242         /* Enable VLAN promisc mode if vlan_filter empty  */
1243         for (i = 0; i < HW_ATL_B0_MAX_VLAN_IDS; i++) {
1244                 if (cfg->vlan_filter[i] != 0)
1245                         break;
1246         }
1247
1248         hw_atl_rpf_vlan_prom_mode_en_set(hw, i == HW_ATL_B0_MAX_VLAN_IDS);
1249
1250         return err;
1251 }
1252
1253 static int
1254 atl_enable_vlan_filter(struct rte_eth_dev *dev, int en)
1255 {
1256         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1257         struct aq_hw_cfg_s *cfg =
1258                 ATL_DEV_PRIVATE_TO_CFG(dev->data->dev_private);
1259         int i;
1260
1261         PMD_INIT_FUNC_TRACE();
1262
1263         for (i = 0; i < HW_ATL_B0_MAX_VLAN_IDS; i++) {
1264                 if (cfg->vlan_filter[i])
1265                         hw_atl_rpf_vlan_flr_en_set(hw, en, i);
1266         }
1267         return 0;
1268 }
1269
1270 static int
1271 atl_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1272 {
1273         struct aq_hw_cfg_s *cfg =
1274                 ATL_DEV_PRIVATE_TO_CFG(dev->data->dev_private);
1275         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1276         int ret = 0;
1277         int i;
1278
1279         PMD_INIT_FUNC_TRACE();
1280
1281         ret = atl_enable_vlan_filter(dev, mask & ETH_VLAN_FILTER_MASK);
1282
1283         cfg->vlan_strip = !!(mask & ETH_VLAN_STRIP_MASK);
1284
1285         for (i = 0; i < dev->data->nb_rx_queues; i++)
1286                 hw_atl_rpo_rx_desc_vlan_stripping_set(hw, cfg->vlan_strip, i);
1287
1288         if (mask & ETH_VLAN_EXTEND_MASK)
1289                 ret = -ENOTSUP;
1290
1291         return ret;
1292 }
1293
1294 static int
1295 atl_vlan_tpid_set(struct rte_eth_dev *dev, enum rte_vlan_type vlan_type,
1296                   uint16_t tpid)
1297 {
1298         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1299         int err = 0;
1300
1301         PMD_INIT_FUNC_TRACE();
1302
1303         switch (vlan_type) {
1304         case ETH_VLAN_TYPE_INNER:
1305                 hw_atl_rpf_vlan_inner_etht_set(hw, tpid);
1306                 break;
1307         case ETH_VLAN_TYPE_OUTER:
1308                 hw_atl_rpf_vlan_outer_etht_set(hw, tpid);
1309                 break;
1310         default:
1311                 PMD_DRV_LOG(ERR, "Unsupported VLAN type");
1312                 err = -ENOTSUP;
1313         }
1314
1315         return err;
1316 }
1317
1318 static void
1319 atl_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue_id, int on)
1320 {
1321         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1322
1323         PMD_INIT_FUNC_TRACE();
1324
1325         if (queue_id > dev->data->nb_rx_queues) {
1326                 PMD_DRV_LOG(ERR, "Invalid queue id");
1327                 return;
1328         }
1329
1330         hw_atl_rpo_rx_desc_vlan_stripping_set(hw, on, queue_id);
1331 }
1332
1333 static int
1334 atl_dev_set_mc_addr_list(struct rte_eth_dev *dev,
1335                           struct ether_addr *mc_addr_set,
1336                           uint32_t nb_mc_addr)
1337 {
1338         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1339         u32 i;
1340
1341         if (nb_mc_addr > AQ_HW_MULTICAST_ADDRESS_MAX - HW_ATL_B0_MAC_MIN)
1342                 return -EINVAL;
1343
1344         /* Update whole uc filters table */
1345         for (i = 0; i < AQ_HW_MULTICAST_ADDRESS_MAX - HW_ATL_B0_MAC_MIN; i++) {
1346                 u8 *mac_addr = NULL;
1347                 u32 l = 0, h = 0;
1348
1349                 if (i < nb_mc_addr) {
1350                         mac_addr = mc_addr_set[i].addr_bytes;
1351                         l = (mac_addr[2] << 24) | (mac_addr[3] << 16) |
1352                                 (mac_addr[4] << 8) | mac_addr[5];
1353                         h = (mac_addr[0] << 8) | mac_addr[1];
1354                 }
1355
1356                 hw_atl_rpfl2_uc_flr_en_set(hw, 0U, HW_ATL_B0_MAC_MIN + i);
1357                 hw_atl_rpfl2unicast_dest_addresslsw_set(hw, l,
1358                                                         HW_ATL_B0_MAC_MIN + i);
1359                 hw_atl_rpfl2unicast_dest_addressmsw_set(hw, h,
1360                                                         HW_ATL_B0_MAC_MIN + i);
1361                 hw_atl_rpfl2_uc_flr_en_set(hw, !!mac_addr,
1362                                            HW_ATL_B0_MAC_MIN + i);
1363         }
1364
1365         return 0;
1366 }
1367
1368 static int
1369 atl_reta_update(struct rte_eth_dev *dev,
1370                    struct rte_eth_rss_reta_entry64 *reta_conf,
1371                    uint16_t reta_size)
1372 {
1373         int i;
1374         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1375         struct aq_hw_cfg_s *cf = ATL_DEV_PRIVATE_TO_CFG(dev->data->dev_private);
1376
1377         for (i = 0; i < reta_size && i < cf->aq_rss.indirection_table_size; i++)
1378                 cf->aq_rss.indirection_table[i] = min(reta_conf->reta[i],
1379                                         dev->data->nb_rx_queues - 1);
1380
1381         hw_atl_b0_hw_rss_set(hw, &cf->aq_rss);
1382         return 0;
1383 }
1384
1385 static int
1386 atl_reta_query(struct rte_eth_dev *dev,
1387                     struct rte_eth_rss_reta_entry64 *reta_conf,
1388                     uint16_t reta_size)
1389 {
1390         int i;
1391         struct aq_hw_cfg_s *cf = ATL_DEV_PRIVATE_TO_CFG(dev->data->dev_private);
1392
1393         for (i = 0; i < reta_size && i < cf->aq_rss.indirection_table_size; i++)
1394                 reta_conf->reta[i] = cf->aq_rss.indirection_table[i];
1395         reta_conf->mask = ~0U;
1396         return 0;
1397 }
1398
1399 static int
1400 atl_rss_hash_update(struct rte_eth_dev *dev,
1401                                  struct rte_eth_rss_conf *rss_conf)
1402 {
1403         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1404         struct aq_hw_cfg_s *cfg =
1405                 ATL_DEV_PRIVATE_TO_CFG(dev->data->dev_private);
1406         static u8 def_rss_key[40] = {
1407                 0x1e, 0xad, 0x71, 0x87, 0x65, 0xfc, 0x26, 0x7d,
1408                 0x0d, 0x45, 0x67, 0x74, 0xcd, 0x06, 0x1a, 0x18,
1409                 0xb6, 0xc1, 0xf0, 0xc7, 0xbb, 0x18, 0xbe, 0xf8,
1410                 0x19, 0x13, 0x4b, 0xa9, 0xd0, 0x3e, 0xfe, 0x70,
1411                 0x25, 0x03, 0xab, 0x50, 0x6a, 0x8b, 0x82, 0x0c
1412         };
1413
1414         cfg->is_rss = !!rss_conf->rss_hf;
1415         if (rss_conf->rss_key) {
1416                 memcpy(cfg->aq_rss.hash_secret_key, rss_conf->rss_key,
1417                        rss_conf->rss_key_len);
1418                 cfg->aq_rss.hash_secret_key_size = rss_conf->rss_key_len;
1419         } else {
1420                 memcpy(cfg->aq_rss.hash_secret_key, def_rss_key,
1421                        sizeof(def_rss_key));
1422                 cfg->aq_rss.hash_secret_key_size = sizeof(def_rss_key);
1423         }
1424
1425         hw_atl_b0_hw_rss_set(hw, &cfg->aq_rss);
1426         hw_atl_b0_hw_rss_hash_set(hw, &cfg->aq_rss);
1427         return 0;
1428 }
1429
1430 static int
1431 atl_rss_hash_conf_get(struct rte_eth_dev *dev,
1432                                  struct rte_eth_rss_conf *rss_conf)
1433 {
1434         struct aq_hw_cfg_s *cfg =
1435                 ATL_DEV_PRIVATE_TO_CFG(dev->data->dev_private);
1436
1437         rss_conf->rss_hf = cfg->is_rss ? ATL_RSS_OFFLOAD_ALL : 0;
1438         if (rss_conf->rss_key) {
1439                 rss_conf->rss_key_len = cfg->aq_rss.hash_secret_key_size;
1440                 memcpy(rss_conf->rss_key, cfg->aq_rss.hash_secret_key,
1441                        rss_conf->rss_key_len);
1442         }
1443
1444         return 0;
1445 }
1446
1447 RTE_PMD_REGISTER_PCI(net_atlantic, rte_atl_pmd);
1448 RTE_PMD_REGISTER_PCI_TABLE(net_atlantic, pci_id_atl_map);
1449 RTE_PMD_REGISTER_KMOD_DEP(net_atlantic, "* igb_uio | uio_pci_generic");
1450
1451 RTE_INIT(atl_init_log)
1452 {
1453         atl_logtype_init = rte_log_register("pmd.net.atlantic.init");
1454         if (atl_logtype_init >= 0)
1455                 rte_log_set_level(atl_logtype_init, RTE_LOG_NOTICE);
1456         atl_logtype_driver = rte_log_register("pmd.net.atlantic.driver");
1457         if (atl_logtype_driver >= 0)
1458                 rte_log_set_level(atl_logtype_driver, RTE_LOG_NOTICE);
1459 }
1460