drivers/net/axgbe/axgbe_ethdev.c

   1 /*   SPDX-License-Identifier: BSD-3-Clause
   2  *   Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved.
   3  *   Copyright(c) 2018 Synopsys, Inc. All rights reserved.
   4  */
   5
   6 #include "axgbe_rxtx.h"
   7 #include "axgbe_ethdev.h"
   8 #include "axgbe_common.h"
   9 #include "axgbe_phy.h"
  10 #include "axgbe_regs.h"
  11 #include "rte_time.h"
  12
  13 static int eth_axgbe_dev_init(struct rte_eth_dev *eth_dev);
  14 static int  axgbe_dev_configure(struct rte_eth_dev *dev);
  15 static int  axgbe_dev_start(struct rte_eth_dev *dev);
  16 static int  axgbe_dev_stop(struct rte_eth_dev *dev);
  17 static void axgbe_dev_interrupt_handler(void *param);
  18 static int axgbe_dev_close(struct rte_eth_dev *dev);
  19 static int axgbe_dev_promiscuous_enable(struct rte_eth_dev *dev);
  20 static int axgbe_dev_promiscuous_disable(struct rte_eth_dev *dev);
  21 static int axgbe_dev_allmulticast_enable(struct rte_eth_dev *dev);
  22 static int axgbe_dev_allmulticast_disable(struct rte_eth_dev *dev);
  23 static int axgbe_dev_mac_addr_set(struct rte_eth_dev *dev,
  24                                   struct rte_ether_addr *mac_addr);
  25 static int axgbe_dev_mac_addr_add(struct rte_eth_dev *dev,
  26                                   struct rte_ether_addr *mac_addr,
  27                                   uint32_t index,
  28                                   uint32_t vmdq);
  29 static void axgbe_dev_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index);
  30 static int axgbe_dev_set_mc_addr_list(struct rte_eth_dev *dev,
  31                                       struct rte_ether_addr *mc_addr_set,
  32                                       uint32_t nb_mc_addr);
  33 static int axgbe_dev_uc_hash_table_set(struct rte_eth_dev *dev,
  34                                        struct rte_ether_addr *mac_addr,
  35                                        uint8_t add);
  36 static int axgbe_dev_uc_all_hash_table_set(struct rte_eth_dev *dev,
  37                                            uint8_t add);
  38 static int axgbe_dev_link_update(struct rte_eth_dev *dev,
  39                                  int wait_to_complete);
  40 static int axgbe_dev_get_regs(struct rte_eth_dev *dev,
  41                               struct rte_dev_reg_info *regs);
  42 static int axgbe_dev_stats_get(struct rte_eth_dev *dev,
  43                                 struct rte_eth_stats *stats);
  44 static int axgbe_dev_stats_reset(struct rte_eth_dev *dev);
  45 static int axgbe_dev_xstats_get(struct rte_eth_dev *dev,
  46                                 struct rte_eth_xstat *stats,
  47                                 unsigned int n);
  48 static int
  49 axgbe_dev_xstats_get_names(struct rte_eth_dev *dev,
  50                            struct rte_eth_xstat_name *xstats_names,
  51                            unsigned int size);
  52 static int
  53 axgbe_dev_xstats_get_by_id(struct rte_eth_dev *dev,
  54                            const uint64_t *ids,
  55                            uint64_t *values,
  56                            unsigned int n);
  57 static int
  58 axgbe_dev_xstats_get_names_by_id(struct rte_eth_dev *dev,
  59                                  struct rte_eth_xstat_name *xstats_names,
  60                                  const uint64_t *ids,
  61                                  unsigned int size);
  62 static int axgbe_dev_xstats_reset(struct rte_eth_dev *dev);
  63 static int axgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
  64                           struct rte_eth_rss_reta_entry64 *reta_conf,
  65                           uint16_t reta_size);
  66 static int axgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
  67                          struct rte_eth_rss_reta_entry64 *reta_conf,
  68                          uint16_t reta_size);
  69 static int axgbe_dev_rss_hash_update(struct rte_eth_dev *dev,
  70                                      struct rte_eth_rss_conf *rss_conf);
  71 static int axgbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
  72                                        struct rte_eth_rss_conf *rss_conf);
  73 static int  axgbe_dev_info_get(struct rte_eth_dev *dev,
  74                                struct rte_eth_dev_info *dev_info);
  75 static int axgbe_flow_ctrl_get(struct rte_eth_dev *dev,
  76                                 struct rte_eth_fc_conf *fc_conf);
  77 static int axgbe_flow_ctrl_set(struct rte_eth_dev *dev,
  78                                 struct rte_eth_fc_conf *fc_conf);
  79 static int axgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev,
  80                                 struct rte_eth_pfc_conf *pfc_conf);
  81 static void axgbe_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
  82         struct rte_eth_rxq_info *qinfo);
  83 static void axgbe_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
  84         struct rte_eth_txq_info *qinfo);
  85 const uint32_t *axgbe_dev_supported_ptypes_get(struct rte_eth_dev *dev);
  86 static int axgb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
  87
  88 static int
  89 axgbe_timesync_enable(struct rte_eth_dev *dev);
  90 static int
  91 axgbe_timesync_disable(struct rte_eth_dev *dev);
  92 static int
  93 axgbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
  94                         struct timespec *timestamp, uint32_t flags);
  95 static int
  96 axgbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
  97                         struct timespec *timestamp);
  98 static int
  99 axgbe_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta);
 100 static int
 101 axgbe_timesync_read_time(struct rte_eth_dev *dev,
 102                         struct timespec *timestamp);
 103 static int
 104 axgbe_timesync_write_time(struct rte_eth_dev *dev,
 105                         const struct timespec *timestamp);
 106 static void
 107 axgbe_set_tstamp_time(struct axgbe_port *pdata, unsigned int sec,
 108                         unsigned int nsec);
 109 static void
 110 axgbe_update_tstamp_addend(struct axgbe_port *pdata,
 111                         unsigned int addend);
 112
 113 struct axgbe_xstats {
 114         char name[RTE_ETH_XSTATS_NAME_SIZE];
 115         int offset;
 116 };
 117
 118 #define AXGMAC_MMC_STAT(_string, _var)                           \
 119         { _string,                                              \
 120           offsetof(struct axgbe_mmc_stats, _var),       \
 121         }
 122
 123 static const struct axgbe_xstats axgbe_xstats_strings[] = {
 124         AXGMAC_MMC_STAT("tx_bytes", txoctetcount_gb),
 125         AXGMAC_MMC_STAT("tx_packets", txframecount_gb),
 126         AXGMAC_MMC_STAT("tx_unicast_packets", txunicastframes_gb),
 127         AXGMAC_MMC_STAT("tx_broadcast_packets", txbroadcastframes_gb),
 128         AXGMAC_MMC_STAT("tx_multicast_packets", txmulticastframes_gb),
 129         AXGMAC_MMC_STAT("tx_vlan_packets", txvlanframes_g),
 130         AXGMAC_MMC_STAT("tx_64_byte_packets", tx64octets_gb),
 131         AXGMAC_MMC_STAT("tx_65_to_127_byte_packets", tx65to127octets_gb),
 132         AXGMAC_MMC_STAT("tx_128_to_255_byte_packets", tx128to255octets_gb),
 133         AXGMAC_MMC_STAT("tx_256_to_511_byte_packets", tx256to511octets_gb),
 134         AXGMAC_MMC_STAT("tx_512_to_1023_byte_packets", tx512to1023octets_gb),
 135         AXGMAC_MMC_STAT("tx_1024_to_max_byte_packets", tx1024tomaxoctets_gb),
 136         AXGMAC_MMC_STAT("tx_underflow_errors", txunderflowerror),
 137         AXGMAC_MMC_STAT("tx_pause_frames", txpauseframes),
 138
 139         AXGMAC_MMC_STAT("rx_bytes", rxoctetcount_gb),
 140         AXGMAC_MMC_STAT("rx_packets", rxframecount_gb),
 141         AXGMAC_MMC_STAT("rx_unicast_packets", rxunicastframes_g),
 142         AXGMAC_MMC_STAT("rx_broadcast_packets", rxbroadcastframes_g),
 143         AXGMAC_MMC_STAT("rx_multicast_packets", rxmulticastframes_g),
 144         AXGMAC_MMC_STAT("rx_vlan_packets", rxvlanframes_gb),
 145         AXGMAC_MMC_STAT("rx_64_byte_packets", rx64octets_gb),
 146         AXGMAC_MMC_STAT("rx_65_to_127_byte_packets", rx65to127octets_gb),
 147         AXGMAC_MMC_STAT("rx_128_to_255_byte_packets", rx128to255octets_gb),
 148         AXGMAC_MMC_STAT("rx_256_to_511_byte_packets", rx256to511octets_gb),
 149         AXGMAC_MMC_STAT("rx_512_to_1023_byte_packets", rx512to1023octets_gb),
 150         AXGMAC_MMC_STAT("rx_1024_to_max_byte_packets", rx1024tomaxoctets_gb),
 151         AXGMAC_MMC_STAT("rx_undersize_packets", rxundersize_g),
 152         AXGMAC_MMC_STAT("rx_oversize_packets", rxoversize_g),
 153         AXGMAC_MMC_STAT("rx_crc_errors", rxcrcerror),
 154         AXGMAC_MMC_STAT("rx_crc_errors_small_packets", rxrunterror),
 155         AXGMAC_MMC_STAT("rx_crc_errors_giant_packets", rxjabbererror),
 156         AXGMAC_MMC_STAT("rx_length_errors", rxlengtherror),
 157         AXGMAC_MMC_STAT("rx_out_of_range_errors", rxoutofrangetype),
 158         AXGMAC_MMC_STAT("rx_fifo_overflow_errors", rxfifooverflow),
 159         AXGMAC_MMC_STAT("rx_watchdog_errors", rxwatchdogerror),
 160         AXGMAC_MMC_STAT("rx_pause_frames", rxpauseframes),
 161 };
 162
 163 #define AXGBE_XSTATS_COUNT        ARRAY_SIZE(axgbe_xstats_strings)
 164
 165 /* The set of PCI devices this driver supports */
 166 #define AMD_PCI_VENDOR_ID       0x1022
 167 #define AMD_PCI_RV_ROOT_COMPLEX_ID      0x15d0
 168 #define AMD_PCI_AXGBE_DEVICE_V2A 0x1458
 169 #define AMD_PCI_AXGBE_DEVICE_V2B 0x1459
 170
 171 static const struct rte_pci_id pci_id_axgbe_map[] = {
 172         {RTE_PCI_DEVICE(AMD_PCI_VENDOR_ID, AMD_PCI_AXGBE_DEVICE_V2A)},
 173         {RTE_PCI_DEVICE(AMD_PCI_VENDOR_ID, AMD_PCI_AXGBE_DEVICE_V2B)},
 174         { .vendor_id = 0, },
 175 };
 176
 177 static struct axgbe_version_data axgbe_v2a = {
 178         .init_function_ptrs_phy_impl    = axgbe_init_function_ptrs_phy_v2,
 179         .xpcs_access                    = AXGBE_XPCS_ACCESS_V2,
 180         .mmc_64bit                      = 1,
 181         .tx_max_fifo_size               = 229376,
 182         .rx_max_fifo_size               = 229376,
 183         .tx_tstamp_workaround           = 1,
 184         .ecc_support                    = 1,
 185         .i2c_support                    = 1,
 186         .an_cdr_workaround              = 1,
 187 };
 188
 189 static struct axgbe_version_data axgbe_v2b = {
 190         .init_function_ptrs_phy_impl    = axgbe_init_function_ptrs_phy_v2,
 191         .xpcs_access                    = AXGBE_XPCS_ACCESS_V2,
 192         .mmc_64bit                      = 1,
 193         .tx_max_fifo_size               = 65536,
 194         .rx_max_fifo_size               = 65536,
 195         .tx_tstamp_workaround           = 1,
 196         .ecc_support                    = 1,
 197         .i2c_support                    = 1,
 198         .an_cdr_workaround              = 1,
 199 };
 200
 201 static const struct rte_eth_desc_lim rx_desc_lim = {
 202         .nb_max = AXGBE_MAX_RING_DESC,
 203         .nb_min = AXGBE_MIN_RING_DESC,
 204         .nb_align = 8,
 205 };
 206
 207 static const struct rte_eth_desc_lim tx_desc_lim = {
 208         .nb_max = AXGBE_MAX_RING_DESC,
 209         .nb_min = AXGBE_MIN_RING_DESC,
 210         .nb_align = 8,
 211 };
 212
 213 static const struct eth_dev_ops axgbe_eth_dev_ops = {
 214         .dev_configure        = axgbe_dev_configure,
 215         .dev_start            = axgbe_dev_start,
 216         .dev_stop             = axgbe_dev_stop,
 217         .dev_close            = axgbe_dev_close,
 218         .promiscuous_enable   = axgbe_dev_promiscuous_enable,
 219         .promiscuous_disable  = axgbe_dev_promiscuous_disable,
 220         .allmulticast_enable  = axgbe_dev_allmulticast_enable,
 221         .allmulticast_disable = axgbe_dev_allmulticast_disable,
 222         .mac_addr_set         = axgbe_dev_mac_addr_set,
 223         .mac_addr_add         = axgbe_dev_mac_addr_add,
 224         .mac_addr_remove      = axgbe_dev_mac_addr_remove,
 225         .set_mc_addr_list     = axgbe_dev_set_mc_addr_list,
 226         .uc_hash_table_set    = axgbe_dev_uc_hash_table_set,
 227         .uc_all_hash_table_set = axgbe_dev_uc_all_hash_table_set,
 228         .link_update          = axgbe_dev_link_update,
 229         .get_reg              = axgbe_dev_get_regs,
 230         .stats_get            = axgbe_dev_stats_get,
 231         .stats_reset          = axgbe_dev_stats_reset,
 232         .xstats_get           = axgbe_dev_xstats_get,
 233         .xstats_reset         = axgbe_dev_xstats_reset,
 234         .xstats_get_names     = axgbe_dev_xstats_get_names,
 235         .xstats_get_names_by_id = axgbe_dev_xstats_get_names_by_id,
 236         .xstats_get_by_id     = axgbe_dev_xstats_get_by_id,
 237         .reta_update          = axgbe_dev_rss_reta_update,
 238         .reta_query           = axgbe_dev_rss_reta_query,
 239         .rss_hash_update      = axgbe_dev_rss_hash_update,
 240         .rss_hash_conf_get    = axgbe_dev_rss_hash_conf_get,
 241         .dev_infos_get        = axgbe_dev_info_get,
 242         .rx_queue_setup       = axgbe_dev_rx_queue_setup,
 243         .rx_queue_release     = axgbe_dev_rx_queue_release,
 244         .tx_queue_setup       = axgbe_dev_tx_queue_setup,
 245         .tx_queue_release     = axgbe_dev_tx_queue_release,
 246         .flow_ctrl_get        = axgbe_flow_ctrl_get,
 247         .flow_ctrl_set        = axgbe_flow_ctrl_set,
 248         .priority_flow_ctrl_set = axgbe_priority_flow_ctrl_set,
 249         .rxq_info_get                 = axgbe_rxq_info_get,
 250         .txq_info_get                 = axgbe_txq_info_get,
 251         .dev_supported_ptypes_get     = axgbe_dev_supported_ptypes_get,
 252         .mtu_set                = axgb_mtu_set,
 253         .timesync_enable              = axgbe_timesync_enable,
 254         .timesync_disable             = axgbe_timesync_disable,
 255         .timesync_read_rx_timestamp   = axgbe_timesync_read_rx_timestamp,
 256         .timesync_read_tx_timestamp   = axgbe_timesync_read_tx_timestamp,
 257         .timesync_adjust_time         = axgbe_timesync_adjust_time,
 258         .timesync_read_time           = axgbe_timesync_read_time,
 259         .timesync_write_time          = axgbe_timesync_write_time,
 260 };
 261
 262 static int axgbe_phy_reset(struct axgbe_port *pdata)
 263 {
 264         pdata->phy_link = -1;
 265         pdata->phy_speed = SPEED_UNKNOWN;
 266         return pdata->phy_if.phy_reset(pdata);
 267 }
 268
 269 /*
 270  * Interrupt handler triggered by NIC  for handling
 271  * specific interrupt.
 272  *
 273  * @param handle
 274  *  Pointer to interrupt handle.
 275  * @param param
 276  *  The address of parameter (struct rte_eth_dev *) regsitered before.
 277  *
 278  * @return
 279  *  void
 280  */
 281 static void
 282 axgbe_dev_interrupt_handler(void *param)
 283 {
 284         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
 285         struct axgbe_port *pdata = dev->data->dev_private;
 286         unsigned int dma_isr, dma_ch_isr;
 287
 288         pdata->phy_if.an_isr(pdata);
 289         /*DMA related interrupts*/
 290         dma_isr = AXGMAC_IOREAD(pdata, DMA_ISR);
 291         PMD_DRV_LOG(DEBUG, "DMA_ISR=%#010x\n", dma_isr);
 292         if (dma_isr) {
 293                 if (dma_isr & 1) {
 294                         dma_ch_isr =
 295                                 AXGMAC_DMA_IOREAD((struct axgbe_rx_queue *)
 296                                                   pdata->rx_queues[0],
 297                                                   DMA_CH_SR);
 298                         PMD_DRV_LOG(DEBUG, "DMA_CH0_ISR=%#010x\n", dma_ch_isr);
 299                         AXGMAC_DMA_IOWRITE((struct axgbe_rx_queue *)
 300                                            pdata->rx_queues[0],
 301                                            DMA_CH_SR, dma_ch_isr);
 302                 }
 303         }
 304         /* Unmask interrupts since disabled after generation */
 305         rte_intr_ack(&pdata->pci_dev->intr_handle);
 306 }
 307
 308 /*
 309  * Configure device link speed and setup link.
 310  * It returns 0 on success.
 311  */
 312 static int
 313 axgbe_dev_configure(struct rte_eth_dev *dev)
 314 {
 315         struct axgbe_port *pdata =  dev->data->dev_private;
 316         /* Checksum offload to hardware */
 317         pdata->rx_csum_enable = dev->data->dev_conf.rxmode.offloads &
 318                                 DEV_RX_OFFLOAD_CHECKSUM;
 319         return 0;
 320 }
 321
 322 static int
 323 axgbe_dev_rx_mq_config(struct rte_eth_dev *dev)
 324 {
 325         struct axgbe_port *pdata = dev->data->dev_private;
 326
 327         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS)
 328                 pdata->rss_enable = 1;
 329         else if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_NONE)
 330                 pdata->rss_enable = 0;
 331         else
 332                 return  -1;
 333         return 0;
 334 }
 335
 336 static int
 337 axgbe_dev_start(struct rte_eth_dev *dev)
 338 {
 339         struct axgbe_port *pdata = dev->data->dev_private;
 340         int ret;
 341         struct rte_eth_dev_data *dev_data = dev->data;
 342         uint16_t max_pkt_len = dev_data->dev_conf.rxmode.max_rx_pkt_len;
 343
 344         dev->dev_ops = &axgbe_eth_dev_ops;
 345
 346         PMD_INIT_FUNC_TRACE();
 347
 348         /* Multiqueue RSS */
 349         ret = axgbe_dev_rx_mq_config(dev);
 350         if (ret) {
 351                 PMD_DRV_LOG(ERR, "Unable to config RX MQ\n");
 352                 return ret;
 353         }
 354         ret = axgbe_phy_reset(pdata);
 355         if (ret) {
 356                 PMD_DRV_LOG(ERR, "phy reset failed\n");
 357                 return ret;
 358         }
 359         ret = pdata->hw_if.init(pdata);
 360         if (ret) {
 361                 PMD_DRV_LOG(ERR, "dev_init failed\n");
 362                 return ret;
 363         }
 364
 365         /* enable uio/vfio intr/eventfd mapping */
 366         rte_intr_enable(&pdata->pci_dev->intr_handle);
 367
 368         /* phy start*/
 369         pdata->phy_if.phy_start(pdata);
 370         axgbe_dev_enable_tx(dev);
 371         axgbe_dev_enable_rx(dev);
 372
 373         rte_bit_relaxed_clear32(AXGBE_STOPPED, &pdata->dev_state);
 374         rte_bit_relaxed_clear32(AXGBE_DOWN, &pdata->dev_state);
 375         if ((dev_data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_SCATTER) ||
 376                                 max_pkt_len > pdata->rx_buf_size)
 377                 dev_data->scattered_rx = 1;
 378
 379         /*  Scatter Rx handling */
 380         if (dev_data->scattered_rx)
 381                 dev->rx_pkt_burst = &eth_axgbe_recv_scattered_pkts;
 382         else
 383                 dev->rx_pkt_burst = &axgbe_recv_pkts;
 384
 385         return 0;
 386 }
 387
 388 /* Stop device: disable rx and tx functions to allow for reconfiguring. */
 389 static int
 390 axgbe_dev_stop(struct rte_eth_dev *dev)
 391 {
 392         struct axgbe_port *pdata = dev->data->dev_private;
 393
 394         PMD_INIT_FUNC_TRACE();
 395
 396         rte_intr_disable(&pdata->pci_dev->intr_handle);
 397
 398         if (rte_bit_relaxed_get32(AXGBE_STOPPED, &pdata->dev_state))
 399                 return 0;
 400
 401         rte_bit_relaxed_set32(AXGBE_STOPPED, &pdata->dev_state);
 402         axgbe_dev_disable_tx(dev);
 403         axgbe_dev_disable_rx(dev);
 404
 405         pdata->phy_if.phy_stop(pdata);
 406         pdata->hw_if.exit(pdata);
 407         memset(&dev->data->dev_link, 0, sizeof(struct rte_eth_link));
 408         rte_bit_relaxed_set32(AXGBE_DOWN, &pdata->dev_state);
 409
 410         return 0;
 411 }
 412
 413 static int
 414 axgbe_dev_promiscuous_enable(struct rte_eth_dev *dev)
 415 {
 416         struct axgbe_port *pdata = dev->data->dev_private;
 417
 418         PMD_INIT_FUNC_TRACE();
 419
 420         AXGMAC_IOWRITE_BITS(pdata, MAC_PFR, PR, 1);
 421
 422         return 0;
 423 }
 424
 425 static int
 426 axgbe_dev_promiscuous_disable(struct rte_eth_dev *dev)
 427 {
 428         struct axgbe_port *pdata = dev->data->dev_private;
 429
 430         PMD_INIT_FUNC_TRACE();
 431
 432         AXGMAC_IOWRITE_BITS(pdata, MAC_PFR, PR, 0);
 433
 434         return 0;
 435 }
 436
 437 static int
 438 axgbe_dev_allmulticast_enable(struct rte_eth_dev *dev)
 439 {
 440         struct axgbe_port *pdata = dev->data->dev_private;
 441
 442         PMD_INIT_FUNC_TRACE();
 443
 444         if (AXGMAC_IOREAD_BITS(pdata, MAC_PFR, PM))
 445                 return 0;
 446         AXGMAC_IOWRITE_BITS(pdata, MAC_PFR, PM, 1);
 447
 448         return 0;
 449 }
 450
 451 static int
 452 axgbe_dev_allmulticast_disable(struct rte_eth_dev *dev)
 453 {
 454         struct axgbe_port *pdata = dev->data->dev_private;
 455
 456         PMD_INIT_FUNC_TRACE();
 457
 458         if (!AXGMAC_IOREAD_BITS(pdata, MAC_PFR, PM))
 459                 return 0;
 460         AXGMAC_IOWRITE_BITS(pdata, MAC_PFR, PM, 0);
 461
 462         return 0;
 463 }
 464
 465 static int
 466 axgbe_dev_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr)
 467 {
 468         struct axgbe_port *pdata = dev->data->dev_private;
 469
 470         /* Set Default MAC Addr */
 471         axgbe_set_mac_addn_addr(pdata, (u8 *)mac_addr, 0);
 472
 473         return 0;
 474 }
 475
 476 static int
 477 axgbe_dev_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
 478                               uint32_t index, uint32_t pool __rte_unused)
 479 {
 480         struct axgbe_port *pdata = dev->data->dev_private;
 481         struct axgbe_hw_features *hw_feat = &pdata->hw_feat;
 482
 483         if (index > hw_feat->addn_mac) {
 484                 PMD_DRV_LOG(ERR, "Invalid Index %d\n", index);
 485                 return -EINVAL;
 486         }
 487         axgbe_set_mac_addn_addr(pdata, (u8 *)mac_addr, index);
 488         return 0;
 489 }
 490
 491 static int
 492 axgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
 493                           struct rte_eth_rss_reta_entry64 *reta_conf,
 494                           uint16_t reta_size)
 495 {
 496         struct axgbe_port *pdata = dev->data->dev_private;
 497         unsigned int i, idx, shift;
 498         int ret;
 499
 500         if (!pdata->rss_enable) {
 501                 PMD_DRV_LOG(ERR, "RSS not enabled\n");
 502                 return -ENOTSUP;
 503         }
 504
 505         if (reta_size == 0 || reta_size > AXGBE_RSS_MAX_TABLE_SIZE) {
 506                 PMD_DRV_LOG(ERR, "reta_size %d is not supported\n", reta_size);
 507                 return -EINVAL;
 508         }
 509
 510         for (i = 0; i < reta_size; i++) {
 511                 idx = i / RTE_RETA_GROUP_SIZE;
 512                 shift = i % RTE_RETA_GROUP_SIZE;
 513                 if ((reta_conf[idx].mask & (1ULL << shift)) == 0)
 514                         continue;
 515                 pdata->rss_table[i] = reta_conf[idx].reta[shift];
 516         }
 517
 518         /* Program the lookup table */
 519         ret = axgbe_write_rss_lookup_table(pdata);
 520         return ret;
 521 }
 522
 523 static int
 524 axgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
 525                          struct rte_eth_rss_reta_entry64 *reta_conf,
 526                          uint16_t reta_size)
 527 {
 528         struct axgbe_port *pdata = dev->data->dev_private;
 529         unsigned int i, idx, shift;
 530
 531         if (!pdata->rss_enable) {
 532                 PMD_DRV_LOG(ERR, "RSS not enabled\n");
 533                 return -ENOTSUP;
 534         }
 535
 536         if (reta_size == 0 || reta_size > AXGBE_RSS_MAX_TABLE_SIZE) {
 537                 PMD_DRV_LOG(ERR, "reta_size %d is not supported\n", reta_size);
 538                 return -EINVAL;
 539         }
 540
 541         for (i = 0; i < reta_size; i++) {
 542                 idx = i / RTE_RETA_GROUP_SIZE;
 543                 shift = i % RTE_RETA_GROUP_SIZE;
 544                 if ((reta_conf[idx].mask & (1ULL << shift)) == 0)
 545                         continue;
 546                 reta_conf[idx].reta[shift] = pdata->rss_table[i];
 547         }
 548         return 0;
 549 }
 550
 551 static int
 552 axgbe_dev_rss_hash_update(struct rte_eth_dev *dev,
 553                           struct rte_eth_rss_conf *rss_conf)
 554 {
 555         struct axgbe_port *pdata = dev->data->dev_private;
 556         int ret;
 557
 558         if (!pdata->rss_enable) {
 559                 PMD_DRV_LOG(ERR, "RSS not enabled\n");
 560                 return -ENOTSUP;
 561         }
 562
 563         if (rss_conf == NULL) {
 564                 PMD_DRV_LOG(ERR, "rss_conf value isn't valid\n");
 565                 return -EINVAL;
 566         }
 567
 568         if (rss_conf->rss_key != NULL &&
 569             rss_conf->rss_key_len == AXGBE_RSS_HASH_KEY_SIZE) {
 570                 rte_memcpy(pdata->rss_key, rss_conf->rss_key,
 571                        AXGBE_RSS_HASH_KEY_SIZE);
 572                 /* Program the hash key */
 573                 ret = axgbe_write_rss_hash_key(pdata);
 574                 if (ret != 0)
 575                         return ret;
 576         }
 577
 578         pdata->rss_hf = rss_conf->rss_hf & AXGBE_RSS_OFFLOAD;
 579
 580         if (pdata->rss_hf & (ETH_RSS_IPV4 | ETH_RSS_IPV6))
 581                 AXGMAC_SET_BITS(pdata->rss_options, MAC_RSSCR, IP2TE, 1);
 582         if (pdata->rss_hf &
 583             (ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_NONFRAG_IPV6_TCP))
 584                 AXGMAC_SET_BITS(pdata->rss_options, MAC_RSSCR, TCP4TE, 1);
 585         if (pdata->rss_hf &
 586             (ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_NONFRAG_IPV6_UDP))
 587                 AXGMAC_SET_BITS(pdata->rss_options, MAC_RSSCR, UDP4TE, 1);
 588
 589         /* Set the RSS options */
 590         AXGMAC_IOWRITE(pdata, MAC_RSSCR, pdata->rss_options);
 591
 592         return 0;
 593 }
 594
 595 static int
 596 axgbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
 597                             struct rte_eth_rss_conf *rss_conf)
 598 {
 599         struct axgbe_port *pdata = dev->data->dev_private;
 600
 601         if (!pdata->rss_enable) {
 602                 PMD_DRV_LOG(ERR, "RSS not enabled\n");
 603                 return -ENOTSUP;
 604         }
 605
 606         if (rss_conf == NULL) {
 607                 PMD_DRV_LOG(ERR, "rss_conf value isn't valid\n");
 608                 return -EINVAL;
 609         }
 610
 611         if (rss_conf->rss_key != NULL &&
 612             rss_conf->rss_key_len >= AXGBE_RSS_HASH_KEY_SIZE) {
 613                 rte_memcpy(rss_conf->rss_key, pdata->rss_key,
 614                        AXGBE_RSS_HASH_KEY_SIZE);
 615         }
 616         rss_conf->rss_key_len = AXGBE_RSS_HASH_KEY_SIZE;
 617         rss_conf->rss_hf = pdata->rss_hf;
 618         return 0;
 619 }
 620
 621 static void
 622 axgbe_dev_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
 623 {
 624         struct axgbe_port *pdata = dev->data->dev_private;
 625         struct axgbe_hw_features *hw_feat = &pdata->hw_feat;
 626
 627         if (index > hw_feat->addn_mac) {
 628                 PMD_DRV_LOG(ERR, "Invalid Index %d\n", index);
 629                 return;
 630         }
 631         axgbe_set_mac_addn_addr(pdata, NULL, index);
 632 }
 633
 634 static int
 635 axgbe_dev_set_mc_addr_list(struct rte_eth_dev *dev,
 636                                       struct rte_ether_addr *mc_addr_set,
 637                                       uint32_t nb_mc_addr)
 638 {
 639         struct axgbe_port *pdata = dev->data->dev_private;
 640         struct axgbe_hw_features *hw_feat = &pdata->hw_feat;
 641         uint32_t index = 1; /* 0 is always default mac */
 642         uint32_t i;
 643
 644         if (nb_mc_addr > hw_feat->addn_mac) {
 645                 PMD_DRV_LOG(ERR, "Invalid Index %d\n", nb_mc_addr);
 646                 return -EINVAL;
 647         }
 648
 649         /* clear unicast addresses */
 650         for (i = 1; i < hw_feat->addn_mac; i++) {
 651                 if (rte_is_zero_ether_addr(&dev->data->mac_addrs[i]))
 652                         continue;
 653                 memset(&dev->data->mac_addrs[i], 0,
 654                        sizeof(struct rte_ether_addr));
 655         }
 656
 657         while (nb_mc_addr--)
 658                 axgbe_set_mac_addn_addr(pdata, (u8 *)mc_addr_set++, index++);
 659
 660         return 0;
 661 }
 662
 663 static int
 664 axgbe_dev_uc_hash_table_set(struct rte_eth_dev *dev,
 665                             struct rte_ether_addr *mac_addr, uint8_t add)
 666 {
 667         struct axgbe_port *pdata = dev->data->dev_private;
 668         struct axgbe_hw_features *hw_feat = &pdata->hw_feat;
 669
 670         if (!hw_feat->hash_table_size) {
 671                 PMD_DRV_LOG(ERR, "MAC Hash Table not supported\n");
 672                 return -ENOTSUP;
 673         }
 674
 675         axgbe_set_mac_hash_table(pdata, (u8 *)mac_addr, add);
 676
 677         if (pdata->uc_hash_mac_addr > 0) {
 678                 AXGMAC_IOWRITE_BITS(pdata, MAC_PFR, HPF, 1);
 679                 AXGMAC_IOWRITE_BITS(pdata, MAC_PFR, HUC, 1);
 680         } else {
 681                 AXGMAC_IOWRITE_BITS(pdata, MAC_PFR, HPF, 0);
 682                 AXGMAC_IOWRITE_BITS(pdata, MAC_PFR, HUC, 0);
 683         }
 684         return 0;
 685 }
 686
 687 static int
 688 axgbe_dev_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t add)
 689 {
 690         struct axgbe_port *pdata = dev->data->dev_private;
 691         struct axgbe_hw_features *hw_feat = &pdata->hw_feat;
 692         uint32_t index;
 693
 694         if (!hw_feat->hash_table_size) {
 695                 PMD_DRV_LOG(ERR, "MAC Hash Table not supported\n");
 696                 return -ENOTSUP;
 697         }
 698
 699         for (index = 0; index < pdata->hash_table_count; index++) {
 700                 if (add)
 701                         pdata->uc_hash_table[index] = ~0;
 702                 else
 703                         pdata->uc_hash_table[index] = 0;
 704
 705                 PMD_DRV_LOG(DEBUG, "%s MAC hash table at Index %#x\n",
 706                             add ? "set" : "clear", index);
 707
 708                 AXGMAC_IOWRITE(pdata, MAC_HTR(index),
 709                                pdata->uc_hash_table[index]);
 710         }
 711
 712         if (add) {
 713                 AXGMAC_IOWRITE_BITS(pdata, MAC_PFR, HPF, 1);
 714                 AXGMAC_IOWRITE_BITS(pdata, MAC_PFR, HUC, 1);
 715         } else {
 716                 AXGMAC_IOWRITE_BITS(pdata, MAC_PFR, HPF, 0);
 717                 AXGMAC_IOWRITE_BITS(pdata, MAC_PFR, HUC, 0);
 718         }
 719         return 0;
 720 }
 721
 722 /* return 0 means link status changed, -1 means not changed */
 723 static int
 724 axgbe_dev_link_update(struct rte_eth_dev *dev,
 725                       int wait_to_complete __rte_unused)
 726 {
 727         struct axgbe_port *pdata = dev->data->dev_private;
 728         struct rte_eth_link link;
 729         int ret = 0;
 730
 731         PMD_INIT_FUNC_TRACE();
 732         rte_delay_ms(800);
 733
 734         pdata->phy_if.phy_status(pdata);
 735
 736         memset(&link, 0, sizeof(struct rte_eth_link));
 737         link.link_duplex = pdata->phy.duplex;
 738         link.link_status = pdata->phy_link;
 739         link.link_speed = pdata->phy_speed;
 740         link.link_autoneg = !(dev->data->dev_conf.link_speeds &
 741                               ETH_LINK_SPEED_FIXED);
 742         ret = rte_eth_linkstatus_set(dev, &link);
 743         if (ret == -1)
 744                 PMD_DRV_LOG(ERR, "No change in link status\n");
 745
 746         return ret;
 747 }
 748
 749 static int
 750 axgbe_dev_get_regs(struct rte_eth_dev *dev, struct rte_dev_reg_info *regs)
 751 {
 752         struct axgbe_port *pdata = dev->data->dev_private;
 753
 754         if (regs->data == NULL) {
 755                 regs->length = axgbe_regs_get_count(pdata);
 756                 regs->width = sizeof(uint32_t);
 757                 return 0;
 758         }
 759
 760         /* Only full register dump is supported */
 761         if (regs->length &&
 762             regs->length != (uint32_t)axgbe_regs_get_count(pdata))
 763                 return -ENOTSUP;
 764
 765         regs->version = pdata->pci_dev->id.vendor_id << 16 |
 766                         pdata->pci_dev->id.device_id;
 767         axgbe_regs_dump(pdata, regs->data);
 768         return 0;
 769 }
 770 static void axgbe_read_mmc_stats(struct axgbe_port *pdata)
 771 {
 772         struct axgbe_mmc_stats *stats = &pdata->mmc_stats;
 773
 774         /* Freeze counters */
 775         AXGMAC_IOWRITE_BITS(pdata, MMC_CR, MCF, 1);
 776
 777         /* Tx counters */
 778         stats->txoctetcount_gb +=
 779                 AXGMAC_IOREAD(pdata, MMC_TXOCTETCOUNT_GB_LO);
 780         stats->txoctetcount_gb +=
 781         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_TXOCTETCOUNT_GB_HI) << 32);
 782
 783         stats->txframecount_gb +=
 784                 AXGMAC_IOREAD(pdata, MMC_TXFRAMECOUNT_GB_LO);
 785         stats->txframecount_gb +=
 786         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_TXFRAMECOUNT_GB_HI) << 32);
 787
 788         stats->txbroadcastframes_g +=
 789                 AXGMAC_IOREAD(pdata, MMC_TXBROADCASTFRAMES_G_LO);
 790         stats->txbroadcastframes_g +=
 791         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_TXBROADCASTFRAMES_G_HI) << 32);
 792
 793         stats->txmulticastframes_g +=
 794                 AXGMAC_IOREAD(pdata, MMC_TXMULTICASTFRAMES_G_LO);
 795         stats->txmulticastframes_g +=
 796         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_TXMULTICASTFRAMES_G_HI) << 32);
 797
 798         stats->tx64octets_gb +=
 799                 AXGMAC_IOREAD(pdata, MMC_TX64OCTETS_GB_LO);
 800         stats->tx64octets_gb +=
 801         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_TX64OCTETS_GB_HI) << 32);
 802
 803         stats->tx65to127octets_gb +=
 804                 AXGMAC_IOREAD(pdata, MMC_TX65TO127OCTETS_GB_LO);
 805         stats->tx65to127octets_gb +=
 806         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_TX65TO127OCTETS_GB_HI) << 32);
 807
 808         stats->tx128to255octets_gb +=
 809                 AXGMAC_IOREAD(pdata, MMC_TX128TO255OCTETS_GB_LO);
 810         stats->tx128to255octets_gb +=
 811         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_TX128TO255OCTETS_GB_HI) << 32);
 812
 813         stats->tx256to511octets_gb +=
 814                 AXGMAC_IOREAD(pdata, MMC_TX256TO511OCTETS_GB_LO);
 815         stats->tx256to511octets_gb +=
 816         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_TX256TO511OCTETS_GB_HI) << 32);
 817
 818         stats->tx512to1023octets_gb +=
 819                 AXGMAC_IOREAD(pdata, MMC_TX512TO1023OCTETS_GB_LO);
 820         stats->tx512to1023octets_gb +=
 821         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_TX512TO1023OCTETS_GB_HI) << 32);
 822
 823         stats->tx1024tomaxoctets_gb +=
 824                 AXGMAC_IOREAD(pdata, MMC_TX1024TOMAXOCTETS_GB_LO);
 825         stats->tx1024tomaxoctets_gb +=
 826         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_TX1024TOMAXOCTETS_GB_HI) << 32);
 827
 828         stats->txunicastframes_gb +=
 829                 AXGMAC_IOREAD(pdata, MMC_TXUNICASTFRAMES_GB_LO);
 830         stats->txunicastframes_gb +=
 831         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_TXUNICASTFRAMES_GB_HI) << 32);
 832
 833         stats->txmulticastframes_gb +=
 834                 AXGMAC_IOREAD(pdata, MMC_TXMULTICASTFRAMES_GB_LO);
 835         stats->txmulticastframes_gb +=
 836         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_TXMULTICASTFRAMES_GB_HI) << 32);
 837
 838         stats->txbroadcastframes_g +=
 839                 AXGMAC_IOREAD(pdata, MMC_TXBROADCASTFRAMES_GB_LO);
 840         stats->txbroadcastframes_g +=
 841         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_TXBROADCASTFRAMES_GB_HI) << 32);
 842
 843         stats->txunderflowerror +=
 844                 AXGMAC_IOREAD(pdata, MMC_TXUNDERFLOWERROR_LO);
 845         stats->txunderflowerror +=
 846         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_TXUNDERFLOWERROR_HI) << 32);
 847
 848         stats->txoctetcount_g +=
 849                 AXGMAC_IOREAD(pdata, MMC_TXOCTETCOUNT_G_LO);
 850         stats->txoctetcount_g +=
 851         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_TXOCTETCOUNT_G_HI) << 32);
 852
 853         stats->txframecount_g +=
 854                 AXGMAC_IOREAD(pdata, MMC_TXFRAMECOUNT_G_LO);
 855         stats->txframecount_g +=
 856         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_TXFRAMECOUNT_G_HI) << 32);
 857
 858         stats->txpauseframes +=
 859                 AXGMAC_IOREAD(pdata, MMC_TXPAUSEFRAMES_LO);
 860         stats->txpauseframes +=
 861         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_TXPAUSEFRAMES_HI) << 32);
 862
 863         stats->txvlanframes_g +=
 864                 AXGMAC_IOREAD(pdata, MMC_TXVLANFRAMES_G_LO);
 865         stats->txvlanframes_g +=
 866         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_TXVLANFRAMES_G_HI) << 32);
 867
 868         /* Rx counters */
 869         stats->rxframecount_gb +=
 870                 AXGMAC_IOREAD(pdata, MMC_RXFRAMECOUNT_GB_LO);
 871         stats->rxframecount_gb +=
 872         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_RXFRAMECOUNT_GB_HI) << 32);
 873
 874         stats->rxoctetcount_gb +=
 875                 AXGMAC_IOREAD(pdata, MMC_RXOCTETCOUNT_GB_LO);
 876         stats->rxoctetcount_gb +=
 877         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_RXOCTETCOUNT_GB_HI) << 32);
 878
 879         stats->rxoctetcount_g +=
 880                 AXGMAC_IOREAD(pdata, MMC_RXOCTETCOUNT_G_LO);
 881         stats->rxoctetcount_g +=
 882         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_RXOCTETCOUNT_G_HI) << 32);
 883
 884         stats->rxbroadcastframes_g +=
 885                 AXGMAC_IOREAD(pdata, MMC_RXBROADCASTFRAMES_G_LO);
 886         stats->rxbroadcastframes_g +=
 887         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_RXBROADCASTFRAMES_G_HI) << 32);
 888
 889         stats->rxmulticastframes_g +=
 890                 AXGMAC_IOREAD(pdata, MMC_RXMULTICASTFRAMES_G_LO);
 891         stats->rxmulticastframes_g +=
 892         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_RXMULTICASTFRAMES_G_HI) << 32);
 893
 894         stats->rxcrcerror +=
 895                 AXGMAC_IOREAD(pdata, MMC_RXCRCERROR_LO);
 896         stats->rxcrcerror +=
 897         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_RXCRCERROR_HI) << 32);
 898
 899         stats->rxrunterror +=
 900                 AXGMAC_IOREAD(pdata, MMC_RXRUNTERROR);
 901
 902         stats->rxjabbererror +=
 903                 AXGMAC_IOREAD(pdata, MMC_RXJABBERERROR);
 904
 905         stats->rxundersize_g +=
 906                 AXGMAC_IOREAD(pdata, MMC_RXUNDERSIZE_G);
 907
 908         stats->rxoversize_g +=
 909                 AXGMAC_IOREAD(pdata, MMC_RXOVERSIZE_G);
 910
 911         stats->rx64octets_gb +=
 912                 AXGMAC_IOREAD(pdata, MMC_RX64OCTETS_GB_LO);
 913         stats->rx64octets_gb +=
 914         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_RX64OCTETS_GB_HI) << 32);
 915
 916         stats->rx65to127octets_gb +=
 917                 AXGMAC_IOREAD(pdata, MMC_RX65TO127OCTETS_GB_LO);
 918         stats->rx65to127octets_gb +=
 919         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_RX65TO127OCTETS_GB_HI) << 32);
 920
 921         stats->rx128to255octets_gb +=
 922                 AXGMAC_IOREAD(pdata, MMC_RX128TO255OCTETS_GB_LO);
 923         stats->rx128to255octets_gb +=
 924         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_RX128TO255OCTETS_GB_HI) << 32);
 925
 926         stats->rx256to511octets_gb +=
 927                 AXGMAC_IOREAD(pdata, MMC_RX256TO511OCTETS_GB_LO);
 928         stats->rx256to511octets_gb +=
 929         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_RX256TO511OCTETS_GB_HI) << 32);
 930
 931         stats->rx512to1023octets_gb +=
 932                 AXGMAC_IOREAD(pdata, MMC_RX512TO1023OCTETS_GB_LO);
 933         stats->rx512to1023octets_gb +=
 934         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_RX512TO1023OCTETS_GB_HI) << 32);
 935
 936         stats->rx1024tomaxoctets_gb +=
 937                 AXGMAC_IOREAD(pdata, MMC_RX1024TOMAXOCTETS_GB_LO);
 938         stats->rx1024tomaxoctets_gb +=
 939         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_RX1024TOMAXOCTETS_GB_HI) << 32);
 940
 941         stats->rxunicastframes_g +=
 942                 AXGMAC_IOREAD(pdata, MMC_RXUNICASTFRAMES_G_LO);
 943         stats->rxunicastframes_g +=
 944         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_RXUNICASTFRAMES_G_HI) << 32);
 945
 946         stats->rxlengtherror +=
 947                 AXGMAC_IOREAD(pdata, MMC_RXLENGTHERROR_LO);
 948         stats->rxlengtherror +=
 949         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_RXLENGTHERROR_HI) << 32);
 950
 951         stats->rxoutofrangetype +=
 952                 AXGMAC_IOREAD(pdata, MMC_RXOUTOFRANGETYPE_LO);
 953         stats->rxoutofrangetype +=
 954         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_RXOUTOFRANGETYPE_HI) << 32);
 955
 956         stats->rxpauseframes +=
 957                 AXGMAC_IOREAD(pdata, MMC_RXPAUSEFRAMES_LO);
 958         stats->rxpauseframes +=
 959         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_RXPAUSEFRAMES_HI) << 32);
 960
 961         stats->rxfifooverflow +=
 962                 AXGMAC_IOREAD(pdata, MMC_RXFIFOOVERFLOW_LO);
 963         stats->rxfifooverflow +=
 964         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_RXFIFOOVERFLOW_HI) << 32);
 965
 966         stats->rxvlanframes_gb +=
 967                 AXGMAC_IOREAD(pdata, MMC_RXVLANFRAMES_GB_LO);
 968         stats->rxvlanframes_gb +=
 969         ((uint64_t)AXGMAC_IOREAD(pdata, MMC_RXVLANFRAMES_GB_HI) << 32);
 970
 971         stats->rxwatchdogerror +=
 972                 AXGMAC_IOREAD(pdata, MMC_RXWATCHDOGERROR);
 973
 974         /* Un-freeze counters */
 975         AXGMAC_IOWRITE_BITS(pdata, MMC_CR, MCF, 0);
 976 }
 977
 978 static int
 979 axgbe_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *stats,
 980                      unsigned int n)
 981 {
 982         struct axgbe_port *pdata = dev->data->dev_private;
 983         unsigned int i;
 984
 985         if (!stats)
 986                 return 0;
 987
 988         axgbe_read_mmc_stats(pdata);
 989
 990         for (i = 0; i < n && i < AXGBE_XSTATS_COUNT; i++) {
 991                 stats[i].id = i;
 992                 stats[i].value = *(u64 *)((uint8_t *)&pdata->mmc_stats +
 993                                 axgbe_xstats_strings[i].offset);
 994         }
 995
 996         return i;
 997 }
 998
 999 static int
1000 axgbe_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1001                            struct rte_eth_xstat_name *xstats_names,
1002                            unsigned int n)
1003 {
1004         unsigned int i;
1005
1006         if (n >= AXGBE_XSTATS_COUNT && xstats_names) {
1007                 for (i = 0; i < AXGBE_XSTATS_COUNT; ++i) {
1008                         snprintf(xstats_names[i].name,
1009                                  RTE_ETH_XSTATS_NAME_SIZE, "%s",
1010                                  axgbe_xstats_strings[i].name);
1011                 }
1012         }
1013
1014         return AXGBE_XSTATS_COUNT;
1015 }
1016
1017 static int
1018 axgbe_dev_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids,
1019                            uint64_t *values, unsigned int n)
1020 {
1021         unsigned int i;
1022         uint64_t values_copy[AXGBE_XSTATS_COUNT];
1023
1024         if (!ids) {
1025                 struct axgbe_port *pdata = dev->data->dev_private;
1026
1027                 if (n < AXGBE_XSTATS_COUNT)
1028                         return AXGBE_XSTATS_COUNT;
1029
1030                 axgbe_read_mmc_stats(pdata);
1031
1032                 for (i = 0; i < AXGBE_XSTATS_COUNT; i++) {
1033                         values[i] = *(u64 *)((uint8_t *)&pdata->mmc_stats +
1034                                         axgbe_xstats_strings[i].offset);
1035                 }
1036
1037                 return i;
1038         }
1039
1040         axgbe_dev_xstats_get_by_id(dev, NULL, values_copy, AXGBE_XSTATS_COUNT);
1041
1042         for (i = 0; i < n; i++) {
1043                 if (ids[i] >= AXGBE_XSTATS_COUNT) {
1044                         PMD_DRV_LOG(ERR, "id value isn't valid\n");
1045                         return -1;
1046                 }
1047                 values[i] = values_copy[ids[i]];
1048         }
1049         return n;
1050 }
1051
1052 static int
1053 axgbe_dev_xstats_get_names_by_id(struct rte_eth_dev *dev,
1054                                  struct rte_eth_xstat_name *xstats_names,
1055                                  const uint64_t *ids,
1056                                  unsigned int size)
1057 {
1058         struct rte_eth_xstat_name xstats_names_copy[AXGBE_XSTATS_COUNT];
1059         unsigned int i;
1060
1061         if (!ids)
1062                 return axgbe_dev_xstats_get_names(dev, xstats_names, size);
1063
1064         axgbe_dev_xstats_get_names(dev, xstats_names_copy, size);
1065
1066         for (i = 0; i < size; i++) {
1067                 if (ids[i] >= AXGBE_XSTATS_COUNT) {
1068                         PMD_DRV_LOG(ERR, "id value isn't valid\n");
1069                         return -1;
1070                 }
1071                 strcpy(xstats_names[i].name, xstats_names_copy[ids[i]].name);
1072         }
1073         return size;
1074 }
1075
1076 static int
1077 axgbe_dev_xstats_reset(struct rte_eth_dev *dev)
1078 {
1079         struct axgbe_port *pdata = dev->data->dev_private;
1080         struct axgbe_mmc_stats *stats = &pdata->mmc_stats;
1081
1082         /* MMC registers are configured for reset on read */
1083         axgbe_read_mmc_stats(pdata);
1084
1085         /* Reset stats */
1086         memset(stats, 0, sizeof(*stats));
1087
1088         return 0;
1089 }
1090
1091 static int
1092 axgbe_dev_stats_get(struct rte_eth_dev *dev,
1093                     struct rte_eth_stats *stats)
1094 {
1095         struct axgbe_rx_queue *rxq;
1096         struct axgbe_tx_queue *txq;
1097         struct axgbe_port *pdata = dev->data->dev_private;
1098         struct axgbe_mmc_stats *mmc_stats = &pdata->mmc_stats;
1099         unsigned int i;
1100
1101         axgbe_read_mmc_stats(pdata);
1102
1103         stats->imissed = mmc_stats->rxfifooverflow;
1104
1105         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1106                 rxq = dev->data->rx_queues[i];
1107                 stats->q_ipackets[i] = rxq->pkts;
1108                 stats->ipackets += rxq->pkts;
1109                 stats->q_ibytes[i] = rxq->bytes;
1110                 stats->ibytes += rxq->bytes;
1111                 stats->rx_nombuf += rxq->rx_mbuf_alloc_failed;
1112                 stats->q_errors[i] = rxq->errors + rxq->rx_mbuf_alloc_failed;
1113                 stats->ierrors += rxq->errors;
1114         }
1115
1116         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1117                 txq = dev->data->tx_queues[i];
1118                 stats->q_opackets[i] = txq->pkts;
1119                 stats->opackets += txq->pkts;
1120                 stats->q_obytes[i] = txq->bytes;
1121                 stats->obytes += txq->bytes;
1122                 stats->oerrors += txq->errors;
1123         }
1124
1125         return 0;
1126 }
1127
1128 static int
1129 axgbe_dev_stats_reset(struct rte_eth_dev *dev)
1130 {
1131         struct axgbe_rx_queue *rxq;
1132         struct axgbe_tx_queue *txq;
1133         unsigned int i;
1134
1135         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1136                 rxq = dev->data->rx_queues[i];
1137                 rxq->pkts = 0;
1138                 rxq->bytes = 0;
1139                 rxq->errors = 0;
1140                 rxq->rx_mbuf_alloc_failed = 0;
1141         }
1142         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1143                 txq = dev->data->tx_queues[i];
1144                 txq->pkts = 0;
1145                 txq->bytes = 0;
1146                 txq->errors = 0;
1147         }
1148
1149         return 0;
1150 }
1151
1152 static int
1153 axgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1154 {
1155         struct axgbe_port *pdata = dev->data->dev_private;
1156
1157         dev_info->max_rx_queues = pdata->rx_ring_count;
1158         dev_info->max_tx_queues = pdata->tx_ring_count;
1159         dev_info->min_rx_bufsize = AXGBE_RX_MIN_BUF_SIZE;
1160         dev_info->max_rx_pktlen = AXGBE_RX_MAX_BUF_SIZE;
1161         dev_info->max_mac_addrs = pdata->hw_feat.addn_mac + 1;
1162         dev_info->max_hash_mac_addrs = pdata->hw_feat.hash_table_size;
1163         dev_info->speed_capa =  ETH_LINK_SPEED_10G;
1164
1165         dev_info->rx_offload_capa =
1166                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1167                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1168                 DEV_RX_OFFLOAD_TCP_CKSUM  |
1169                 DEV_RX_OFFLOAD_JUMBO_FRAME      |
1170                 DEV_RX_OFFLOAD_SCATTER    |
1171                 DEV_RX_OFFLOAD_KEEP_CRC;
1172
1173         dev_info->tx_offload_capa =
1174                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1175                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1176                 DEV_TX_OFFLOAD_TCP_CKSUM;
1177
1178         if (pdata->hw_feat.rss) {
1179                 dev_info->flow_type_rss_offloads = AXGBE_RSS_OFFLOAD;
1180                 dev_info->reta_size = pdata->hw_feat.hash_table_size;
1181                 dev_info->hash_key_size =  AXGBE_RSS_HASH_KEY_SIZE;
1182         }
1183
1184         dev_info->rx_desc_lim = rx_desc_lim;
1185         dev_info->tx_desc_lim = tx_desc_lim;
1186
1187         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1188                 .rx_free_thresh = AXGBE_RX_FREE_THRESH,
1189         };
1190
1191         dev_info->default_txconf = (struct rte_eth_txconf) {
1192                 .tx_free_thresh = AXGBE_TX_FREE_THRESH,
1193         };
1194
1195         return 0;
1196 }
1197
1198 static int
1199 axgbe_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1200 {
1201         struct axgbe_port *pdata = dev->data->dev_private;
1202         struct xgbe_fc_info fc = pdata->fc;
1203         unsigned int reg, reg_val = 0;
1204
1205         reg = MAC_Q0TFCR;
1206         reg_val = AXGMAC_IOREAD(pdata, reg);
1207         fc.low_water[0] =  AXGMAC_MTL_IOREAD_BITS(pdata, 0, MTL_Q_RQFCR, RFA);
1208         fc.high_water[0] =  AXGMAC_MTL_IOREAD_BITS(pdata, 0, MTL_Q_RQFCR, RFD);
1209         fc.pause_time[0] = AXGMAC_GET_BITS(reg_val, MAC_Q0TFCR, PT);
1210         fc.autoneg = pdata->pause_autoneg;
1211
1212         if (pdata->rx_pause && pdata->tx_pause)
1213                 fc.mode = RTE_FC_FULL;
1214         else if (pdata->rx_pause)
1215                 fc.mode = RTE_FC_RX_PAUSE;
1216         else if (pdata->tx_pause)
1217                 fc.mode = RTE_FC_TX_PAUSE;
1218         else
1219                 fc.mode = RTE_FC_NONE;
1220
1221         fc_conf->high_water =  (1024 + (fc.low_water[0] << 9)) / 1024;
1222         fc_conf->low_water =  (1024 + (fc.high_water[0] << 9)) / 1024;
1223         fc_conf->pause_time = fc.pause_time[0];
1224         fc_conf->send_xon = fc.send_xon;
1225         fc_conf->mode = fc.mode;
1226
1227         return 0;
1228 }
1229
1230 static int
1231 axgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1232 {
1233         struct axgbe_port *pdata = dev->data->dev_private;
1234         struct xgbe_fc_info fc = pdata->fc;
1235         unsigned int reg, reg_val = 0;
1236         reg = MAC_Q0TFCR;
1237
1238         pdata->pause_autoneg = fc_conf->autoneg;
1239         pdata->phy.pause_autoneg = pdata->pause_autoneg;
1240         fc.send_xon = fc_conf->send_xon;
1241         AXGMAC_MTL_IOWRITE_BITS(pdata, 0, MTL_Q_RQFCR, RFA,
1242                         AXGMAC_FLOW_CONTROL_VALUE(1024 * fc_conf->high_water));
1243         AXGMAC_MTL_IOWRITE_BITS(pdata, 0, MTL_Q_RQFCR, RFD,
1244                         AXGMAC_FLOW_CONTROL_VALUE(1024 * fc_conf->low_water));
1245         AXGMAC_SET_BITS(reg_val, MAC_Q0TFCR, PT, fc_conf->pause_time);
1246         AXGMAC_IOWRITE(pdata, reg, reg_val);
1247         fc.mode = fc_conf->mode;
1248
1249         if (fc.mode == RTE_FC_FULL) {
1250                 pdata->tx_pause = 1;
1251                 pdata->rx_pause = 1;
1252         } else if (fc.mode == RTE_FC_RX_PAUSE) {
1253                 pdata->tx_pause = 0;
1254                 pdata->rx_pause = 1;
1255         } else if (fc.mode == RTE_FC_TX_PAUSE) {
1256                 pdata->tx_pause = 1;
1257                 pdata->rx_pause = 0;
1258         } else {
1259                 pdata->tx_pause = 0;
1260                 pdata->rx_pause = 0;
1261         }
1262
1263         if (pdata->tx_pause != (unsigned int)pdata->phy.tx_pause)
1264                 pdata->hw_if.config_tx_flow_control(pdata);
1265
1266         if (pdata->rx_pause != (unsigned int)pdata->phy.rx_pause)
1267                 pdata->hw_if.config_rx_flow_control(pdata);
1268
1269         pdata->hw_if.config_flow_control(pdata);
1270         pdata->phy.tx_pause = pdata->tx_pause;
1271         pdata->phy.rx_pause = pdata->rx_pause;
1272
1273         return 0;
1274 }
1275
1276 static int
1277 axgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev,
1278                 struct rte_eth_pfc_conf *pfc_conf)
1279 {
1280         struct axgbe_port *pdata = dev->data->dev_private;
1281         struct xgbe_fc_info fc = pdata->fc;
1282         uint8_t tc_num;
1283
1284         tc_num = pdata->pfc_map[pfc_conf->priority];
1285
1286         if (pfc_conf->priority >= pdata->hw_feat.tc_cnt) {
1287                 PMD_INIT_LOG(ERR, "Max supported  traffic class: %d\n",
1288                                 pdata->hw_feat.tc_cnt);
1289         return -EINVAL;
1290         }
1291
1292         pdata->pause_autoneg = pfc_conf->fc.autoneg;
1293         pdata->phy.pause_autoneg = pdata->pause_autoneg;
1294         fc.send_xon = pfc_conf->fc.send_xon;
1295         AXGMAC_MTL_IOWRITE_BITS(pdata, tc_num, MTL_Q_RQFCR, RFA,
1296                 AXGMAC_FLOW_CONTROL_VALUE(1024 * pfc_conf->fc.high_water));
1297         AXGMAC_MTL_IOWRITE_BITS(pdata, tc_num, MTL_Q_RQFCR, RFD,
1298                 AXGMAC_FLOW_CONTROL_VALUE(1024 * pfc_conf->fc.low_water));
1299
1300         switch (tc_num) {
1301         case 0:
1302                 AXGMAC_IOWRITE_BITS(pdata, MTL_TCPM0R,
1303                                 PSTC0, pfc_conf->fc.pause_time);
1304                 break;
1305         case 1:
1306                 AXGMAC_IOWRITE_BITS(pdata, MTL_TCPM0R,
1307                                 PSTC1, pfc_conf->fc.pause_time);
1308                 break;
1309         case 2:
1310                 AXGMAC_IOWRITE_BITS(pdata, MTL_TCPM0R,
1311                                 PSTC2, pfc_conf->fc.pause_time);
1312                 break;
1313         case 3:
1314                 AXGMAC_IOWRITE_BITS(pdata, MTL_TCPM0R,
1315                                 PSTC3, pfc_conf->fc.pause_time);
1316                 break;
1317         case 4:
1318                 AXGMAC_IOWRITE_BITS(pdata, MTL_TCPM1R,
1319                                 PSTC4, pfc_conf->fc.pause_time);
1320                 break;
1321         case 5:
1322                 AXGMAC_IOWRITE_BITS(pdata, MTL_TCPM1R,
1323                                 PSTC5, pfc_conf->fc.pause_time);
1324                 break;
1325         case 7:
1326                 AXGMAC_IOWRITE_BITS(pdata, MTL_TCPM1R,
1327                                 PSTC6, pfc_conf->fc.pause_time);
1328                 break;
1329         case 6:
1330                 AXGMAC_IOWRITE_BITS(pdata, MTL_TCPM1R,
1331                                 PSTC7, pfc_conf->fc.pause_time);
1332                 break;
1333         }
1334
1335         fc.mode = pfc_conf->fc.mode;
1336
1337         if (fc.mode == RTE_FC_FULL) {
1338                 pdata->tx_pause = 1;
1339                 pdata->rx_pause = 1;
1340                 AXGMAC_IOWRITE_BITS(pdata, MAC_RFCR, PFCE, 1);
1341         } else if (fc.mode == RTE_FC_RX_PAUSE) {
1342                 pdata->tx_pause = 0;
1343                 pdata->rx_pause = 1;
1344                 AXGMAC_IOWRITE_BITS(pdata, MAC_RFCR, PFCE, 1);
1345         } else if (fc.mode == RTE_FC_TX_PAUSE) {
1346                 pdata->tx_pause = 1;
1347                 pdata->rx_pause = 0;
1348                 AXGMAC_IOWRITE_BITS(pdata, MAC_RFCR, PFCE, 0);
1349         } else {
1350                 pdata->tx_pause = 0;
1351                 pdata->rx_pause = 0;
1352                 AXGMAC_IOWRITE_BITS(pdata, MAC_RFCR, PFCE, 0);
1353         }
1354
1355         if (pdata->tx_pause != (unsigned int)pdata->phy.tx_pause)
1356                 pdata->hw_if.config_tx_flow_control(pdata);
1357
1358         if (pdata->rx_pause != (unsigned int)pdata->phy.rx_pause)
1359                 pdata->hw_if.config_rx_flow_control(pdata);
1360         pdata->hw_if.config_flow_control(pdata);
1361         pdata->phy.tx_pause = pdata->tx_pause;
1362         pdata->phy.rx_pause = pdata->rx_pause;
1363
1364         return 0;
1365 }
1366
1367 void
1368 axgbe_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
1369         struct rte_eth_rxq_info *qinfo)
1370 {
1371         struct   axgbe_rx_queue *rxq;
1372
1373         rxq = dev->data->rx_queues[queue_id];
1374         qinfo->mp = rxq->mb_pool;
1375         qinfo->scattered_rx = dev->data->scattered_rx;
1376         qinfo->nb_desc = rxq->nb_desc;
1377         qinfo->conf.rx_free_thresh = rxq->free_thresh;
1378 }
1379
1380 void
1381 axgbe_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
1382         struct rte_eth_txq_info *qinfo)
1383 {
1384         struct  axgbe_tx_queue *txq;
1385
1386         txq = dev->data->tx_queues[queue_id];
1387         qinfo->nb_desc = txq->nb_desc;
1388         qinfo->conf.tx_free_thresh = txq->free_thresh;
1389 }
1390 const uint32_t *
1391 axgbe_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1392 {
1393         static const uint32_t ptypes[] = {
1394                 RTE_PTYPE_L2_ETHER,
1395                 RTE_PTYPE_L2_ETHER_TIMESYNC,
1396                 RTE_PTYPE_L2_ETHER_LLDP,
1397                 RTE_PTYPE_L2_ETHER_ARP,
1398                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
1399                 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
1400                 RTE_PTYPE_L4_FRAG,
1401                 RTE_PTYPE_L4_ICMP,
1402                 RTE_PTYPE_L4_NONFRAG,
1403                 RTE_PTYPE_L4_SCTP,
1404                 RTE_PTYPE_L4_TCP,
1405                 RTE_PTYPE_L4_UDP,
1406                 RTE_PTYPE_TUNNEL_GRENAT,
1407                 RTE_PTYPE_TUNNEL_IP,
1408                 RTE_PTYPE_INNER_L2_ETHER,
1409                 RTE_PTYPE_INNER_L2_ETHER_VLAN,
1410                 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
1411                 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
1412                 RTE_PTYPE_INNER_L4_FRAG,
1413                 RTE_PTYPE_INNER_L4_ICMP,
1414                 RTE_PTYPE_INNER_L4_NONFRAG,
1415                 RTE_PTYPE_INNER_L4_SCTP,
1416                 RTE_PTYPE_INNER_L4_TCP,
1417                 RTE_PTYPE_INNER_L4_UDP,
1418                 RTE_PTYPE_UNKNOWN
1419         };
1420
1421         if (dev->rx_pkt_burst == axgbe_recv_pkts)
1422                 return ptypes;
1423         return NULL;
1424 }
1425
1426 static int axgb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
1427 {
1428         struct rte_eth_dev_info dev_info;
1429         struct axgbe_port *pdata = dev->data->dev_private;
1430         uint32_t frame_size = mtu + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
1431         unsigned int val = 0;
1432         axgbe_dev_info_get(dev, &dev_info);
1433         /* check that mtu is within the allowed range */
1434         if (mtu < RTE_ETHER_MIN_MTU || frame_size > dev_info.max_rx_pktlen)
1435                 return -EINVAL;
1436         /* mtu setting is forbidden if port is start */
1437         if (dev->data->dev_started) {
1438                 PMD_DRV_LOG(ERR, "port %d must be stopped before configuration",
1439                                 dev->data->port_id);
1440                 return -EBUSY;
1441         }
1442         if (frame_size > RTE_ETHER_MAX_LEN) {
1443                 dev->data->dev_conf.rxmode.offloads |=
1444                         DEV_RX_OFFLOAD_JUMBO_FRAME;
1445                 val = 1;
1446         } else {
1447                 dev->data->dev_conf.rxmode.offloads &=
1448                         ~DEV_RX_OFFLOAD_JUMBO_FRAME;
1449                 val = 0;
1450         }
1451         AXGMAC_IOWRITE_BITS(pdata, MAC_RCR, JE, val);
1452         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
1453         return 0;
1454 }
1455
1456 static void
1457 axgbe_update_tstamp_time(struct axgbe_port *pdata,
1458                 unsigned int sec, unsigned int nsec, int addsub)
1459 {
1460         unsigned int count = 100;
1461         uint32_t sub_val = 0;
1462         uint32_t sub_val_sec = 0xFFFFFFFF;
1463         uint32_t sub_val_nsec = 0x3B9ACA00;
1464
1465         if (addsub) {
1466                 if (sec)
1467                         sub_val = sub_val_sec - (sec - 1);
1468                 else
1469                         sub_val = sec;
1470
1471                 AXGMAC_IOWRITE(pdata, MAC_STSUR, sub_val);
1472                 sub_val = sub_val_nsec - nsec;
1473                 AXGMAC_IOWRITE(pdata, MAC_STNUR, sub_val);
1474                 AXGMAC_IOWRITE_BITS(pdata, MAC_STNUR, ADDSUB, 1);
1475         } else {
1476                 AXGMAC_IOWRITE(pdata, MAC_STSUR, sec);
1477                 AXGMAC_IOWRITE_BITS(pdata, MAC_STNUR, ADDSUB, 0);
1478                 AXGMAC_IOWRITE(pdata, MAC_STNUR, nsec);
1479         }
1480         AXGMAC_IOWRITE_BITS(pdata, MAC_TSCR, TSUPDT, 1);
1481         /* Wait for time update to complete */
1482         while (--count && AXGMAC_IOREAD_BITS(pdata, MAC_TSCR, TSUPDT))
1483                 rte_delay_ms(1);
1484 }
1485
1486 static inline uint64_t
1487 div_u64_rem(uint64_t dividend, uint32_t divisor, uint32_t *remainder)
1488 {
1489         *remainder = dividend % divisor;
1490         return dividend / divisor;
1491 }
1492
1493 static inline uint64_t
1494 div_u64(uint64_t dividend, uint32_t divisor)
1495 {
1496         uint32_t remainder;
1497         return div_u64_rem(dividend, divisor, &remainder);
1498 }
1499
1500 static int
1501 axgbe_adjfreq(struct axgbe_port *pdata, int64_t delta)
1502 {
1503         uint64_t adjust;
1504         uint32_t addend, diff;
1505         unsigned int neg_adjust = 0;
1506
1507         if (delta < 0) {
1508                 neg_adjust = 1;
1509                 delta = -delta;
1510         }
1511         adjust = (uint64_t)pdata->tstamp_addend;
1512         adjust *= delta;
1513         diff = (uint32_t)div_u64(adjust, 1000000000UL);
1514         addend = (neg_adjust) ? pdata->tstamp_addend - diff :
1515                                 pdata->tstamp_addend + diff;
1516         pdata->tstamp_addend = addend;
1517         axgbe_update_tstamp_addend(pdata, addend);
1518         return 0;
1519 }
1520
1521 static int
1522 axgbe_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta)
1523 {
1524         struct axgbe_port *pdata = dev->data->dev_private;
1525         struct timespec timestamp_delta;
1526
1527         axgbe_adjfreq(pdata, delta);
1528         pdata->systime_tc.nsec += delta;
1529
1530         if (delta < 0) {
1531                 delta = -delta;
1532                 timestamp_delta = rte_ns_to_timespec(delta);
1533                 axgbe_update_tstamp_time(pdata, timestamp_delta.tv_sec,
1534                                 timestamp_delta.tv_nsec, 1);
1535         } else {
1536                 timestamp_delta = rte_ns_to_timespec(delta);
1537                 axgbe_update_tstamp_time(pdata, timestamp_delta.tv_sec,
1538                                 timestamp_delta.tv_nsec, 0);
1539         }
1540         return 0;
1541 }
1542
1543 static int
1544 axgbe_timesync_read_time(struct rte_eth_dev *dev,
1545                 struct timespec *timestamp)
1546 {
1547         uint64_t nsec;
1548         struct axgbe_port *pdata = dev->data->dev_private;
1549
1550         nsec = AXGMAC_IOREAD(pdata, MAC_STSR);
1551         nsec *= NSEC_PER_SEC;
1552         nsec += AXGMAC_IOREAD(pdata, MAC_STNR);
1553         *timestamp = rte_ns_to_timespec(nsec);
1554         return 0;
1555 }
1556 static int
1557 axgbe_timesync_write_time(struct rte_eth_dev *dev,
1558                                     const struct timespec *timestamp)
1559 {
1560         unsigned int count = 100;
1561         struct axgbe_port *pdata = dev->data->dev_private;
1562
1563         AXGMAC_IOWRITE(pdata, MAC_STSUR, timestamp->tv_sec);
1564         AXGMAC_IOWRITE(pdata, MAC_STNUR, timestamp->tv_nsec);
1565         AXGMAC_IOWRITE_BITS(pdata, MAC_TSCR, TSUPDT, 1);
1566         /* Wait for time update to complete */
1567         while (--count && AXGMAC_IOREAD_BITS(pdata, MAC_TSCR, TSUPDT))
1568                 rte_delay_ms(1);
1569         if (!count)
1570                 PMD_DRV_LOG(ERR, "Timed out update timestamp\n");
1571         return 0;
1572 }
1573
1574 static void
1575 axgbe_update_tstamp_addend(struct axgbe_port *pdata,
1576                 uint32_t addend)
1577 {
1578         unsigned int count = 100;
1579
1580         AXGMAC_IOWRITE(pdata, MAC_TSAR, addend);
1581         AXGMAC_IOWRITE_BITS(pdata, MAC_TSCR, TSADDREG, 1);
1582
1583         /* Wait for addend update to complete */
1584         while (--count && AXGMAC_IOREAD_BITS(pdata, MAC_TSCR, TSADDREG))
1585                 rte_delay_ms(1);
1586         if (!count)
1587                 PMD_DRV_LOG(ERR, "Timed out updating timestamp addend register\n");
1588 }
1589
1590 static void
1591 axgbe_set_tstamp_time(struct axgbe_port *pdata, unsigned int sec,
1592                 unsigned int nsec)
1593 {
1594         unsigned int count = 100;
1595
1596         /*System Time Sec Update*/
1597         AXGMAC_IOWRITE(pdata, MAC_STSUR, sec);
1598         /*System Time nanoSec Update*/
1599         AXGMAC_IOWRITE(pdata, MAC_STNUR, nsec);
1600         /*Initialize Timestamp*/
1601         AXGMAC_IOWRITE_BITS(pdata, MAC_TSCR, TSINIT, 1);
1602
1603         /* Wait for time update to complete */
1604         while (--count && AXGMAC_IOREAD_BITS(pdata, MAC_TSCR, TSINIT))
1605                 rte_delay_ms(1);
1606         if (!count)
1607                 PMD_DRV_LOG(ERR, "Timed out initializing timestamp\n");
1608 }
1609
1610 static int
1611 axgbe_timesync_enable(struct rte_eth_dev *dev)
1612 {
1613         struct axgbe_port *pdata = dev->data->dev_private;
1614         unsigned int mac_tscr = 0;
1615         uint64_t dividend;
1616         struct timespec timestamp;
1617         uint64_t nsec;
1618
1619         /* Set one nano-second accuracy */
1620         AXGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSCTRLSSR, 1);
1621
1622         /* Set fine timestamp update */
1623         AXGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSCFUPDT, 1);
1624
1625         /* Overwrite earlier timestamps */
1626         AXGMAC_SET_BITS(mac_tscr, MAC_TSCR, TXTSSTSM, 1);
1627
1628         AXGMAC_IOWRITE(pdata, MAC_TSCR, mac_tscr);
1629
1630         /* Enabling processing of ptp over eth pkt */
1631         AXGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSIPENA, 1);
1632         AXGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSVER2ENA, 1);
1633         /* Enable timestamp for all pkts*/
1634         AXGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSENALL, 1);
1635
1636         /* enabling timestamp */
1637         AXGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSENA, 1);
1638         AXGMAC_IOWRITE(pdata, MAC_TSCR, mac_tscr);
1639
1640         /* Exit if timestamping is not enabled */
1641         if (!AXGMAC_GET_BITS(mac_tscr, MAC_TSCR, TSENA)) {
1642                 PMD_DRV_LOG(ERR, "Exiting as timestamp is not enabled\n");
1643                 return 0;
1644         }
1645
1646         /* Sub-second Increment Value*/
1647         AXGMAC_IOWRITE_BITS(pdata, MAC_SSIR, SSINC, AXGBE_TSTAMP_SSINC);
1648         /* Sub-nanosecond Increment Value */
1649         AXGMAC_IOWRITE_BITS(pdata, MAC_SSIR, SNSINC, AXGBE_TSTAMP_SNSINC);
1650
1651         pdata->ptpclk_rate = AXGBE_V2_PTP_CLOCK_FREQ;
1652         dividend = 50000000;
1653         dividend <<= 32;
1654         pdata->tstamp_addend = div_u64(dividend, pdata->ptpclk_rate);
1655
1656         axgbe_update_tstamp_addend(pdata, pdata->tstamp_addend);
1657         axgbe_set_tstamp_time(pdata, 0, 0);
1658
1659         /* Initialize the timecounter */
1660         memset(&pdata->systime_tc, 0, sizeof(struct rte_timecounter));
1661
1662         pdata->systime_tc.cc_mask = AXGBE_CYCLECOUNTER_MASK;
1663         pdata->systime_tc.cc_shift = 0;
1664         pdata->systime_tc.nsec_mask = 0;
1665
1666         PMD_DRV_LOG(DEBUG, "Initializing system time counter with realtime\n");
1667
1668         /* Updating the counter once with clock real time */
1669         clock_gettime(CLOCK_REALTIME, &timestamp);
1670         nsec = rte_timespec_to_ns(&timestamp);
1671         nsec = rte_timecounter_update(&pdata->systime_tc, nsec);
1672         axgbe_set_tstamp_time(pdata, timestamp.tv_sec, timestamp.tv_nsec);
1673         return 0;
1674 }
1675
1676 static int
1677 axgbe_timesync_disable(struct rte_eth_dev *dev)
1678 {
1679         struct axgbe_port *pdata = dev->data->dev_private;
1680         unsigned int mac_tscr = 0;
1681
1682         /*disable timestamp for all pkts*/
1683         AXGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSENALL, 0);
1684         /*disable the addened register*/
1685         AXGMAC_IOWRITE_BITS(pdata, MAC_TSCR, TSADDREG, 0);
1686         /* disable timestamp update */
1687         AXGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSCFUPDT, 0);
1688         /*disable time stamp*/
1689         AXGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSENA, 0);
1690         return 0;
1691 }
1692
1693 static int
1694 axgbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
1695                                 struct timespec *timestamp, uint32_t flags)
1696 {
1697         uint64_t nsec = 0;
1698         volatile union axgbe_rx_desc *desc;
1699         uint16_t idx, pmt;
1700         struct axgbe_rx_queue *rxq = *dev->data->rx_queues;
1701
1702         idx = AXGBE_GET_DESC_IDX(rxq, rxq->cur);
1703         desc = &rxq->desc[idx];
1704
1705         while (AXGMAC_GET_BITS_LE(desc->write.desc3, RX_NORMAL_DESC3, OWN))
1706                 rte_delay_ms(1);
1707         if (AXGMAC_GET_BITS_LE(desc->write.desc3, RX_NORMAL_DESC3, CTXT)) {
1708                 if (AXGMAC_GET_BITS_LE(desc->write.desc3, RX_CONTEXT_DESC3, TSA) &&
1709                                 !AXGMAC_GET_BITS_LE(desc->write.desc3,
1710                                         RX_CONTEXT_DESC3, TSD)) {
1711                         pmt = AXGMAC_GET_BITS_LE(desc->write.desc3,
1712                                         RX_CONTEXT_DESC3, PMT);
1713                         nsec = rte_le_to_cpu_32(desc->write.desc1);
1714                         nsec *= NSEC_PER_SEC;
1715                         nsec += rte_le_to_cpu_32(desc->write.desc0);
1716                         if (nsec != 0xffffffffffffffffULL) {
1717                                 if (pmt == 0x01)
1718                                         *timestamp = rte_ns_to_timespec(nsec);
1719                                 PMD_DRV_LOG(DEBUG,
1720                                         "flags = 0x%x nsec = %"PRIu64"\n",
1721                                         flags, nsec);
1722                         }
1723                 }
1724         }
1725
1726         return 0;
1727 }
1728
1729 static int
1730 axgbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
1731                                 struct timespec *timestamp)
1732 {
1733         uint64_t nsec;
1734         struct axgbe_port *pdata = dev->data->dev_private;
1735         unsigned int tx_snr, tx_ssr;
1736
1737         rte_delay_us(5);
1738         if (pdata->vdata->tx_tstamp_workaround) {
1739                 tx_snr = AXGMAC_IOREAD(pdata, MAC_TXSNR);
1740                 tx_ssr = AXGMAC_IOREAD(pdata, MAC_TXSSR);
1741
1742         } else {
1743                 tx_ssr = AXGMAC_IOREAD(pdata, MAC_TXSSR);
1744                 tx_snr = AXGMAC_IOREAD(pdata, MAC_TXSNR);
1745         }
1746         if (AXGMAC_GET_BITS(tx_snr, MAC_TXSNR, TXTSSTSMIS)) {
1747                 PMD_DRV_LOG(DEBUG, "Waiting for TXTSSTSMIS\n");
1748                 return 0;
1749         }
1750         nsec = tx_ssr;
1751         nsec *= NSEC_PER_SEC;
1752         nsec += tx_snr;
1753         PMD_DRV_LOG(DEBUG, "nsec = %"PRIu64" tx_ssr = %d tx_snr = %d\n",
1754                         nsec, tx_ssr, tx_snr);
1755         *timestamp = rte_ns_to_timespec(nsec);
1756         return 0;
1757 }
1758
1759 static void axgbe_get_all_hw_features(struct axgbe_port *pdata)
1760 {
1761         unsigned int mac_hfr0, mac_hfr1, mac_hfr2;
1762         struct axgbe_hw_features *hw_feat = &pdata->hw_feat;
1763
1764         mac_hfr0 = AXGMAC_IOREAD(pdata, MAC_HWF0R);
1765         mac_hfr1 = AXGMAC_IOREAD(pdata, MAC_HWF1R);
1766         mac_hfr2 = AXGMAC_IOREAD(pdata, MAC_HWF2R);
1767
1768         memset(hw_feat, 0, sizeof(*hw_feat));
1769
1770         hw_feat->version = AXGMAC_IOREAD(pdata, MAC_VR);
1771
1772         /* Hardware feature register 0 */
1773         hw_feat->gmii        = AXGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, GMIISEL);
1774         hw_feat->vlhash      = AXGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, VLHASH);
1775         hw_feat->sma         = AXGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, SMASEL);
1776         hw_feat->rwk         = AXGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, RWKSEL);
1777         hw_feat->mgk         = AXGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, MGKSEL);
1778         hw_feat->mmc         = AXGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, MMCSEL);
1779         hw_feat->aoe         = AXGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, ARPOFFSEL);
1780         hw_feat->ts          = AXGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, TSSEL);
1781         hw_feat->eee         = AXGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, EEESEL);
1782         hw_feat->tx_coe      = AXGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, TXCOESEL);
1783         hw_feat->rx_coe      = AXGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, RXCOESEL);
1784         hw_feat->addn_mac    = AXGMAC_GET_BITS(mac_hfr0, MAC_HWF0R,
1785                                               ADDMACADRSEL);
1786         hw_feat->ts_src      = AXGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, TSSTSSEL);
1787         hw_feat->sa_vlan_ins = AXGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, SAVLANINS);
1788
1789         /* Hardware feature register 1 */
1790         hw_feat->rx_fifo_size  = AXGMAC_GET_BITS(mac_hfr1, MAC_HWF1R,
1791                                                 RXFIFOSIZE);
1792         hw_feat->tx_fifo_size  = AXGMAC_GET_BITS(mac_hfr1, MAC_HWF1R,
1793                                                 TXFIFOSIZE);
1794         hw_feat->adv_ts_hi     = AXGMAC_GET_BITS(mac_hfr1,
1795                                                  MAC_HWF1R, ADVTHWORD);
1796         hw_feat->dma_width     = AXGMAC_GET_BITS(mac_hfr1, MAC_HWF1R, ADDR64);
1797         hw_feat->dcb           = AXGMAC_GET_BITS(mac_hfr1, MAC_HWF1R, DCBEN);
1798         hw_feat->sph           = AXGMAC_GET_BITS(mac_hfr1, MAC_HWF1R, SPHEN);
1799         hw_feat->tso           = AXGMAC_GET_BITS(mac_hfr1, MAC_HWF1R, TSOEN);
1800         hw_feat->dma_debug     = AXGMAC_GET_BITS(mac_hfr1, MAC_HWF1R, DBGMEMA);
1801         hw_feat->rss           = AXGMAC_GET_BITS(mac_hfr1, MAC_HWF1R, RSSEN);
1802         hw_feat->tc_cnt        = AXGMAC_GET_BITS(mac_hfr1, MAC_HWF1R, NUMTC);
1803         hw_feat->hash_table_size = AXGMAC_GET_BITS(mac_hfr1, MAC_HWF1R,
1804                                                   HASHTBLSZ);
1805         hw_feat->l3l4_filter_num = AXGMAC_GET_BITS(mac_hfr1, MAC_HWF1R,
1806                                                   L3L4FNUM);
1807
1808         /* Hardware feature register 2 */
1809         hw_feat->rx_q_cnt     = AXGMAC_GET_BITS(mac_hfr2, MAC_HWF2R, RXQCNT);
1810         hw_feat->tx_q_cnt     = AXGMAC_GET_BITS(mac_hfr2, MAC_HWF2R, TXQCNT);
1811         hw_feat->rx_ch_cnt    = AXGMAC_GET_BITS(mac_hfr2, MAC_HWF2R, RXCHCNT);
1812         hw_feat->tx_ch_cnt    = AXGMAC_GET_BITS(mac_hfr2, MAC_HWF2R, TXCHCNT);
1813         hw_feat->pps_out_num  = AXGMAC_GET_BITS(mac_hfr2, MAC_HWF2R, PPSOUTNUM);
1814         hw_feat->aux_snap_num = AXGMAC_GET_BITS(mac_hfr2, MAC_HWF2R,
1815                                                 AUXSNAPNUM);
1816
1817         /* Translate the Hash Table size into actual number */
1818         switch (hw_feat->hash_table_size) {
1819         case 0:
1820                 break;
1821         case 1:
1822                 hw_feat->hash_table_size = 64;
1823                 break;
1824         case 2:
1825                 hw_feat->hash_table_size = 128;
1826                 break;
1827         case 3:
1828                 hw_feat->hash_table_size = 256;
1829                 break;
1830         }
1831
1832         /* Translate the address width setting into actual number */
1833         switch (hw_feat->dma_width) {
1834         case 0:
1835                 hw_feat->dma_width = 32;
1836                 break;
1837         case 1:
1838                 hw_feat->dma_width = 40;
1839                 break;
1840         case 2:
1841                 hw_feat->dma_width = 48;
1842                 break;
1843         default:
1844                 hw_feat->dma_width = 32;
1845         }
1846
1847         /* The Queue, Channel and TC counts are zero based so increment them
1848          * to get the actual number
1849          */
1850         hw_feat->rx_q_cnt++;
1851         hw_feat->tx_q_cnt++;
1852         hw_feat->rx_ch_cnt++;
1853         hw_feat->tx_ch_cnt++;
1854         hw_feat->tc_cnt++;
1855
1856         /* Translate the fifo sizes into actual numbers */
1857         hw_feat->rx_fifo_size = 1 << (hw_feat->rx_fifo_size + 7);
1858         hw_feat->tx_fifo_size = 1 << (hw_feat->tx_fifo_size + 7);
1859 }
1860
1861 static void axgbe_init_all_fptrs(struct axgbe_port *pdata)
1862 {
1863         axgbe_init_function_ptrs_dev(&pdata->hw_if);
1864         axgbe_init_function_ptrs_phy(&pdata->phy_if);
1865         axgbe_init_function_ptrs_i2c(&pdata->i2c_if);
1866         pdata->vdata->init_function_ptrs_phy_impl(&pdata->phy_if);
1867 }
1868
1869 static void axgbe_set_counts(struct axgbe_port *pdata)
1870 {
1871         /* Set all the function pointers */
1872         axgbe_init_all_fptrs(pdata);
1873
1874         /* Populate the hardware features */
1875         axgbe_get_all_hw_features(pdata);
1876
1877         /* Set default max values if not provided */
1878         if (!pdata->tx_max_channel_count)
1879                 pdata->tx_max_channel_count = pdata->hw_feat.tx_ch_cnt;
1880         if (!pdata->rx_max_channel_count)
1881                 pdata->rx_max_channel_count = pdata->hw_feat.rx_ch_cnt;
1882
1883         if (!pdata->tx_max_q_count)
1884                 pdata->tx_max_q_count = pdata->hw_feat.tx_q_cnt;
1885         if (!pdata->rx_max_q_count)
1886                 pdata->rx_max_q_count = pdata->hw_feat.rx_q_cnt;
1887
1888         /* Calculate the number of Tx and Rx rings to be created
1889          *  -Tx (DMA) Channels map 1-to-1 to Tx Queues so set
1890          *   the number of Tx queues to the number of Tx channels
1891          *   enabled
1892          *  -Rx (DMA) Channels do not map 1-to-1 so use the actual
1893          *   number of Rx queues or maximum allowed
1894          */
1895         pdata->tx_ring_count = RTE_MIN(pdata->hw_feat.tx_ch_cnt,
1896                                      pdata->tx_max_channel_count);
1897         pdata->tx_ring_count = RTE_MIN(pdata->tx_ring_count,
1898                                      pdata->tx_max_q_count);
1899
1900         pdata->tx_q_count = pdata->tx_ring_count;
1901
1902         pdata->rx_ring_count = RTE_MIN(pdata->hw_feat.rx_ch_cnt,
1903                                      pdata->rx_max_channel_count);
1904
1905         pdata->rx_q_count = RTE_MIN(pdata->hw_feat.rx_q_cnt,
1906                                   pdata->rx_max_q_count);
1907 }
1908
1909 static void axgbe_default_config(struct axgbe_port *pdata)
1910 {
1911         pdata->pblx8 = DMA_PBL_X8_ENABLE;
1912         pdata->tx_sf_mode = MTL_TSF_ENABLE;
1913         pdata->tx_threshold = MTL_TX_THRESHOLD_64;
1914         pdata->tx_pbl = DMA_PBL_32;
1915         pdata->tx_osp_mode = DMA_OSP_ENABLE;
1916         pdata->rx_sf_mode = MTL_RSF_ENABLE;
1917         pdata->rx_threshold = MTL_RX_THRESHOLD_64;
1918         pdata->rx_pbl = DMA_PBL_32;
1919         pdata->pause_autoneg = 1;
1920         pdata->tx_pause = 0;
1921         pdata->rx_pause = 0;
1922         pdata->phy_speed = SPEED_UNKNOWN;
1923         pdata->power_down = 0;
1924 }
1925
1926 static int
1927 pci_device_cmp(const struct rte_device *dev, const void *_pci_id)
1928 {
1929         const struct rte_pci_device *pdev = RTE_DEV_TO_PCI_CONST(dev);
1930         const struct rte_pci_id *pcid = _pci_id;
1931
1932         if (pdev->id.vendor_id == AMD_PCI_VENDOR_ID &&
1933                         pdev->id.device_id == pcid->device_id)
1934                 return 0;
1935         return 1;
1936 }
1937
1938 static bool
1939 pci_search_device(int device_id)
1940 {
1941         struct rte_bus *pci_bus;
1942         struct rte_pci_id dev_id;
1943
1944         dev_id.device_id = device_id;
1945         pci_bus = rte_bus_find_by_name("pci");
1946         return (pci_bus != NULL) &&
1947                 (pci_bus->find_device(NULL, pci_device_cmp, &dev_id) != NULL);
1948 }
1949
1950 /*
1951  * It returns 0 on success.
1952  */
1953 static int
1954 eth_axgbe_dev_init(struct rte_eth_dev *eth_dev)
1955 {
1956         PMD_INIT_FUNC_TRACE();
1957         struct axgbe_port *pdata;
1958         struct rte_pci_device *pci_dev;
1959         uint32_t reg, mac_lo, mac_hi;
1960         uint32_t len;
1961         int ret;
1962
1963         eth_dev->dev_ops = &axgbe_eth_dev_ops;
1964
1965         eth_dev->rx_descriptor_status = axgbe_dev_rx_descriptor_status;
1966         eth_dev->tx_descriptor_status = axgbe_dev_tx_descriptor_status;
1967
1968         /*
1969          * For secondary processes, we don't initialise any further as primary
1970          * has already done this work.
1971          */
1972         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1973                 return 0;
1974
1975         eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
1976
1977         pdata = eth_dev->data->dev_private;
1978         /* initial state */
1979         rte_bit_relaxed_set32(AXGBE_DOWN, &pdata->dev_state);
1980         rte_bit_relaxed_set32(AXGBE_STOPPED, &pdata->dev_state);
1981         pdata->eth_dev = eth_dev;
1982
1983         pci_dev = RTE_DEV_TO_PCI(eth_dev->device);
1984         pdata->pci_dev = pci_dev;
1985
1986         /*
1987          * Use root complex device ID to differentiate RV AXGBE vs SNOWY AXGBE
1988          */
1989         if (pci_search_device(AMD_PCI_RV_ROOT_COMPLEX_ID)) {
1990                 pdata->xpcs_window_def_reg = PCS_V2_RV_WINDOW_DEF;
1991                 pdata->xpcs_window_sel_reg = PCS_V2_RV_WINDOW_SELECT;
1992         } else {
1993                 pdata->xpcs_window_def_reg = PCS_V2_WINDOW_DEF;
1994                 pdata->xpcs_window_sel_reg = PCS_V2_WINDOW_SELECT;
1995         }
1996
1997         pdata->xgmac_regs =
1998                 (void *)pci_dev->mem_resource[AXGBE_AXGMAC_BAR].addr;
1999         pdata->xprop_regs = (void *)((uint8_t *)pdata->xgmac_regs
2000                                      + AXGBE_MAC_PROP_OFFSET);
2001         pdata->xi2c_regs = (void *)((uint8_t *)pdata->xgmac_regs
2002                                     + AXGBE_I2C_CTRL_OFFSET);
2003         pdata->xpcs_regs = (void *)pci_dev->mem_resource[AXGBE_XPCS_BAR].addr;
2004
2005         /* version specific driver data*/
2006         if (pci_dev->id.device_id == AMD_PCI_AXGBE_DEVICE_V2A)
2007                 pdata->vdata = &axgbe_v2a;
2008         else
2009                 pdata->vdata = &axgbe_v2b;
2010
2011         /* Configure the PCS indirect addressing support */
2012         reg = XPCS32_IOREAD(pdata, pdata->xpcs_window_def_reg);
2013         pdata->xpcs_window = XPCS_GET_BITS(reg, PCS_V2_WINDOW_DEF, OFFSET);
2014         pdata->xpcs_window <<= 6;
2015         pdata->xpcs_window_size = XPCS_GET_BITS(reg, PCS_V2_WINDOW_DEF, SIZE);
2016         pdata->xpcs_window_size = 1 << (pdata->xpcs_window_size + 7);
2017         pdata->xpcs_window_mask = pdata->xpcs_window_size - 1;
2018
2019         PMD_INIT_LOG(DEBUG,
2020                      "xpcs window :%x, size :%x, mask :%x ", pdata->xpcs_window,
2021                      pdata->xpcs_window_size, pdata->xpcs_window_mask);
2022         XP_IOWRITE(pdata, XP_INT_EN, 0x1fffff);
2023
2024         /* Retrieve the MAC address */
2025         mac_lo = XP_IOREAD(pdata, XP_MAC_ADDR_LO);
2026         mac_hi = XP_IOREAD(pdata, XP_MAC_ADDR_HI);
2027         pdata->mac_addr.addr_bytes[0] = mac_lo & 0xff;
2028         pdata->mac_addr.addr_bytes[1] = (mac_lo >> 8) & 0xff;
2029         pdata->mac_addr.addr_bytes[2] = (mac_lo >> 16) & 0xff;
2030         pdata->mac_addr.addr_bytes[3] = (mac_lo >> 24) & 0xff;
2031         pdata->mac_addr.addr_bytes[4] = mac_hi & 0xff;
2032         pdata->mac_addr.addr_bytes[5] = (mac_hi >> 8)  &  0xff;
2033
2034         len = RTE_ETHER_ADDR_LEN * AXGBE_MAX_MAC_ADDRS;
2035         eth_dev->data->mac_addrs = rte_zmalloc("axgbe_mac_addr", len, 0);
2036
2037         if (!eth_dev->data->mac_addrs) {
2038                 PMD_INIT_LOG(ERR,
2039                              "Failed to alloc %u bytes needed to "
2040                              "store MAC addresses", len);
2041                 return -ENOMEM;
2042         }
2043
2044         /* Allocate memory for storing hash filter MAC addresses */
2045         len = RTE_ETHER_ADDR_LEN * AXGBE_MAX_HASH_MAC_ADDRS;
2046         eth_dev->data->hash_mac_addrs = rte_zmalloc("axgbe_hash_mac_addr",
2047                                                     len, 0);
2048
2049         if (eth_dev->data->hash_mac_addrs == NULL) {
2050                 PMD_INIT_LOG(ERR,
2051                              "Failed to allocate %d bytes needed to "
2052                              "store MAC addresses", len);
2053                 return -ENOMEM;
2054         }
2055
2056         if (!rte_is_valid_assigned_ether_addr(&pdata->mac_addr))
2057                 rte_eth_random_addr(pdata->mac_addr.addr_bytes);
2058
2059         /* Copy the permanent MAC address */
2060         rte_ether_addr_copy(&pdata->mac_addr, &eth_dev->data->mac_addrs[0]);
2061
2062         /* Clock settings */
2063         pdata->sysclk_rate = AXGBE_V2_DMA_CLOCK_FREQ;
2064         pdata->ptpclk_rate = AXGBE_V2_PTP_CLOCK_FREQ;
2065
2066         /* Set the DMA coherency values */
2067         pdata->coherent = 1;
2068         pdata->axdomain = AXGBE_DMA_OS_AXDOMAIN;
2069         pdata->arcache = AXGBE_DMA_OS_ARCACHE;
2070         pdata->awcache = AXGBE_DMA_OS_AWCACHE;
2071
2072         /* Set the maximum channels and queues */
2073         reg = XP_IOREAD(pdata, XP_PROP_1);
2074         pdata->tx_max_channel_count = XP_GET_BITS(reg, XP_PROP_1, MAX_TX_DMA);
2075         pdata->rx_max_channel_count = XP_GET_BITS(reg, XP_PROP_1, MAX_RX_DMA);
2076         pdata->tx_max_q_count = XP_GET_BITS(reg, XP_PROP_1, MAX_TX_QUEUES);
2077         pdata->rx_max_q_count = XP_GET_BITS(reg, XP_PROP_1, MAX_RX_QUEUES);
2078
2079         /* Set the hardware channel and queue counts */
2080         axgbe_set_counts(pdata);
2081
2082         /* Set the maximum fifo amounts */
2083         reg = XP_IOREAD(pdata, XP_PROP_2);
2084         pdata->tx_max_fifo_size = XP_GET_BITS(reg, XP_PROP_2, TX_FIFO_SIZE);
2085         pdata->tx_max_fifo_size *= 16384;
2086         pdata->tx_max_fifo_size = RTE_MIN(pdata->tx_max_fifo_size,
2087                                           pdata->vdata->tx_max_fifo_size);
2088         pdata->rx_max_fifo_size = XP_GET_BITS(reg, XP_PROP_2, RX_FIFO_SIZE);
2089         pdata->rx_max_fifo_size *= 16384;
2090         pdata->rx_max_fifo_size = RTE_MIN(pdata->rx_max_fifo_size,
2091                                           pdata->vdata->rx_max_fifo_size);
2092         /* Issue software reset to DMA */
2093         ret = pdata->hw_if.exit(pdata);
2094         if (ret)
2095                 PMD_DRV_LOG(ERR, "hw_if->exit EBUSY error\n");
2096
2097         /* Set default configuration data */
2098         axgbe_default_config(pdata);
2099
2100         /* Set default max values if not provided */
2101         if (!pdata->tx_max_fifo_size)
2102                 pdata->tx_max_fifo_size = pdata->hw_feat.tx_fifo_size;
2103         if (!pdata->rx_max_fifo_size)
2104                 pdata->rx_max_fifo_size = pdata->hw_feat.rx_fifo_size;
2105
2106         pdata->tx_desc_count = AXGBE_MAX_RING_DESC;
2107         pdata->rx_desc_count = AXGBE_MAX_RING_DESC;
2108         pthread_mutex_init(&pdata->xpcs_mutex, NULL);
2109         pthread_mutex_init(&pdata->i2c_mutex, NULL);
2110         pthread_mutex_init(&pdata->an_mutex, NULL);
2111         pthread_mutex_init(&pdata->phy_mutex, NULL);
2112
2113         ret = pdata->phy_if.phy_init(pdata);
2114         if (ret) {
2115                 rte_free(eth_dev->data->mac_addrs);
2116                 eth_dev->data->mac_addrs = NULL;
2117                 return ret;
2118         }
2119
2120         rte_intr_callback_register(&pci_dev->intr_handle,
2121                                    axgbe_dev_interrupt_handler,
2122                                    (void *)eth_dev);
2123         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
2124                      eth_dev->data->port_id, pci_dev->id.vendor_id,
2125                      pci_dev->id.device_id);
2126
2127         return 0;
2128 }
2129
2130 static int
2131 axgbe_dev_close(struct rte_eth_dev *eth_dev)
2132 {
2133         struct rte_pci_device *pci_dev;
2134
2135         PMD_INIT_FUNC_TRACE();
2136
2137         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2138                 return 0;
2139
2140         pci_dev = RTE_DEV_TO_PCI(eth_dev->device);
2141         axgbe_dev_clear_queues(eth_dev);
2142
2143         /* disable uio intr before callback unregister */
2144         rte_intr_disable(&pci_dev->intr_handle);
2145         rte_intr_callback_unregister(&pci_dev->intr_handle,
2146                                      axgbe_dev_interrupt_handler,
2147                                      (void *)eth_dev);
2148
2149         return 0;
2150 }
2151
2152 static int eth_axgbe_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
2153         struct rte_pci_device *pci_dev)
2154 {
2155         return rte_eth_dev_pci_generic_probe(pci_dev,
2156                 sizeof(struct axgbe_port), eth_axgbe_dev_init);
2157 }
2158
2159 static int eth_axgbe_pci_remove(struct rte_pci_device *pci_dev)
2160 {
2161         return rte_eth_dev_pci_generic_remove(pci_dev, axgbe_dev_close);
2162 }
2163
2164 static struct rte_pci_driver rte_axgbe_pmd = {
2165         .id_table = pci_id_axgbe_map,
2166         .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
2167         .probe = eth_axgbe_pci_probe,
2168         .remove = eth_axgbe_pci_remove,
2169 };
2170
2171 RTE_PMD_REGISTER_PCI(net_axgbe, rte_axgbe_pmd);
2172 RTE_PMD_REGISTER_PCI_TABLE(net_axgbe, pci_id_axgbe_map);
2173 RTE_PMD_REGISTER_KMOD_DEP(net_axgbe, "* igb_uio | uio_pci_generic | vfio-pci");
2174 RTE_LOG_REGISTER(axgbe_logtype_init, pmd.net.axgbe.init, NOTICE);
2175 RTE_LOG_REGISTER(axgbe_logtype_driver, pmd.net.axgbe.driver, NOTICE);