drivers/net/fm10k/fm10k_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2013-2016 Intel Corporation
   3  */
   4
   5 #include <rte_ethdev_driver.h>
   6 #include <rte_ethdev_pci.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_string_fns.h>
  10 #include <rte_dev.h>
  11 #include <rte_spinlock.h>
  12 #include <rte_kvargs.h>
  13
  14 #include "fm10k.h"
  15 #include "base/fm10k_api.h"
  16
  17 /* Default delay to acquire mailbox lock */
  18 #define FM10K_MBXLOCK_DELAY_US 20
  19 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  20
  21 #define MAIN_VSI_POOL_NUMBER 0
  22
  23 /* Max try times to acquire switch status */
  24 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  25 /* Wait interval to get switch status */
  26 #define WAIT_SWITCH_MSG_US    100000
  27 /* A period of quiescence for switch */
  28 #define FM10K_SWITCH_QUIESCE_US 100000
  29 /* Number of chars per uint32 type */
  30 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  31 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  32
  33 /* default 1:1 map from queue ID to interrupt vector ID */
  34 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  35
  36 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  37 #define MAX_LPORT_NUM    128
  38 #define GLORT_FD_Q_BASE  0x40
  39 #define GLORT_PF_MASK    0xFFC0
  40 #define GLORT_FD_MASK    GLORT_PF_MASK
  41 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  42
  43 int fm10k_logtype_init;
  44 int fm10k_logtype_driver;
  45
  46 #ifdef RTE_LIBRTE_FM10K_DEBUG_RX
  47 int fm10k_logtype_rx;
  48 #endif
  49 #ifdef RTE_LIBRTE_FM10K_DEBUG_TX
  50 int fm10k_logtype_tx;
  51 #endif
  52 #ifdef RTE_LIBRTE_FM10K_DEBUG_TX_FREE
  53 int fm10k_logtype_tx_free;
  54 #endif
  55
  56 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  57 static int fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  58 static int fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  59 static int fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  60 static int fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  61 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  62 static int
  63 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  64 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  65         const u8 *mac, bool add, uint32_t pool);
  66 static void fm10k_tx_queue_release(void *queue);
  67 static void fm10k_rx_queue_release(void *queue);
  68 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  69 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  70 static int fm10k_check_ftag(struct rte_devargs *devargs);
  71 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  72
  73 static int fm10k_dev_infos_get(struct rte_eth_dev *dev,
  74                                struct rte_eth_dev_info *dev_info);
  75 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev);
  76 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev);
  77 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev);
  78 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev);
  79
  80 struct fm10k_xstats_name_off {
  81         char name[RTE_ETH_XSTATS_NAME_SIZE];
  82         unsigned offset;
  83 };
  84
  85 static const struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  86         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  87         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  88         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  89         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  90         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  91         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  92         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  93         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  94                 nodesc_drop)},
  95 };
  96
  97 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  98                 sizeof(fm10k_hw_stats_strings[0]))
  99
 100 static const struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
 101         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
 102         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
 103         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
 104 };
 105
 106 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
 107                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
 108
 109 static const struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 110         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 111         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 112 };
 113
 114 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 115                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 116
 117 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 118                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 119 static int
 120 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 121
 122 static void
 123 fm10k_mbx_initlock(struct fm10k_hw *hw)
 124 {
 125         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 126 }
 127
 128 static void
 129 fm10k_mbx_lock(struct fm10k_hw *hw)
 130 {
 131         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 132                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 133 }
 134
 135 static void
 136 fm10k_mbx_unlock(struct fm10k_hw *hw)
 137 {
 138         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 139 }
 140
 141 /* Stubs needed for linkage when vPMD is disabled */
 142 __rte_weak int
 143 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 144 {
 145         return -1;
 146 }
 147
 148 __rte_weak uint16_t
 149 fm10k_recv_pkts_vec(
 150         __rte_unused void *rx_queue,
 151         __rte_unused struct rte_mbuf **rx_pkts,
 152         __rte_unused uint16_t nb_pkts)
 153 {
 154         return 0;
 155 }
 156
 157 __rte_weak uint16_t
 158 fm10k_recv_scattered_pkts_vec(
 159                 __rte_unused void *rx_queue,
 160                 __rte_unused struct rte_mbuf **rx_pkts,
 161                 __rte_unused uint16_t nb_pkts)
 162 {
 163         return 0;
 164 }
 165
 166 __rte_weak int
 167 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 168
 169 {
 170         return -1;
 171 }
 172
 173 __rte_weak void
 174 fm10k_rx_queue_release_mbufs_vec(
 175                 __rte_unused struct fm10k_rx_queue *rxq)
 176 {
 177         return;
 178 }
 179
 180 __rte_weak void
 181 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 182 {
 183         return;
 184 }
 185
 186 __rte_weak int
 187 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 188 {
 189         return -1;
 190 }
 191
 192 __rte_weak uint16_t
 193 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 194                            __rte_unused struct rte_mbuf **tx_pkts,
 195                            __rte_unused uint16_t nb_pkts)
 196 {
 197         return 0;
 198 }
 199
 200 /*
 201  * reset queue to initial state, allocate software buffers used when starting
 202  * device.
 203  * return 0 on success
 204  * return -ENOMEM if buffers cannot be allocated
 205  * return -EINVAL if buffers do not satisfy alignment condition
 206  */
 207 static inline int
 208 rx_queue_reset(struct fm10k_rx_queue *q)
 209 {
 210         static const union fm10k_rx_desc zero = {{0} };
 211         uint64_t dma_addr;
 212         int i, diag;
 213         PMD_INIT_FUNC_TRACE();
 214
 215         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 216         if (diag != 0)
 217                 return -ENOMEM;
 218
 219         for (i = 0; i < q->nb_desc; ++i) {
 220                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 221                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 222                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 223                                                 q->nb_desc);
 224                         return -EINVAL;
 225                 }
 226                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 227                 q->hw_ring[i].q.pkt_addr = dma_addr;
 228                 q->hw_ring[i].q.hdr_addr = dma_addr;
 229         }
 230
 231         /* initialize extra software ring entries. Space for these extra
 232          * entries is always allocated.
 233          */
 234         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 235         for (i = 0; i < q->nb_fake_desc; ++i) {
 236                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 237                 q->hw_ring[q->nb_desc + i] = zero;
 238         }
 239
 240         q->next_dd = 0;
 241         q->next_alloc = 0;
 242         q->next_trigger = q->alloc_thresh - 1;
 243         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 244         q->rxrearm_start = 0;
 245         q->rxrearm_nb = 0;
 246
 247         return 0;
 248 }
 249
 250 /*
 251  * clean queue, descriptor rings, free software buffers used when stopping
 252  * device.
 253  */
 254 static inline void
 255 rx_queue_clean(struct fm10k_rx_queue *q)
 256 {
 257         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 258         uint32_t i;
 259         PMD_INIT_FUNC_TRACE();
 260
 261         /* zero descriptor rings */
 262         for (i = 0; i < q->nb_desc; ++i)
 263                 q->hw_ring[i] = zero;
 264
 265         /* zero faked descriptors */
 266         for (i = 0; i < q->nb_fake_desc; ++i)
 267                 q->hw_ring[q->nb_desc + i] = zero;
 268
 269         /* vPMD driver has a different way of releasing mbufs. */
 270         if (q->rx_using_sse) {
 271                 fm10k_rx_queue_release_mbufs_vec(q);
 272                 return;
 273         }
 274
 275         /* free software buffers */
 276         for (i = 0; i < q->nb_desc; ++i) {
 277                 if (q->sw_ring[i]) {
 278                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 279                         q->sw_ring[i] = NULL;
 280                 }
 281         }
 282 }
 283
 284 /*
 285  * free all queue memory used when releasing the queue (i.e. configure)
 286  */
 287 static inline void
 288 rx_queue_free(struct fm10k_rx_queue *q)
 289 {
 290         PMD_INIT_FUNC_TRACE();
 291         if (q) {
 292                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 293                 rx_queue_clean(q);
 294                 if (q->sw_ring) {
 295                         rte_free(q->sw_ring);
 296                         q->sw_ring = NULL;
 297                 }
 298                 rte_free(q);
 299                 q = NULL;
 300         }
 301 }
 302
 303 /*
 304  * disable RX queue, wait unitl HW finished necessary flush operation
 305  */
 306 static inline int
 307 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 308 {
 309         uint32_t reg, i;
 310
 311         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 312         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 313                         reg & ~FM10K_RXQCTL_ENABLE);
 314
 315         /* Wait 100us at most */
 316         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 317                 rte_delay_us(1);
 318                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 319                 if (!(reg & FM10K_RXQCTL_ENABLE))
 320                         break;
 321         }
 322
 323         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 324                 return -1;
 325
 326         return 0;
 327 }
 328
 329 /*
 330  * reset queue to initial state, allocate software buffers used when starting
 331  * device
 332  */
 333 static inline void
 334 tx_queue_reset(struct fm10k_tx_queue *q)
 335 {
 336         PMD_INIT_FUNC_TRACE();
 337         q->last_free = 0;
 338         q->next_free = 0;
 339         q->nb_used = 0;
 340         q->nb_free = q->nb_desc - 1;
 341         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 342         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 343 }
 344
 345 /*
 346  * clean queue, descriptor rings, free software buffers used when stopping
 347  * device
 348  */
 349 static inline void
 350 tx_queue_clean(struct fm10k_tx_queue *q)
 351 {
 352         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 353         uint32_t i;
 354         PMD_INIT_FUNC_TRACE();
 355
 356         /* zero descriptor rings */
 357         for (i = 0; i < q->nb_desc; ++i)
 358                 q->hw_ring[i] = zero;
 359
 360         /* free software buffers */
 361         for (i = 0; i < q->nb_desc; ++i) {
 362                 if (q->sw_ring[i]) {
 363                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 364                         q->sw_ring[i] = NULL;
 365                 }
 366         }
 367 }
 368
 369 /*
 370  * free all queue memory used when releasing the queue (i.e. configure)
 371  */
 372 static inline void
 373 tx_queue_free(struct fm10k_tx_queue *q)
 374 {
 375         PMD_INIT_FUNC_TRACE();
 376         if (q) {
 377                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 378                 tx_queue_clean(q);
 379                 if (q->rs_tracker.list) {
 380                         rte_free(q->rs_tracker.list);
 381                         q->rs_tracker.list = NULL;
 382                 }
 383                 if (q->sw_ring) {
 384                         rte_free(q->sw_ring);
 385                         q->sw_ring = NULL;
 386                 }
 387                 rte_free(q);
 388                 q = NULL;
 389         }
 390 }
 391
 392 /*
 393  * disable TX queue, wait unitl HW finished necessary flush operation
 394  */
 395 static inline int
 396 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 397 {
 398         uint32_t reg, i;
 399
 400         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 401         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 402                         reg & ~FM10K_TXDCTL_ENABLE);
 403
 404         /* Wait 100us at most */
 405         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 406                 rte_delay_us(1);
 407                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 408                 if (!(reg & FM10K_TXDCTL_ENABLE))
 409                         break;
 410         }
 411
 412         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 413                 return -1;
 414
 415         return 0;
 416 }
 417
 418 static int
 419 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 420 {
 421         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 422         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 423         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 424         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 425
 426         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 427
 428         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 429                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 430                 return -EINVAL;
 431         }
 432
 433         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 434                 return 0;
 435
 436         if (hw->mac.type == fm10k_mac_vf) {
 437                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 438                 return -EINVAL;
 439         }
 440
 441         /* Check VMDQ queue pool number */
 442         if (vmdq_conf->nb_queue_pools >
 443                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 444                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 445                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 446                         vmdq_conf->nb_queue_pools);
 447                 return -EINVAL;
 448         }
 449
 450         return 0;
 451 }
 452
 453 static const struct fm10k_txq_ops def_txq_ops = {
 454         .reset = tx_queue_reset,
 455 };
 456
 457 static int
 458 fm10k_dev_configure(struct rte_eth_dev *dev)
 459 {
 460         int ret;
 461
 462         PMD_INIT_FUNC_TRACE();
 463
 464         dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH;
 465
 466         /* multipe queue mode checking */
 467         ret  = fm10k_check_mq_mode(dev);
 468         if (ret != 0) {
 469                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 470                             ret);
 471                 return ret;
 472         }
 473
 474         dev->data->scattered_rx = 0;
 475
 476         return 0;
 477 }
 478
 479 static void
 480 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 481 {
 482         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 483         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 484         uint32_t i;
 485
 486         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 487
 488         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 489                 if (!vmdq_conf->pool_map[i].pools)
 490                         continue;
 491                 fm10k_mbx_lock(hw);
 492                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 493                 fm10k_mbx_unlock(hw);
 494         }
 495 }
 496
 497 static void
 498 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 499 {
 500         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 501
 502         /* Add default mac address */
 503         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 504                 MAIN_VSI_POOL_NUMBER);
 505 }
 506
 507 static void
 508 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 509 {
 510         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 511         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 512         uint32_t mrqc, *key, i, reta, j;
 513         uint64_t hf;
 514
 515 #define RSS_KEY_SIZE 40
 516         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 517                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 518                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 519                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 520                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 521                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 522         };
 523
 524         if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 525                 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 526                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 527                 return;
 528         }
 529
 530         /* random key is rss_intel_key (default) or user provided (rss_key) */
 531         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 532                 key = (uint32_t *)rss_intel_key;
 533         else
 534                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 535
 536         /* Now fill our hash function seeds, 4 bytes at a time */
 537         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 538                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 539
 540         /*
 541          * Fill in redirection table
 542          * The byte-swap is needed because NIC registers are in
 543          * little-endian order.
 544          */
 545         reta = 0;
 546         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 547                 if (j == dev->data->nb_rx_queues)
 548                         j = 0;
 549                 reta = (reta << CHAR_BIT) | j;
 550                 if ((i & 3) == 3)
 551                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 552                                         rte_bswap32(reta));
 553         }
 554
 555         /*
 556          * Generate RSS hash based on packet types, TCP/UDP
 557          * port numbers and/or IPv4/v6 src and dst addresses
 558          */
 559         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 560         mrqc = 0;
 561         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 562         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 563         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 564         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 565         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 566         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 567         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 568         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 569         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 570
 571         if (mrqc == 0) {
 572                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 573                         "supported", hf);
 574                 return;
 575         }
 576
 577         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 578 }
 579
 580 static void
 581 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 582 {
 583         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 584         uint32_t i;
 585
 586         for (i = 0; i < nb_lport_new; i++) {
 587                 /* Set unicast mode by default. App can change
 588                  * to other mode in other API func.
 589                  */
 590                 fm10k_mbx_lock(hw);
 591                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 592                         FM10K_XCAST_MODE_NONE);
 593                 fm10k_mbx_unlock(hw);
 594         }
 595 }
 596
 597 static void
 598 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 599 {
 600         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 601         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 602         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 603         struct fm10k_macvlan_filter_info *macvlan;
 604         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 605         uint16_t nb_lport_new;
 606
 607         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 608         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 609
 610         fm10k_dev_rss_configure(dev);
 611
 612         /* only PF supports VMDQ */
 613         if (hw->mac.type != fm10k_mac_pf)
 614                 return;
 615
 616         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 617                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 618
 619         /* no pool number change, no need to update logic port and VLAN/MAC */
 620         if (macvlan->nb_queue_pools == nb_queue_pools)
 621                 return;
 622
 623         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 624         fm10k_dev_logic_port_update(dev, nb_lport_new);
 625
 626         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 627         memset(dev->data->mac_addrs, 0,
 628                 RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 629         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
 630                 &dev->data->mac_addrs[0]);
 631         memset(macvlan, 0, sizeof(*macvlan));
 632         macvlan->nb_queue_pools = nb_queue_pools;
 633
 634         if (nb_queue_pools)
 635                 fm10k_dev_vmdq_rx_configure(dev);
 636         else
 637                 fm10k_dev_pf_main_vsi_reset(dev);
 638 }
 639
 640 static int
 641 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 642 {
 643         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 644         int i, ret;
 645         struct fm10k_tx_queue *txq;
 646         uint64_t base_addr;
 647         uint32_t size;
 648
 649         /* Disable TXINT to avoid possible interrupt */
 650         for (i = 0; i < hw->mac.max_queues; i++)
 651                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 652                                 3 << FM10K_TXINT_TIMER_SHIFT);
 653
 654         /* Setup TX queue */
 655         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 656                 txq = dev->data->tx_queues[i];
 657                 base_addr = txq->hw_ring_phys_addr;
 658                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 659
 660                 /* disable queue to avoid issues while updating state */
 661                 ret = tx_queue_disable(hw, i);
 662                 if (ret) {
 663                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 664                         return -1;
 665                 }
 666                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 667                  * register is read-only for VF.
 668                  */
 669                 if (fm10k_check_ftag(dev->device->devargs)) {
 670                         if (hw->mac.type == fm10k_mac_pf) {
 671                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 672                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 673                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 674                         } else {
 675                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 676                                 return -ENOTSUP;
 677                         }
 678                 }
 679
 680                 /* set location and size for descriptor ring */
 681                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 682                                 base_addr & UINT64_LOWER_32BITS_MASK);
 683                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 684                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 685                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 686
 687                 /* assign default SGLORT for each TX queue by PF */
 688                 if (hw->mac.type == fm10k_mac_pf)
 689                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 690         }
 691
 692         /* set up vector or scalar TX function as appropriate */
 693         fm10k_set_tx_function(dev);
 694
 695         return 0;
 696 }
 697
 698 static int
 699 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 700 {
 701         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 702         struct fm10k_macvlan_filter_info *macvlan;
 703         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 704         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 705         int i, ret;
 706         struct fm10k_rx_queue *rxq;
 707         uint64_t base_addr;
 708         uint32_t size;
 709         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 710         uint32_t logic_port = hw->mac.dglort_map;
 711         uint16_t buf_size;
 712         uint16_t queue_stride = 0;
 713
 714         /* enable RXINT for interrupt mode */
 715         i = 0;
 716         if (rte_intr_dp_is_en(intr_handle)) {
 717                 for (; i < dev->data->nb_rx_queues; i++) {
 718                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 719                         if (hw->mac.type == fm10k_mac_pf)
 720                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 721                                         FM10K_ITR_AUTOMASK |
 722                                         FM10K_ITR_MASK_CLEAR);
 723                         else
 724                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 725                                         FM10K_ITR_AUTOMASK |
 726                                         FM10K_ITR_MASK_CLEAR);
 727                 }
 728         }
 729         /* Disable other RXINT to avoid possible interrupt */
 730         for (; i < hw->mac.max_queues; i++)
 731                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 732                         3 << FM10K_RXINT_TIMER_SHIFT);
 733
 734         /* Setup RX queues */
 735         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 736                 rxq = dev->data->rx_queues[i];
 737                 base_addr = rxq->hw_ring_phys_addr;
 738                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 739
 740                 /* disable queue to avoid issues while updating state */
 741                 ret = rx_queue_disable(hw, i);
 742                 if (ret) {
 743                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 744                         return -1;
 745                 }
 746
 747                 /* Setup the Base and Length of the Rx Descriptor Ring */
 748                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 749                                 base_addr & UINT64_LOWER_32BITS_MASK);
 750                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 751                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 752                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 753
 754                 /* Configure the Rx buffer size for one buff without split */
 755                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 756                         RTE_PKTMBUF_HEADROOM);
 757                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 758                  * reserved for this purpose, and the worst case could be 511B.
 759                  * But SRR reg assumes all buffers have the same size. In order
 760                  * to fill the gap, we'll have to consider the worst case and
 761                  * assume 512B is reserved. If we don't do so, it's possible
 762                  * for HW to overwrite data to next mbuf.
 763                  */
 764                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 765
 766                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 767                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 768                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 769
 770                 /* It adds dual VLAN length for supporting dual VLAN */
 771                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 772                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 773                         rxq->offloads & DEV_RX_OFFLOAD_SCATTER) {
 774                         uint32_t reg;
 775                         dev->data->scattered_rx = 1;
 776                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 777                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 778                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 779                 }
 780
 781                 /* Enable drop on empty, it's RO for VF */
 782                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 783                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 784
 785                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 786                 FM10K_WRITE_FLUSH(hw);
 787         }
 788
 789         /* Configure VMDQ/RSS if applicable */
 790         fm10k_dev_mq_rx_configure(dev);
 791
 792         /* Decide the best RX function */
 793         fm10k_set_rx_function(dev);
 794
 795         /* update RX_SGLORT for loopback suppress*/
 796         if (hw->mac.type != fm10k_mac_pf)
 797                 return 0;
 798         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 799         if (macvlan->nb_queue_pools)
 800                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 801         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 802                 if (i && queue_stride && !(i % queue_stride))
 803                         logic_port++;
 804                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 805         }
 806
 807         return 0;
 808 }
 809
 810 static int
 811 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 812 {
 813         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 814         int err;
 815         uint32_t reg;
 816         struct fm10k_rx_queue *rxq;
 817
 818         PMD_INIT_FUNC_TRACE();
 819
 820         rxq = dev->data->rx_queues[rx_queue_id];
 821         err = rx_queue_reset(rxq);
 822         if (err == -ENOMEM) {
 823                 PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 824                 return err;
 825         } else if (err == -EINVAL) {
 826                 PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 827                         " %d", err);
 828                 return err;
 829         }
 830
 831         /* Setup the HW Rx Head and Tail Descriptor Pointers
 832          * Note: this must be done AFTER the queue is enabled on real
 833          * hardware, but BEFORE the queue is enabled when using the
 834          * emulation platform. Do it in both places for now and remove
 835          * this comment and the following two register writes when the
 836          * emulation platform is no longer being used.
 837          */
 838         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 839         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 840
 841         /* Set PF ownership flag for PF devices */
 842         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 843         if (hw->mac.type == fm10k_mac_pf)
 844                 reg |= FM10K_RXQCTL_PF;
 845         reg |= FM10K_RXQCTL_ENABLE;
 846         /* enable RX queue */
 847         FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 848         FM10K_WRITE_FLUSH(hw);
 849
 850         /* Setup the HW Rx Head and Tail Descriptor Pointers
 851          * Note: this must be done AFTER the queue is enabled
 852          */
 853         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 854         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 855         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 856
 857         return 0;
 858 }
 859
 860 static int
 861 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 862 {
 863         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 864
 865         PMD_INIT_FUNC_TRACE();
 866
 867         /* Disable RX queue */
 868         rx_queue_disable(hw, rx_queue_id);
 869
 870         /* Free mbuf and clean HW ring */
 871         rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 872         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 873
 874         return 0;
 875 }
 876
 877 static int
 878 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 879 {
 880         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 881         /** @todo - this should be defined in the shared code */
 882 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 883         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 884         struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 885
 886         PMD_INIT_FUNC_TRACE();
 887
 888         q->ops->reset(q);
 889
 890         /* reset head and tail pointers */
 891         FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 892         FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 893
 894         /* enable TX queue */
 895         FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 896                                 FM10K_TXDCTL_ENABLE | txdctl);
 897         FM10K_WRITE_FLUSH(hw);
 898         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 899
 900         return 0;
 901 }
 902
 903 static int
 904 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 905 {
 906         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 907
 908         PMD_INIT_FUNC_TRACE();
 909
 910         tx_queue_disable(hw, tx_queue_id);
 911         tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 912         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 913
 914         return 0;
 915 }
 916
 917 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 918 {
 919         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 920                 != FM10K_DGLORTMAP_NONE);
 921 }
 922
 923 static int
 924 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 925 {
 926         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 927         int status;
 928
 929         PMD_INIT_FUNC_TRACE();
 930
 931         /* Return if it didn't acquire valid glort range */
 932         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 933                 return 0;
 934
 935         fm10k_mbx_lock(hw);
 936         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 937                                 FM10K_XCAST_MODE_PROMISC);
 938         fm10k_mbx_unlock(hw);
 939
 940         if (status != FM10K_SUCCESS) {
 941                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 942                 return -EAGAIN;
 943         }
 944
 945         return 0;
 946 }
 947
 948 static int
 949 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 950 {
 951         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 952         uint8_t mode;
 953         int status;
 954
 955         PMD_INIT_FUNC_TRACE();
 956
 957         /* Return if it didn't acquire valid glort range */
 958         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 959                 return 0;
 960
 961         if (dev->data->all_multicast == 1)
 962                 mode = FM10K_XCAST_MODE_ALLMULTI;
 963         else
 964                 mode = FM10K_XCAST_MODE_NONE;
 965
 966         fm10k_mbx_lock(hw);
 967         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 968                                 mode);
 969         fm10k_mbx_unlock(hw);
 970
 971         if (status != FM10K_SUCCESS) {
 972                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 973                 return -EAGAIN;
 974         }
 975
 976         return 0;
 977 }
 978
 979 static int
 980 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 981 {
 982         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 983         int status;
 984
 985         PMD_INIT_FUNC_TRACE();
 986
 987         /* Return if it didn't acquire valid glort range */
 988         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 989                 return 0;
 990
 991         /* If promiscuous mode is enabled, it doesn't make sense to enable
 992          * allmulticast and disable promiscuous since fm10k only can select
 993          * one of the modes.
 994          */
 995         if (dev->data->promiscuous) {
 996                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 997                         "needn't enable allmulticast");
 998                 return 0;
 999         }
1000
1001         fm10k_mbx_lock(hw);
1002         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1003                                 FM10K_XCAST_MODE_ALLMULTI);
1004         fm10k_mbx_unlock(hw);
1005
1006         if (status != FM10K_SUCCESS) {
1007                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
1008                 return -EAGAIN;
1009         }
1010
1011         return 0;
1012 }
1013
1014 static int
1015 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1016 {
1017         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1018         int status;
1019
1020         PMD_INIT_FUNC_TRACE();
1021
1022         /* Return if it didn't acquire valid glort range */
1023         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1024                 return 0;
1025
1026         if (dev->data->promiscuous) {
1027                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1028                         "since promisc mode is enabled");
1029                 return -EINVAL;
1030         }
1031
1032         fm10k_mbx_lock(hw);
1033         /* Change mode to unicast mode */
1034         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1035                                 FM10K_XCAST_MODE_NONE);
1036         fm10k_mbx_unlock(hw);
1037
1038         if (status != FM10K_SUCCESS) {
1039                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1040                 return -EAGAIN;
1041         }
1042
1043         return 0;
1044 }
1045
1046 static void
1047 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1048 {
1049         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1050         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1051         uint16_t nb_queue_pools;
1052         struct fm10k_macvlan_filter_info *macvlan;
1053
1054         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1055         nb_queue_pools = macvlan->nb_queue_pools;
1056         pool_len = nb_queue_pools ? rte_fls_u32(nb_queue_pools - 1) : 0;
1057         rss_len = rte_fls_u32(dev->data->nb_rx_queues - 1) - pool_len;
1058
1059         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1060         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1061         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1062                         hw->mac.dglort_map;
1063         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1064         /* Configure VMDQ/RSS DGlort Decoder */
1065         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1066
1067         /* Flow Director configurations, only queue number is valid. */
1068         dglortdec = rte_fls_u32(dev->data->nb_rx_queues - 1);
1069         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1070                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1071         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1072         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1073
1074         /* Invalidate all other GLORT entries */
1075         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1076                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1077                                 FM10K_DGLORTMAP_NONE);
1078 }
1079
1080 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1081 static int
1082 fm10k_dev_start(struct rte_eth_dev *dev)
1083 {
1084         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1085         int i, diag;
1086
1087         PMD_INIT_FUNC_TRACE();
1088
1089         /* stop, init, then start the hw */
1090         diag = fm10k_stop_hw(hw);
1091         if (diag != FM10K_SUCCESS) {
1092                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1093                 return -EIO;
1094         }
1095
1096         diag = fm10k_init_hw(hw);
1097         if (diag != FM10K_SUCCESS) {
1098                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1099                 return -EIO;
1100         }
1101
1102         diag = fm10k_start_hw(hw);
1103         if (diag != FM10K_SUCCESS) {
1104                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1105                 return -EIO;
1106         }
1107
1108         diag = fm10k_dev_tx_init(dev);
1109         if (diag) {
1110                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1111                 return diag;
1112         }
1113
1114         if (fm10k_dev_rxq_interrupt_setup(dev))
1115                 return -EIO;
1116
1117         diag = fm10k_dev_rx_init(dev);
1118         if (diag) {
1119                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1120                 return diag;
1121         }
1122
1123         if (hw->mac.type == fm10k_mac_pf)
1124                 fm10k_dev_dglort_map_configure(dev);
1125
1126         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1127                 struct fm10k_rx_queue *rxq;
1128                 rxq = dev->data->rx_queues[i];
1129
1130                 if (rxq->rx_deferred_start)
1131                         continue;
1132                 diag = fm10k_dev_rx_queue_start(dev, i);
1133                 if (diag != 0) {
1134                         int j;
1135                         for (j = 0; j < i; ++j)
1136                                 rx_queue_clean(dev->data->rx_queues[j]);
1137                         return diag;
1138                 }
1139         }
1140
1141         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1142                 struct fm10k_tx_queue *txq;
1143                 txq = dev->data->tx_queues[i];
1144
1145                 if (txq->tx_deferred_start)
1146                         continue;
1147                 diag = fm10k_dev_tx_queue_start(dev, i);
1148                 if (diag != 0) {
1149                         int j;
1150                         for (j = 0; j < i; ++j)
1151                                 tx_queue_clean(dev->data->tx_queues[j]);
1152                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1153                                 rx_queue_clean(dev->data->rx_queues[j]);
1154                         return diag;
1155                 }
1156         }
1157
1158         /* Update default vlan when not in VMDQ mode */
1159         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1160                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1161
1162         fm10k_link_update(dev, 0);
1163
1164         return 0;
1165 }
1166
1167 static void
1168 fm10k_dev_stop(struct rte_eth_dev *dev)
1169 {
1170         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1171         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1172         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1173         int i;
1174
1175         PMD_INIT_FUNC_TRACE();
1176
1177         if (dev->data->tx_queues)
1178                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1179                         fm10k_dev_tx_queue_stop(dev, i);
1180
1181         if (dev->data->rx_queues)
1182                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1183                         fm10k_dev_rx_queue_stop(dev, i);
1184
1185         /* Disable datapath event */
1186         if (rte_intr_dp_is_en(intr_handle)) {
1187                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1188                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1189                                 3 << FM10K_RXINT_TIMER_SHIFT);
1190                         if (hw->mac.type == fm10k_mac_pf)
1191                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1192                                         FM10K_ITR_MASK_SET);
1193                         else
1194                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1195                                         FM10K_ITR_MASK_SET);
1196                 }
1197         }
1198         /* Clean datapath event and queue/vec mapping */
1199         rte_intr_efd_disable(intr_handle);
1200         rte_free(intr_handle->intr_vec);
1201         intr_handle->intr_vec = NULL;
1202 }
1203
1204 static void
1205 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1206 {
1207         int i;
1208
1209         PMD_INIT_FUNC_TRACE();
1210
1211         if (dev->data->tx_queues) {
1212                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1213                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1214
1215                         tx_queue_free(txq);
1216                 }
1217         }
1218
1219         if (dev->data->rx_queues) {
1220                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1221                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1222         }
1223 }
1224
1225 static int
1226 fm10k_link_update(struct rte_eth_dev *dev,
1227         __rte_unused int wait_to_complete)
1228 {
1229         struct fm10k_dev_info *dev_info =
1230                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1231         PMD_INIT_FUNC_TRACE();
1232
1233         dev->data->dev_link.link_speed  = ETH_SPEED_NUM_50G;
1234         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1235         dev->data->dev_link.link_status =
1236                 dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP;
1237         dev->data->dev_link.link_autoneg = ETH_LINK_FIXED;
1238
1239         return 0;
1240 }
1241
1242 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1243         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1244 {
1245         unsigned i, q;
1246         unsigned count = 0;
1247
1248         if (xstats_names != NULL) {
1249                 /* Note: limit checked in rte_eth_xstats_names() */
1250
1251                 /* Global stats */
1252                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1253                         snprintf(xstats_names[count].name,
1254                                 sizeof(xstats_names[count].name),
1255                                 "%s", fm10k_hw_stats_strings[count].name);
1256                         count++;
1257                 }
1258
1259                 /* PF queue stats */
1260                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1261                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1262                                 snprintf(xstats_names[count].name,
1263                                         sizeof(xstats_names[count].name),
1264                                         "rx_q%u_%s", q,
1265                                         fm10k_hw_stats_rx_q_strings[i].name);
1266                                 count++;
1267                         }
1268                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1269                                 snprintf(xstats_names[count].name,
1270                                         sizeof(xstats_names[count].name),
1271                                         "tx_q%u_%s", q,
1272                                         fm10k_hw_stats_tx_q_strings[i].name);
1273                                 count++;
1274                         }
1275                 }
1276         }
1277         return FM10K_NB_XSTATS;
1278 }
1279
1280 static int
1281 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1282                  unsigned n)
1283 {
1284         struct fm10k_hw_stats *hw_stats =
1285                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1286         unsigned i, q, count = 0;
1287
1288         if (n < FM10K_NB_XSTATS)
1289                 return FM10K_NB_XSTATS;
1290
1291         /* Global stats */
1292         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1293                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1294                         fm10k_hw_stats_strings[count].offset);
1295                 xstats[count].id = count;
1296                 count++;
1297         }
1298
1299         /* PF queue stats */
1300         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1301                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1302                         xstats[count].value =
1303                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1304                                 fm10k_hw_stats_rx_q_strings[i].offset);
1305                         xstats[count].id = count;
1306                         count++;
1307                 }
1308                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1309                         xstats[count].value =
1310                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1311                                 fm10k_hw_stats_tx_q_strings[i].offset);
1312                         xstats[count].id = count;
1313                         count++;
1314                 }
1315         }
1316
1317         return FM10K_NB_XSTATS;
1318 }
1319
1320 static int
1321 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1322 {
1323         uint64_t ipackets, opackets, ibytes, obytes, imissed;
1324         struct fm10k_hw *hw =
1325                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1326         struct fm10k_hw_stats *hw_stats =
1327                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1328         int i;
1329
1330         PMD_INIT_FUNC_TRACE();
1331
1332         fm10k_update_hw_stats(hw, hw_stats);
1333
1334         ipackets = opackets = ibytes = obytes = imissed = 0;
1335         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1336                 (i < hw->mac.max_queues); ++i) {
1337                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1338                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1339                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1340                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1341                 stats->q_errors[i]   = hw_stats->q[i].rx_drops.count;
1342                 ipackets += stats->q_ipackets[i];
1343                 opackets += stats->q_opackets[i];
1344                 ibytes   += stats->q_ibytes[i];
1345                 obytes   += stats->q_obytes[i];
1346                 imissed  += stats->q_errors[i];
1347         }
1348         stats->ipackets = ipackets;
1349         stats->opackets = opackets;
1350         stats->ibytes = ibytes;
1351         stats->obytes = obytes;
1352         stats->imissed = imissed;
1353         return 0;
1354 }
1355
1356 static int
1357 fm10k_stats_reset(struct rte_eth_dev *dev)
1358 {
1359         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1360         struct fm10k_hw_stats *hw_stats =
1361                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1362
1363         PMD_INIT_FUNC_TRACE();
1364
1365         memset(hw_stats, 0, sizeof(*hw_stats));
1366         fm10k_rebind_hw_stats(hw, hw_stats);
1367
1368         return 0;
1369 }
1370
1371 static int
1372 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1373         struct rte_eth_dev_info *dev_info)
1374 {
1375         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1376         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1377
1378         PMD_INIT_FUNC_TRACE();
1379
1380         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1381         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1382         dev_info->max_rx_queues      = hw->mac.max_queues;
1383         dev_info->max_tx_queues      = hw->mac.max_queues;
1384         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1385         dev_info->max_hash_mac_addrs = 0;
1386         dev_info->max_vfs            = pdev->max_vfs;
1387         dev_info->vmdq_pool_base     = 0;
1388         dev_info->vmdq_queue_base    = 0;
1389         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1390         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1391         dev_info->rx_queue_offload_capa = fm10k_get_rx_queue_offloads_capa(dev);
1392         dev_info->rx_offload_capa = fm10k_get_rx_port_offloads_capa(dev) |
1393                                     dev_info->rx_queue_offload_capa;
1394         dev_info->tx_queue_offload_capa = fm10k_get_tx_queue_offloads_capa(dev);
1395         dev_info->tx_offload_capa = fm10k_get_tx_port_offloads_capa(dev) |
1396                                     dev_info->tx_queue_offload_capa;
1397
1398         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1399         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1400         dev_info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1401                                         ETH_RSS_IPV6 |
1402                                         ETH_RSS_IPV6_EX |
1403                                         ETH_RSS_NONFRAG_IPV4_TCP |
1404                                         ETH_RSS_NONFRAG_IPV6_TCP |
1405                                         ETH_RSS_IPV6_TCP_EX |
1406                                         ETH_RSS_NONFRAG_IPV4_UDP |
1407                                         ETH_RSS_NONFRAG_IPV6_UDP |
1408                                         ETH_RSS_IPV6_UDP_EX;
1409
1410         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1411                 .rx_thresh = {
1412                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1413                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1414                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1415                 },
1416                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1417                 .rx_drop_en = 0,
1418                 .offloads = 0,
1419         };
1420
1421         dev_info->default_txconf = (struct rte_eth_txconf) {
1422                 .tx_thresh = {
1423                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1424                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1425                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1426                 },
1427                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1428                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1429                 .offloads = 0,
1430         };
1431
1432         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1433                 .nb_max = FM10K_MAX_RX_DESC,
1434                 .nb_min = FM10K_MIN_RX_DESC,
1435                 .nb_align = FM10K_MULT_RX_DESC,
1436         };
1437
1438         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1439                 .nb_max = FM10K_MAX_TX_DESC,
1440                 .nb_min = FM10K_MIN_TX_DESC,
1441                 .nb_align = FM10K_MULT_TX_DESC,
1442                 .nb_seg_max = FM10K_TX_MAX_SEG,
1443                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1444         };
1445
1446         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1447                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1448                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1449
1450         return 0;
1451 }
1452
1453 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1454 static const uint32_t *
1455 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1456 {
1457         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1458             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1459                 static uint32_t ptypes[] = {
1460                         /* refers to rx_desc_to_ol_flags() */
1461                         RTE_PTYPE_L2_ETHER,
1462                         RTE_PTYPE_L3_IPV4,
1463                         RTE_PTYPE_L3_IPV4_EXT,
1464                         RTE_PTYPE_L3_IPV6,
1465                         RTE_PTYPE_L3_IPV6_EXT,
1466                         RTE_PTYPE_L4_TCP,
1467                         RTE_PTYPE_L4_UDP,
1468                         RTE_PTYPE_UNKNOWN
1469                 };
1470
1471                 return ptypes;
1472         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1473                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1474                 static uint32_t ptypes_vec[] = {
1475                         /* refers to fm10k_desc_to_pktype_v() */
1476                         RTE_PTYPE_L3_IPV4,
1477                         RTE_PTYPE_L3_IPV4_EXT,
1478                         RTE_PTYPE_L3_IPV6,
1479                         RTE_PTYPE_L3_IPV6_EXT,
1480                         RTE_PTYPE_L4_TCP,
1481                         RTE_PTYPE_L4_UDP,
1482                         RTE_PTYPE_TUNNEL_GENEVE,
1483                         RTE_PTYPE_TUNNEL_NVGRE,
1484                         RTE_PTYPE_TUNNEL_VXLAN,
1485                         RTE_PTYPE_TUNNEL_GRE,
1486                         RTE_PTYPE_UNKNOWN
1487                 };
1488
1489                 return ptypes_vec;
1490         }
1491
1492         return NULL;
1493 }
1494 #else
1495 static const uint32_t *
1496 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1497 {
1498         return NULL;
1499 }
1500 #endif
1501
1502 static int
1503 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1504 {
1505         s32 result;
1506         uint16_t mac_num = 0;
1507         uint32_t vid_idx, vid_bit, mac_index;
1508         struct fm10k_hw *hw;
1509         struct fm10k_macvlan_filter_info *macvlan;
1510         struct rte_eth_dev_data *data = dev->data;
1511
1512         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1513         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1514
1515         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1516                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1517                 return -EINVAL;
1518         }
1519
1520         if (vlan_id > ETH_VLAN_ID_MAX) {
1521                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1522                 return -EINVAL;
1523         }
1524
1525         vid_idx = FM10K_VFTA_IDX(vlan_id);
1526         vid_bit = FM10K_VFTA_BIT(vlan_id);
1527         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1528         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1529                 return 0;
1530         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1531         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1532                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1533                         "in the VLAN filter table");
1534                 return -EINVAL;
1535         }
1536
1537         fm10k_mbx_lock(hw);
1538         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1539         fm10k_mbx_unlock(hw);
1540         if (result != FM10K_SUCCESS) {
1541                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1542                 return -EIO;
1543         }
1544
1545         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1546                         (result == FM10K_SUCCESS); mac_index++) {
1547                 if (rte_is_zero_ether_addr(&data->mac_addrs[mac_index]))
1548                         continue;
1549                 if (mac_num > macvlan->mac_num - 1) {
1550                         PMD_INIT_LOG(ERR, "MAC address number "
1551                                         "not match");
1552                         break;
1553                 }
1554                 fm10k_mbx_lock(hw);
1555                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1556                         data->mac_addrs[mac_index].addr_bytes,
1557                         vlan_id, on, 0);
1558                 fm10k_mbx_unlock(hw);
1559                 mac_num++;
1560         }
1561         if (result != FM10K_SUCCESS) {
1562                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1563                 return -EIO;
1564         }
1565
1566         if (on) {
1567                 macvlan->vlan_num++;
1568                 macvlan->vfta[vid_idx] |= vid_bit;
1569         } else {
1570                 macvlan->vlan_num--;
1571                 macvlan->vfta[vid_idx] &= ~vid_bit;
1572         }
1573         return 0;
1574 }
1575
1576 static int
1577 fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1578 {
1579         if (mask & ETH_VLAN_STRIP_MASK) {
1580                 if (!(dev->data->dev_conf.rxmode.offloads &
1581                         DEV_RX_OFFLOAD_VLAN_STRIP))
1582                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1583                                         "always on in fm10k");
1584         }
1585
1586         if (mask & ETH_VLAN_EXTEND_MASK) {
1587                 if (dev->data->dev_conf.rxmode.offloads &
1588                         DEV_RX_OFFLOAD_VLAN_EXTEND)
1589                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1590                                         "supported in fm10k");
1591         }
1592
1593         if (mask & ETH_VLAN_FILTER_MASK) {
1594                 if (!(dev->data->dev_conf.rxmode.offloads &
1595                         DEV_RX_OFFLOAD_VLAN_FILTER))
1596                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1597         }
1598
1599         return 0;
1600 }
1601
1602 /* Add/Remove a MAC address, and update filters to main VSI */
1603 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1604                 const u8 *mac, bool add, uint32_t pool)
1605 {
1606         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1607         struct fm10k_macvlan_filter_info *macvlan;
1608         uint32_t i, j, k;
1609
1610         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1611
1612         if (pool != MAIN_VSI_POOL_NUMBER) {
1613                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1614                         "mac to pool %u", pool);
1615                 return;
1616         }
1617         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1618                 if (!macvlan->vfta[j])
1619                         continue;
1620                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1621                         if (!(macvlan->vfta[j] & (1 << k)))
1622                                 continue;
1623                         if (i + 1 > macvlan->vlan_num) {
1624                                 PMD_INIT_LOG(ERR, "vlan number not match");
1625                                 return;
1626                         }
1627                         fm10k_mbx_lock(hw);
1628                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1629                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1630                         fm10k_mbx_unlock(hw);
1631                         i++;
1632                 }
1633         }
1634 }
1635
1636 /* Add/Remove a MAC address, and update filters to VMDQ */
1637 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1638                 const u8 *mac, bool add, uint32_t pool)
1639 {
1640         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1641         struct fm10k_macvlan_filter_info *macvlan;
1642         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1643         uint32_t i;
1644
1645         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1646         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1647
1648         if (pool > macvlan->nb_queue_pools) {
1649                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1650                         " Max pool is %u",
1651                         pool, macvlan->nb_queue_pools);
1652                 return;
1653         }
1654         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1655                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1656                         continue;
1657                 fm10k_mbx_lock(hw);
1658                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1659                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1660                 fm10k_mbx_unlock(hw);
1661         }
1662 }
1663
1664 /* Add/Remove a MAC address, and update filters */
1665 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1666                 const u8 *mac, bool add, uint32_t pool)
1667 {
1668         struct fm10k_macvlan_filter_info *macvlan;
1669
1670         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1671
1672         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1673                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1674         else
1675                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1676
1677         if (add)
1678                 macvlan->mac_num++;
1679         else
1680                 macvlan->mac_num--;
1681 }
1682
1683 /* Add a MAC address, and update filters */
1684 static int
1685 fm10k_macaddr_add(struct rte_eth_dev *dev,
1686                 struct rte_ether_addr *mac_addr,
1687                 uint32_t index,
1688                 uint32_t pool)
1689 {
1690         struct fm10k_macvlan_filter_info *macvlan;
1691
1692         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1693         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1694         macvlan->mac_vmdq_id[index] = pool;
1695         return 0;
1696 }
1697
1698 /* Remove a MAC address, and update filters */
1699 static void
1700 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1701 {
1702         struct rte_eth_dev_data *data = dev->data;
1703         struct fm10k_macvlan_filter_info *macvlan;
1704
1705         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1706         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1707                         FALSE, macvlan->mac_vmdq_id[index]);
1708         macvlan->mac_vmdq_id[index] = 0;
1709 }
1710
1711 static inline int
1712 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1713 {
1714         if ((request < min) || (request > max) || ((request % mult) != 0))
1715                 return -1;
1716         else
1717                 return 0;
1718 }
1719
1720
1721 static inline int
1722 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1723 {
1724         if ((request < min) || (request > max) || ((div % request) != 0))
1725                 return -1;
1726         else
1727                 return 0;
1728 }
1729
1730 static inline int
1731 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1732 {
1733         uint16_t rx_free_thresh;
1734
1735         if (conf->rx_free_thresh == 0)
1736                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1737         else
1738                 rx_free_thresh = conf->rx_free_thresh;
1739
1740         /* make sure the requested threshold satisfies the constraints */
1741         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1742                         FM10K_RX_FREE_THRESH_MAX(q),
1743                         FM10K_RX_FREE_THRESH_DIV(q),
1744                         rx_free_thresh)) {
1745                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1746                         "less than or equal to %u, "
1747                         "greater than or equal to %u, "
1748                         "and a divisor of %u",
1749                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1750                         FM10K_RX_FREE_THRESH_MIN(q),
1751                         FM10K_RX_FREE_THRESH_DIV(q));
1752                 return -EINVAL;
1753         }
1754
1755         q->alloc_thresh = rx_free_thresh;
1756         q->drop_en = conf->rx_drop_en;
1757         q->rx_deferred_start = conf->rx_deferred_start;
1758
1759         return 0;
1760 }
1761
1762 /*
1763  * Hardware requires specific alignment for Rx packet buffers. At
1764  * least one of the following two conditions must be satisfied.
1765  *  1. Address is 512B aligned
1766  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1767  *
1768  * As such, the driver may need to adjust the DMA address within the
1769  * buffer by up to 512B.
1770  *
1771  * return 1 if the element size is valid, otherwise return 0.
1772  */
1773 static int
1774 mempool_element_size_valid(struct rte_mempool *mp)
1775 {
1776         uint32_t min_size;
1777
1778         /* elt_size includes mbuf header and headroom */
1779         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1780                         RTE_PKTMBUF_HEADROOM;
1781
1782         /* account for up to 512B of alignment */
1783         min_size -= FM10K_RX_DATABUF_ALIGN;
1784
1785         /* sanity check for overflow */
1786         if (min_size > mp->elt_size)
1787                 return 0;
1788
1789         /* size is valid */
1790         return 1;
1791 }
1792
1793 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev)
1794 {
1795         RTE_SET_USED(dev);
1796
1797         return (uint64_t)(DEV_RX_OFFLOAD_SCATTER);
1798 }
1799
1800 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
1801 {
1802         RTE_SET_USED(dev);
1803
1804         return  (uint64_t)(DEV_RX_OFFLOAD_VLAN_STRIP  |
1805                            DEV_RX_OFFLOAD_VLAN_FILTER |
1806                            DEV_RX_OFFLOAD_IPV4_CKSUM  |
1807                            DEV_RX_OFFLOAD_UDP_CKSUM   |
1808                            DEV_RX_OFFLOAD_TCP_CKSUM   |
1809                            DEV_RX_OFFLOAD_JUMBO_FRAME |
1810                            DEV_RX_OFFLOAD_HEADER_SPLIT |
1811                            DEV_RX_OFFLOAD_RSS_HASH);
1812 }
1813
1814 static int
1815 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1816         uint16_t nb_desc, unsigned int socket_id,
1817         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1818 {
1819         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1820         struct fm10k_dev_info *dev_info =
1821                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1822         struct fm10k_rx_queue *q;
1823         const struct rte_memzone *mz;
1824         uint64_t offloads;
1825
1826         PMD_INIT_FUNC_TRACE();
1827
1828         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1829
1830         /* make sure the mempool element size can account for alignment. */
1831         if (!mempool_element_size_valid(mp)) {
1832                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1833                 return -EINVAL;
1834         }
1835
1836         /* make sure a valid number of descriptors have been requested */
1837         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1838                                 FM10K_MULT_RX_DESC, nb_desc)) {
1839                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1840                         "less than or equal to %"PRIu32", "
1841                         "greater than or equal to %u, "
1842                         "and a multiple of %u",
1843                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1844                         FM10K_MULT_RX_DESC);
1845                 return -EINVAL;
1846         }
1847
1848         /*
1849          * if this queue existed already, free the associated memory. The
1850          * queue cannot be reused in case we need to allocate memory on
1851          * different socket than was previously used.
1852          */
1853         if (dev->data->rx_queues[queue_id] != NULL) {
1854                 rx_queue_free(dev->data->rx_queues[queue_id]);
1855                 dev->data->rx_queues[queue_id] = NULL;
1856         }
1857
1858         /* allocate memory for the queue structure */
1859         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1860                                 socket_id);
1861         if (q == NULL) {
1862                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1863                 return -ENOMEM;
1864         }
1865
1866         /* setup queue */
1867         q->mp = mp;
1868         q->nb_desc = nb_desc;
1869         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1870         q->port_id = dev->data->port_id;
1871         q->queue_id = queue_id;
1872         q->tail_ptr = (volatile uint32_t *)
1873                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1874         q->offloads = offloads;
1875         if (handle_rxconf(q, conf))
1876                 return -EINVAL;
1877
1878         /* allocate memory for the software ring */
1879         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1880                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1881                         RTE_CACHE_LINE_SIZE, socket_id);
1882         if (q->sw_ring == NULL) {
1883                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1884                 rte_free(q);
1885                 return -ENOMEM;
1886         }
1887
1888         /*
1889          * allocate memory for the hardware descriptor ring. A memzone large
1890          * enough to hold the maximum ring size is requested to allow for
1891          * resizing in later calls to the queue setup function.
1892          */
1893         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1894                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1895                                       socket_id);
1896         if (mz == NULL) {
1897                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1898                 rte_free(q->sw_ring);
1899                 rte_free(q);
1900                 return -ENOMEM;
1901         }
1902         q->hw_ring = mz->addr;
1903         q->hw_ring_phys_addr = mz->iova;
1904
1905         /* Check if number of descs satisfied Vector requirement */
1906         if (!rte_is_power_of_2(nb_desc)) {
1907                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1908                                     "preconditions - canceling the feature for "
1909                                     "the whole port[%d]",
1910                              q->queue_id, q->port_id);
1911                 dev_info->rx_vec_allowed = false;
1912         } else
1913                 fm10k_rxq_vec_setup(q);
1914
1915         dev->data->rx_queues[queue_id] = q;
1916         return 0;
1917 }
1918
1919 static void
1920 fm10k_rx_queue_release(void *queue)
1921 {
1922         PMD_INIT_FUNC_TRACE();
1923
1924         rx_queue_free(queue);
1925 }
1926
1927 static inline int
1928 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1929 {
1930         uint16_t tx_free_thresh;
1931         uint16_t tx_rs_thresh;
1932
1933         /* constraint MACROs require that tx_free_thresh is configured
1934          * before tx_rs_thresh */
1935         if (conf->tx_free_thresh == 0)
1936                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1937         else
1938                 tx_free_thresh = conf->tx_free_thresh;
1939
1940         /* make sure the requested threshold satisfies the constraints */
1941         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1942                         FM10K_TX_FREE_THRESH_MAX(q),
1943                         FM10K_TX_FREE_THRESH_DIV(q),
1944                         tx_free_thresh)) {
1945                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1946                         "less than or equal to %u, "
1947                         "greater than or equal to %u, "
1948                         "and a divisor of %u",
1949                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1950                         FM10K_TX_FREE_THRESH_MIN(q),
1951                         FM10K_TX_FREE_THRESH_DIV(q));
1952                 return -EINVAL;
1953         }
1954
1955         q->free_thresh = tx_free_thresh;
1956
1957         if (conf->tx_rs_thresh == 0)
1958                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1959         else
1960                 tx_rs_thresh = conf->tx_rs_thresh;
1961
1962         q->tx_deferred_start = conf->tx_deferred_start;
1963
1964         /* make sure the requested threshold satisfies the constraints */
1965         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1966                         FM10K_TX_RS_THRESH_MAX(q),
1967                         FM10K_TX_RS_THRESH_DIV(q),
1968                         tx_rs_thresh)) {
1969                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1970                         "less than or equal to %u, "
1971                         "greater than or equal to %u, "
1972                         "and a divisor of %u",
1973                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1974                         FM10K_TX_RS_THRESH_MIN(q),
1975                         FM10K_TX_RS_THRESH_DIV(q));
1976                 return -EINVAL;
1977         }
1978
1979         q->rs_thresh = tx_rs_thresh;
1980
1981         return 0;
1982 }
1983
1984 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev)
1985 {
1986         RTE_SET_USED(dev);
1987
1988         return 0;
1989 }
1990
1991 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
1992 {
1993         RTE_SET_USED(dev);
1994
1995         return (uint64_t)(DEV_TX_OFFLOAD_VLAN_INSERT |
1996                           DEV_TX_OFFLOAD_MULTI_SEGS  |
1997                           DEV_TX_OFFLOAD_IPV4_CKSUM  |
1998                           DEV_TX_OFFLOAD_UDP_CKSUM   |
1999                           DEV_TX_OFFLOAD_TCP_CKSUM   |
2000                           DEV_TX_OFFLOAD_TCP_TSO);
2001 }
2002
2003 static int
2004 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
2005         uint16_t nb_desc, unsigned int socket_id,
2006         const struct rte_eth_txconf *conf)
2007 {
2008         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2009         struct fm10k_tx_queue *q;
2010         const struct rte_memzone *mz;
2011         uint64_t offloads;
2012
2013         PMD_INIT_FUNC_TRACE();
2014
2015         offloads = conf->offloads | dev->data->dev_conf.txmode.offloads;
2016
2017         /* make sure a valid number of descriptors have been requested */
2018         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
2019                                 FM10K_MULT_TX_DESC, nb_desc)) {
2020                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
2021                         "less than or equal to %"PRIu32", "
2022                         "greater than or equal to %u, "
2023                         "and a multiple of %u",
2024                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
2025                         FM10K_MULT_TX_DESC);
2026                 return -EINVAL;
2027         }
2028
2029         /*
2030          * if this queue existed already, free the associated memory. The
2031          * queue cannot be reused in case we need to allocate memory on
2032          * different socket than was previously used.
2033          */
2034         if (dev->data->tx_queues[queue_id] != NULL) {
2035                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2036
2037                 tx_queue_free(txq);
2038                 dev->data->tx_queues[queue_id] = NULL;
2039         }
2040
2041         /* allocate memory for the queue structure */
2042         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2043                                 socket_id);
2044         if (q == NULL) {
2045                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2046                 return -ENOMEM;
2047         }
2048
2049         /* setup queue */
2050         q->nb_desc = nb_desc;
2051         q->port_id = dev->data->port_id;
2052         q->queue_id = queue_id;
2053         q->offloads = offloads;
2054         q->ops = &def_txq_ops;
2055         q->tail_ptr = (volatile uint32_t *)
2056                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2057         if (handle_txconf(q, conf))
2058                 return -EINVAL;
2059
2060         /* allocate memory for the software ring */
2061         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2062                                         nb_desc * sizeof(struct rte_mbuf *),
2063                                         RTE_CACHE_LINE_SIZE, socket_id);
2064         if (q->sw_ring == NULL) {
2065                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2066                 rte_free(q);
2067                 return -ENOMEM;
2068         }
2069
2070         /*
2071          * allocate memory for the hardware descriptor ring. A memzone large
2072          * enough to hold the maximum ring size is requested to allow for
2073          * resizing in later calls to the queue setup function.
2074          */
2075         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2076                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2077                                       socket_id);
2078         if (mz == NULL) {
2079                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2080                 rte_free(q->sw_ring);
2081                 rte_free(q);
2082                 return -ENOMEM;
2083         }
2084         q->hw_ring = mz->addr;
2085         q->hw_ring_phys_addr = mz->iova;
2086
2087         /*
2088          * allocate memory for the RS bit tracker. Enough slots to hold the
2089          * descriptor index for each RS bit needing to be set are required.
2090          */
2091         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2092                                 ((nb_desc + 1) / q->rs_thresh) *
2093                                 sizeof(uint16_t),
2094                                 RTE_CACHE_LINE_SIZE, socket_id);
2095         if (q->rs_tracker.list == NULL) {
2096                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2097                 rte_free(q->sw_ring);
2098                 rte_free(q);
2099                 return -ENOMEM;
2100         }
2101
2102         dev->data->tx_queues[queue_id] = q;
2103         return 0;
2104 }
2105
2106 static void
2107 fm10k_tx_queue_release(void *queue)
2108 {
2109         struct fm10k_tx_queue *q = queue;
2110         PMD_INIT_FUNC_TRACE();
2111
2112         tx_queue_free(q);
2113 }
2114
2115 static int
2116 fm10k_reta_update(struct rte_eth_dev *dev,
2117                         struct rte_eth_rss_reta_entry64 *reta_conf,
2118                         uint16_t reta_size)
2119 {
2120         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2121         uint16_t i, j, idx, shift;
2122         uint8_t mask;
2123         uint32_t reta;
2124
2125         PMD_INIT_FUNC_TRACE();
2126
2127         if (reta_size > FM10K_MAX_RSS_INDICES) {
2128                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2129                         "(%d) doesn't match the number hardware can supported "
2130                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2131                 return -EINVAL;
2132         }
2133
2134         /*
2135          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2136          * 128-entries in 32 registers
2137          */
2138         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2139                 idx = i / RTE_RETA_GROUP_SIZE;
2140                 shift = i % RTE_RETA_GROUP_SIZE;
2141                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2142                                 BIT_MASK_PER_UINT32);
2143                 if (mask == 0)
2144                         continue;
2145
2146                 reta = 0;
2147                 if (mask != BIT_MASK_PER_UINT32)
2148                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2149
2150                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2151                         if (mask & (0x1 << j)) {
2152                                 if (mask != 0xF)
2153                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2154                                 reta |= reta_conf[idx].reta[shift + j] <<
2155                                                 (CHAR_BIT * j);
2156                         }
2157                 }
2158                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2159         }
2160
2161         return 0;
2162 }
2163
2164 static int
2165 fm10k_reta_query(struct rte_eth_dev *dev,
2166                         struct rte_eth_rss_reta_entry64 *reta_conf,
2167                         uint16_t reta_size)
2168 {
2169         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2170         uint16_t i, j, idx, shift;
2171         uint8_t mask;
2172         uint32_t reta;
2173
2174         PMD_INIT_FUNC_TRACE();
2175
2176         if (reta_size < FM10K_MAX_RSS_INDICES) {
2177                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2178                         "(%d) doesn't match the number hardware can supported "
2179                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2180                 return -EINVAL;
2181         }
2182
2183         /*
2184          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2185          * 128-entries in 32 registers
2186          */
2187         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2188                 idx = i / RTE_RETA_GROUP_SIZE;
2189                 shift = i % RTE_RETA_GROUP_SIZE;
2190                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2191                                 BIT_MASK_PER_UINT32);
2192                 if (mask == 0)
2193                         continue;
2194
2195                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2196                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2197                         if (mask & (0x1 << j))
2198                                 reta_conf[idx].reta[shift + j] = ((reta >>
2199                                         CHAR_BIT * j) & UINT8_MAX);
2200                 }
2201         }
2202
2203         return 0;
2204 }
2205
2206 static int
2207 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2208         struct rte_eth_rss_conf *rss_conf)
2209 {
2210         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2211         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2212         uint32_t mrqc;
2213         uint64_t hf = rss_conf->rss_hf;
2214         int i;
2215
2216         PMD_INIT_FUNC_TRACE();
2217
2218         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2219                                 FM10K_RSSRK_ENTRIES_PER_REG))
2220                 return -EINVAL;
2221
2222         if (hf == 0)
2223                 return -EINVAL;
2224
2225         mrqc = 0;
2226         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2227         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2228         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2229         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2230         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2231         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2232         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2233         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2234         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2235
2236         /* If the mapping doesn't fit any supported, return */
2237         if (mrqc == 0)
2238                 return -EINVAL;
2239
2240         if (key != NULL)
2241                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2242                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2243
2244         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2245
2246         return 0;
2247 }
2248
2249 static int
2250 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2251         struct rte_eth_rss_conf *rss_conf)
2252 {
2253         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2254         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2255         uint32_t mrqc;
2256         uint64_t hf;
2257         int i;
2258
2259         PMD_INIT_FUNC_TRACE();
2260
2261         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2262                                 FM10K_RSSRK_ENTRIES_PER_REG))
2263                 return -EINVAL;
2264
2265         if (key != NULL)
2266                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2267                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2268
2269         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2270         hf = 0;
2271         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2272         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2273         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2274         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2275         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2276         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2277         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2278         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2279         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2280
2281         rss_conf->rss_hf = hf;
2282
2283         return 0;
2284 }
2285
2286 static void
2287 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2288 {
2289         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2290         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2291
2292         /* Bind all local non-queue interrupt to vector 0 */
2293         int_map |= FM10K_MISC_VEC_ID;
2294
2295         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2296         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2297         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2298         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2299         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2300         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2301
2302         /* Enable misc causes */
2303         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2304                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2305                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2306                                 FM10K_EIMR_ENABLE(MAILBOX) |
2307                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2308                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2309                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2310                                 FM10K_EIMR_ENABLE(VFLR));
2311
2312         /* Enable ITR 0 */
2313         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2314                                         FM10K_ITR_MASK_CLEAR);
2315         FM10K_WRITE_FLUSH(hw);
2316 }
2317
2318 static void
2319 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2320 {
2321         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2322         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2323
2324         int_map |= FM10K_MISC_VEC_ID;
2325
2326         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2327         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2328         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2329         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2330         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2331         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2332
2333         /* Disable misc causes */
2334         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2335                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2336                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2337                                 FM10K_EIMR_DISABLE(MAILBOX) |
2338                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2339                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2340                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2341                                 FM10K_EIMR_DISABLE(VFLR));
2342
2343         /* Disable ITR 0 */
2344         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2345         FM10K_WRITE_FLUSH(hw);
2346 }
2347
2348 static void
2349 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2350 {
2351         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2352         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2353
2354         /* Bind all local non-queue interrupt to vector 0 */
2355         int_map |= FM10K_MISC_VEC_ID;
2356
2357         /* Only INT 0 available, other 15 are reserved. */
2358         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2359
2360         /* Enable ITR 0 */
2361         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2362                                         FM10K_ITR_MASK_CLEAR);
2363         FM10K_WRITE_FLUSH(hw);
2364 }
2365
2366 static void
2367 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2368 {
2369         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2370         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2371
2372         int_map |= FM10K_MISC_VEC_ID;
2373
2374         /* Only INT 0 available, other 15 are reserved. */
2375         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2376
2377         /* Disable ITR 0 */
2378         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2379         FM10K_WRITE_FLUSH(hw);
2380 }
2381
2382 static int
2383 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2384 {
2385         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2386         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2387
2388         /* Enable ITR */
2389         if (hw->mac.type == fm10k_mac_pf)
2390                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2391                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2392         else
2393                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2394                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2395         rte_intr_ack(&pdev->intr_handle);
2396         return 0;
2397 }
2398
2399 static int
2400 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2401 {
2402         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2403         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2404
2405         /* Disable ITR */
2406         if (hw->mac.type == fm10k_mac_pf)
2407                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2408                         FM10K_ITR_MASK_SET);
2409         else
2410                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2411                         FM10K_ITR_MASK_SET);
2412         return 0;
2413 }
2414
2415 static int
2416 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2417 {
2418         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2419         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2420         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2421         uint32_t intr_vector, vec;
2422         uint16_t queue_id;
2423         int result = 0;
2424
2425         /* fm10k needs one separate interrupt for mailbox,
2426          * so only drivers which support multiple interrupt vectors
2427          * e.g. vfio-pci can work for fm10k interrupt mode
2428          */
2429         if (!rte_intr_cap_multiple(intr_handle) ||
2430                         dev->data->dev_conf.intr_conf.rxq == 0)
2431                 return result;
2432
2433         intr_vector = dev->data->nb_rx_queues;
2434
2435         /* disable interrupt first */
2436         rte_intr_disable(intr_handle);
2437         if (hw->mac.type == fm10k_mac_pf)
2438                 fm10k_dev_disable_intr_pf(dev);
2439         else
2440                 fm10k_dev_disable_intr_vf(dev);
2441
2442         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2443                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2444                 result = -EIO;
2445         }
2446
2447         if (rte_intr_dp_is_en(intr_handle) && !result) {
2448                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2449                         dev->data->nb_rx_queues * sizeof(int), 0);
2450                 if (intr_handle->intr_vec) {
2451                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2452                                         queue_id < dev->data->nb_rx_queues;
2453                                         queue_id++) {
2454                                 intr_handle->intr_vec[queue_id] = vec;
2455                                 if (vec < intr_handle->nb_efd - 1
2456                                                 + FM10K_RX_VEC_START)
2457                                         vec++;
2458                         }
2459                 } else {
2460                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2461                                 " intr_vec", dev->data->nb_rx_queues);
2462                         rte_intr_efd_disable(intr_handle);
2463                         result = -ENOMEM;
2464                 }
2465         }
2466
2467         if (hw->mac.type == fm10k_mac_pf)
2468                 fm10k_dev_enable_intr_pf(dev);
2469         else
2470                 fm10k_dev_enable_intr_vf(dev);
2471         rte_intr_enable(intr_handle);
2472         hw->mac.ops.update_int_moderator(hw);
2473         return result;
2474 }
2475
2476 static int
2477 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2478 {
2479         struct fm10k_fault fault;
2480         int err;
2481         const char *estr = "Unknown error";
2482
2483         /* Process PCA fault */
2484         if (eicr & FM10K_EICR_PCA_FAULT) {
2485                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2486                 if (err)
2487                         goto error;
2488                 switch (fault.type) {
2489                 case PCA_NO_FAULT:
2490                         estr = "PCA_NO_FAULT"; break;
2491                 case PCA_UNMAPPED_ADDR:
2492                         estr = "PCA_UNMAPPED_ADDR"; break;
2493                 case PCA_BAD_QACCESS_PF:
2494                         estr = "PCA_BAD_QACCESS_PF"; break;
2495                 case PCA_BAD_QACCESS_VF:
2496                         estr = "PCA_BAD_QACCESS_VF"; break;
2497                 case PCA_MALICIOUS_REQ:
2498                         estr = "PCA_MALICIOUS_REQ"; break;
2499                 case PCA_POISONED_TLP:
2500                         estr = "PCA_POISONED_TLP"; break;
2501                 case PCA_TLP_ABORT:
2502                         estr = "PCA_TLP_ABORT"; break;
2503                 default:
2504                         goto error;
2505                 }
2506                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2507                         estr, fault.func ? "VF" : "PF", fault.func,
2508                         fault.address, fault.specinfo);
2509         }
2510
2511         /* Process THI fault */
2512         if (eicr & FM10K_EICR_THI_FAULT) {
2513                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2514                 if (err)
2515                         goto error;
2516                 switch (fault.type) {
2517                 case THI_NO_FAULT:
2518                         estr = "THI_NO_FAULT"; break;
2519                 case THI_MAL_DIS_Q_FAULT:
2520                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2521                 default:
2522                         goto error;
2523                 }
2524                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2525                         estr, fault.func ? "VF" : "PF", fault.func,
2526                         fault.address, fault.specinfo);
2527         }
2528
2529         /* Process FUM fault */
2530         if (eicr & FM10K_EICR_FUM_FAULT) {
2531                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2532                 if (err)
2533                         goto error;
2534                 switch (fault.type) {
2535                 case FUM_NO_FAULT:
2536                         estr = "FUM_NO_FAULT"; break;
2537                 case FUM_UNMAPPED_ADDR:
2538                         estr = "FUM_UNMAPPED_ADDR"; break;
2539                 case FUM_POISONED_TLP:
2540                         estr = "FUM_POISONED_TLP"; break;
2541                 case FUM_BAD_VF_QACCESS:
2542                         estr = "FUM_BAD_VF_QACCESS"; break;
2543                 case FUM_ADD_DECODE_ERR:
2544                         estr = "FUM_ADD_DECODE_ERR"; break;
2545                 case FUM_RO_ERROR:
2546                         estr = "FUM_RO_ERROR"; break;
2547                 case FUM_QPRC_CRC_ERROR:
2548                         estr = "FUM_QPRC_CRC_ERROR"; break;
2549                 case FUM_CSR_TIMEOUT:
2550                         estr = "FUM_CSR_TIMEOUT"; break;
2551                 case FUM_INVALID_TYPE:
2552                         estr = "FUM_INVALID_TYPE"; break;
2553                 case FUM_INVALID_LENGTH:
2554                         estr = "FUM_INVALID_LENGTH"; break;
2555                 case FUM_INVALID_BE:
2556                         estr = "FUM_INVALID_BE"; break;
2557                 case FUM_INVALID_ALIGN:
2558                         estr = "FUM_INVALID_ALIGN"; break;
2559                 default:
2560                         goto error;
2561                 }
2562                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2563                         estr, fault.func ? "VF" : "PF", fault.func,
2564                         fault.address, fault.specinfo);
2565         }
2566
2567         return 0;
2568 error:
2569         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2570         return err;
2571 }
2572
2573 /**
2574  * PF interrupt handler triggered by NIC for handling specific interrupt.
2575  *
2576  * @param handle
2577  *  Pointer to interrupt handle.
2578  * @param param
2579  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2580  *
2581  * @return
2582  *  void
2583  */
2584 static void
2585 fm10k_dev_interrupt_handler_pf(void *param)
2586 {
2587         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2588         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2589         uint32_t cause, status;
2590         struct fm10k_dev_info *dev_info =
2591                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2592         int status_mbx;
2593         s32 err;
2594
2595         if (hw->mac.type != fm10k_mac_pf)
2596                 return;
2597
2598         cause = FM10K_READ_REG(hw, FM10K_EICR);
2599
2600         /* Handle PCI fault cases */
2601         if (cause & FM10K_EICR_FAULT_MASK) {
2602                 PMD_INIT_LOG(ERR, "INT: find fault!");
2603                 fm10k_dev_handle_fault(hw, cause);
2604         }
2605
2606         /* Handle switch up/down */
2607         if (cause & FM10K_EICR_SWITCHNOTREADY)
2608                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2609
2610         if (cause & FM10K_EICR_SWITCHREADY) {
2611                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2612                 if (dev_info->sm_down == 1) {
2613                         fm10k_mbx_lock(hw);
2614
2615                         /* For recreating logical ports */
2616                         status_mbx = hw->mac.ops.update_lport_state(hw,
2617                                         hw->mac.dglort_map, MAX_LPORT_NUM, 1);
2618                         if (status_mbx == FM10K_SUCCESS)
2619                                 PMD_INIT_LOG(INFO,
2620                                         "INT: Recreated Logical port");
2621                         else
2622                                 PMD_INIT_LOG(INFO,
2623                                         "INT: Logical ports weren't recreated");
2624
2625                         status_mbx = hw->mac.ops.update_xcast_mode(hw,
2626                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2627                         if (status_mbx != FM10K_SUCCESS)
2628                                 PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2629
2630                         fm10k_mbx_unlock(hw);
2631
2632                         /* first clear the internal SW recording structure */
2633                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2634                                                 ETH_MQ_RX_VMDQ_FLAG))
2635                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2636                                         false);
2637
2638                         fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2639                                         MAIN_VSI_POOL_NUMBER);
2640
2641                         /*
2642                          * Add default mac address and vlan for the logical
2643                          * ports that have been created, leave to the
2644                          * application to fully recover Rx filtering.
2645                          */
2646                         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2647                                         MAIN_VSI_POOL_NUMBER);
2648
2649                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2650                                                 ETH_MQ_RX_VMDQ_FLAG))
2651                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2652                                         true);
2653
2654                         dev_info->sm_down = 0;
2655                         _rte_eth_dev_callback_process(dev,
2656                                         RTE_ETH_EVENT_INTR_LSC,
2657                                         NULL);
2658                 }
2659         }
2660
2661         /* Handle mailbox message */
2662         fm10k_mbx_lock(hw);
2663         err = hw->mbx.ops.process(hw, &hw->mbx);
2664         fm10k_mbx_unlock(hw);
2665
2666         if (err == FM10K_ERR_RESET_REQUESTED) {
2667                 PMD_INIT_LOG(INFO, "INT: Switch is down");
2668                 dev_info->sm_down = 1;
2669                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2670                                 NULL);
2671         }
2672
2673         /* Handle SRAM error */
2674         if (cause & FM10K_EICR_SRAMERROR) {
2675                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2676
2677                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2678                 /* Write to clear pending bits */
2679                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2680
2681                 /* Todo: print out error message after shared code  updates */
2682         }
2683
2684         /* Clear these 3 events if having any */
2685         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2686                  FM10K_EICR_SWITCHREADY;
2687         if (cause)
2688                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2689
2690         /* Re-enable interrupt from device side */
2691         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2692                                         FM10K_ITR_MASK_CLEAR);
2693         /* Re-enable interrupt from host side */
2694         rte_intr_ack(dev->intr_handle);
2695 }
2696
2697 /**
2698  * VF interrupt handler triggered by NIC for handling specific interrupt.
2699  *
2700  * @param handle
2701  *  Pointer to interrupt handle.
2702  * @param param
2703  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2704  *
2705  * @return
2706  *  void
2707  */
2708 static void
2709 fm10k_dev_interrupt_handler_vf(void *param)
2710 {
2711         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2712         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2713         struct fm10k_mbx_info *mbx = &hw->mbx;
2714         struct fm10k_dev_info *dev_info =
2715                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2716         const enum fm10k_mbx_state state = mbx->state;
2717         int status_mbx;
2718
2719         if (hw->mac.type != fm10k_mac_vf)
2720                 return;
2721
2722         /* Handle mailbox message if lock is acquired */
2723         fm10k_mbx_lock(hw);
2724         hw->mbx.ops.process(hw, &hw->mbx);
2725         fm10k_mbx_unlock(hw);
2726
2727         if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) {
2728                 PMD_INIT_LOG(INFO, "INT: Switch has gone down");
2729
2730                 fm10k_mbx_lock(hw);
2731                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2732                                 MAX_LPORT_NUM, 1);
2733                 fm10k_mbx_unlock(hw);
2734
2735                 /* Setting reset flag */
2736                 dev_info->sm_down = 1;
2737                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2738                                 NULL);
2739         }
2740
2741         if (dev_info->sm_down == 1 &&
2742                         hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) {
2743                 PMD_INIT_LOG(INFO, "INT: Switch has gone up");
2744                 fm10k_mbx_lock(hw);
2745                 status_mbx = hw->mac.ops.update_xcast_mode(hw,
2746                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2747                 if (status_mbx != FM10K_SUCCESS)
2748                         PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2749                 fm10k_mbx_unlock(hw);
2750
2751                 /* first clear the internal SW recording structure */
2752                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false);
2753                 fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2754                                 MAIN_VSI_POOL_NUMBER);
2755
2756                 /*
2757                  * Add default mac address and vlan for the logical ports that
2758                  * have been created, leave to the application to fully recover
2759                  * Rx filtering.
2760                  */
2761                 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2762                                 MAIN_VSI_POOL_NUMBER);
2763                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
2764
2765                 dev_info->sm_down = 0;
2766                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2767                                 NULL);
2768         }
2769
2770         /* Re-enable interrupt from device side */
2771         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2772                                         FM10K_ITR_MASK_CLEAR);
2773         /* Re-enable interrupt from host side */
2774         rte_intr_ack(dev->intr_handle);
2775 }
2776
2777 /* Mailbox message handler in VF */
2778 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2779         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2780         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2781         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2782         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2783 };
2784
2785 static int
2786 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2787 {
2788         int err = 0;
2789
2790         /* Initialize mailbox lock */
2791         fm10k_mbx_initlock(hw);
2792
2793         /* Replace default message handler with new ones */
2794         if (hw->mac.type == fm10k_mac_vf)
2795                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2796
2797         if (err) {
2798                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2799                                 err);
2800                 return err;
2801         }
2802         /* Connect to SM for PF device or PF for VF device */
2803         return hw->mbx.ops.connect(hw, &hw->mbx);
2804 }
2805
2806 static void
2807 fm10k_close_mbx_service(struct fm10k_hw *hw)
2808 {
2809         /* Disconnect from SM for PF device or PF for VF device */
2810         hw->mbx.ops.disconnect(hw, &hw->mbx);
2811 }
2812
2813 static void
2814 fm10k_dev_close(struct rte_eth_dev *dev)
2815 {
2816         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2817         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2818         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2819
2820         PMD_INIT_FUNC_TRACE();
2821
2822         fm10k_mbx_lock(hw);
2823         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2824                 MAX_LPORT_NUM, false);
2825         fm10k_mbx_unlock(hw);
2826
2827         /* allow 100ms for device to quiesce */
2828         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
2829
2830         /* Stop mailbox service first */
2831         fm10k_close_mbx_service(hw);
2832         fm10k_dev_stop(dev);
2833         fm10k_dev_queue_release(dev);
2834         fm10k_stop_hw(hw);
2835
2836         dev->dev_ops = NULL;
2837         dev->rx_pkt_burst = NULL;
2838         dev->tx_pkt_burst = NULL;
2839
2840         /* disable uio/vfio intr */
2841         rte_intr_disable(intr_handle);
2842
2843         /*PF/VF has different interrupt handling mechanism */
2844         if (hw->mac.type == fm10k_mac_pf) {
2845                 /* disable interrupt */
2846                 fm10k_dev_disable_intr_pf(dev);
2847
2848                 /* unregister callback func to eal lib */
2849                 rte_intr_callback_unregister(intr_handle,
2850                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2851         } else {
2852                 /* disable interrupt */
2853                 fm10k_dev_disable_intr_vf(dev);
2854
2855                 rte_intr_callback_unregister(intr_handle,
2856                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2857         }
2858 }
2859
2860 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2861         .dev_configure          = fm10k_dev_configure,
2862         .dev_start              = fm10k_dev_start,
2863         .dev_stop               = fm10k_dev_stop,
2864         .dev_close              = fm10k_dev_close,
2865         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2866         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2867         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2868         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2869         .stats_get              = fm10k_stats_get,
2870         .xstats_get             = fm10k_xstats_get,
2871         .xstats_get_names       = fm10k_xstats_get_names,
2872         .stats_reset            = fm10k_stats_reset,
2873         .xstats_reset           = fm10k_stats_reset,
2874         .link_update            = fm10k_link_update,
2875         .dev_infos_get          = fm10k_dev_infos_get,
2876         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2877         .vlan_filter_set        = fm10k_vlan_filter_set,
2878         .vlan_offload_set       = fm10k_vlan_offload_set,
2879         .mac_addr_add           = fm10k_macaddr_add,
2880         .mac_addr_remove        = fm10k_macaddr_remove,
2881         .rx_queue_start         = fm10k_dev_rx_queue_start,
2882         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2883         .tx_queue_start         = fm10k_dev_tx_queue_start,
2884         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2885         .rx_queue_setup         = fm10k_rx_queue_setup,
2886         .rx_queue_release       = fm10k_rx_queue_release,
2887         .tx_queue_setup         = fm10k_tx_queue_setup,
2888         .tx_queue_release       = fm10k_tx_queue_release,
2889         .rx_queue_count         = fm10k_dev_rx_queue_count,
2890         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2891         .rx_descriptor_status = fm10k_dev_rx_descriptor_status,
2892         .tx_descriptor_status = fm10k_dev_tx_descriptor_status,
2893         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2894         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2895         .reta_update            = fm10k_reta_update,
2896         .reta_query             = fm10k_reta_query,
2897         .rss_hash_update        = fm10k_rss_hash_update,
2898         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2899 };
2900
2901 static int ftag_check_handler(__rte_unused const char *key,
2902                 const char *value, __rte_unused void *opaque)
2903 {
2904         if (strcmp(value, "1"))
2905                 return -1;
2906
2907         return 0;
2908 }
2909
2910 static int
2911 fm10k_check_ftag(struct rte_devargs *devargs)
2912 {
2913         struct rte_kvargs *kvlist;
2914         const char *ftag_key = "enable_ftag";
2915
2916         if (devargs == NULL)
2917                 return 0;
2918
2919         kvlist = rte_kvargs_parse(devargs->args, NULL);
2920         if (kvlist == NULL)
2921                 return 0;
2922
2923         if (!rte_kvargs_count(kvlist, ftag_key)) {
2924                 rte_kvargs_free(kvlist);
2925                 return 0;
2926         }
2927         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2928         if (rte_kvargs_process(kvlist, ftag_key,
2929                                 ftag_check_handler, NULL) < 0) {
2930                 rte_kvargs_free(kvlist);
2931                 return 0;
2932         }
2933         rte_kvargs_free(kvlist);
2934
2935         return 1;
2936 }
2937
2938 static uint16_t
2939 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2940                     uint16_t nb_pkts)
2941 {
2942         uint16_t nb_tx = 0;
2943         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2944
2945         while (nb_pkts) {
2946                 uint16_t ret, num;
2947
2948                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2949                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2950                                                  num);
2951                 nb_tx += ret;
2952                 nb_pkts -= ret;
2953                 if (ret < num)
2954                         break;
2955         }
2956
2957         return nb_tx;
2958 }
2959
2960 static void __attribute__((cold))
2961 fm10k_set_tx_function(struct rte_eth_dev *dev)
2962 {
2963         struct fm10k_tx_queue *txq;
2964         int i;
2965         int use_sse = 1;
2966         uint16_t tx_ftag_en = 0;
2967
2968         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2969                 /* primary process has set the ftag flag and offloads */
2970                 txq = dev->data->tx_queues[0];
2971                 if (fm10k_tx_vec_condition_check(txq)) {
2972                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2973                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2974                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2975                 } else {
2976                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2977                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2978                         dev->tx_pkt_prepare = NULL;
2979                 }
2980                 return;
2981         }
2982
2983         if (fm10k_check_ftag(dev->device->devargs))
2984                 tx_ftag_en = 1;
2985
2986         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2987                 txq = dev->data->tx_queues[i];
2988                 txq->tx_ftag_en = tx_ftag_en;
2989                 /* Check if Vector Tx is satisfied */
2990                 if (fm10k_tx_vec_condition_check(txq))
2991                         use_sse = 0;
2992         }
2993
2994         if (use_sse) {
2995                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2996                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2997                         txq = dev->data->tx_queues[i];
2998                         fm10k_txq_vec_setup(txq);
2999                 }
3000                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
3001                 dev->tx_pkt_prepare = NULL;
3002         } else {
3003                 dev->tx_pkt_burst = fm10k_xmit_pkts;
3004                 dev->tx_pkt_prepare = fm10k_prep_pkts;
3005                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
3006         }
3007 }
3008
3009 static void __attribute__((cold))
3010 fm10k_set_rx_function(struct rte_eth_dev *dev)
3011 {
3012         struct fm10k_dev_info *dev_info =
3013                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3014         uint16_t i, rx_using_sse;
3015         uint16_t rx_ftag_en = 0;
3016
3017         if (fm10k_check_ftag(dev->device->devargs))
3018                 rx_ftag_en = 1;
3019
3020         /* In order to allow Vector Rx there are a few configuration
3021          * conditions to be met.
3022          */
3023         if (!fm10k_rx_vec_condition_check(dev) &&
3024                         dev_info->rx_vec_allowed && !rx_ftag_en) {
3025                 if (dev->data->scattered_rx)
3026                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
3027                 else
3028                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
3029         } else if (dev->data->scattered_rx)
3030                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
3031         else
3032                 dev->rx_pkt_burst = fm10k_recv_pkts;
3033
3034         rx_using_sse =
3035                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
3036                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
3037
3038         if (rx_using_sse)
3039                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
3040         else
3041                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
3042
3043         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3044                 return;
3045
3046         for (i = 0; i < dev->data->nb_rx_queues; i++) {
3047                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
3048
3049                 rxq->rx_using_sse = rx_using_sse;
3050                 rxq->rx_ftag_en = rx_ftag_en;
3051         }
3052 }
3053
3054 static void
3055 fm10k_params_init(struct rte_eth_dev *dev)
3056 {
3057         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3058         struct fm10k_dev_info *info =
3059                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3060
3061         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
3062          * there is no way to get link status without reading BAR4.  Until this
3063          * works, assume we have maximum bandwidth.
3064          * @todo - fix bus info
3065          */
3066         hw->bus_caps.speed = fm10k_bus_speed_8000;
3067         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
3068         hw->bus_caps.payload = fm10k_bus_payload_512;
3069         hw->bus.speed = fm10k_bus_speed_8000;
3070         hw->bus.width = fm10k_bus_width_pcie_x8;
3071         hw->bus.payload = fm10k_bus_payload_256;
3072
3073         info->rx_vec_allowed = true;
3074         info->sm_down = false;
3075 }
3076
3077 static int
3078 eth_fm10k_dev_init(struct rte_eth_dev *dev)
3079 {
3080         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3081         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3082         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3083         int diag, i;
3084         struct fm10k_macvlan_filter_info *macvlan;
3085
3086         PMD_INIT_FUNC_TRACE();
3087
3088         dev->dev_ops = &fm10k_eth_dev_ops;
3089         dev->rx_pkt_burst = &fm10k_recv_pkts;
3090         dev->tx_pkt_burst = &fm10k_xmit_pkts;
3091         dev->tx_pkt_prepare = &fm10k_prep_pkts;
3092
3093         /*
3094          * Primary process does the whole initialization, for secondary
3095          * processes, we just select the same Rx and Tx function as primary.
3096          */
3097         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3098                 fm10k_set_rx_function(dev);
3099                 fm10k_set_tx_function(dev);
3100                 return 0;
3101         }
3102
3103         rte_eth_copy_pci_info(dev, pdev);
3104
3105         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
3106         memset(macvlan, 0, sizeof(*macvlan));
3107         /* Vendor and Device ID need to be set before init of shared code */
3108         memset(hw, 0, sizeof(*hw));
3109         hw->device_id = pdev->id.device_id;
3110         hw->vendor_id = pdev->id.vendor_id;
3111         hw->subsystem_device_id = pdev->id.subsystem_device_id;
3112         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
3113         hw->revision_id = 0;
3114         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
3115         if (hw->hw_addr == NULL) {
3116                 PMD_INIT_LOG(ERR, "Bad mem resource."
3117                         " Try to blacklist unused devices.");
3118                 return -EIO;
3119         }
3120
3121         /* Store fm10k_adapter pointer */
3122         hw->back = dev->data->dev_private;
3123
3124         /* Initialize the shared code */
3125         diag = fm10k_init_shared_code(hw);
3126         if (diag != FM10K_SUCCESS) {
3127                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
3128                 return -EIO;
3129         }
3130
3131         /* Initialize parameters */
3132         fm10k_params_init(dev);
3133
3134         /* Initialize the hw */
3135         diag = fm10k_init_hw(hw);
3136         if (diag != FM10K_SUCCESS) {
3137                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
3138                 return -EIO;
3139         }
3140
3141         /* Initialize MAC address(es) */
3142         dev->data->mac_addrs = rte_zmalloc("fm10k",
3143                         RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
3144         if (dev->data->mac_addrs == NULL) {
3145                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
3146                 return -ENOMEM;
3147         }
3148
3149         diag = fm10k_read_mac_addr(hw);
3150
3151         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3152                         &dev->data->mac_addrs[0]);
3153
3154         if (diag != FM10K_SUCCESS ||
3155                 !rte_is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
3156
3157                 /* Generate a random addr */
3158                 rte_eth_random_addr(hw->mac.addr);
3159                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
3160                 rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3161                 &dev->data->mac_addrs[0]);
3162         }
3163
3164         /* Pass the information to the rte_eth_dev_close() that it should also
3165          * release the private port resources.
3166          */
3167         dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
3168
3169         /* Reset the hw statistics */
3170         diag = fm10k_stats_reset(dev);
3171         if (diag != 0) {
3172                 PMD_INIT_LOG(ERR, "Stats reset failed: %d", diag);
3173                 return diag;
3174         }
3175
3176         /* Reset the hw */
3177         diag = fm10k_reset_hw(hw);
3178         if (diag != FM10K_SUCCESS) {
3179                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
3180                 return -EIO;
3181         }
3182
3183         /* Setup mailbox service */
3184         diag = fm10k_setup_mbx_service(hw);
3185         if (diag != FM10K_SUCCESS) {
3186                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
3187                 return -EIO;
3188         }
3189
3190         /*PF/VF has different interrupt handling mechanism */
3191         if (hw->mac.type == fm10k_mac_pf) {
3192                 /* register callback func to eal lib */
3193                 rte_intr_callback_register(intr_handle,
3194                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3195
3196                 /* enable MISC interrupt */
3197                 fm10k_dev_enable_intr_pf(dev);
3198         } else { /* VF */
3199                 rte_intr_callback_register(intr_handle,
3200                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3201
3202                 fm10k_dev_enable_intr_vf(dev);
3203         }
3204
3205         /* Enable intr after callback registered */
3206         rte_intr_enable(intr_handle);
3207
3208         hw->mac.ops.update_int_moderator(hw);
3209
3210         /* Make sure Switch Manager is ready before going forward. */
3211         if (hw->mac.type == fm10k_mac_pf) {
3212                 int switch_ready = 0;
3213
3214                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3215                         fm10k_mbx_lock(hw);
3216                         hw->mac.ops.get_host_state(hw, &switch_ready);
3217                         fm10k_mbx_unlock(hw);
3218                         if (switch_ready)
3219                                 break;
3220                         /* Delay some time to acquire async LPORT_MAP info. */
3221                         rte_delay_us(WAIT_SWITCH_MSG_US);
3222                 }
3223
3224                 if (switch_ready == 0) {
3225                         PMD_INIT_LOG(ERR, "switch is not ready");
3226                         return -1;
3227                 }
3228         }
3229
3230         /*
3231          * Below function will trigger operations on mailbox, acquire lock to
3232          * avoid race condition from interrupt handler. Operations on mailbox
3233          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3234          * will handle and generate an interrupt to our side. Then,  FIFO in
3235          * mailbox will be touched.
3236          */
3237         fm10k_mbx_lock(hw);
3238         /* Enable port first */
3239         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3240                                         MAX_LPORT_NUM, 1);
3241
3242         /* Set unicast mode by default. App can change to other mode in other
3243          * API func.
3244          */
3245         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3246                                         FM10K_XCAST_MODE_NONE);
3247
3248         fm10k_mbx_unlock(hw);
3249
3250         /* Make sure default VID is ready before going forward. */
3251         if (hw->mac.type == fm10k_mac_pf) {
3252                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3253                         if (hw->mac.default_vid)
3254                                 break;
3255                         /* Delay some time to acquire async port VLAN info. */
3256                         rte_delay_us(WAIT_SWITCH_MSG_US);
3257                 }
3258
3259                 if (!hw->mac.default_vid) {
3260                         PMD_INIT_LOG(ERR, "default VID is not ready");
3261                         return -1;
3262                 }
3263         }
3264
3265         /* Add default mac address */
3266         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3267                 MAIN_VSI_POOL_NUMBER);
3268
3269         return 0;
3270 }
3271
3272 static int
3273 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3274 {
3275         PMD_INIT_FUNC_TRACE();
3276
3277         /* only uninitialize in the primary process */
3278         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3279                 return 0;
3280
3281         /* safe to close dev here */
3282         fm10k_dev_close(dev);
3283
3284         return 0;
3285 }
3286
3287 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3288         struct rte_pci_device *pci_dev)
3289 {
3290         return rte_eth_dev_pci_generic_probe(pci_dev,
3291                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3292 }
3293
3294 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3295 {
3296         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3297 }
3298
3299 /*
3300  * The set of PCI devices this driver supports. This driver will enable both PF
3301  * and SRIOV-VF devices.
3302  */
3303 static const struct rte_pci_id pci_id_fm10k_map[] = {
3304         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3305         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3306         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3307         { .vendor_id = 0, /* sentinel */ },
3308 };
3309
3310 static struct rte_pci_driver rte_pmd_fm10k = {
3311         .id_table = pci_id_fm10k_map,
3312         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
3313         .probe = eth_fm10k_pci_probe,
3314         .remove = eth_fm10k_pci_remove,
3315 };
3316
3317 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3318 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3319 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");
3320
3321 RTE_INIT(fm10k_init_log)
3322 {
3323         fm10k_logtype_init = rte_log_register("pmd.net.fm10k.init");
3324         if (fm10k_logtype_init >= 0)
3325                 rte_log_set_level(fm10k_logtype_init, RTE_LOG_NOTICE);
3326         fm10k_logtype_driver = rte_log_register("pmd.net.fm10k.driver");
3327         if (fm10k_logtype_driver >= 0)
3328                 rte_log_set_level(fm10k_logtype_driver, RTE_LOG_NOTICE);
3329
3330 #ifdef RTE_LIBRTE_FM10K_DEBUG_RX
3331         fm10k_logtype_rx = rte_log_register("pmd.net.fm10k.rx");
3332         if (fm10k_logtype_rx >= 0)
3333                 rte_log_set_level(fm10k_logtype_rx, RTE_LOG_DEBUG);
3334 #endif
3335
3336 #ifdef RTE_LIBRTE_FM10K_DEBUG_TX
3337         fm10k_logtype_tx = rte_log_register("pmd.net.fm10k.tx");
3338         if (fm10k_logtype_tx >= 0)
3339                 rte_log_set_level(fm10k_logtype_tx, RTE_LOG_DEBUG);
3340 #endif
3341
3342 #ifdef RTE_LIBRTE_FM10K_DEBUG_TX_FREE
3343         fm10k_logtype_tx_free = rte_log_register("pmd.net.fm10k.tx_free");
3344         if (fm10k_logtype_tx_free >= 0)
3345                 rte_log_set_level(fm10k_logtype_tx_free, RTE_LOG_DEBUG);
3346 #endif
3347 }