drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 /* Default delay to acquire mailbox lock */
  45 #define FM10K_MBXLOCK_DELAY_US 20
  46 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  47
  48 #define MAIN_VSI_POOL_NUMBER 0
  49
  50 /* Max try times to acquire switch status */
  51 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  52 /* Wait interval to get switch status */
  53 #define WAIT_SWITCH_MSG_US    100000
  54 /* Number of chars per uint32 type */
  55 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  56 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  57
  58 #define FM10K_SIMPLE_TX_FLAG ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \
  59                                 ETH_TXQ_FLAGS_NOOFFLOADS)
  60
  61 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  62 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  63 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  64 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  65 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  66 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  67 static int
  68 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  69 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  70         const u8 *mac, bool add, uint32_t pool);
  71 static void fm10k_tx_queue_release(void *queue);
  72 static void fm10k_rx_queue_release(void *queue);
  73 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  74 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  75
  76 struct fm10k_xstats_name_off {
  77         char name[RTE_ETH_XSTATS_NAME_SIZE];
  78         unsigned offset;
  79 };
  80
  81 struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  82         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  83         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  84         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  85         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  86         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  87         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  88         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  89         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  90                 nodesc_drop)},
  91 };
  92
  93 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  94                 sizeof(fm10k_hw_stats_strings[0]))
  95
  96 struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
  97         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
  98         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
  99         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
 100 };
 101
 102 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
 103                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
 104
 105 struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 106         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 107         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 108 };
 109
 110 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 111                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 112
 113 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 114                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 115
 116 static void
 117 fm10k_mbx_initlock(struct fm10k_hw *hw)
 118 {
 119         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 120 }
 121
 122 static void
 123 fm10k_mbx_lock(struct fm10k_hw *hw)
 124 {
 125         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 126                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 127 }
 128
 129 static void
 130 fm10k_mbx_unlock(struct fm10k_hw *hw)
 131 {
 132         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 133 }
 134
 135 /*
 136  * reset queue to initial state, allocate software buffers used when starting
 137  * device.
 138  * return 0 on success
 139  * return -ENOMEM if buffers cannot be allocated
 140  * return -EINVAL if buffers do not satisfy alignment condition
 141  */
 142 static inline int
 143 rx_queue_reset(struct fm10k_rx_queue *q)
 144 {
 145         static const union fm10k_rx_desc zero = {{0} };
 146         uint64_t dma_addr;
 147         int i, diag;
 148         PMD_INIT_FUNC_TRACE();
 149
 150         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 151         if (diag != 0)
 152                 return -ENOMEM;
 153
 154         for (i = 0; i < q->nb_desc; ++i) {
 155                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 156                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 157                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 158                                                 q->nb_desc);
 159                         return -EINVAL;
 160                 }
 161                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 162                 q->hw_ring[i].q.pkt_addr = dma_addr;
 163                 q->hw_ring[i].q.hdr_addr = dma_addr;
 164         }
 165
 166         /* initialize extra software ring entries. Space for these extra
 167          * entries is always allocated.
 168          */
 169         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 170         for (i = 0; i < q->nb_fake_desc; ++i) {
 171                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 172                 q->hw_ring[q->nb_desc + i] = zero;
 173         }
 174
 175         q->next_dd = 0;
 176         q->next_alloc = 0;
 177         q->next_trigger = q->alloc_thresh - 1;
 178         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 179         q->rxrearm_start = 0;
 180         q->rxrearm_nb = 0;
 181
 182         return 0;
 183 }
 184
 185 /*
 186  * clean queue, descriptor rings, free software buffers used when stopping
 187  * device.
 188  */
 189 static inline void
 190 rx_queue_clean(struct fm10k_rx_queue *q)
 191 {
 192         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 193         uint32_t i;
 194         PMD_INIT_FUNC_TRACE();
 195
 196         /* zero descriptor rings */
 197         for (i = 0; i < q->nb_desc; ++i)
 198                 q->hw_ring[i] = zero;
 199
 200         /* zero faked descriptors */
 201         for (i = 0; i < q->nb_fake_desc; ++i)
 202                 q->hw_ring[q->nb_desc + i] = zero;
 203
 204         /* vPMD driver has a different way of releasing mbufs. */
 205         if (q->rx_using_sse) {
 206                 fm10k_rx_queue_release_mbufs_vec(q);
 207                 return;
 208         }
 209
 210         /* free software buffers */
 211         for (i = 0; i < q->nb_desc; ++i) {
 212                 if (q->sw_ring[i]) {
 213                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 214                         q->sw_ring[i] = NULL;
 215                 }
 216         }
 217 }
 218
 219 /*
 220  * free all queue memory used when releasing the queue (i.e. configure)
 221  */
 222 static inline void
 223 rx_queue_free(struct fm10k_rx_queue *q)
 224 {
 225         PMD_INIT_FUNC_TRACE();
 226         if (q) {
 227                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 228                 rx_queue_clean(q);
 229                 if (q->sw_ring) {
 230                         rte_free(q->sw_ring);
 231                         q->sw_ring = NULL;
 232                 }
 233                 rte_free(q);
 234                 q = NULL;
 235         }
 236 }
 237
 238 /*
 239  * disable RX queue, wait unitl HW finished necessary flush operation
 240  */
 241 static inline int
 242 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 243 {
 244         uint32_t reg, i;
 245
 246         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 247         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 248                         reg & ~FM10K_RXQCTL_ENABLE);
 249
 250         /* Wait 100us at most */
 251         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 252                 rte_delay_us(1);
 253                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 254                 if (!(reg & FM10K_RXQCTL_ENABLE))
 255                         break;
 256         }
 257
 258         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 259                 return -1;
 260
 261         return 0;
 262 }
 263
 264 /*
 265  * reset queue to initial state, allocate software buffers used when starting
 266  * device
 267  */
 268 static inline void
 269 tx_queue_reset(struct fm10k_tx_queue *q)
 270 {
 271         PMD_INIT_FUNC_TRACE();
 272         q->last_free = 0;
 273         q->next_free = 0;
 274         q->nb_used = 0;
 275         q->nb_free = q->nb_desc - 1;
 276         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 277         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 278 }
 279
 280 /*
 281  * clean queue, descriptor rings, free software buffers used when stopping
 282  * device
 283  */
 284 static inline void
 285 tx_queue_clean(struct fm10k_tx_queue *q)
 286 {
 287         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 288         uint32_t i;
 289         PMD_INIT_FUNC_TRACE();
 290
 291         /* zero descriptor rings */
 292         for (i = 0; i < q->nb_desc; ++i)
 293                 q->hw_ring[i] = zero;
 294
 295         /* free software buffers */
 296         for (i = 0; i < q->nb_desc; ++i) {
 297                 if (q->sw_ring[i]) {
 298                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 299                         q->sw_ring[i] = NULL;
 300                 }
 301         }
 302 }
 303
 304 /*
 305  * free all queue memory used when releasing the queue (i.e. configure)
 306  */
 307 static inline void
 308 tx_queue_free(struct fm10k_tx_queue *q)
 309 {
 310         PMD_INIT_FUNC_TRACE();
 311         if (q) {
 312                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 313                 tx_queue_clean(q);
 314                 if (q->rs_tracker.list) {
 315                         rte_free(q->rs_tracker.list);
 316                         q->rs_tracker.list = NULL;
 317                 }
 318                 if (q->sw_ring) {
 319                         rte_free(q->sw_ring);
 320                         q->sw_ring = NULL;
 321                 }
 322                 rte_free(q);
 323                 q = NULL;
 324         }
 325 }
 326
 327 /*
 328  * disable TX queue, wait unitl HW finished necessary flush operation
 329  */
 330 static inline int
 331 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 332 {
 333         uint32_t reg, i;
 334
 335         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 336         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 337                         reg & ~FM10K_TXDCTL_ENABLE);
 338
 339         /* Wait 100us at most */
 340         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 341                 rte_delay_us(1);
 342                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 343                 if (!(reg & FM10K_TXDCTL_ENABLE))
 344                         break;
 345         }
 346
 347         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 348                 return -1;
 349
 350         return 0;
 351 }
 352
 353 static int
 354 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 355 {
 356         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 357         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 358         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 359         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 360
 361         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 362
 363         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 364                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 365                 return -EINVAL;
 366         }
 367
 368         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 369                 return 0;
 370
 371         if (hw->mac.type == fm10k_mac_vf) {
 372                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 373                 return -EINVAL;
 374         }
 375
 376         /* Check VMDQ queue pool number */
 377         if (vmdq_conf->nb_queue_pools >
 378                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 379                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 380                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 381                         vmdq_conf->nb_queue_pools);
 382                 return -EINVAL;
 383         }
 384
 385         return 0;
 386 }
 387
 388 static const struct fm10k_txq_ops def_txq_ops = {
 389         .release_mbufs = tx_queue_free,
 390         .reset = tx_queue_reset,
 391 };
 392
 393 static int
 394 fm10k_dev_configure(struct rte_eth_dev *dev)
 395 {
 396         int ret;
 397
 398         PMD_INIT_FUNC_TRACE();
 399
 400         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 401                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 402         /* multipe queue mode checking */
 403         ret  = fm10k_check_mq_mode(dev);
 404         if (ret != 0) {
 405                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 406                             ret);
 407                 return ret;
 408         }
 409
 410         return 0;
 411 }
 412
 413 /* fls = find last set bit = 32 minus the number of leading zeros */
 414 #ifndef fls
 415 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 416 #endif
 417
 418 static void
 419 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 420 {
 421         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 422         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 423         uint32_t i;
 424
 425         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 426
 427         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 428                 if (!vmdq_conf->pool_map[i].pools)
 429                         continue;
 430                 fm10k_mbx_lock(hw);
 431                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 432                 fm10k_mbx_unlock(hw);
 433         }
 434 }
 435
 436 static void
 437 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 438 {
 439         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 440
 441         /* Add default mac address */
 442         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 443                 MAIN_VSI_POOL_NUMBER);
 444 }
 445
 446 static void
 447 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 448 {
 449         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 450         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 451         uint32_t mrqc, *key, i, reta, j;
 452         uint64_t hf;
 453
 454 #define RSS_KEY_SIZE 40
 455         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 456                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 457                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 458                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 459                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 460                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 461         };
 462
 463         if (dev->data->nb_rx_queues == 1 ||
 464             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 465             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 466                 return;
 467
 468         /* random key is rss_intel_key (default) or user provided (rss_key) */
 469         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 470                 key = (uint32_t *)rss_intel_key;
 471         else
 472                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 473
 474         /* Now fill our hash function seeds, 4 bytes at a time */
 475         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 476                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 477
 478         /*
 479          * Fill in redirection table
 480          * The byte-swap is needed because NIC registers are in
 481          * little-endian order.
 482          */
 483         reta = 0;
 484         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 485                 if (j == dev->data->nb_rx_queues)
 486                         j = 0;
 487                 reta = (reta << CHAR_BIT) | j;
 488                 if ((i & 3) == 3)
 489                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 490                                         rte_bswap32(reta));
 491         }
 492
 493         /*
 494          * Generate RSS hash based on packet types, TCP/UDP
 495          * port numbers and/or IPv4/v6 src and dst addresses
 496          */
 497         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 498         mrqc = 0;
 499         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 500         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 501         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 502         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 503         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 504         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 505         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 506         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 507         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 508
 509         if (mrqc == 0) {
 510                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 511                         "supported", hf);
 512                 return;
 513         }
 514
 515         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 516 }
 517
 518 static void
 519 fm10k_dev_logic_port_update(struct rte_eth_dev *dev,
 520         uint16_t nb_lport_old, uint16_t nb_lport_new)
 521 {
 522         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 523         uint32_t i;
 524
 525         fm10k_mbx_lock(hw);
 526         /* Disable previous logic ports */
 527         if (nb_lport_old)
 528                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
 529                         nb_lport_old, false);
 530         /* Enable new logic ports */
 531         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
 532                 nb_lport_new, true);
 533         fm10k_mbx_unlock(hw);
 534
 535         for (i = 0; i < nb_lport_new; i++) {
 536                 /* Set unicast mode by default. App can change
 537                  * to other mode in other API func.
 538                  */
 539                 fm10k_mbx_lock(hw);
 540                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 541                         FM10K_XCAST_MODE_NONE);
 542                 fm10k_mbx_unlock(hw);
 543         }
 544 }
 545
 546 static void
 547 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 548 {
 549         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 550         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 551         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 552         struct fm10k_macvlan_filter_info *macvlan;
 553         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 554         uint16_t nb_lport_new, nb_lport_old;
 555
 556         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 557         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 558
 559         fm10k_dev_rss_configure(dev);
 560
 561         /* only PF supports VMDQ */
 562         if (hw->mac.type != fm10k_mac_pf)
 563                 return;
 564
 565         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 566                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 567
 568         /* no pool number change, no need to update logic port and VLAN/MAC */
 569         if (macvlan->nb_queue_pools == nb_queue_pools)
 570                 return;
 571
 572         nb_lport_old = macvlan->nb_queue_pools ? macvlan->nb_queue_pools : 1;
 573         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 574         fm10k_dev_logic_port_update(dev, nb_lport_old, nb_lport_new);
 575
 576         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 577         memset(dev->data->mac_addrs, 0,
 578                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 579         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 580                 &dev->data->mac_addrs[0]);
 581         memset(macvlan, 0, sizeof(*macvlan));
 582         macvlan->nb_queue_pools = nb_queue_pools;
 583
 584         if (nb_queue_pools)
 585                 fm10k_dev_vmdq_rx_configure(dev);
 586         else
 587                 fm10k_dev_pf_main_vsi_reset(dev);
 588 }
 589
 590 static int
 591 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 592 {
 593         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 594         int i, ret;
 595         struct fm10k_tx_queue *txq;
 596         uint64_t base_addr;
 597         uint32_t size;
 598
 599         /* Disable TXINT to avoid possible interrupt */
 600         for (i = 0; i < hw->mac.max_queues; i++)
 601                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 602                                 3 << FM10K_TXINT_TIMER_SHIFT);
 603
 604         /* Setup TX queue */
 605         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 606                 txq = dev->data->tx_queues[i];
 607                 base_addr = txq->hw_ring_phys_addr;
 608                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 609
 610                 /* disable queue to avoid issues while updating state */
 611                 ret = tx_queue_disable(hw, i);
 612                 if (ret) {
 613                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 614                         return -1;
 615                 }
 616
 617                 /* set location and size for descriptor ring */
 618                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 619                                 base_addr & UINT64_LOWER_32BITS_MASK);
 620                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 621                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 622                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 623         }
 624
 625         /* set up vector or scalar TX function as appropriate */
 626         fm10k_set_tx_function(dev);
 627
 628         return 0;
 629 }
 630
 631 static int
 632 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 633 {
 634         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 635         int i, ret;
 636         struct fm10k_rx_queue *rxq;
 637         uint64_t base_addr;
 638         uint32_t size;
 639         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 640         uint16_t buf_size;
 641
 642         /* Disable RXINT to avoid possible interrupt */
 643         for (i = 0; i < hw->mac.max_queues; i++)
 644                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 645                                 3 << FM10K_RXINT_TIMER_SHIFT);
 646
 647         /* Setup RX queues */
 648         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 649                 rxq = dev->data->rx_queues[i];
 650                 base_addr = rxq->hw_ring_phys_addr;
 651                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 652
 653                 /* disable queue to avoid issues while updating state */
 654                 ret = rx_queue_disable(hw, i);
 655                 if (ret) {
 656                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 657                         return -1;
 658                 }
 659
 660                 /* Setup the Base and Length of the Rx Descriptor Ring */
 661                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 662                                 base_addr & UINT64_LOWER_32BITS_MASK);
 663                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 664                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 665                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 666
 667                 /* Configure the Rx buffer size for one buff without split */
 668                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 669                         RTE_PKTMBUF_HEADROOM);
 670                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 671                  * reserved for this purpose, and the worst case could be 511B.
 672                  * But SRR reg assumes all buffers have the same size. In order
 673                  * to fill the gap, we'll have to consider the worst case and
 674                  * assume 512B is reserved. If we don't do so, it's possible
 675                  * for HW to overwrite data to next mbuf.
 676                  */
 677                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 678
 679                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 680                                 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
 681
 682                 /* It adds dual VLAN length for supporting dual VLAN */
 683                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 684                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 685                         dev->data->dev_conf.rxmode.enable_scatter) {
 686                         uint32_t reg;
 687                         dev->data->scattered_rx = 1;
 688                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 689                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 690                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 691                 }
 692
 693                 /* Enable drop on empty, it's RO for VF */
 694                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 695                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 696
 697                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 698                 FM10K_WRITE_FLUSH(hw);
 699         }
 700
 701         /* Configure VMDQ/RSS if applicable */
 702         fm10k_dev_mq_rx_configure(dev);
 703
 704         /* Decide the best RX function */
 705         fm10k_set_rx_function(dev);
 706
 707         return 0;
 708 }
 709
 710 static int
 711 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 712 {
 713         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 714         int err = -1;
 715         uint32_t reg;
 716         struct fm10k_rx_queue *rxq;
 717
 718         PMD_INIT_FUNC_TRACE();
 719
 720         if (rx_queue_id < dev->data->nb_rx_queues) {
 721                 rxq = dev->data->rx_queues[rx_queue_id];
 722                 err = rx_queue_reset(rxq);
 723                 if (err == -ENOMEM) {
 724                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 725                         return err;
 726                 } else if (err == -EINVAL) {
 727                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 728                                 " %d", err);
 729                         return err;
 730                 }
 731
 732                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 733                  * Note: this must be done AFTER the queue is enabled on real
 734                  * hardware, but BEFORE the queue is enabled when using the
 735                  * emulation platform. Do it in both places for now and remove
 736                  * this comment and the following two register writes when the
 737                  * emulation platform is no longer being used.
 738                  */
 739                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 740                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 741
 742                 /* Set PF ownership flag for PF devices */
 743                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 744                 if (hw->mac.type == fm10k_mac_pf)
 745                         reg |= FM10K_RXQCTL_PF;
 746                 reg |= FM10K_RXQCTL_ENABLE;
 747                 /* enable RX queue */
 748                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 749                 FM10K_WRITE_FLUSH(hw);
 750
 751                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 752                  * Note: this must be done AFTER the queue is enabled
 753                  */
 754                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 755                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 756                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 757         }
 758
 759         return err;
 760 }
 761
 762 static int
 763 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 764 {
 765         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 766
 767         PMD_INIT_FUNC_TRACE();
 768
 769         if (rx_queue_id < dev->data->nb_rx_queues) {
 770                 /* Disable RX queue */
 771                 rx_queue_disable(hw, rx_queue_id);
 772
 773                 /* Free mbuf and clean HW ring */
 774                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 775                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 776         }
 777
 778         return 0;
 779 }
 780
 781 static int
 782 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 783 {
 784         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 785         /** @todo - this should be defined in the shared code */
 786 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 787         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 788         int err = 0;
 789
 790         PMD_INIT_FUNC_TRACE();
 791
 792         if (tx_queue_id < dev->data->nb_tx_queues) {
 793                 struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 794
 795                 q->ops->reset(q);
 796
 797                 /* reset head and tail pointers */
 798                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 799                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 800
 801                 /* enable TX queue */
 802                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 803                                         FM10K_TXDCTL_ENABLE | txdctl);
 804                 FM10K_WRITE_FLUSH(hw);
 805                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 806         } else
 807                 err = -1;
 808
 809         return err;
 810 }
 811
 812 static int
 813 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 814 {
 815         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 816
 817         PMD_INIT_FUNC_TRACE();
 818
 819         if (tx_queue_id < dev->data->nb_tx_queues) {
 820                 tx_queue_disable(hw, tx_queue_id);
 821                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 822                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 823         }
 824
 825         return 0;
 826 }
 827
 828 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 829 {
 830         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 831                 != FM10K_DGLORTMAP_NONE);
 832 }
 833
 834 static void
 835 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 836 {
 837         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 838         int status;
 839
 840         PMD_INIT_FUNC_TRACE();
 841
 842         /* Return if it didn't acquire valid glort range */
 843         if (!fm10k_glort_valid(hw))
 844                 return;
 845
 846         fm10k_mbx_lock(hw);
 847         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 848                                 FM10K_XCAST_MODE_PROMISC);
 849         fm10k_mbx_unlock(hw);
 850
 851         if (status != FM10K_SUCCESS)
 852                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 853 }
 854
 855 static void
 856 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 857 {
 858         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 859         uint8_t mode;
 860         int status;
 861
 862         PMD_INIT_FUNC_TRACE();
 863
 864         /* Return if it didn't acquire valid glort range */
 865         if (!fm10k_glort_valid(hw))
 866                 return;
 867
 868         if (dev->data->all_multicast == 1)
 869                 mode = FM10K_XCAST_MODE_ALLMULTI;
 870         else
 871                 mode = FM10K_XCAST_MODE_NONE;
 872
 873         fm10k_mbx_lock(hw);
 874         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 875                                 mode);
 876         fm10k_mbx_unlock(hw);
 877
 878         if (status != FM10K_SUCCESS)
 879                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 880 }
 881
 882 static void
 883 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 884 {
 885         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 886         int status;
 887
 888         PMD_INIT_FUNC_TRACE();
 889
 890         /* Return if it didn't acquire valid glort range */
 891         if (!fm10k_glort_valid(hw))
 892                 return;
 893
 894         /* If promiscuous mode is enabled, it doesn't make sense to enable
 895          * allmulticast and disable promiscuous since fm10k only can select
 896          * one of the modes.
 897          */
 898         if (dev->data->promiscuous) {
 899                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 900                         "needn't enable allmulticast");
 901                 return;
 902         }
 903
 904         fm10k_mbx_lock(hw);
 905         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 906                                 FM10K_XCAST_MODE_ALLMULTI);
 907         fm10k_mbx_unlock(hw);
 908
 909         if (status != FM10K_SUCCESS)
 910                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 911 }
 912
 913 static void
 914 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 915 {
 916         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 917         int status;
 918
 919         PMD_INIT_FUNC_TRACE();
 920
 921         /* Return if it didn't acquire valid glort range */
 922         if (!fm10k_glort_valid(hw))
 923                 return;
 924
 925         if (dev->data->promiscuous) {
 926                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
 927                         "since promisc mode is enabled");
 928                 return;
 929         }
 930
 931         fm10k_mbx_lock(hw);
 932         /* Change mode to unicast mode */
 933         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 934                                 FM10K_XCAST_MODE_NONE);
 935         fm10k_mbx_unlock(hw);
 936
 937         if (status != FM10K_SUCCESS)
 938                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
 939 }
 940
 941 static void
 942 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
 943 {
 944         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 945         uint32_t dglortdec, pool_len, rss_len, i;
 946         uint16_t nb_queue_pools;
 947         struct fm10k_macvlan_filter_info *macvlan;
 948
 949         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 950         nb_queue_pools = macvlan->nb_queue_pools;
 951         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
 952         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
 953         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
 954
 955         /* Establish only MAP 0 as valid */
 956         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
 957
 958         /* Configure VMDQ/RSS DGlort Decoder */
 959         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
 960
 961         /* Invalidate all other GLORT entries */
 962         for (i = 1; i < FM10K_DGLORT_COUNT; i++)
 963                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
 964                                 FM10K_DGLORTMAP_NONE);
 965 }
 966
 967 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
 968 static int
 969 fm10k_dev_start(struct rte_eth_dev *dev)
 970 {
 971         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 972         int i, diag;
 973
 974         PMD_INIT_FUNC_TRACE();
 975
 976         /* stop, init, then start the hw */
 977         diag = fm10k_stop_hw(hw);
 978         if (diag != FM10K_SUCCESS) {
 979                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
 980                 return -EIO;
 981         }
 982
 983         diag = fm10k_init_hw(hw);
 984         if (diag != FM10K_SUCCESS) {
 985                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
 986                 return -EIO;
 987         }
 988
 989         diag = fm10k_start_hw(hw);
 990         if (diag != FM10K_SUCCESS) {
 991                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
 992                 return -EIO;
 993         }
 994
 995         diag = fm10k_dev_tx_init(dev);
 996         if (diag) {
 997                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
 998                 return diag;
 999         }
1000
1001         diag = fm10k_dev_rx_init(dev);
1002         if (diag) {
1003                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1004                 return diag;
1005         }
1006
1007         if (hw->mac.type == fm10k_mac_pf)
1008                 fm10k_dev_dglort_map_configure(dev);
1009
1010         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1011                 struct fm10k_rx_queue *rxq;
1012                 rxq = dev->data->rx_queues[i];
1013
1014                 if (rxq->rx_deferred_start)
1015                         continue;
1016                 diag = fm10k_dev_rx_queue_start(dev, i);
1017                 if (diag != 0) {
1018                         int j;
1019                         for (j = 0; j < i; ++j)
1020                                 rx_queue_clean(dev->data->rx_queues[j]);
1021                         return diag;
1022                 }
1023         }
1024
1025         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1026                 struct fm10k_tx_queue *txq;
1027                 txq = dev->data->tx_queues[i];
1028
1029                 if (txq->tx_deferred_start)
1030                         continue;
1031                 diag = fm10k_dev_tx_queue_start(dev, i);
1032                 if (diag != 0) {
1033                         int j;
1034                         for (j = 0; j < i; ++j)
1035                                 tx_queue_clean(dev->data->tx_queues[j]);
1036                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1037                                 rx_queue_clean(dev->data->rx_queues[j]);
1038                         return diag;
1039                 }
1040         }
1041
1042         /* Update default vlan when not in VMDQ mode */
1043         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1044                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1045
1046         return 0;
1047 }
1048
1049 static void
1050 fm10k_dev_stop(struct rte_eth_dev *dev)
1051 {
1052         int i;
1053
1054         PMD_INIT_FUNC_TRACE();
1055
1056         if (dev->data->tx_queues)
1057                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1058                         fm10k_dev_tx_queue_stop(dev, i);
1059
1060         if (dev->data->rx_queues)
1061                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1062                         fm10k_dev_rx_queue_stop(dev, i);
1063 }
1064
1065 static void
1066 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1067 {
1068         int i;
1069
1070         PMD_INIT_FUNC_TRACE();
1071
1072         if (dev->data->tx_queues) {
1073                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1074                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1075
1076                         txq->ops->release_mbufs(txq);
1077                 }
1078         }
1079
1080         if (dev->data->rx_queues) {
1081                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1082                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1083         }
1084 }
1085
1086 static void
1087 fm10k_dev_close(struct rte_eth_dev *dev)
1088 {
1089         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1090         uint16_t nb_lport;
1091         struct fm10k_macvlan_filter_info *macvlan;
1092
1093         PMD_INIT_FUNC_TRACE();
1094
1095         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1096         nb_lport = macvlan->nb_queue_pools ? macvlan->nb_queue_pools : 1;
1097         fm10k_mbx_lock(hw);
1098         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1099                 nb_lport, false);
1100         fm10k_mbx_unlock(hw);
1101
1102         /* Stop mailbox service first */
1103         fm10k_close_mbx_service(hw);
1104         fm10k_dev_stop(dev);
1105         fm10k_dev_queue_release(dev);
1106         fm10k_stop_hw(hw);
1107 }
1108
1109 static int
1110 fm10k_link_update(struct rte_eth_dev *dev,
1111         __rte_unused int wait_to_complete)
1112 {
1113         PMD_INIT_FUNC_TRACE();
1114
1115         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
1116          * x8 PCIe interface. For now, we leave the speed undefined since there
1117          * is no 50Gbps Ethernet. */
1118         dev->data->dev_link.link_speed  = 0;
1119         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1120         dev->data->dev_link.link_status = 1;
1121
1122         return 0;
1123 }
1124
1125 static int
1126 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstats *xstats,
1127                  unsigned n)
1128 {
1129         struct fm10k_hw_stats *hw_stats =
1130                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1131         unsigned i, q, count = 0;
1132
1133         if (n < FM10K_NB_XSTATS)
1134                 return FM10K_NB_XSTATS;
1135
1136         /* Global stats */
1137         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1138                 snprintf(xstats[count].name, sizeof(xstats[count].name),
1139                          "%s", fm10k_hw_stats_strings[count].name);
1140                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1141                         fm10k_hw_stats_strings[count].offset);
1142                 count++;
1143         }
1144
1145         /* PF queue stats */
1146         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1147                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1148                         snprintf(xstats[count].name, sizeof(xstats[count].name),
1149                                  "rx_q%u_%s", q,
1150                                  fm10k_hw_stats_rx_q_strings[i].name);
1151                         xstats[count].value =
1152                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1153                                 fm10k_hw_stats_rx_q_strings[i].offset);
1154                         count++;
1155                 }
1156                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1157                         snprintf(xstats[count].name, sizeof(xstats[count].name),
1158                                  "tx_q%u_%s", q,
1159                                  fm10k_hw_stats_tx_q_strings[i].name);
1160                         xstats[count].value =
1161                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1162                                 fm10k_hw_stats_tx_q_strings[i].offset);
1163                         count++;
1164                 }
1165         }
1166
1167         return FM10K_NB_XSTATS;
1168 }
1169
1170 static void
1171 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1172 {
1173         uint64_t ipackets, opackets, ibytes, obytes;
1174         struct fm10k_hw *hw =
1175                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1176         struct fm10k_hw_stats *hw_stats =
1177                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1178         int i;
1179
1180         PMD_INIT_FUNC_TRACE();
1181
1182         fm10k_update_hw_stats(hw, hw_stats);
1183
1184         ipackets = opackets = ibytes = obytes = 0;
1185         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1186                 (i < hw->mac.max_queues); ++i) {
1187                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1188                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1189                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1190                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1191                 ipackets += stats->q_ipackets[i];
1192                 opackets += stats->q_opackets[i];
1193                 ibytes   += stats->q_ibytes[i];
1194                 obytes   += stats->q_obytes[i];
1195         }
1196         stats->ipackets = ipackets;
1197         stats->opackets = opackets;
1198         stats->ibytes = ibytes;
1199         stats->obytes = obytes;
1200 }
1201
1202 static void
1203 fm10k_stats_reset(struct rte_eth_dev *dev)
1204 {
1205         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1206         struct fm10k_hw_stats *hw_stats =
1207                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1208
1209         PMD_INIT_FUNC_TRACE();
1210
1211         memset(hw_stats, 0, sizeof(*hw_stats));
1212         fm10k_rebind_hw_stats(hw, hw_stats);
1213 }
1214
1215 static void
1216 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1217         struct rte_eth_dev_info *dev_info)
1218 {
1219         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1220
1221         PMD_INIT_FUNC_TRACE();
1222
1223         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1224         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1225         dev_info->max_rx_queues      = hw->mac.max_queues;
1226         dev_info->max_tx_queues      = hw->mac.max_queues;
1227         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1228         dev_info->max_hash_mac_addrs = 0;
1229         dev_info->max_vfs            = dev->pci_dev->max_vfs;
1230         dev_info->vmdq_pool_base     = 0;
1231         dev_info->vmdq_queue_base    = 0;
1232         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1233         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1234         dev_info->rx_offload_capa =
1235                 DEV_RX_OFFLOAD_VLAN_STRIP |
1236                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1237                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1238                 DEV_RX_OFFLOAD_TCP_CKSUM;
1239         dev_info->tx_offload_capa =
1240                 DEV_TX_OFFLOAD_VLAN_INSERT |
1241                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1242                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1243                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1244                 DEV_TX_OFFLOAD_TCP_TSO;
1245
1246         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1247         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1248
1249         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1250                 .rx_thresh = {
1251                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1252                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1253                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1254                 },
1255                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1256                 .rx_drop_en = 0,
1257         };
1258
1259         dev_info->default_txconf = (struct rte_eth_txconf) {
1260                 .tx_thresh = {
1261                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1262                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1263                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1264                 },
1265                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1266                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1267                 .txq_flags = FM10K_SIMPLE_TX_FLAG,
1268         };
1269
1270         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1271                 .nb_max = FM10K_MAX_RX_DESC,
1272                 .nb_min = FM10K_MIN_RX_DESC,
1273                 .nb_align = FM10K_MULT_RX_DESC,
1274         };
1275
1276         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1277                 .nb_max = FM10K_MAX_TX_DESC,
1278                 .nb_min = FM10K_MIN_TX_DESC,
1279                 .nb_align = FM10K_MULT_TX_DESC,
1280         };
1281 }
1282
1283 static int
1284 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1285 {
1286         s32 result;
1287         uint16_t mac_num = 0;
1288         uint32_t vid_idx, vid_bit, mac_index;
1289         struct fm10k_hw *hw;
1290         struct fm10k_macvlan_filter_info *macvlan;
1291         struct rte_eth_dev_data *data = dev->data;
1292
1293         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1294         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1295
1296         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1297                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1298                 return (-EINVAL);
1299         }
1300
1301         if (vlan_id > ETH_VLAN_ID_MAX) {
1302                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1303                 return (-EINVAL);
1304         }
1305
1306         vid_idx = FM10K_VFTA_IDX(vlan_id);
1307         vid_bit = FM10K_VFTA_BIT(vlan_id);
1308         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1309         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1310                 return 0;
1311         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1312         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1313                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1314                         "in the VLAN filter table");
1315                 return (-EINVAL);
1316         }
1317
1318         fm10k_mbx_lock(hw);
1319         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1320         fm10k_mbx_unlock(hw);
1321         if (result != FM10K_SUCCESS) {
1322                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1323                 return (-EIO);
1324         }
1325
1326         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1327                         (result == FM10K_SUCCESS); mac_index++) {
1328                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1329                         continue;
1330                 if (mac_num > macvlan->mac_num - 1) {
1331                         PMD_INIT_LOG(ERR, "MAC address number "
1332                                         "not match");
1333                         break;
1334                 }
1335                 fm10k_mbx_lock(hw);
1336                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1337                         data->mac_addrs[mac_index].addr_bytes,
1338                         vlan_id, on, 0);
1339                 fm10k_mbx_unlock(hw);
1340                 mac_num++;
1341         }
1342         if (result != FM10K_SUCCESS) {
1343                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1344                 return (-EIO);
1345         }
1346
1347         if (on) {
1348                 macvlan->vlan_num++;
1349                 macvlan->vfta[vid_idx] |= vid_bit;
1350         } else {
1351                 macvlan->vlan_num--;
1352                 macvlan->vfta[vid_idx] &= ~vid_bit;
1353         }
1354         return 0;
1355 }
1356
1357 static void
1358 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1359 {
1360         if (mask & ETH_VLAN_STRIP_MASK) {
1361                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1362                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1363                                         "always on in fm10k");
1364         }
1365
1366         if (mask & ETH_VLAN_EXTEND_MASK) {
1367                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1368                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1369                                         "supported in fm10k");
1370         }
1371
1372         if (mask & ETH_VLAN_FILTER_MASK) {
1373                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1374                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1375         }
1376 }
1377
1378 /* Add/Remove a MAC address, and update filters to main VSI */
1379 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1380                 const u8 *mac, bool add, uint32_t pool)
1381 {
1382         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1383         struct fm10k_macvlan_filter_info *macvlan;
1384         uint32_t i, j, k;
1385
1386         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1387
1388         if (pool != MAIN_VSI_POOL_NUMBER) {
1389                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1390                         "mac to pool %u", pool);
1391                 return;
1392         }
1393         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1394                 if (!macvlan->vfta[j])
1395                         continue;
1396                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1397                         if (!(macvlan->vfta[j] & (1 << k)))
1398                                 continue;
1399                         if (i + 1 > macvlan->vlan_num) {
1400                                 PMD_INIT_LOG(ERR, "vlan number not match");
1401                                 return;
1402                         }
1403                         fm10k_mbx_lock(hw);
1404                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1405                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1406                         fm10k_mbx_unlock(hw);
1407                         i++;
1408                 }
1409         }
1410 }
1411
1412 /* Add/Remove a MAC address, and update filters to VMDQ */
1413 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1414                 const u8 *mac, bool add, uint32_t pool)
1415 {
1416         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1417         struct fm10k_macvlan_filter_info *macvlan;
1418         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1419         uint32_t i;
1420
1421         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1422         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1423
1424         if (pool > macvlan->nb_queue_pools) {
1425                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1426                         " Max pool is %u",
1427                         pool, macvlan->nb_queue_pools);
1428                 return;
1429         }
1430         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1431                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1432                         continue;
1433                 fm10k_mbx_lock(hw);
1434                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1435                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1436                 fm10k_mbx_unlock(hw);
1437         }
1438 }
1439
1440 /* Add/Remove a MAC address, and update filters */
1441 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1442                 const u8 *mac, bool add, uint32_t pool)
1443 {
1444         struct fm10k_macvlan_filter_info *macvlan;
1445
1446         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1447
1448         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1449                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1450         else
1451                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1452
1453         if (add)
1454                 macvlan->mac_num++;
1455         else
1456                 macvlan->mac_num--;
1457 }
1458
1459 /* Add a MAC address, and update filters */
1460 static void
1461 fm10k_macaddr_add(struct rte_eth_dev *dev,
1462                 struct ether_addr *mac_addr,
1463                 uint32_t index,
1464                 uint32_t pool)
1465 {
1466         struct fm10k_macvlan_filter_info *macvlan;
1467
1468         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1469         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1470         macvlan->mac_vmdq_id[index] = pool;
1471 }
1472
1473 /* Remove a MAC address, and update filters */
1474 static void
1475 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1476 {
1477         struct rte_eth_dev_data *data = dev->data;
1478         struct fm10k_macvlan_filter_info *macvlan;
1479
1480         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1481         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1482                         FALSE, macvlan->mac_vmdq_id[index]);
1483         macvlan->mac_vmdq_id[index] = 0;
1484 }
1485
1486 static inline int
1487 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1488 {
1489         if ((request < min) || (request > max) || ((request % mult) != 0))
1490                 return -1;
1491         else
1492                 return 0;
1493 }
1494
1495 /*
1496  * Create a memzone for hardware descriptor rings. Malloc cannot be used since
1497  * the physical address is required. If the memzone is already created, then
1498  * this function returns a pointer to the existing memzone.
1499  */
1500 static inline const struct rte_memzone *
1501 allocate_hw_ring(const char *driver_name, const char *ring_name,
1502         uint8_t port_id, uint16_t queue_id, int socket_id,
1503         uint32_t size, uint32_t align)
1504 {
1505         char name[RTE_MEMZONE_NAMESIZE];
1506         const struct rte_memzone *mz;
1507
1508         snprintf(name, sizeof(name), "%s_%s_%d_%d_%d",
1509                  driver_name, ring_name, port_id, queue_id, socket_id);
1510
1511         /* return the memzone if it already exists */
1512         mz = rte_memzone_lookup(name);
1513         if (mz)
1514                 return mz;
1515
1516 #ifdef RTE_LIBRTE_XEN_DOM0
1517         return rte_memzone_reserve_bounded(name, size, socket_id, 0, align,
1518                                            RTE_PGSIZE_2M);
1519 #else
1520         return rte_memzone_reserve_aligned(name, size, socket_id, 0, align);
1521 #endif
1522 }
1523
1524 static inline int
1525 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1526 {
1527         if ((request < min) || (request > max) || ((div % request) != 0))
1528                 return -1;
1529         else
1530                 return 0;
1531 }
1532
1533 static inline int
1534 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1535 {
1536         uint16_t rx_free_thresh;
1537
1538         if (conf->rx_free_thresh == 0)
1539                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1540         else
1541                 rx_free_thresh = conf->rx_free_thresh;
1542
1543         /* make sure the requested threshold satisfies the constraints */
1544         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1545                         FM10K_RX_FREE_THRESH_MAX(q),
1546                         FM10K_RX_FREE_THRESH_DIV(q),
1547                         rx_free_thresh)) {
1548                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1549                         "less than or equal to %u, "
1550                         "greater than or equal to %u, "
1551                         "and a divisor of %u",
1552                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1553                         FM10K_RX_FREE_THRESH_MIN(q),
1554                         FM10K_RX_FREE_THRESH_DIV(q));
1555                 return (-EINVAL);
1556         }
1557
1558         q->alloc_thresh = rx_free_thresh;
1559         q->drop_en = conf->rx_drop_en;
1560         q->rx_deferred_start = conf->rx_deferred_start;
1561
1562         return 0;
1563 }
1564
1565 /*
1566  * Hardware requires specific alignment for Rx packet buffers. At
1567  * least one of the following two conditions must be satisfied.
1568  *  1. Address is 512B aligned
1569  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1570  *
1571  * As such, the driver may need to adjust the DMA address within the
1572  * buffer by up to 512B.
1573  *
1574  * return 1 if the element size is valid, otherwise return 0.
1575  */
1576 static int
1577 mempool_element_size_valid(struct rte_mempool *mp)
1578 {
1579         uint32_t min_size;
1580
1581         /* elt_size includes mbuf header and headroom */
1582         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1583                         RTE_PKTMBUF_HEADROOM;
1584
1585         /* account for up to 512B of alignment */
1586         min_size -= FM10K_RX_DATABUF_ALIGN;
1587
1588         /* sanity check for overflow */
1589         if (min_size > mp->elt_size)
1590                 return 0;
1591
1592         /* size is valid */
1593         return 1;
1594 }
1595
1596 static int
1597 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1598         uint16_t nb_desc, unsigned int socket_id,
1599         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1600 {
1601         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1602         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
1603         struct fm10k_rx_queue *q;
1604         const struct rte_memzone *mz;
1605
1606         PMD_INIT_FUNC_TRACE();
1607
1608         /* make sure the mempool element size can account for alignment. */
1609         if (!mempool_element_size_valid(mp)) {
1610                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1611                 return (-EINVAL);
1612         }
1613
1614         /* make sure a valid number of descriptors have been requested */
1615         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1616                                 FM10K_MULT_RX_DESC, nb_desc)) {
1617                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1618                         "less than or equal to %"PRIu32", "
1619                         "greater than or equal to %u, "
1620                         "and a multiple of %u",
1621                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1622                         FM10K_MULT_RX_DESC);
1623                 return (-EINVAL);
1624         }
1625
1626         /*
1627          * if this queue existed already, free the associated memory. The
1628          * queue cannot be reused in case we need to allocate memory on
1629          * different socket than was previously used.
1630          */
1631         if (dev->data->rx_queues[queue_id] != NULL) {
1632                 rx_queue_free(dev->data->rx_queues[queue_id]);
1633                 dev->data->rx_queues[queue_id] = NULL;
1634         }
1635
1636         /* allocate memory for the queue structure */
1637         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1638                                 socket_id);
1639         if (q == NULL) {
1640                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1641                 return (-ENOMEM);
1642         }
1643
1644         /* setup queue */
1645         q->mp = mp;
1646         q->nb_desc = nb_desc;
1647         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1648         q->port_id = dev->data->port_id;
1649         q->queue_id = queue_id;
1650         q->tail_ptr = (volatile uint32_t *)
1651                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1652         if (handle_rxconf(q, conf))
1653                 return (-EINVAL);
1654
1655         /* allocate memory for the software ring */
1656         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1657                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1658                         RTE_CACHE_LINE_SIZE, socket_id);
1659         if (q->sw_ring == NULL) {
1660                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1661                 rte_free(q);
1662                 return (-ENOMEM);
1663         }
1664
1665         /*
1666          * allocate memory for the hardware descriptor ring. A memzone large
1667          * enough to hold the maximum ring size is requested to allow for
1668          * resizing in later calls to the queue setup function.
1669          */
1670         mz = allocate_hw_ring(dev->driver->pci_drv.name, "rx_ring",
1671                                 dev->data->port_id, queue_id, socket_id,
1672                                 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC);
1673         if (mz == NULL) {
1674                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1675                 rte_free(q->sw_ring);
1676                 rte_free(q);
1677                 return (-ENOMEM);
1678         }
1679         q->hw_ring = mz->addr;
1680 #ifdef RTE_LIBRTE_XEN_DOM0
1681         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1682 #else
1683         q->hw_ring_phys_addr = mz->phys_addr;
1684 #endif
1685
1686         /* Check if number of descs satisfied Vector requirement */
1687         if (!rte_is_power_of_2(nb_desc)) {
1688                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1689                                     "preconditions - canceling the feature for "
1690                                     "the whole port[%d]",
1691                              q->queue_id, q->port_id);
1692                 dev_info->rx_vec_allowed = false;
1693         } else
1694                 fm10k_rxq_vec_setup(q);
1695
1696         dev->data->rx_queues[queue_id] = q;
1697         return 0;
1698 }
1699
1700 static void
1701 fm10k_rx_queue_release(void *queue)
1702 {
1703         PMD_INIT_FUNC_TRACE();
1704
1705         rx_queue_free(queue);
1706 }
1707
1708 static inline int
1709 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1710 {
1711         uint16_t tx_free_thresh;
1712         uint16_t tx_rs_thresh;
1713
1714         /* constraint MACROs require that tx_free_thresh is configured
1715          * before tx_rs_thresh */
1716         if (conf->tx_free_thresh == 0)
1717                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1718         else
1719                 tx_free_thresh = conf->tx_free_thresh;
1720
1721         /* make sure the requested threshold satisfies the constraints */
1722         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1723                         FM10K_TX_FREE_THRESH_MAX(q),
1724                         FM10K_TX_FREE_THRESH_DIV(q),
1725                         tx_free_thresh)) {
1726                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1727                         "less than or equal to %u, "
1728                         "greater than or equal to %u, "
1729                         "and a divisor of %u",
1730                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1731                         FM10K_TX_FREE_THRESH_MIN(q),
1732                         FM10K_TX_FREE_THRESH_DIV(q));
1733                 return (-EINVAL);
1734         }
1735
1736         q->free_thresh = tx_free_thresh;
1737
1738         if (conf->tx_rs_thresh == 0)
1739                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1740         else
1741                 tx_rs_thresh = conf->tx_rs_thresh;
1742
1743         q->tx_deferred_start = conf->tx_deferred_start;
1744
1745         /* make sure the requested threshold satisfies the constraints */
1746         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1747                         FM10K_TX_RS_THRESH_MAX(q),
1748                         FM10K_TX_RS_THRESH_DIV(q),
1749                         tx_rs_thresh)) {
1750                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1751                         "less than or equal to %u, "
1752                         "greater than or equal to %u, "
1753                         "and a divisor of %u",
1754                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1755                         FM10K_TX_RS_THRESH_MIN(q),
1756                         FM10K_TX_RS_THRESH_DIV(q));
1757                 return (-EINVAL);
1758         }
1759
1760         q->rs_thresh = tx_rs_thresh;
1761
1762         return 0;
1763 }
1764
1765 static int
1766 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1767         uint16_t nb_desc, unsigned int socket_id,
1768         const struct rte_eth_txconf *conf)
1769 {
1770         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1771         struct fm10k_tx_queue *q;
1772         const struct rte_memzone *mz;
1773
1774         PMD_INIT_FUNC_TRACE();
1775
1776         /* make sure a valid number of descriptors have been requested */
1777         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1778                                 FM10K_MULT_TX_DESC, nb_desc)) {
1779                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1780                         "less than or equal to %"PRIu32", "
1781                         "greater than or equal to %u, "
1782                         "and a multiple of %u",
1783                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1784                         FM10K_MULT_TX_DESC);
1785                 return (-EINVAL);
1786         }
1787
1788         /*
1789          * if this queue existed already, free the associated memory. The
1790          * queue cannot be reused in case we need to allocate memory on
1791          * different socket than was previously used.
1792          */
1793         if (dev->data->tx_queues[queue_id] != NULL) {
1794                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
1795
1796                 txq->ops->release_mbufs(txq);
1797                 dev->data->tx_queues[queue_id] = NULL;
1798         }
1799
1800         /* allocate memory for the queue structure */
1801         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1802                                 socket_id);
1803         if (q == NULL) {
1804                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1805                 return (-ENOMEM);
1806         }
1807
1808         /* setup queue */
1809         q->nb_desc = nb_desc;
1810         q->port_id = dev->data->port_id;
1811         q->queue_id = queue_id;
1812         q->txq_flags = conf->txq_flags;
1813         q->ops = &def_txq_ops;
1814         q->tail_ptr = (volatile uint32_t *)
1815                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1816         if (handle_txconf(q, conf))
1817                 return (-EINVAL);
1818
1819         /* allocate memory for the software ring */
1820         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1821                                         nb_desc * sizeof(struct rte_mbuf *),
1822                                         RTE_CACHE_LINE_SIZE, socket_id);
1823         if (q->sw_ring == NULL) {
1824                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1825                 rte_free(q);
1826                 return (-ENOMEM);
1827         }
1828
1829         /*
1830          * allocate memory for the hardware descriptor ring. A memzone large
1831          * enough to hold the maximum ring size is requested to allow for
1832          * resizing in later calls to the queue setup function.
1833          */
1834         mz = allocate_hw_ring(dev->driver->pci_drv.name, "tx_ring",
1835                                 dev->data->port_id, queue_id, socket_id,
1836                                 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC);
1837         if (mz == NULL) {
1838                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1839                 rte_free(q->sw_ring);
1840                 rte_free(q);
1841                 return (-ENOMEM);
1842         }
1843         q->hw_ring = mz->addr;
1844 #ifdef RTE_LIBRTE_XEN_DOM0
1845         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1846 #else
1847         q->hw_ring_phys_addr = mz->phys_addr;
1848 #endif
1849
1850         /*
1851          * allocate memory for the RS bit tracker. Enough slots to hold the
1852          * descriptor index for each RS bit needing to be set are required.
1853          */
1854         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1855                                 ((nb_desc + 1) / q->rs_thresh) *
1856                                 sizeof(uint16_t),
1857                                 RTE_CACHE_LINE_SIZE, socket_id);
1858         if (q->rs_tracker.list == NULL) {
1859                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1860                 rte_free(q->sw_ring);
1861                 rte_free(q);
1862                 return (-ENOMEM);
1863         }
1864
1865         dev->data->tx_queues[queue_id] = q;
1866         return 0;
1867 }
1868
1869 static void
1870 fm10k_tx_queue_release(void *queue)
1871 {
1872         struct fm10k_tx_queue *q = queue;
1873         PMD_INIT_FUNC_TRACE();
1874
1875         q->ops->release_mbufs(q);
1876 }
1877
1878 static int
1879 fm10k_reta_update(struct rte_eth_dev *dev,
1880                         struct rte_eth_rss_reta_entry64 *reta_conf,
1881                         uint16_t reta_size)
1882 {
1883         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1884         uint16_t i, j, idx, shift;
1885         uint8_t mask;
1886         uint32_t reta;
1887
1888         PMD_INIT_FUNC_TRACE();
1889
1890         if (reta_size > FM10K_MAX_RSS_INDICES) {
1891                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1892                         "(%d) doesn't match the number hardware can supported "
1893                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1894                 return -EINVAL;
1895         }
1896
1897         /*
1898          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1899          * 128-entries in 32 registers
1900          */
1901         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1902                 idx = i / RTE_RETA_GROUP_SIZE;
1903                 shift = i % RTE_RETA_GROUP_SIZE;
1904                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1905                                 BIT_MASK_PER_UINT32);
1906                 if (mask == 0)
1907                         continue;
1908
1909                 reta = 0;
1910                 if (mask != BIT_MASK_PER_UINT32)
1911                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1912
1913                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1914                         if (mask & (0x1 << j)) {
1915                                 if (mask != 0xF)
1916                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1917                                 reta |= reta_conf[idx].reta[shift + j] <<
1918                                                 (CHAR_BIT * j);
1919                         }
1920                 }
1921                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1922         }
1923
1924         return 0;
1925 }
1926
1927 static int
1928 fm10k_reta_query(struct rte_eth_dev *dev,
1929                         struct rte_eth_rss_reta_entry64 *reta_conf,
1930                         uint16_t reta_size)
1931 {
1932         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1933         uint16_t i, j, idx, shift;
1934         uint8_t mask;
1935         uint32_t reta;
1936
1937         PMD_INIT_FUNC_TRACE();
1938
1939         if (reta_size < FM10K_MAX_RSS_INDICES) {
1940                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1941                         "(%d) doesn't match the number hardware can supported "
1942                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1943                 return -EINVAL;
1944         }
1945
1946         /*
1947          * Read Redirection Table RETA[n], n=0..31. The redirection table has
1948          * 128-entries in 32 registers
1949          */
1950         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1951                 idx = i / RTE_RETA_GROUP_SIZE;
1952                 shift = i % RTE_RETA_GROUP_SIZE;
1953                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1954                                 BIT_MASK_PER_UINT32);
1955                 if (mask == 0)
1956                         continue;
1957
1958                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1959                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1960                         if (mask & (0x1 << j))
1961                                 reta_conf[idx].reta[shift + j] = ((reta >>
1962                                         CHAR_BIT * j) & UINT8_MAX);
1963                 }
1964         }
1965
1966         return 0;
1967 }
1968
1969 static int
1970 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1971         struct rte_eth_rss_conf *rss_conf)
1972 {
1973         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1974         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1975         uint32_t mrqc;
1976         uint64_t hf = rss_conf->rss_hf;
1977         int i;
1978
1979         PMD_INIT_FUNC_TRACE();
1980
1981         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1982                 FM10K_RSSRK_ENTRIES_PER_REG)
1983                 return -EINVAL;
1984
1985         if (hf == 0)
1986                 return -EINVAL;
1987
1988         mrqc = 0;
1989         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
1990         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
1991         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
1992         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
1993         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
1994         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
1995         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
1996         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
1997         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
1998
1999         /* If the mapping doesn't fit any supported, return */
2000         if (mrqc == 0)
2001                 return -EINVAL;
2002
2003         if (key != NULL)
2004                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2005                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2006
2007         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2008
2009         return 0;
2010 }
2011
2012 static int
2013 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2014         struct rte_eth_rss_conf *rss_conf)
2015 {
2016         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2017         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2018         uint32_t mrqc;
2019         uint64_t hf;
2020         int i;
2021
2022         PMD_INIT_FUNC_TRACE();
2023
2024         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2025                                 FM10K_RSSRK_ENTRIES_PER_REG)
2026                 return -EINVAL;
2027
2028         if (key != NULL)
2029                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2030                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2031
2032         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2033         hf = 0;
2034         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2035         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2036         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2037         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2038         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2039         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2040         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2041         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2042         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2043
2044         rss_conf->rss_hf = hf;
2045
2046         return 0;
2047 }
2048
2049 static void
2050 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2051 {
2052         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2053         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2054
2055         /* Bind all local non-queue interrupt to vector 0 */
2056         int_map |= 0;
2057
2058         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
2059         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
2060         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
2061         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
2062         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
2063         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
2064
2065         /* Enable misc causes */
2066         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2067                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2068                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2069                                 FM10K_EIMR_ENABLE(MAILBOX) |
2070                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2071                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2072                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2073                                 FM10K_EIMR_ENABLE(VFLR));
2074
2075         /* Enable ITR 0 */
2076         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2077                                         FM10K_ITR_MASK_CLEAR);
2078         FM10K_WRITE_FLUSH(hw);
2079 }
2080
2081 static void
2082 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2083 {
2084         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2085         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2086
2087         int_map |= 0;
2088
2089         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
2090         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
2091         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
2092         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
2093         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
2094         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
2095
2096         /* Disable misc causes */
2097         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2098                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2099                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2100                                 FM10K_EIMR_DISABLE(MAILBOX) |
2101                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2102                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2103                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2104                                 FM10K_EIMR_DISABLE(VFLR));
2105
2106         /* Disable ITR 0 */
2107         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2108         FM10K_WRITE_FLUSH(hw);
2109 }
2110
2111 static void
2112 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2113 {
2114         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2115         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2116
2117         /* Bind all local non-queue interrupt to vector 0 */
2118         int_map |= 0;
2119
2120         /* Only INT 0 available, other 15 are reserved. */
2121         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2122
2123         /* Enable ITR 0 */
2124         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2125                                         FM10K_ITR_MASK_CLEAR);
2126         FM10K_WRITE_FLUSH(hw);
2127 }
2128
2129 static void
2130 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2131 {
2132         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2133         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2134
2135         int_map |= 0;
2136
2137         /* Only INT 0 available, other 15 are reserved. */
2138         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2139
2140         /* Disable ITR 0 */
2141         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2142         FM10K_WRITE_FLUSH(hw);
2143 }
2144
2145 static int
2146 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2147 {
2148         struct fm10k_fault fault;
2149         int err;
2150         const char *estr = "Unknown error";
2151
2152         /* Process PCA fault */
2153         if (eicr & FM10K_EICR_PCA_FAULT) {
2154                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2155                 if (err)
2156                         goto error;
2157                 switch (fault.type) {
2158                 case PCA_NO_FAULT:
2159                         estr = "PCA_NO_FAULT"; break;
2160                 case PCA_UNMAPPED_ADDR:
2161                         estr = "PCA_UNMAPPED_ADDR"; break;
2162                 case PCA_BAD_QACCESS_PF:
2163                         estr = "PCA_BAD_QACCESS_PF"; break;
2164                 case PCA_BAD_QACCESS_VF:
2165                         estr = "PCA_BAD_QACCESS_VF"; break;
2166                 case PCA_MALICIOUS_REQ:
2167                         estr = "PCA_MALICIOUS_REQ"; break;
2168                 case PCA_POISONED_TLP:
2169                         estr = "PCA_POISONED_TLP"; break;
2170                 case PCA_TLP_ABORT:
2171                         estr = "PCA_TLP_ABORT"; break;
2172                 default:
2173                         goto error;
2174                 }
2175                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2176                         estr, fault.func ? "VF" : "PF", fault.func,
2177                         fault.address, fault.specinfo);
2178         }
2179
2180         /* Process THI fault */
2181         if (eicr & FM10K_EICR_THI_FAULT) {
2182                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2183                 if (err)
2184                         goto error;
2185                 switch (fault.type) {
2186                 case THI_NO_FAULT:
2187                         estr = "THI_NO_FAULT"; break;
2188                 case THI_MAL_DIS_Q_FAULT:
2189                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2190                 default:
2191                         goto error;
2192                 }
2193                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2194                         estr, fault.func ? "VF" : "PF", fault.func,
2195                         fault.address, fault.specinfo);
2196         }
2197
2198         /* Process FUM fault */
2199         if (eicr & FM10K_EICR_FUM_FAULT) {
2200                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2201                 if (err)
2202                         goto error;
2203                 switch (fault.type) {
2204                 case FUM_NO_FAULT:
2205                         estr = "FUM_NO_FAULT"; break;
2206                 case FUM_UNMAPPED_ADDR:
2207                         estr = "FUM_UNMAPPED_ADDR"; break;
2208                 case FUM_POISONED_TLP:
2209                         estr = "FUM_POISONED_TLP"; break;
2210                 case FUM_BAD_VF_QACCESS:
2211                         estr = "FUM_BAD_VF_QACCESS"; break;
2212                 case FUM_ADD_DECODE_ERR:
2213                         estr = "FUM_ADD_DECODE_ERR"; break;
2214                 case FUM_RO_ERROR:
2215                         estr = "FUM_RO_ERROR"; break;
2216                 case FUM_QPRC_CRC_ERROR:
2217                         estr = "FUM_QPRC_CRC_ERROR"; break;
2218                 case FUM_CSR_TIMEOUT:
2219                         estr = "FUM_CSR_TIMEOUT"; break;
2220                 case FUM_INVALID_TYPE:
2221                         estr = "FUM_INVALID_TYPE"; break;
2222                 case FUM_INVALID_LENGTH:
2223                         estr = "FUM_INVALID_LENGTH"; break;
2224                 case FUM_INVALID_BE:
2225                         estr = "FUM_INVALID_BE"; break;
2226                 case FUM_INVALID_ALIGN:
2227                         estr = "FUM_INVALID_ALIGN"; break;
2228                 default:
2229                         goto error;
2230                 }
2231                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2232                         estr, fault.func ? "VF" : "PF", fault.func,
2233                         fault.address, fault.specinfo);
2234         }
2235
2236         return 0;
2237 error:
2238         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2239         return err;
2240 }
2241
2242 /**
2243  * PF interrupt handler triggered by NIC for handling specific interrupt.
2244  *
2245  * @param handle
2246  *  Pointer to interrupt handle.
2247  * @param param
2248  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2249  *
2250  * @return
2251  *  void
2252  */
2253 static void
2254 fm10k_dev_interrupt_handler_pf(
2255                         __rte_unused struct rte_intr_handle *handle,
2256                         void *param)
2257 {
2258         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2259         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2260         uint32_t cause, status;
2261
2262         if (hw->mac.type != fm10k_mac_pf)
2263                 return;
2264
2265         cause = FM10K_READ_REG(hw, FM10K_EICR);
2266
2267         /* Handle PCI fault cases */
2268         if (cause & FM10K_EICR_FAULT_MASK) {
2269                 PMD_INIT_LOG(ERR, "INT: find fault!");
2270                 fm10k_dev_handle_fault(hw, cause);
2271         }
2272
2273         /* Handle switch up/down */
2274         if (cause & FM10K_EICR_SWITCHNOTREADY)
2275                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2276
2277         if (cause & FM10K_EICR_SWITCHREADY)
2278                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2279
2280         /* Handle mailbox message */
2281         fm10k_mbx_lock(hw);
2282         hw->mbx.ops.process(hw, &hw->mbx);
2283         fm10k_mbx_unlock(hw);
2284
2285         /* Handle SRAM error */
2286         if (cause & FM10K_EICR_SRAMERROR) {
2287                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2288
2289                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2290                 /* Write to clear pending bits */
2291                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2292
2293                 /* Todo: print out error message after shared code  updates */
2294         }
2295
2296         /* Clear these 3 events if having any */
2297         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2298                  FM10K_EICR_SWITCHREADY;
2299         if (cause)
2300                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2301
2302         /* Re-enable interrupt from device side */
2303         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2304                                         FM10K_ITR_MASK_CLEAR);
2305         /* Re-enable interrupt from host side */
2306         rte_intr_enable(&(dev->pci_dev->intr_handle));
2307 }
2308
2309 /**
2310  * VF interrupt handler triggered by NIC for handling specific interrupt.
2311  *
2312  * @param handle
2313  *  Pointer to interrupt handle.
2314  * @param param
2315  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2316  *
2317  * @return
2318  *  void
2319  */
2320 static void
2321 fm10k_dev_interrupt_handler_vf(
2322                         __rte_unused struct rte_intr_handle *handle,
2323                         void *param)
2324 {
2325         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2326         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2327
2328         if (hw->mac.type != fm10k_mac_vf)
2329                 return;
2330
2331         /* Handle mailbox message if lock is acquired */
2332         fm10k_mbx_lock(hw);
2333         hw->mbx.ops.process(hw, &hw->mbx);
2334         fm10k_mbx_unlock(hw);
2335
2336         /* Re-enable interrupt from device side */
2337         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2338                                         FM10K_ITR_MASK_CLEAR);
2339         /* Re-enable interrupt from host side */
2340         rte_intr_enable(&(dev->pci_dev->intr_handle));
2341 }
2342
2343 /* Mailbox message handler in VF */
2344 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2345         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2346         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2347         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2348         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2349 };
2350
2351 /* Mailbox message handler in PF */
2352 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
2353         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
2354         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
2355         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
2356         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
2357         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
2358         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
2359         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2360 };
2361
2362 static int
2363 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2364 {
2365         int err;
2366
2367         /* Initialize mailbox lock */
2368         fm10k_mbx_initlock(hw);
2369
2370         /* Replace default message handler with new ones */
2371         if (hw->mac.type == fm10k_mac_pf)
2372                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
2373         else
2374                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2375
2376         if (err) {
2377                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2378                                 err);
2379                 return err;
2380         }
2381         /* Connect to SM for PF device or PF for VF device */
2382         return hw->mbx.ops.connect(hw, &hw->mbx);
2383 }
2384
2385 static void
2386 fm10k_close_mbx_service(struct fm10k_hw *hw)
2387 {
2388         /* Disconnect from SM for PF device or PF for VF device */
2389         hw->mbx.ops.disconnect(hw, &hw->mbx);
2390 }
2391
2392 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2393         .dev_configure          = fm10k_dev_configure,
2394         .dev_start              = fm10k_dev_start,
2395         .dev_stop               = fm10k_dev_stop,
2396         .dev_close              = fm10k_dev_close,
2397         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2398         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2399         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2400         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2401         .stats_get              = fm10k_stats_get,
2402         .xstats_get             = fm10k_xstats_get,
2403         .stats_reset            = fm10k_stats_reset,
2404         .xstats_reset           = fm10k_stats_reset,
2405         .link_update            = fm10k_link_update,
2406         .dev_infos_get          = fm10k_dev_infos_get,
2407         .vlan_filter_set        = fm10k_vlan_filter_set,
2408         .vlan_offload_set       = fm10k_vlan_offload_set,
2409         .mac_addr_add           = fm10k_macaddr_add,
2410         .mac_addr_remove        = fm10k_macaddr_remove,
2411         .rx_queue_start         = fm10k_dev_rx_queue_start,
2412         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2413         .tx_queue_start         = fm10k_dev_tx_queue_start,
2414         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2415         .rx_queue_setup         = fm10k_rx_queue_setup,
2416         .rx_queue_release       = fm10k_rx_queue_release,
2417         .tx_queue_setup         = fm10k_tx_queue_setup,
2418         .tx_queue_release       = fm10k_tx_queue_release,
2419         .reta_update            = fm10k_reta_update,
2420         .reta_query             = fm10k_reta_query,
2421         .rss_hash_update        = fm10k_rss_hash_update,
2422         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2423 };
2424
2425 static void __attribute__((cold))
2426 fm10k_set_tx_function(struct rte_eth_dev *dev)
2427 {
2428         struct fm10k_tx_queue *txq;
2429         int i;
2430         int use_sse = 1;
2431
2432         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2433                 txq = dev->data->tx_queues[i];
2434                 if ((txq->txq_flags & FM10K_SIMPLE_TX_FLAG) !=
2435                         FM10K_SIMPLE_TX_FLAG) {
2436                         use_sse = 0;
2437                         break;
2438                 }
2439         }
2440
2441         if (use_sse) {
2442                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2443                         txq = dev->data->tx_queues[i];
2444                         fm10k_txq_vec_setup(txq);
2445                 }
2446                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2447         } else
2448                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2449 }
2450
2451 static void __attribute__((cold))
2452 fm10k_set_rx_function(struct rte_eth_dev *dev)
2453 {
2454         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2455         uint16_t i, rx_using_sse;
2456
2457         /* In order to allow Vector Rx there are a few configuration
2458          * conditions to be met.
2459          */
2460         if (!fm10k_rx_vec_condition_check(dev) && dev_info->rx_vec_allowed) {
2461                 if (dev->data->scattered_rx)
2462                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2463                 else
2464                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2465         } else if (dev->data->scattered_rx)
2466                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2467
2468         rx_using_sse =
2469                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2470                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2471
2472         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2473                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2474
2475                 rxq->rx_using_sse = rx_using_sse;
2476         }
2477 }
2478
2479 static void
2480 fm10k_params_init(struct rte_eth_dev *dev)
2481 {
2482         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2483         struct fm10k_dev_info *info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2484
2485         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2486          * there is no way to get link status without reading BAR4.  Until this
2487          * works, assume we have maximum bandwidth.
2488          * @todo - fix bus info
2489          */
2490         hw->bus_caps.speed = fm10k_bus_speed_8000;
2491         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2492         hw->bus_caps.payload = fm10k_bus_payload_512;
2493         hw->bus.speed = fm10k_bus_speed_8000;
2494         hw->bus.width = fm10k_bus_width_pcie_x8;
2495         hw->bus.payload = fm10k_bus_payload_256;
2496
2497         info->rx_vec_allowed = true;
2498 }
2499
2500 static int
2501 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2502 {
2503         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2504         int diag;
2505         struct fm10k_macvlan_filter_info *macvlan;
2506
2507         PMD_INIT_FUNC_TRACE();
2508
2509         dev->dev_ops = &fm10k_eth_dev_ops;
2510         dev->rx_pkt_burst = &fm10k_recv_pkts;
2511         dev->tx_pkt_burst = &fm10k_xmit_pkts;
2512
2513         /* only initialize in the primary process */
2514         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2515                 return 0;
2516
2517         rte_eth_copy_pci_info(dev, dev->pci_dev);
2518
2519         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2520         memset(macvlan, 0, sizeof(*macvlan));
2521         /* Vendor and Device ID need to be set before init of shared code */
2522         memset(hw, 0, sizeof(*hw));
2523         hw->device_id = dev->pci_dev->id.device_id;
2524         hw->vendor_id = dev->pci_dev->id.vendor_id;
2525         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
2526         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
2527         hw->revision_id = 0;
2528         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2529         if (hw->hw_addr == NULL) {
2530                 PMD_INIT_LOG(ERR, "Bad mem resource."
2531                         " Try to blacklist unused devices.");
2532                 return -EIO;
2533         }
2534
2535         /* Store fm10k_adapter pointer */
2536         hw->back = dev->data->dev_private;
2537
2538         /* Initialize the shared code */
2539         diag = fm10k_init_shared_code(hw);
2540         if (diag != FM10K_SUCCESS) {
2541                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2542                 return -EIO;
2543         }
2544
2545         /* Initialize parameters */
2546         fm10k_params_init(dev);
2547
2548         /* Initialize the hw */
2549         diag = fm10k_init_hw(hw);
2550         if (diag != FM10K_SUCCESS) {
2551                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2552                 return -EIO;
2553         }
2554
2555         /* Initialize MAC address(es) */
2556         dev->data->mac_addrs = rte_zmalloc("fm10k",
2557                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2558         if (dev->data->mac_addrs == NULL) {
2559                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2560                 return -ENOMEM;
2561         }
2562
2563         diag = fm10k_read_mac_addr(hw);
2564
2565         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2566                         &dev->data->mac_addrs[0]);
2567
2568         if (diag != FM10K_SUCCESS ||
2569                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2570
2571                 /* Generate a random addr */
2572                 eth_random_addr(hw->mac.addr);
2573                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2574                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2575                 &dev->data->mac_addrs[0]);
2576         }
2577
2578         /* Reset the hw statistics */
2579         fm10k_stats_reset(dev);
2580
2581         /* Reset the hw */
2582         diag = fm10k_reset_hw(hw);
2583         if (diag != FM10K_SUCCESS) {
2584                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2585                 return -EIO;
2586         }
2587
2588         /* Setup mailbox service */
2589         diag = fm10k_setup_mbx_service(hw);
2590         if (diag != FM10K_SUCCESS) {
2591                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2592                 return -EIO;
2593         }
2594
2595         /*PF/VF has different interrupt handling mechanism */
2596         if (hw->mac.type == fm10k_mac_pf) {
2597                 /* register callback func to eal lib */
2598                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2599                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2600
2601                 /* enable MISC interrupt */
2602                 fm10k_dev_enable_intr_pf(dev);
2603         } else { /* VF */
2604                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2605                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2606
2607                 fm10k_dev_enable_intr_vf(dev);
2608         }
2609
2610         /* Enable uio intr after callback registered */
2611         rte_intr_enable(&(dev->pci_dev->intr_handle));
2612
2613         hw->mac.ops.update_int_moderator(hw);
2614
2615         /* Make sure Switch Manager is ready before going forward. */
2616         if (hw->mac.type == fm10k_mac_pf) {
2617                 int switch_ready = 0;
2618                 int i;
2619
2620                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2621                         fm10k_mbx_lock(hw);
2622                         hw->mac.ops.get_host_state(hw, &switch_ready);
2623                         fm10k_mbx_unlock(hw);
2624                         if (switch_ready)
2625                                 break;
2626                         /* Delay some time to acquire async LPORT_MAP info. */
2627                         rte_delay_us(WAIT_SWITCH_MSG_US);
2628                 }
2629
2630                 if (switch_ready == 0) {
2631                         PMD_INIT_LOG(ERR, "switch is not ready");
2632                         return -1;
2633                 }
2634         }
2635
2636         /*
2637          * Below function will trigger operations on mailbox, acquire lock to
2638          * avoid race condition from interrupt handler. Operations on mailbox
2639          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2640          * will handle and generate an interrupt to our side. Then,  FIFO in
2641          * mailbox will be touched.
2642          */
2643         fm10k_mbx_lock(hw);
2644         /* Enable port first */
2645         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, 1, 1);
2646
2647         /* Set unicast mode by default. App can change to other mode in other
2648          * API func.
2649          */
2650         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2651                                         FM10K_XCAST_MODE_NONE);
2652
2653         fm10k_mbx_unlock(hw);
2654
2655         /* Add default mac address */
2656         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2657                 MAIN_VSI_POOL_NUMBER);
2658
2659         return 0;
2660 }
2661
2662 static int
2663 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
2664 {
2665         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2666
2667         PMD_INIT_FUNC_TRACE();
2668
2669         /* only uninitialize in the primary process */
2670         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2671                 return 0;
2672
2673         /* safe to close dev here */
2674         fm10k_dev_close(dev);
2675
2676         dev->dev_ops = NULL;
2677         dev->rx_pkt_burst = NULL;
2678         dev->tx_pkt_burst = NULL;
2679
2680         /* disable uio/vfio intr */
2681         rte_intr_disable(&(dev->pci_dev->intr_handle));
2682
2683         /*PF/VF has different interrupt handling mechanism */
2684         if (hw->mac.type == fm10k_mac_pf) {
2685                 /* disable interrupt */
2686                 fm10k_dev_disable_intr_pf(dev);
2687
2688                 /* unregister callback func to eal lib */
2689                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2690                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2691         } else {
2692                 /* disable interrupt */
2693                 fm10k_dev_disable_intr_vf(dev);
2694
2695                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2696                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2697         }
2698
2699         /* free mac memory */
2700         if (dev->data->mac_addrs) {
2701                 rte_free(dev->data->mac_addrs);
2702                 dev->data->mac_addrs = NULL;
2703         }
2704
2705         memset(hw, 0, sizeof(*hw));
2706
2707         return 0;
2708 }
2709
2710 /*
2711  * The set of PCI devices this driver supports. This driver will enable both PF
2712  * and SRIOV-VF devices.
2713  */
2714 static const struct rte_pci_id pci_id_fm10k_map[] = {
2715 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2716 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2717 #include "rte_pci_dev_ids.h"
2718         { .vendor_id = 0, /* sentinel */ },
2719 };
2720
2721 static struct eth_driver rte_pmd_fm10k = {
2722         .pci_drv = {
2723                 .name = "rte_pmd_fm10k",
2724                 .id_table = pci_id_fm10k_map,
2725                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
2726         },
2727         .eth_dev_init = eth_fm10k_dev_init,
2728         .eth_dev_uninit = eth_fm10k_dev_uninit,
2729         .dev_private_size = sizeof(struct fm10k_adapter),
2730 };
2731
2732 /*
2733  * Driver initialization routine.
2734  * Invoked once at EAL init time.
2735  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2736  */
2737 static int
2738 rte_pmd_fm10k_init(__rte_unused const char *name,
2739         __rte_unused const char *params)
2740 {
2741         PMD_INIT_FUNC_TRACE();
2742         rte_eth_driver_register(&rte_pmd_fm10k);
2743         return 0;
2744 }
2745
2746 static struct rte_driver rte_fm10k_driver = {
2747         .type = PMD_PDEV,
2748         .init = rte_pmd_fm10k_init,
2749 };
2750
2751 PMD_REGISTER_DRIVER(rte_fm10k_driver);