drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 /* Default delay to acquire mailbox lock */
  45 #define FM10K_MBXLOCK_DELAY_US 20
  46 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  47
  48 #define MAIN_VSI_POOL_NUMBER 0
  49
  50 /* Max try times to acquire switch status */
  51 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  52 /* Wait interval to get switch status */
  53 #define WAIT_SWITCH_MSG_US    100000
  54 /* Number of chars per uint32 type */
  55 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  56 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  57
  58 #define FM10K_SIMPLE_TX_FLAG ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \
  59                                 ETH_TXQ_FLAGS_NOOFFLOADS)
  60
  61 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  62 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  63 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  64 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  65 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  66 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  67 static int
  68 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  69 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  70         const u8 *mac, bool add, uint32_t pool);
  71 static void fm10k_tx_queue_release(void *queue);
  72 static void fm10k_rx_queue_release(void *queue);
  73 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  74 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  75
  76 struct fm10k_xstats_name_off {
  77         char name[RTE_ETH_XSTATS_NAME_SIZE];
  78         unsigned offset;
  79 };
  80
  81 struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  82         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  83         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  84         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  85         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  86         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  87         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  88         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  89         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  90                 nodesc_drop)},
  91 };
  92
  93 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  94                 sizeof(fm10k_hw_stats_strings[0]))
  95
  96 struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
  97         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
  98         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
  99         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
 100 };
 101
 102 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
 103                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
 104
 105 struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 106         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 107         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 108 };
 109
 110 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 111                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 112
 113 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 114                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 115
 116 static void
 117 fm10k_mbx_initlock(struct fm10k_hw *hw)
 118 {
 119         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 120 }
 121
 122 static void
 123 fm10k_mbx_lock(struct fm10k_hw *hw)
 124 {
 125         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 126                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 127 }
 128
 129 static void
 130 fm10k_mbx_unlock(struct fm10k_hw *hw)
 131 {
 132         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 133 }
 134
 135 /*
 136  * reset queue to initial state, allocate software buffers used when starting
 137  * device.
 138  * return 0 on success
 139  * return -ENOMEM if buffers cannot be allocated
 140  * return -EINVAL if buffers do not satisfy alignment condition
 141  */
 142 static inline int
 143 rx_queue_reset(struct fm10k_rx_queue *q)
 144 {
 145         static const union fm10k_rx_desc zero = {{0} };
 146         uint64_t dma_addr;
 147         int i, diag;
 148         PMD_INIT_FUNC_TRACE();
 149
 150         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 151         if (diag != 0)
 152                 return -ENOMEM;
 153
 154         for (i = 0; i < q->nb_desc; ++i) {
 155                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 156                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 157                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 158                                                 q->nb_desc);
 159                         return -EINVAL;
 160                 }
 161                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 162                 q->hw_ring[i].q.pkt_addr = dma_addr;
 163                 q->hw_ring[i].q.hdr_addr = dma_addr;
 164         }
 165
 166         /* initialize extra software ring entries. Space for these extra
 167          * entries is always allocated.
 168          */
 169         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 170         for (i = 0; i < q->nb_fake_desc; ++i) {
 171                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 172                 q->hw_ring[q->nb_desc + i] = zero;
 173         }
 174
 175         q->next_dd = 0;
 176         q->next_alloc = 0;
 177         q->next_trigger = q->alloc_thresh - 1;
 178         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 179         q->rxrearm_start = 0;
 180         q->rxrearm_nb = 0;
 181
 182         return 0;
 183 }
 184
 185 /*
 186  * clean queue, descriptor rings, free software buffers used when stopping
 187  * device.
 188  */
 189 static inline void
 190 rx_queue_clean(struct fm10k_rx_queue *q)
 191 {
 192         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 193         uint32_t i;
 194         PMD_INIT_FUNC_TRACE();
 195
 196         /* zero descriptor rings */
 197         for (i = 0; i < q->nb_desc; ++i)
 198                 q->hw_ring[i] = zero;
 199
 200         /* zero faked descriptors */
 201         for (i = 0; i < q->nb_fake_desc; ++i)
 202                 q->hw_ring[q->nb_desc + i] = zero;
 203
 204         /* vPMD driver has a different way of releasing mbufs. */
 205         if (q->rx_using_sse) {
 206                 fm10k_rx_queue_release_mbufs_vec(q);
 207                 return;
 208         }
 209
 210         /* free software buffers */
 211         for (i = 0; i < q->nb_desc; ++i) {
 212                 if (q->sw_ring[i]) {
 213                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 214                         q->sw_ring[i] = NULL;
 215                 }
 216         }
 217 }
 218
 219 /*
 220  * free all queue memory used when releasing the queue (i.e. configure)
 221  */
 222 static inline void
 223 rx_queue_free(struct fm10k_rx_queue *q)
 224 {
 225         PMD_INIT_FUNC_TRACE();
 226         if (q) {
 227                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 228                 rx_queue_clean(q);
 229                 if (q->sw_ring) {
 230                         rte_free(q->sw_ring);
 231                         q->sw_ring = NULL;
 232                 }
 233                 rte_free(q);
 234                 q = NULL;
 235         }
 236 }
 237
 238 /*
 239  * disable RX queue, wait unitl HW finished necessary flush operation
 240  */
 241 static inline int
 242 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 243 {
 244         uint32_t reg, i;
 245
 246         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 247         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 248                         reg & ~FM10K_RXQCTL_ENABLE);
 249
 250         /* Wait 100us at most */
 251         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 252                 rte_delay_us(1);
 253                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 254                 if (!(reg & FM10K_RXQCTL_ENABLE))
 255                         break;
 256         }
 257
 258         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 259                 return -1;
 260
 261         return 0;
 262 }
 263
 264 /*
 265  * reset queue to initial state, allocate software buffers used when starting
 266  * device
 267  */
 268 static inline void
 269 tx_queue_reset(struct fm10k_tx_queue *q)
 270 {
 271         PMD_INIT_FUNC_TRACE();
 272         q->last_free = 0;
 273         q->next_free = 0;
 274         q->nb_used = 0;
 275         q->nb_free = q->nb_desc - 1;
 276         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 277         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 278 }
 279
 280 /*
 281  * clean queue, descriptor rings, free software buffers used when stopping
 282  * device
 283  */
 284 static inline void
 285 tx_queue_clean(struct fm10k_tx_queue *q)
 286 {
 287         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 288         uint32_t i;
 289         PMD_INIT_FUNC_TRACE();
 290
 291         /* zero descriptor rings */
 292         for (i = 0; i < q->nb_desc; ++i)
 293                 q->hw_ring[i] = zero;
 294
 295         /* free software buffers */
 296         for (i = 0; i < q->nb_desc; ++i) {
 297                 if (q->sw_ring[i]) {
 298                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 299                         q->sw_ring[i] = NULL;
 300                 }
 301         }
 302 }
 303
 304 /*
 305  * free all queue memory used when releasing the queue (i.e. configure)
 306  */
 307 static inline void
 308 tx_queue_free(struct fm10k_tx_queue *q)
 309 {
 310         PMD_INIT_FUNC_TRACE();
 311         if (q) {
 312                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 313                 tx_queue_clean(q);
 314                 if (q->rs_tracker.list) {
 315                         rte_free(q->rs_tracker.list);
 316                         q->rs_tracker.list = NULL;
 317                 }
 318                 if (q->sw_ring) {
 319                         rte_free(q->sw_ring);
 320                         q->sw_ring = NULL;
 321                 }
 322                 rte_free(q);
 323                 q = NULL;
 324         }
 325 }
 326
 327 /*
 328  * disable TX queue, wait unitl HW finished necessary flush operation
 329  */
 330 static inline int
 331 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 332 {
 333         uint32_t reg, i;
 334
 335         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 336         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 337                         reg & ~FM10K_TXDCTL_ENABLE);
 338
 339         /* Wait 100us at most */
 340         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 341                 rte_delay_us(1);
 342                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 343                 if (!(reg & FM10K_TXDCTL_ENABLE))
 344                         break;
 345         }
 346
 347         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 348                 return -1;
 349
 350         return 0;
 351 }
 352
 353 static int
 354 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 355 {
 356         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 357         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 358         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 359         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 360
 361         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 362
 363         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 364                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 365                 return -EINVAL;
 366         }
 367
 368         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 369                 return 0;
 370
 371         if (hw->mac.type == fm10k_mac_vf) {
 372                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 373                 return -EINVAL;
 374         }
 375
 376         /* Check VMDQ queue pool number */
 377         if (vmdq_conf->nb_queue_pools >
 378                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 379                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 380                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 381                         vmdq_conf->nb_queue_pools);
 382                 return -EINVAL;
 383         }
 384
 385         return 0;
 386 }
 387
 388 static const struct fm10k_txq_ops def_txq_ops = {
 389         .release_mbufs = tx_queue_free,
 390         .reset = tx_queue_reset,
 391 };
 392
 393 static int
 394 fm10k_dev_configure(struct rte_eth_dev *dev)
 395 {
 396         int ret;
 397
 398         PMD_INIT_FUNC_TRACE();
 399
 400         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 401                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 402         /* multipe queue mode checking */
 403         ret  = fm10k_check_mq_mode(dev);
 404         if (ret != 0) {
 405                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 406                             ret);
 407                 return ret;
 408         }
 409
 410         return 0;
 411 }
 412
 413 /* fls = find last set bit = 32 minus the number of leading zeros */
 414 #ifndef fls
 415 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 416 #endif
 417
 418 static void
 419 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 420 {
 421         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 422         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 423         uint32_t i;
 424
 425         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 426
 427         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 428                 if (!vmdq_conf->pool_map[i].pools)
 429                         continue;
 430                 fm10k_mbx_lock(hw);
 431                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 432                 fm10k_mbx_unlock(hw);
 433         }
 434 }
 435
 436 static void
 437 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 438 {
 439         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 440
 441         /* Add default mac address */
 442         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 443                 MAIN_VSI_POOL_NUMBER);
 444 }
 445
 446 static void
 447 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 448 {
 449         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 450         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 451         uint32_t mrqc, *key, i, reta, j;
 452         uint64_t hf;
 453
 454 #define RSS_KEY_SIZE 40
 455         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 456                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 457                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 458                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 459                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 460                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 461         };
 462
 463         if (dev->data->nb_rx_queues == 1 ||
 464             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 465             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 466                 return;
 467
 468         /* random key is rss_intel_key (default) or user provided (rss_key) */
 469         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 470                 key = (uint32_t *)rss_intel_key;
 471         else
 472                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 473
 474         /* Now fill our hash function seeds, 4 bytes at a time */
 475         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 476                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 477
 478         /*
 479          * Fill in redirection table
 480          * The byte-swap is needed because NIC registers are in
 481          * little-endian order.
 482          */
 483         reta = 0;
 484         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 485                 if (j == dev->data->nb_rx_queues)
 486                         j = 0;
 487                 reta = (reta << CHAR_BIT) | j;
 488                 if ((i & 3) == 3)
 489                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 490                                         rte_bswap32(reta));
 491         }
 492
 493         /*
 494          * Generate RSS hash based on packet types, TCP/UDP
 495          * port numbers and/or IPv4/v6 src and dst addresses
 496          */
 497         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 498         mrqc = 0;
 499         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 500         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 501         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 502         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 503         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 504         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 505         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 506         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 507         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 508
 509         if (mrqc == 0) {
 510                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 511                         "supported", hf);
 512                 return;
 513         }
 514
 515         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 516 }
 517
 518 static void
 519 fm10k_dev_logic_port_update(struct rte_eth_dev *dev,
 520         uint16_t nb_lport_old, uint16_t nb_lport_new)
 521 {
 522         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 523         uint32_t i;
 524
 525         fm10k_mbx_lock(hw);
 526         /* Disable previous logic ports */
 527         if (nb_lport_old)
 528                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
 529                         nb_lport_old, false);
 530         /* Enable new logic ports */
 531         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
 532                 nb_lport_new, true);
 533         fm10k_mbx_unlock(hw);
 534
 535         for (i = 0; i < nb_lport_new; i++) {
 536                 /* Set unicast mode by default. App can change
 537                  * to other mode in other API func.
 538                  */
 539                 fm10k_mbx_lock(hw);
 540                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 541                         FM10K_XCAST_MODE_NONE);
 542                 fm10k_mbx_unlock(hw);
 543         }
 544 }
 545
 546 static void
 547 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 548 {
 549         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 550         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 551         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 552         struct fm10k_macvlan_filter_info *macvlan;
 553         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 554         uint16_t nb_lport_new, nb_lport_old;
 555
 556         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 557         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 558
 559         fm10k_dev_rss_configure(dev);
 560
 561         /* only PF supports VMDQ */
 562         if (hw->mac.type != fm10k_mac_pf)
 563                 return;
 564
 565         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 566                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 567
 568         /* no pool number change, no need to update logic port and VLAN/MAC */
 569         if (macvlan->nb_queue_pools == nb_queue_pools)
 570                 return;
 571
 572         nb_lport_old = macvlan->nb_queue_pools ? macvlan->nb_queue_pools : 1;
 573         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 574         fm10k_dev_logic_port_update(dev, nb_lport_old, nb_lport_new);
 575
 576         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 577         memset(dev->data->mac_addrs, 0,
 578                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 579         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 580                 &dev->data->mac_addrs[0]);
 581         memset(macvlan, 0, sizeof(*macvlan));
 582         macvlan->nb_queue_pools = nb_queue_pools;
 583
 584         if (nb_queue_pools)
 585                 fm10k_dev_vmdq_rx_configure(dev);
 586         else
 587                 fm10k_dev_pf_main_vsi_reset(dev);
 588 }
 589
 590 static int
 591 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 592 {
 593         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 594         int i, ret;
 595         struct fm10k_tx_queue *txq;
 596         uint64_t base_addr;
 597         uint32_t size;
 598
 599         /* Disable TXINT to avoid possible interrupt */
 600         for (i = 0; i < hw->mac.max_queues; i++)
 601                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 602                                 3 << FM10K_TXINT_TIMER_SHIFT);
 603
 604         /* Setup TX queue */
 605         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 606                 txq = dev->data->tx_queues[i];
 607                 base_addr = txq->hw_ring_phys_addr;
 608                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 609
 610                 /* disable queue to avoid issues while updating state */
 611                 ret = tx_queue_disable(hw, i);
 612                 if (ret) {
 613                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 614                         return -1;
 615                 }
 616
 617                 /* set location and size for descriptor ring */
 618                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 619                                 base_addr & UINT64_LOWER_32BITS_MASK);
 620                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 621                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 622                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 623         }
 624
 625         /* set up vector or scalar TX function as appropriate */
 626         fm10k_set_tx_function(dev);
 627
 628         return 0;
 629 }
 630
 631 static int
 632 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 633 {
 634         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 635         int i, ret;
 636         struct fm10k_rx_queue *rxq;
 637         uint64_t base_addr;
 638         uint32_t size;
 639         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 640         uint16_t buf_size;
 641
 642         /* Disable RXINT to avoid possible interrupt */
 643         for (i = 0; i < hw->mac.max_queues; i++)
 644                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 645                                 3 << FM10K_RXINT_TIMER_SHIFT);
 646
 647         /* Setup RX queues */
 648         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 649                 rxq = dev->data->rx_queues[i];
 650                 base_addr = rxq->hw_ring_phys_addr;
 651                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 652
 653                 /* disable queue to avoid issues while updating state */
 654                 ret = rx_queue_disable(hw, i);
 655                 if (ret) {
 656                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 657                         return -1;
 658                 }
 659
 660                 /* Setup the Base and Length of the Rx Descriptor Ring */
 661                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 662                                 base_addr & UINT64_LOWER_32BITS_MASK);
 663                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 664                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 665                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 666
 667                 /* Configure the Rx buffer size for one buff without split */
 668                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 669                         RTE_PKTMBUF_HEADROOM);
 670                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 671                  * reserved for this purpose, and the worst case could be 511B.
 672                  * But SRR reg assumes all buffers have the same size. In order
 673                  * to fill the gap, we'll have to consider the worst case and
 674                  * assume 512B is reserved. If we don't do so, it's possible
 675                  * for HW to overwrite data to next mbuf.
 676                  */
 677                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 678
 679                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 680                                 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
 681
 682                 /* It adds dual VLAN length for supporting dual VLAN */
 683                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 684                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 685                         dev->data->dev_conf.rxmode.enable_scatter) {
 686                         uint32_t reg;
 687                         dev->data->scattered_rx = 1;
 688                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 689                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 690                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 691                 }
 692
 693                 /* Enable drop on empty, it's RO for VF */
 694                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 695                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 696
 697                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 698                 FM10K_WRITE_FLUSH(hw);
 699         }
 700
 701         /* Configure VMDQ/RSS if applicable */
 702         fm10k_dev_mq_rx_configure(dev);
 703
 704         /* Decide the best RX function */
 705         fm10k_set_rx_function(dev);
 706
 707         return 0;
 708 }
 709
 710 static int
 711 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 712 {
 713         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 714         int err = -1;
 715         uint32_t reg;
 716         struct fm10k_rx_queue *rxq;
 717
 718         PMD_INIT_FUNC_TRACE();
 719
 720         if (rx_queue_id < dev->data->nb_rx_queues) {
 721                 rxq = dev->data->rx_queues[rx_queue_id];
 722                 err = rx_queue_reset(rxq);
 723                 if (err == -ENOMEM) {
 724                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 725                         return err;
 726                 } else if (err == -EINVAL) {
 727                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 728                                 " %d", err);
 729                         return err;
 730                 }
 731
 732                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 733                  * Note: this must be done AFTER the queue is enabled on real
 734                  * hardware, but BEFORE the queue is enabled when using the
 735                  * emulation platform. Do it in both places for now and remove
 736                  * this comment and the following two register writes when the
 737                  * emulation platform is no longer being used.
 738                  */
 739                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 740                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 741
 742                 /* Set PF ownership flag for PF devices */
 743                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 744                 if (hw->mac.type == fm10k_mac_pf)
 745                         reg |= FM10K_RXQCTL_PF;
 746                 reg |= FM10K_RXQCTL_ENABLE;
 747                 /* enable RX queue */
 748                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 749                 FM10K_WRITE_FLUSH(hw);
 750
 751                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 752                  * Note: this must be done AFTER the queue is enabled
 753                  */
 754                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 755                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 756                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 757         }
 758
 759         return err;
 760 }
 761
 762 static int
 763 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 764 {
 765         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 766
 767         PMD_INIT_FUNC_TRACE();
 768
 769         if (rx_queue_id < dev->data->nb_rx_queues) {
 770                 /* Disable RX queue */
 771                 rx_queue_disable(hw, rx_queue_id);
 772
 773                 /* Free mbuf and clean HW ring */
 774                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 775                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 776         }
 777
 778         return 0;
 779 }
 780
 781 static int
 782 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 783 {
 784         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 785         /** @todo - this should be defined in the shared code */
 786 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 787         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 788         int err = 0;
 789
 790         PMD_INIT_FUNC_TRACE();
 791
 792         if (tx_queue_id < dev->data->nb_tx_queues) {
 793                 struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 794
 795                 q->ops->reset(q);
 796
 797                 /* reset head and tail pointers */
 798                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 799                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 800
 801                 /* enable TX queue */
 802                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 803                                         FM10K_TXDCTL_ENABLE | txdctl);
 804                 FM10K_WRITE_FLUSH(hw);
 805                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 806         } else
 807                 err = -1;
 808
 809         return err;
 810 }
 811
 812 static int
 813 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 814 {
 815         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 816
 817         PMD_INIT_FUNC_TRACE();
 818
 819         if (tx_queue_id < dev->data->nb_tx_queues) {
 820                 tx_queue_disable(hw, tx_queue_id);
 821                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 822                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 823         }
 824
 825         return 0;
 826 }
 827
 828 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 829 {
 830         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 831                 != FM10K_DGLORTMAP_NONE);
 832 }
 833
 834 static void
 835 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 836 {
 837         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 838         int status;
 839
 840         PMD_INIT_FUNC_TRACE();
 841
 842         /* Return if it didn't acquire valid glort range */
 843         if (!fm10k_glort_valid(hw))
 844                 return;
 845
 846         fm10k_mbx_lock(hw);
 847         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 848                                 FM10K_XCAST_MODE_PROMISC);
 849         fm10k_mbx_unlock(hw);
 850
 851         if (status != FM10K_SUCCESS)
 852                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 853 }
 854
 855 static void
 856 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 857 {
 858         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 859         uint8_t mode;
 860         int status;
 861
 862         PMD_INIT_FUNC_TRACE();
 863
 864         /* Return if it didn't acquire valid glort range */
 865         if (!fm10k_glort_valid(hw))
 866                 return;
 867
 868         if (dev->data->all_multicast == 1)
 869                 mode = FM10K_XCAST_MODE_ALLMULTI;
 870         else
 871                 mode = FM10K_XCAST_MODE_NONE;
 872
 873         fm10k_mbx_lock(hw);
 874         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 875                                 mode);
 876         fm10k_mbx_unlock(hw);
 877
 878         if (status != FM10K_SUCCESS)
 879                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 880 }
 881
 882 static void
 883 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 884 {
 885         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 886         int status;
 887
 888         PMD_INIT_FUNC_TRACE();
 889
 890         /* Return if it didn't acquire valid glort range */
 891         if (!fm10k_glort_valid(hw))
 892                 return;
 893
 894         /* If promiscuous mode is enabled, it doesn't make sense to enable
 895          * allmulticast and disable promiscuous since fm10k only can select
 896          * one of the modes.
 897          */
 898         if (dev->data->promiscuous) {
 899                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 900                         "needn't enable allmulticast");
 901                 return;
 902         }
 903
 904         fm10k_mbx_lock(hw);
 905         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 906                                 FM10K_XCAST_MODE_ALLMULTI);
 907         fm10k_mbx_unlock(hw);
 908
 909         if (status != FM10K_SUCCESS)
 910                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 911 }
 912
 913 static void
 914 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 915 {
 916         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 917         int status;
 918
 919         PMD_INIT_FUNC_TRACE();
 920
 921         /* Return if it didn't acquire valid glort range */
 922         if (!fm10k_glort_valid(hw))
 923                 return;
 924
 925         if (dev->data->promiscuous) {
 926                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
 927                         "since promisc mode is enabled");
 928                 return;
 929         }
 930
 931         fm10k_mbx_lock(hw);
 932         /* Change mode to unicast mode */
 933         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 934                                 FM10K_XCAST_MODE_NONE);
 935         fm10k_mbx_unlock(hw);
 936
 937         if (status != FM10K_SUCCESS)
 938                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
 939 }
 940
 941 static void
 942 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
 943 {
 944         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 945         uint32_t dglortdec, pool_len, rss_len, i;
 946         uint16_t nb_queue_pools;
 947         struct fm10k_macvlan_filter_info *macvlan;
 948
 949         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 950         nb_queue_pools = macvlan->nb_queue_pools;
 951         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
 952         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
 953         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
 954
 955         /* Establish only MAP 0 as valid */
 956         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
 957
 958         /* Configure VMDQ/RSS DGlort Decoder */
 959         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
 960
 961         /* Invalidate all other GLORT entries */
 962         for (i = 1; i < FM10K_DGLORT_COUNT; i++)
 963                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
 964                                 FM10K_DGLORTMAP_NONE);
 965 }
 966
 967 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
 968 static int
 969 fm10k_dev_start(struct rte_eth_dev *dev)
 970 {
 971         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 972         int i, diag;
 973
 974         PMD_INIT_FUNC_TRACE();
 975
 976         /* stop, init, then start the hw */
 977         diag = fm10k_stop_hw(hw);
 978         if (diag != FM10K_SUCCESS) {
 979                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
 980                 return -EIO;
 981         }
 982
 983         diag = fm10k_init_hw(hw);
 984         if (diag != FM10K_SUCCESS) {
 985                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
 986                 return -EIO;
 987         }
 988
 989         diag = fm10k_start_hw(hw);
 990         if (diag != FM10K_SUCCESS) {
 991                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
 992                 return -EIO;
 993         }
 994
 995         diag = fm10k_dev_tx_init(dev);
 996         if (diag) {
 997                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
 998                 return diag;
 999         }
1000
1001         diag = fm10k_dev_rx_init(dev);
1002         if (diag) {
1003                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1004                 return diag;
1005         }
1006
1007         if (hw->mac.type == fm10k_mac_pf)
1008                 fm10k_dev_dglort_map_configure(dev);
1009
1010         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1011                 struct fm10k_rx_queue *rxq;
1012                 rxq = dev->data->rx_queues[i];
1013
1014                 if (rxq->rx_deferred_start)
1015                         continue;
1016                 diag = fm10k_dev_rx_queue_start(dev, i);
1017                 if (diag != 0) {
1018                         int j;
1019                         for (j = 0; j < i; ++j)
1020                                 rx_queue_clean(dev->data->rx_queues[j]);
1021                         return diag;
1022                 }
1023         }
1024
1025         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1026                 struct fm10k_tx_queue *txq;
1027                 txq = dev->data->tx_queues[i];
1028
1029                 if (txq->tx_deferred_start)
1030                         continue;
1031                 diag = fm10k_dev_tx_queue_start(dev, i);
1032                 if (diag != 0) {
1033                         int j;
1034                         for (j = 0; j < i; ++j)
1035                                 tx_queue_clean(dev->data->tx_queues[j]);
1036                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1037                                 rx_queue_clean(dev->data->rx_queues[j]);
1038                         return diag;
1039                 }
1040         }
1041
1042         /* Update default vlan when not in VMDQ mode */
1043         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1044                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1045
1046         return 0;
1047 }
1048
1049 static void
1050 fm10k_dev_stop(struct rte_eth_dev *dev)
1051 {
1052         int i;
1053
1054         PMD_INIT_FUNC_TRACE();
1055
1056         if (dev->data->tx_queues)
1057                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1058                         fm10k_dev_tx_queue_stop(dev, i);
1059
1060         if (dev->data->rx_queues)
1061                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1062                         fm10k_dev_rx_queue_stop(dev, i);
1063 }
1064
1065 static void
1066 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1067 {
1068         int i;
1069
1070         PMD_INIT_FUNC_TRACE();
1071
1072         if (dev->data->tx_queues) {
1073                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1074                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1075
1076                         txq->ops->release_mbufs(txq);
1077                 }
1078         }
1079
1080         if (dev->data->rx_queues) {
1081                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1082                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1083         }
1084 }
1085
1086 static void
1087 fm10k_dev_close(struct rte_eth_dev *dev)
1088 {
1089         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1090         uint16_t nb_lport;
1091         struct fm10k_macvlan_filter_info *macvlan;
1092
1093         PMD_INIT_FUNC_TRACE();
1094
1095         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1096         nb_lport = macvlan->nb_queue_pools ? macvlan->nb_queue_pools : 1;
1097         fm10k_mbx_lock(hw);
1098         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1099                 nb_lport, false);
1100         fm10k_mbx_unlock(hw);
1101
1102         /* Stop mailbox service first */
1103         fm10k_close_mbx_service(hw);
1104         fm10k_dev_stop(dev);
1105         fm10k_dev_queue_release(dev);
1106         fm10k_stop_hw(hw);
1107 }
1108
1109 static int
1110 fm10k_link_update(struct rte_eth_dev *dev,
1111         __rte_unused int wait_to_complete)
1112 {
1113         PMD_INIT_FUNC_TRACE();
1114
1115         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
1116          * x8 PCIe interface. For now, we leave the speed undefined since there
1117          * is no 50Gbps Ethernet. */
1118         dev->data->dev_link.link_speed  = 0;
1119         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1120         dev->data->dev_link.link_status = 1;
1121
1122         return 0;
1123 }
1124
1125 static int
1126 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstats *xstats,
1127                  unsigned n)
1128 {
1129         struct fm10k_hw_stats *hw_stats =
1130                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1131         unsigned i, q, count = 0;
1132
1133         if (n < FM10K_NB_XSTATS)
1134                 return FM10K_NB_XSTATS;
1135
1136         /* Global stats */
1137         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1138                 snprintf(xstats[count].name, sizeof(xstats[count].name),
1139                          "%s", fm10k_hw_stats_strings[count].name);
1140                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1141                         fm10k_hw_stats_strings[count].offset);
1142                 count++;
1143         }
1144
1145         /* PF queue stats */
1146         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1147                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1148                         snprintf(xstats[count].name, sizeof(xstats[count].name),
1149                                  "rx_q%u_%s", q,
1150                                  fm10k_hw_stats_rx_q_strings[i].name);
1151                         xstats[count].value =
1152                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1153                                 fm10k_hw_stats_rx_q_strings[i].offset);
1154                         count++;
1155                 }
1156                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1157                         snprintf(xstats[count].name, sizeof(xstats[count].name),
1158                                  "tx_q%u_%s", q,
1159                                  fm10k_hw_stats_tx_q_strings[i].name);
1160                         xstats[count].value =
1161                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1162                                 fm10k_hw_stats_tx_q_strings[i].offset);
1163                         count++;
1164                 }
1165         }
1166
1167         return FM10K_NB_XSTATS;
1168 }
1169
1170 static void
1171 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1172 {
1173         uint64_t ipackets, opackets, ibytes, obytes;
1174         struct fm10k_hw *hw =
1175                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1176         struct fm10k_hw_stats *hw_stats =
1177                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1178         int i;
1179
1180         PMD_INIT_FUNC_TRACE();
1181
1182         fm10k_update_hw_stats(hw, hw_stats);
1183
1184         ipackets = opackets = ibytes = obytes = 0;
1185         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1186                 (i < hw->mac.max_queues); ++i) {
1187                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1188                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1189                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1190                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1191                 ipackets += stats->q_ipackets[i];
1192                 opackets += stats->q_opackets[i];
1193                 ibytes   += stats->q_ibytes[i];
1194                 obytes   += stats->q_obytes[i];
1195         }
1196         stats->ipackets = ipackets;
1197         stats->opackets = opackets;
1198         stats->ibytes = ibytes;
1199         stats->obytes = obytes;
1200 }
1201
1202 static void
1203 fm10k_stats_reset(struct rte_eth_dev *dev)
1204 {
1205         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1206         struct fm10k_hw_stats *hw_stats =
1207                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1208
1209         PMD_INIT_FUNC_TRACE();
1210
1211         memset(hw_stats, 0, sizeof(*hw_stats));
1212         fm10k_rebind_hw_stats(hw, hw_stats);
1213 }
1214
1215 static void
1216 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1217         struct rte_eth_dev_info *dev_info)
1218 {
1219         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1220
1221         PMD_INIT_FUNC_TRACE();
1222
1223         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1224         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1225         dev_info->max_rx_queues      = hw->mac.max_queues;
1226         dev_info->max_tx_queues      = hw->mac.max_queues;
1227         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1228         dev_info->max_hash_mac_addrs = 0;
1229         dev_info->max_vfs            = dev->pci_dev->max_vfs;
1230         dev_info->vmdq_pool_base     = 0;
1231         dev_info->vmdq_queue_base    = 0;
1232         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1233         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1234         dev_info->rx_offload_capa =
1235                 DEV_RX_OFFLOAD_VLAN_STRIP |
1236                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1237                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1238                 DEV_RX_OFFLOAD_TCP_CKSUM;
1239         dev_info->tx_offload_capa =
1240                 DEV_TX_OFFLOAD_VLAN_INSERT |
1241                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1242                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1243                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1244                 DEV_TX_OFFLOAD_TCP_TSO;
1245
1246         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1247         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1248
1249         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1250                 .rx_thresh = {
1251                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1252                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1253                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1254                 },
1255                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1256                 .rx_drop_en = 0,
1257         };
1258
1259         dev_info->default_txconf = (struct rte_eth_txconf) {
1260                 .tx_thresh = {
1261                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1262                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1263                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1264                 },
1265                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1266                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1267                 .txq_flags = FM10K_SIMPLE_TX_FLAG,
1268         };
1269
1270         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1271                 .nb_max = FM10K_MAX_RX_DESC,
1272                 .nb_min = FM10K_MIN_RX_DESC,
1273                 .nb_align = FM10K_MULT_RX_DESC,
1274         };
1275
1276         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1277                 .nb_max = FM10K_MAX_TX_DESC,
1278                 .nb_min = FM10K_MIN_TX_DESC,
1279                 .nb_align = FM10K_MULT_TX_DESC,
1280         };
1281 }
1282
1283 static int
1284 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1285 {
1286         s32 result;
1287         uint16_t mac_num = 0;
1288         uint32_t vid_idx, vid_bit, mac_index;
1289         struct fm10k_hw *hw;
1290         struct fm10k_macvlan_filter_info *macvlan;
1291         struct rte_eth_dev_data *data = dev->data;
1292
1293         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1294         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1295
1296         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1297                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1298                 return (-EINVAL);
1299         }
1300
1301         if (vlan_id > ETH_VLAN_ID_MAX) {
1302                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1303                 return (-EINVAL);
1304         }
1305
1306         vid_idx = FM10K_VFTA_IDX(vlan_id);
1307         vid_bit = FM10K_VFTA_BIT(vlan_id);
1308         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1309         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1310                 return 0;
1311         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1312         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1313                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1314                         "in the VLAN filter table");
1315                 return (-EINVAL);
1316         }
1317
1318         fm10k_mbx_lock(hw);
1319         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1320         fm10k_mbx_unlock(hw);
1321         if (result != FM10K_SUCCESS) {
1322                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1323                 return (-EIO);
1324         }
1325
1326         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1327                         (result == FM10K_SUCCESS); mac_index++) {
1328                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1329                         continue;
1330                 if (mac_num > macvlan->mac_num - 1) {
1331                         PMD_INIT_LOG(ERR, "MAC address number "
1332                                         "not match");
1333                         break;
1334                 }
1335                 fm10k_mbx_lock(hw);
1336                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1337                         data->mac_addrs[mac_index].addr_bytes,
1338                         vlan_id, on, 0);
1339                 fm10k_mbx_unlock(hw);
1340                 mac_num++;
1341         }
1342         if (result != FM10K_SUCCESS) {
1343                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1344                 return (-EIO);
1345         }
1346
1347         if (on) {
1348                 macvlan->vlan_num++;
1349                 macvlan->vfta[vid_idx] |= vid_bit;
1350         } else {
1351                 macvlan->vlan_num--;
1352                 macvlan->vfta[vid_idx] &= ~vid_bit;
1353         }
1354         return 0;
1355 }
1356
1357 static void
1358 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1359 {
1360         if (mask & ETH_VLAN_STRIP_MASK) {
1361                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1362                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1363                                         "always on in fm10k");
1364         }
1365
1366         if (mask & ETH_VLAN_EXTEND_MASK) {
1367                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1368                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1369                                         "supported in fm10k");
1370         }
1371
1372         if (mask & ETH_VLAN_FILTER_MASK) {
1373                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1374                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1375         }
1376 }
1377
1378 /* Add/Remove a MAC address, and update filters to main VSI */
1379 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1380                 const u8 *mac, bool add, uint32_t pool)
1381 {
1382         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1383         struct fm10k_macvlan_filter_info *macvlan;
1384         uint32_t i, j, k;
1385
1386         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1387
1388         if (pool != MAIN_VSI_POOL_NUMBER) {
1389                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1390                         "mac to pool %u", pool);
1391                 return;
1392         }
1393         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1394                 if (!macvlan->vfta[j])
1395                         continue;
1396                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1397                         if (!(macvlan->vfta[j] & (1 << k)))
1398                                 continue;
1399                         if (i + 1 > macvlan->vlan_num) {
1400                                 PMD_INIT_LOG(ERR, "vlan number not match");
1401                                 return;
1402                         }
1403                         fm10k_mbx_lock(hw);
1404                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1405                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1406                         fm10k_mbx_unlock(hw);
1407                         i++;
1408                 }
1409         }
1410 }
1411
1412 /* Add/Remove a MAC address, and update filters to VMDQ */
1413 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1414                 const u8 *mac, bool add, uint32_t pool)
1415 {
1416         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1417         struct fm10k_macvlan_filter_info *macvlan;
1418         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1419         uint32_t i;
1420
1421         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1422         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1423
1424         if (pool > macvlan->nb_queue_pools) {
1425                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1426                         " Max pool is %u",
1427                         pool, macvlan->nb_queue_pools);
1428                 return;
1429         }
1430         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1431                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1432                         continue;
1433                 fm10k_mbx_lock(hw);
1434                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1435                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1436                 fm10k_mbx_unlock(hw);
1437         }
1438 }
1439
1440 /* Add/Remove a MAC address, and update filters */
1441 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1442                 const u8 *mac, bool add, uint32_t pool)
1443 {
1444         struct fm10k_macvlan_filter_info *macvlan;
1445
1446         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1447
1448         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1449                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1450         else
1451                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1452
1453         if (add)
1454                 macvlan->mac_num++;
1455         else
1456                 macvlan->mac_num--;
1457 }
1458
1459 /* Add a MAC address, and update filters */
1460 static void
1461 fm10k_macaddr_add(struct rte_eth_dev *dev,
1462                 struct ether_addr *mac_addr,
1463                 uint32_t index,
1464                 uint32_t pool)
1465 {
1466         struct fm10k_macvlan_filter_info *macvlan;
1467
1468         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1469         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1470         macvlan->mac_vmdq_id[index] = pool;
1471 }
1472
1473 /* Remove a MAC address, and update filters */
1474 static void
1475 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1476 {
1477         struct rte_eth_dev_data *data = dev->data;
1478         struct fm10k_macvlan_filter_info *macvlan;
1479
1480         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1481         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1482                         FALSE, macvlan->mac_vmdq_id[index]);
1483         macvlan->mac_vmdq_id[index] = 0;
1484 }
1485
1486 static inline int
1487 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1488 {
1489         if ((request < min) || (request > max) || ((request % mult) != 0))
1490                 return -1;
1491         else
1492                 return 0;
1493 }
1494
1495
1496 static inline int
1497 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1498 {
1499         if ((request < min) || (request > max) || ((div % request) != 0))
1500                 return -1;
1501         else
1502                 return 0;
1503 }
1504
1505 static inline int
1506 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1507 {
1508         uint16_t rx_free_thresh;
1509
1510         if (conf->rx_free_thresh == 0)
1511                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1512         else
1513                 rx_free_thresh = conf->rx_free_thresh;
1514
1515         /* make sure the requested threshold satisfies the constraints */
1516         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1517                         FM10K_RX_FREE_THRESH_MAX(q),
1518                         FM10K_RX_FREE_THRESH_DIV(q),
1519                         rx_free_thresh)) {
1520                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1521                         "less than or equal to %u, "
1522                         "greater than or equal to %u, "
1523                         "and a divisor of %u",
1524                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1525                         FM10K_RX_FREE_THRESH_MIN(q),
1526                         FM10K_RX_FREE_THRESH_DIV(q));
1527                 return (-EINVAL);
1528         }
1529
1530         q->alloc_thresh = rx_free_thresh;
1531         q->drop_en = conf->rx_drop_en;
1532         q->rx_deferred_start = conf->rx_deferred_start;
1533
1534         return 0;
1535 }
1536
1537 /*
1538  * Hardware requires specific alignment for Rx packet buffers. At
1539  * least one of the following two conditions must be satisfied.
1540  *  1. Address is 512B aligned
1541  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1542  *
1543  * As such, the driver may need to adjust the DMA address within the
1544  * buffer by up to 512B.
1545  *
1546  * return 1 if the element size is valid, otherwise return 0.
1547  */
1548 static int
1549 mempool_element_size_valid(struct rte_mempool *mp)
1550 {
1551         uint32_t min_size;
1552
1553         /* elt_size includes mbuf header and headroom */
1554         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1555                         RTE_PKTMBUF_HEADROOM;
1556
1557         /* account for up to 512B of alignment */
1558         min_size -= FM10K_RX_DATABUF_ALIGN;
1559
1560         /* sanity check for overflow */
1561         if (min_size > mp->elt_size)
1562                 return 0;
1563
1564         /* size is valid */
1565         return 1;
1566 }
1567
1568 static int
1569 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1570         uint16_t nb_desc, unsigned int socket_id,
1571         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1572 {
1573         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1574         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
1575         struct fm10k_rx_queue *q;
1576         const struct rte_memzone *mz;
1577
1578         PMD_INIT_FUNC_TRACE();
1579
1580         /* make sure the mempool element size can account for alignment. */
1581         if (!mempool_element_size_valid(mp)) {
1582                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1583                 return (-EINVAL);
1584         }
1585
1586         /* make sure a valid number of descriptors have been requested */
1587         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1588                                 FM10K_MULT_RX_DESC, nb_desc)) {
1589                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1590                         "less than or equal to %"PRIu32", "
1591                         "greater than or equal to %u, "
1592                         "and a multiple of %u",
1593                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1594                         FM10K_MULT_RX_DESC);
1595                 return (-EINVAL);
1596         }
1597
1598         /*
1599          * if this queue existed already, free the associated memory. The
1600          * queue cannot be reused in case we need to allocate memory on
1601          * different socket than was previously used.
1602          */
1603         if (dev->data->rx_queues[queue_id] != NULL) {
1604                 rx_queue_free(dev->data->rx_queues[queue_id]);
1605                 dev->data->rx_queues[queue_id] = NULL;
1606         }
1607
1608         /* allocate memory for the queue structure */
1609         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1610                                 socket_id);
1611         if (q == NULL) {
1612                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1613                 return (-ENOMEM);
1614         }
1615
1616         /* setup queue */
1617         q->mp = mp;
1618         q->nb_desc = nb_desc;
1619         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1620         q->port_id = dev->data->port_id;
1621         q->queue_id = queue_id;
1622         q->tail_ptr = (volatile uint32_t *)
1623                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1624         if (handle_rxconf(q, conf))
1625                 return (-EINVAL);
1626
1627         /* allocate memory for the software ring */
1628         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1629                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1630                         RTE_CACHE_LINE_SIZE, socket_id);
1631         if (q->sw_ring == NULL) {
1632                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1633                 rte_free(q);
1634                 return (-ENOMEM);
1635         }
1636
1637         /*
1638          * allocate memory for the hardware descriptor ring. A memzone large
1639          * enough to hold the maximum ring size is requested to allow for
1640          * resizing in later calls to the queue setup function.
1641          */
1642         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1643                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1644                                       socket_id);
1645         if (mz == NULL) {
1646                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1647                 rte_free(q->sw_ring);
1648                 rte_free(q);
1649                 return (-ENOMEM);
1650         }
1651         q->hw_ring = mz->addr;
1652         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1653
1654         /* Check if number of descs satisfied Vector requirement */
1655         if (!rte_is_power_of_2(nb_desc)) {
1656                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1657                                     "preconditions - canceling the feature for "
1658                                     "the whole port[%d]",
1659                              q->queue_id, q->port_id);
1660                 dev_info->rx_vec_allowed = false;
1661         } else
1662                 fm10k_rxq_vec_setup(q);
1663
1664         dev->data->rx_queues[queue_id] = q;
1665         return 0;
1666 }
1667
1668 static void
1669 fm10k_rx_queue_release(void *queue)
1670 {
1671         PMD_INIT_FUNC_TRACE();
1672
1673         rx_queue_free(queue);
1674 }
1675
1676 static inline int
1677 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1678 {
1679         uint16_t tx_free_thresh;
1680         uint16_t tx_rs_thresh;
1681
1682         /* constraint MACROs require that tx_free_thresh is configured
1683          * before tx_rs_thresh */
1684         if (conf->tx_free_thresh == 0)
1685                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1686         else
1687                 tx_free_thresh = conf->tx_free_thresh;
1688
1689         /* make sure the requested threshold satisfies the constraints */
1690         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1691                         FM10K_TX_FREE_THRESH_MAX(q),
1692                         FM10K_TX_FREE_THRESH_DIV(q),
1693                         tx_free_thresh)) {
1694                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1695                         "less than or equal to %u, "
1696                         "greater than or equal to %u, "
1697                         "and a divisor of %u",
1698                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1699                         FM10K_TX_FREE_THRESH_MIN(q),
1700                         FM10K_TX_FREE_THRESH_DIV(q));
1701                 return (-EINVAL);
1702         }
1703
1704         q->free_thresh = tx_free_thresh;
1705
1706         if (conf->tx_rs_thresh == 0)
1707                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1708         else
1709                 tx_rs_thresh = conf->tx_rs_thresh;
1710
1711         q->tx_deferred_start = conf->tx_deferred_start;
1712
1713         /* make sure the requested threshold satisfies the constraints */
1714         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1715                         FM10K_TX_RS_THRESH_MAX(q),
1716                         FM10K_TX_RS_THRESH_DIV(q),
1717                         tx_rs_thresh)) {
1718                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1719                         "less than or equal to %u, "
1720                         "greater than or equal to %u, "
1721                         "and a divisor of %u",
1722                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1723                         FM10K_TX_RS_THRESH_MIN(q),
1724                         FM10K_TX_RS_THRESH_DIV(q));
1725                 return (-EINVAL);
1726         }
1727
1728         q->rs_thresh = tx_rs_thresh;
1729
1730         return 0;
1731 }
1732
1733 static int
1734 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1735         uint16_t nb_desc, unsigned int socket_id,
1736         const struct rte_eth_txconf *conf)
1737 {
1738         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1739         struct fm10k_tx_queue *q;
1740         const struct rte_memzone *mz;
1741
1742         PMD_INIT_FUNC_TRACE();
1743
1744         /* make sure a valid number of descriptors have been requested */
1745         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1746                                 FM10K_MULT_TX_DESC, nb_desc)) {
1747                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1748                         "less than or equal to %"PRIu32", "
1749                         "greater than or equal to %u, "
1750                         "and a multiple of %u",
1751                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1752                         FM10K_MULT_TX_DESC);
1753                 return (-EINVAL);
1754         }
1755
1756         /*
1757          * if this queue existed already, free the associated memory. The
1758          * queue cannot be reused in case we need to allocate memory on
1759          * different socket than was previously used.
1760          */
1761         if (dev->data->tx_queues[queue_id] != NULL) {
1762                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
1763
1764                 txq->ops->release_mbufs(txq);
1765                 dev->data->tx_queues[queue_id] = NULL;
1766         }
1767
1768         /* allocate memory for the queue structure */
1769         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1770                                 socket_id);
1771         if (q == NULL) {
1772                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1773                 return (-ENOMEM);
1774         }
1775
1776         /* setup queue */
1777         q->nb_desc = nb_desc;
1778         q->port_id = dev->data->port_id;
1779         q->queue_id = queue_id;
1780         q->txq_flags = conf->txq_flags;
1781         q->ops = &def_txq_ops;
1782         q->tail_ptr = (volatile uint32_t *)
1783                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1784         if (handle_txconf(q, conf))
1785                 return (-EINVAL);
1786
1787         /* allocate memory for the software ring */
1788         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1789                                         nb_desc * sizeof(struct rte_mbuf *),
1790                                         RTE_CACHE_LINE_SIZE, socket_id);
1791         if (q->sw_ring == NULL) {
1792                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1793                 rte_free(q);
1794                 return (-ENOMEM);
1795         }
1796
1797         /*
1798          * allocate memory for the hardware descriptor ring. A memzone large
1799          * enough to hold the maximum ring size is requested to allow for
1800          * resizing in later calls to the queue setup function.
1801          */
1802         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
1803                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
1804                                       socket_id);
1805         if (mz == NULL) {
1806                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1807                 rte_free(q->sw_ring);
1808                 rte_free(q);
1809                 return (-ENOMEM);
1810         }
1811         q->hw_ring = mz->addr;
1812         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1813
1814         /*
1815          * allocate memory for the RS bit tracker. Enough slots to hold the
1816          * descriptor index for each RS bit needing to be set are required.
1817          */
1818         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1819                                 ((nb_desc + 1) / q->rs_thresh) *
1820                                 sizeof(uint16_t),
1821                                 RTE_CACHE_LINE_SIZE, socket_id);
1822         if (q->rs_tracker.list == NULL) {
1823                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1824                 rte_free(q->sw_ring);
1825                 rte_free(q);
1826                 return (-ENOMEM);
1827         }
1828
1829         dev->data->tx_queues[queue_id] = q;
1830         return 0;
1831 }
1832
1833 static void
1834 fm10k_tx_queue_release(void *queue)
1835 {
1836         struct fm10k_tx_queue *q = queue;
1837         PMD_INIT_FUNC_TRACE();
1838
1839         q->ops->release_mbufs(q);
1840 }
1841
1842 static int
1843 fm10k_reta_update(struct rte_eth_dev *dev,
1844                         struct rte_eth_rss_reta_entry64 *reta_conf,
1845                         uint16_t reta_size)
1846 {
1847         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1848         uint16_t i, j, idx, shift;
1849         uint8_t mask;
1850         uint32_t reta;
1851
1852         PMD_INIT_FUNC_TRACE();
1853
1854         if (reta_size > FM10K_MAX_RSS_INDICES) {
1855                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1856                         "(%d) doesn't match the number hardware can supported "
1857                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1858                 return -EINVAL;
1859         }
1860
1861         /*
1862          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1863          * 128-entries in 32 registers
1864          */
1865         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1866                 idx = i / RTE_RETA_GROUP_SIZE;
1867                 shift = i % RTE_RETA_GROUP_SIZE;
1868                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1869                                 BIT_MASK_PER_UINT32);
1870                 if (mask == 0)
1871                         continue;
1872
1873                 reta = 0;
1874                 if (mask != BIT_MASK_PER_UINT32)
1875                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1876
1877                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1878                         if (mask & (0x1 << j)) {
1879                                 if (mask != 0xF)
1880                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1881                                 reta |= reta_conf[idx].reta[shift + j] <<
1882                                                 (CHAR_BIT * j);
1883                         }
1884                 }
1885                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1886         }
1887
1888         return 0;
1889 }
1890
1891 static int
1892 fm10k_reta_query(struct rte_eth_dev *dev,
1893                         struct rte_eth_rss_reta_entry64 *reta_conf,
1894                         uint16_t reta_size)
1895 {
1896         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1897         uint16_t i, j, idx, shift;
1898         uint8_t mask;
1899         uint32_t reta;
1900
1901         PMD_INIT_FUNC_TRACE();
1902
1903         if (reta_size < FM10K_MAX_RSS_INDICES) {
1904                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1905                         "(%d) doesn't match the number hardware can supported "
1906                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1907                 return -EINVAL;
1908         }
1909
1910         /*
1911          * Read Redirection Table RETA[n], n=0..31. The redirection table has
1912          * 128-entries in 32 registers
1913          */
1914         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1915                 idx = i / RTE_RETA_GROUP_SIZE;
1916                 shift = i % RTE_RETA_GROUP_SIZE;
1917                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1918                                 BIT_MASK_PER_UINT32);
1919                 if (mask == 0)
1920                         continue;
1921
1922                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1923                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1924                         if (mask & (0x1 << j))
1925                                 reta_conf[idx].reta[shift + j] = ((reta >>
1926                                         CHAR_BIT * j) & UINT8_MAX);
1927                 }
1928         }
1929
1930         return 0;
1931 }
1932
1933 static int
1934 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1935         struct rte_eth_rss_conf *rss_conf)
1936 {
1937         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1938         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1939         uint32_t mrqc;
1940         uint64_t hf = rss_conf->rss_hf;
1941         int i;
1942
1943         PMD_INIT_FUNC_TRACE();
1944
1945         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1946                 FM10K_RSSRK_ENTRIES_PER_REG)
1947                 return -EINVAL;
1948
1949         if (hf == 0)
1950                 return -EINVAL;
1951
1952         mrqc = 0;
1953         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
1954         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
1955         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
1956         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
1957         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
1958         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
1959         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
1960         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
1961         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
1962
1963         /* If the mapping doesn't fit any supported, return */
1964         if (mrqc == 0)
1965                 return -EINVAL;
1966
1967         if (key != NULL)
1968                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1969                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
1970
1971         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
1972
1973         return 0;
1974 }
1975
1976 static int
1977 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
1978         struct rte_eth_rss_conf *rss_conf)
1979 {
1980         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1981         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1982         uint32_t mrqc;
1983         uint64_t hf;
1984         int i;
1985
1986         PMD_INIT_FUNC_TRACE();
1987
1988         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1989                                 FM10K_RSSRK_ENTRIES_PER_REG)
1990                 return -EINVAL;
1991
1992         if (key != NULL)
1993                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1994                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
1995
1996         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
1997         hf = 0;
1998         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
1999         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2000         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2001         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2002         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2003         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2004         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2005         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2006         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2007
2008         rss_conf->rss_hf = hf;
2009
2010         return 0;
2011 }
2012
2013 static void
2014 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2015 {
2016         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2017         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2018
2019         /* Bind all local non-queue interrupt to vector 0 */
2020         int_map |= 0;
2021
2022         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
2023         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
2024         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
2025         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
2026         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
2027         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
2028
2029         /* Enable misc causes */
2030         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2031                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2032                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2033                                 FM10K_EIMR_ENABLE(MAILBOX) |
2034                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2035                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2036                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2037                                 FM10K_EIMR_ENABLE(VFLR));
2038
2039         /* Enable ITR 0 */
2040         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2041                                         FM10K_ITR_MASK_CLEAR);
2042         FM10K_WRITE_FLUSH(hw);
2043 }
2044
2045 static void
2046 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2047 {
2048         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2049         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2050
2051         int_map |= 0;
2052
2053         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
2054         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
2055         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
2056         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
2057         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
2058         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
2059
2060         /* Disable misc causes */
2061         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2062                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2063                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2064                                 FM10K_EIMR_DISABLE(MAILBOX) |
2065                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2066                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2067                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2068                                 FM10K_EIMR_DISABLE(VFLR));
2069
2070         /* Disable ITR 0 */
2071         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2072         FM10K_WRITE_FLUSH(hw);
2073 }
2074
2075 static void
2076 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2077 {
2078         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2079         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2080
2081         /* Bind all local non-queue interrupt to vector 0 */
2082         int_map |= 0;
2083
2084         /* Only INT 0 available, other 15 are reserved. */
2085         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2086
2087         /* Enable ITR 0 */
2088         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2089                                         FM10K_ITR_MASK_CLEAR);
2090         FM10K_WRITE_FLUSH(hw);
2091 }
2092
2093 static void
2094 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2095 {
2096         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2097         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2098
2099         int_map |= 0;
2100
2101         /* Only INT 0 available, other 15 are reserved. */
2102         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2103
2104         /* Disable ITR 0 */
2105         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2106         FM10K_WRITE_FLUSH(hw);
2107 }
2108
2109 static int
2110 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2111 {
2112         struct fm10k_fault fault;
2113         int err;
2114         const char *estr = "Unknown error";
2115
2116         /* Process PCA fault */
2117         if (eicr & FM10K_EICR_PCA_FAULT) {
2118                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2119                 if (err)
2120                         goto error;
2121                 switch (fault.type) {
2122                 case PCA_NO_FAULT:
2123                         estr = "PCA_NO_FAULT"; break;
2124                 case PCA_UNMAPPED_ADDR:
2125                         estr = "PCA_UNMAPPED_ADDR"; break;
2126                 case PCA_BAD_QACCESS_PF:
2127                         estr = "PCA_BAD_QACCESS_PF"; break;
2128                 case PCA_BAD_QACCESS_VF:
2129                         estr = "PCA_BAD_QACCESS_VF"; break;
2130                 case PCA_MALICIOUS_REQ:
2131                         estr = "PCA_MALICIOUS_REQ"; break;
2132                 case PCA_POISONED_TLP:
2133                         estr = "PCA_POISONED_TLP"; break;
2134                 case PCA_TLP_ABORT:
2135                         estr = "PCA_TLP_ABORT"; break;
2136                 default:
2137                         goto error;
2138                 }
2139                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2140                         estr, fault.func ? "VF" : "PF", fault.func,
2141                         fault.address, fault.specinfo);
2142         }
2143
2144         /* Process THI fault */
2145         if (eicr & FM10K_EICR_THI_FAULT) {
2146                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2147                 if (err)
2148                         goto error;
2149                 switch (fault.type) {
2150                 case THI_NO_FAULT:
2151                         estr = "THI_NO_FAULT"; break;
2152                 case THI_MAL_DIS_Q_FAULT:
2153                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2154                 default:
2155                         goto error;
2156                 }
2157                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2158                         estr, fault.func ? "VF" : "PF", fault.func,
2159                         fault.address, fault.specinfo);
2160         }
2161
2162         /* Process FUM fault */
2163         if (eicr & FM10K_EICR_FUM_FAULT) {
2164                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2165                 if (err)
2166                         goto error;
2167                 switch (fault.type) {
2168                 case FUM_NO_FAULT:
2169                         estr = "FUM_NO_FAULT"; break;
2170                 case FUM_UNMAPPED_ADDR:
2171                         estr = "FUM_UNMAPPED_ADDR"; break;
2172                 case FUM_POISONED_TLP:
2173                         estr = "FUM_POISONED_TLP"; break;
2174                 case FUM_BAD_VF_QACCESS:
2175                         estr = "FUM_BAD_VF_QACCESS"; break;
2176                 case FUM_ADD_DECODE_ERR:
2177                         estr = "FUM_ADD_DECODE_ERR"; break;
2178                 case FUM_RO_ERROR:
2179                         estr = "FUM_RO_ERROR"; break;
2180                 case FUM_QPRC_CRC_ERROR:
2181                         estr = "FUM_QPRC_CRC_ERROR"; break;
2182                 case FUM_CSR_TIMEOUT:
2183                         estr = "FUM_CSR_TIMEOUT"; break;
2184                 case FUM_INVALID_TYPE:
2185                         estr = "FUM_INVALID_TYPE"; break;
2186                 case FUM_INVALID_LENGTH:
2187                         estr = "FUM_INVALID_LENGTH"; break;
2188                 case FUM_INVALID_BE:
2189                         estr = "FUM_INVALID_BE"; break;
2190                 case FUM_INVALID_ALIGN:
2191                         estr = "FUM_INVALID_ALIGN"; break;
2192                 default:
2193                         goto error;
2194                 }
2195                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2196                         estr, fault.func ? "VF" : "PF", fault.func,
2197                         fault.address, fault.specinfo);
2198         }
2199
2200         return 0;
2201 error:
2202         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2203         return err;
2204 }
2205
2206 /**
2207  * PF interrupt handler triggered by NIC for handling specific interrupt.
2208  *
2209  * @param handle
2210  *  Pointer to interrupt handle.
2211  * @param param
2212  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2213  *
2214  * @return
2215  *  void
2216  */
2217 static void
2218 fm10k_dev_interrupt_handler_pf(
2219                         __rte_unused struct rte_intr_handle *handle,
2220                         void *param)
2221 {
2222         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2223         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2224         uint32_t cause, status;
2225
2226         if (hw->mac.type != fm10k_mac_pf)
2227                 return;
2228
2229         cause = FM10K_READ_REG(hw, FM10K_EICR);
2230
2231         /* Handle PCI fault cases */
2232         if (cause & FM10K_EICR_FAULT_MASK) {
2233                 PMD_INIT_LOG(ERR, "INT: find fault!");
2234                 fm10k_dev_handle_fault(hw, cause);
2235         }
2236
2237         /* Handle switch up/down */
2238         if (cause & FM10K_EICR_SWITCHNOTREADY)
2239                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2240
2241         if (cause & FM10K_EICR_SWITCHREADY)
2242                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2243
2244         /* Handle mailbox message */
2245         fm10k_mbx_lock(hw);
2246         hw->mbx.ops.process(hw, &hw->mbx);
2247         fm10k_mbx_unlock(hw);
2248
2249         /* Handle SRAM error */
2250         if (cause & FM10K_EICR_SRAMERROR) {
2251                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2252
2253                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2254                 /* Write to clear pending bits */
2255                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2256
2257                 /* Todo: print out error message after shared code  updates */
2258         }
2259
2260         /* Clear these 3 events if having any */
2261         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2262                  FM10K_EICR_SWITCHREADY;
2263         if (cause)
2264                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2265
2266         /* Re-enable interrupt from device side */
2267         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2268                                         FM10K_ITR_MASK_CLEAR);
2269         /* Re-enable interrupt from host side */
2270         rte_intr_enable(&(dev->pci_dev->intr_handle));
2271 }
2272
2273 /**
2274  * VF interrupt handler triggered by NIC for handling specific interrupt.
2275  *
2276  * @param handle
2277  *  Pointer to interrupt handle.
2278  * @param param
2279  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2280  *
2281  * @return
2282  *  void
2283  */
2284 static void
2285 fm10k_dev_interrupt_handler_vf(
2286                         __rte_unused struct rte_intr_handle *handle,
2287                         void *param)
2288 {
2289         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2290         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2291
2292         if (hw->mac.type != fm10k_mac_vf)
2293                 return;
2294
2295         /* Handle mailbox message if lock is acquired */
2296         fm10k_mbx_lock(hw);
2297         hw->mbx.ops.process(hw, &hw->mbx);
2298         fm10k_mbx_unlock(hw);
2299
2300         /* Re-enable interrupt from device side */
2301         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2302                                         FM10K_ITR_MASK_CLEAR);
2303         /* Re-enable interrupt from host side */
2304         rte_intr_enable(&(dev->pci_dev->intr_handle));
2305 }
2306
2307 /* Mailbox message handler in VF */
2308 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2309         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2310         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2311         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2312         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2313 };
2314
2315 /* Mailbox message handler in PF */
2316 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
2317         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
2318         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
2319         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
2320         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
2321         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
2322         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
2323         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2324 };
2325
2326 static int
2327 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2328 {
2329         int err;
2330
2331         /* Initialize mailbox lock */
2332         fm10k_mbx_initlock(hw);
2333
2334         /* Replace default message handler with new ones */
2335         if (hw->mac.type == fm10k_mac_pf)
2336                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
2337         else
2338                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2339
2340         if (err) {
2341                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2342                                 err);
2343                 return err;
2344         }
2345         /* Connect to SM for PF device or PF for VF device */
2346         return hw->mbx.ops.connect(hw, &hw->mbx);
2347 }
2348
2349 static void
2350 fm10k_close_mbx_service(struct fm10k_hw *hw)
2351 {
2352         /* Disconnect from SM for PF device or PF for VF device */
2353         hw->mbx.ops.disconnect(hw, &hw->mbx);
2354 }
2355
2356 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2357         .dev_configure          = fm10k_dev_configure,
2358         .dev_start              = fm10k_dev_start,
2359         .dev_stop               = fm10k_dev_stop,
2360         .dev_close              = fm10k_dev_close,
2361         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2362         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2363         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2364         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2365         .stats_get              = fm10k_stats_get,
2366         .xstats_get             = fm10k_xstats_get,
2367         .stats_reset            = fm10k_stats_reset,
2368         .xstats_reset           = fm10k_stats_reset,
2369         .link_update            = fm10k_link_update,
2370         .dev_infos_get          = fm10k_dev_infos_get,
2371         .vlan_filter_set        = fm10k_vlan_filter_set,
2372         .vlan_offload_set       = fm10k_vlan_offload_set,
2373         .mac_addr_add           = fm10k_macaddr_add,
2374         .mac_addr_remove        = fm10k_macaddr_remove,
2375         .rx_queue_start         = fm10k_dev_rx_queue_start,
2376         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2377         .tx_queue_start         = fm10k_dev_tx_queue_start,
2378         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2379         .rx_queue_setup         = fm10k_rx_queue_setup,
2380         .rx_queue_release       = fm10k_rx_queue_release,
2381         .tx_queue_setup         = fm10k_tx_queue_setup,
2382         .tx_queue_release       = fm10k_tx_queue_release,
2383         .reta_update            = fm10k_reta_update,
2384         .reta_query             = fm10k_reta_query,
2385         .rss_hash_update        = fm10k_rss_hash_update,
2386         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2387 };
2388
2389 static void __attribute__((cold))
2390 fm10k_set_tx_function(struct rte_eth_dev *dev)
2391 {
2392         struct fm10k_tx_queue *txq;
2393         int i;
2394         int use_sse = 1;
2395
2396         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2397                 txq = dev->data->tx_queues[i];
2398                 if ((txq->txq_flags & FM10K_SIMPLE_TX_FLAG) !=
2399                         FM10K_SIMPLE_TX_FLAG) {
2400                         use_sse = 0;
2401                         break;
2402                 }
2403         }
2404
2405         if (use_sse) {
2406                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2407                         txq = dev->data->tx_queues[i];
2408                         fm10k_txq_vec_setup(txq);
2409                 }
2410                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2411         } else
2412                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2413 }
2414
2415 static void __attribute__((cold))
2416 fm10k_set_rx_function(struct rte_eth_dev *dev)
2417 {
2418         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2419         uint16_t i, rx_using_sse;
2420
2421         /* In order to allow Vector Rx there are a few configuration
2422          * conditions to be met.
2423          */
2424         if (!fm10k_rx_vec_condition_check(dev) && dev_info->rx_vec_allowed) {
2425                 if (dev->data->scattered_rx)
2426                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2427                 else
2428                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2429         } else if (dev->data->scattered_rx)
2430                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2431
2432         rx_using_sse =
2433                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2434                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2435
2436         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2437                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2438
2439                 rxq->rx_using_sse = rx_using_sse;
2440         }
2441 }
2442
2443 static void
2444 fm10k_params_init(struct rte_eth_dev *dev)
2445 {
2446         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2447         struct fm10k_dev_info *info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2448
2449         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2450          * there is no way to get link status without reading BAR4.  Until this
2451          * works, assume we have maximum bandwidth.
2452          * @todo - fix bus info
2453          */
2454         hw->bus_caps.speed = fm10k_bus_speed_8000;
2455         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2456         hw->bus_caps.payload = fm10k_bus_payload_512;
2457         hw->bus.speed = fm10k_bus_speed_8000;
2458         hw->bus.width = fm10k_bus_width_pcie_x8;
2459         hw->bus.payload = fm10k_bus_payload_256;
2460
2461         info->rx_vec_allowed = true;
2462 }
2463
2464 static int
2465 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2466 {
2467         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2468         int diag;
2469         struct fm10k_macvlan_filter_info *macvlan;
2470
2471         PMD_INIT_FUNC_TRACE();
2472
2473         dev->dev_ops = &fm10k_eth_dev_ops;
2474         dev->rx_pkt_burst = &fm10k_recv_pkts;
2475         dev->tx_pkt_burst = &fm10k_xmit_pkts;
2476
2477         /* only initialize in the primary process */
2478         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2479                 return 0;
2480
2481         rte_eth_copy_pci_info(dev, dev->pci_dev);
2482
2483         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2484         memset(macvlan, 0, sizeof(*macvlan));
2485         /* Vendor and Device ID need to be set before init of shared code */
2486         memset(hw, 0, sizeof(*hw));
2487         hw->device_id = dev->pci_dev->id.device_id;
2488         hw->vendor_id = dev->pci_dev->id.vendor_id;
2489         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
2490         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
2491         hw->revision_id = 0;
2492         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2493         if (hw->hw_addr == NULL) {
2494                 PMD_INIT_LOG(ERR, "Bad mem resource."
2495                         " Try to blacklist unused devices.");
2496                 return -EIO;
2497         }
2498
2499         /* Store fm10k_adapter pointer */
2500         hw->back = dev->data->dev_private;
2501
2502         /* Initialize the shared code */
2503         diag = fm10k_init_shared_code(hw);
2504         if (diag != FM10K_SUCCESS) {
2505                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2506                 return -EIO;
2507         }
2508
2509         /* Initialize parameters */
2510         fm10k_params_init(dev);
2511
2512         /* Initialize the hw */
2513         diag = fm10k_init_hw(hw);
2514         if (diag != FM10K_SUCCESS) {
2515                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2516                 return -EIO;
2517         }
2518
2519         /* Initialize MAC address(es) */
2520         dev->data->mac_addrs = rte_zmalloc("fm10k",
2521                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2522         if (dev->data->mac_addrs == NULL) {
2523                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2524                 return -ENOMEM;
2525         }
2526
2527         diag = fm10k_read_mac_addr(hw);
2528
2529         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2530                         &dev->data->mac_addrs[0]);
2531
2532         if (diag != FM10K_SUCCESS ||
2533                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2534
2535                 /* Generate a random addr */
2536                 eth_random_addr(hw->mac.addr);
2537                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2538                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2539                 &dev->data->mac_addrs[0]);
2540         }
2541
2542         /* Reset the hw statistics */
2543         fm10k_stats_reset(dev);
2544
2545         /* Reset the hw */
2546         diag = fm10k_reset_hw(hw);
2547         if (diag != FM10K_SUCCESS) {
2548                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2549                 return -EIO;
2550         }
2551
2552         /* Setup mailbox service */
2553         diag = fm10k_setup_mbx_service(hw);
2554         if (diag != FM10K_SUCCESS) {
2555                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2556                 return -EIO;
2557         }
2558
2559         /*PF/VF has different interrupt handling mechanism */
2560         if (hw->mac.type == fm10k_mac_pf) {
2561                 /* register callback func to eal lib */
2562                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2563                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2564
2565                 /* enable MISC interrupt */
2566                 fm10k_dev_enable_intr_pf(dev);
2567         } else { /* VF */
2568                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2569                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2570
2571                 fm10k_dev_enable_intr_vf(dev);
2572         }
2573
2574         /* Enable uio intr after callback registered */
2575         rte_intr_enable(&(dev->pci_dev->intr_handle));
2576
2577         hw->mac.ops.update_int_moderator(hw);
2578
2579         /* Make sure Switch Manager is ready before going forward. */
2580         if (hw->mac.type == fm10k_mac_pf) {
2581                 int switch_ready = 0;
2582                 int i;
2583
2584                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2585                         fm10k_mbx_lock(hw);
2586                         hw->mac.ops.get_host_state(hw, &switch_ready);
2587                         fm10k_mbx_unlock(hw);
2588                         if (switch_ready)
2589                                 break;
2590                         /* Delay some time to acquire async LPORT_MAP info. */
2591                         rte_delay_us(WAIT_SWITCH_MSG_US);
2592                 }
2593
2594                 if (switch_ready == 0) {
2595                         PMD_INIT_LOG(ERR, "switch is not ready");
2596                         return -1;
2597                 }
2598         }
2599
2600         /*
2601          * Below function will trigger operations on mailbox, acquire lock to
2602          * avoid race condition from interrupt handler. Operations on mailbox
2603          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2604          * will handle and generate an interrupt to our side. Then,  FIFO in
2605          * mailbox will be touched.
2606          */
2607         fm10k_mbx_lock(hw);
2608         /* Enable port first */
2609         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, 1, 1);
2610
2611         /* Set unicast mode by default. App can change to other mode in other
2612          * API func.
2613          */
2614         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2615                                         FM10K_XCAST_MODE_NONE);
2616
2617         fm10k_mbx_unlock(hw);
2618
2619         /* Add default mac address */
2620         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2621                 MAIN_VSI_POOL_NUMBER);
2622
2623         return 0;
2624 }
2625
2626 static int
2627 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
2628 {
2629         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2630
2631         PMD_INIT_FUNC_TRACE();
2632
2633         /* only uninitialize in the primary process */
2634         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2635                 return 0;
2636
2637         /* safe to close dev here */
2638         fm10k_dev_close(dev);
2639
2640         dev->dev_ops = NULL;
2641         dev->rx_pkt_burst = NULL;
2642         dev->tx_pkt_burst = NULL;
2643
2644         /* disable uio/vfio intr */
2645         rte_intr_disable(&(dev->pci_dev->intr_handle));
2646
2647         /*PF/VF has different interrupt handling mechanism */
2648         if (hw->mac.type == fm10k_mac_pf) {
2649                 /* disable interrupt */
2650                 fm10k_dev_disable_intr_pf(dev);
2651
2652                 /* unregister callback func to eal lib */
2653                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2654                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2655         } else {
2656                 /* disable interrupt */
2657                 fm10k_dev_disable_intr_vf(dev);
2658
2659                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2660                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2661         }
2662
2663         /* free mac memory */
2664         if (dev->data->mac_addrs) {
2665                 rte_free(dev->data->mac_addrs);
2666                 dev->data->mac_addrs = NULL;
2667         }
2668
2669         memset(hw, 0, sizeof(*hw));
2670
2671         return 0;
2672 }
2673
2674 /*
2675  * The set of PCI devices this driver supports. This driver will enable both PF
2676  * and SRIOV-VF devices.
2677  */
2678 static const struct rte_pci_id pci_id_fm10k_map[] = {
2679 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2680 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2681 #include "rte_pci_dev_ids.h"
2682         { .vendor_id = 0, /* sentinel */ },
2683 };
2684
2685 static struct eth_driver rte_pmd_fm10k = {
2686         .pci_drv = {
2687                 .name = "rte_pmd_fm10k",
2688                 .id_table = pci_id_fm10k_map,
2689                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
2690         },
2691         .eth_dev_init = eth_fm10k_dev_init,
2692         .eth_dev_uninit = eth_fm10k_dev_uninit,
2693         .dev_private_size = sizeof(struct fm10k_adapter),
2694 };
2695
2696 /*
2697  * Driver initialization routine.
2698  * Invoked once at EAL init time.
2699  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2700  */
2701 static int
2702 rte_pmd_fm10k_init(__rte_unused const char *name,
2703         __rte_unused const char *params)
2704 {
2705         PMD_INIT_FUNC_TRACE();
2706         rte_eth_driver_register(&rte_pmd_fm10k);
2707         return 0;
2708 }
2709
2710 static struct rte_driver rte_fm10k_driver = {
2711         .type = PMD_PDEV,
2712         .init = rte_pmd_fm10k_init,
2713 };
2714
2715 PMD_REGISTER_DRIVER(rte_fm10k_driver);