drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_ethdev_pci.h>
  36 #include <rte_malloc.h>
  37 #include <rte_memzone.h>
  38 #include <rte_string_fns.h>
  39 #include <rte_dev.h>
  40 #include <rte_spinlock.h>
  41 #include <rte_kvargs.h>
  42
  43 #include "fm10k.h"
  44 #include "base/fm10k_api.h"
  45
  46 /* Default delay to acquire mailbox lock */
  47 #define FM10K_MBXLOCK_DELAY_US 20
  48 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  49
  50 #define MAIN_VSI_POOL_NUMBER 0
  51
  52 /* Max try times to acquire switch status */
  53 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  54 /* Wait interval to get switch status */
  55 #define WAIT_SWITCH_MSG_US    100000
  56 /* A period of quiescence for switch */
  57 #define FM10K_SWITCH_QUIESCE_US 10000
  58 /* Number of chars per uint32 type */
  59 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  60 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  61
  62 /* default 1:1 map from queue ID to interrupt vector ID */
  63 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  64
  65 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  66 #define MAX_LPORT_NUM    128
  67 #define GLORT_FD_Q_BASE  0x40
  68 #define GLORT_PF_MASK    0xFFC0
  69 #define GLORT_FD_MASK    GLORT_PF_MASK
  70 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  71
  72 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  73 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  74 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  75 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  76 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  77 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  78 static int
  79 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  80 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  81         const u8 *mac, bool add, uint32_t pool);
  82 static void fm10k_tx_queue_release(void *queue);
  83 static void fm10k_rx_queue_release(void *queue);
  84 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  85 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  86 static int fm10k_check_ftag(struct rte_devargs *devargs);
  87 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  88
  89 struct fm10k_xstats_name_off {
  90         char name[RTE_ETH_XSTATS_NAME_SIZE];
  91         unsigned offset;
  92 };
  93
  94 struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  95         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  96         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  97         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  98         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  99         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
 100         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
 101         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
 102         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
 103                 nodesc_drop)},
 104 };
 105
 106 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
 107                 sizeof(fm10k_hw_stats_strings[0]))
 108
 109 struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
 110         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
 111         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
 112         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
 113 };
 114
 115 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
 116                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
 117
 118 struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 119         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 120         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 121 };
 122
 123 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 124                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 125
 126 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 127                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 128 static int
 129 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 130
 131 static void
 132 fm10k_mbx_initlock(struct fm10k_hw *hw)
 133 {
 134         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 135 }
 136
 137 static void
 138 fm10k_mbx_lock(struct fm10k_hw *hw)
 139 {
 140         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 141                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 142 }
 143
 144 static void
 145 fm10k_mbx_unlock(struct fm10k_hw *hw)
 146 {
 147         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 148 }
 149
 150 /* Stubs needed for linkage when vPMD is disabled */
 151 int __attribute__((weak))
 152 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 153 {
 154         return -1;
 155 }
 156
 157 uint16_t __attribute__((weak))
 158 fm10k_recv_pkts_vec(
 159         __rte_unused void *rx_queue,
 160         __rte_unused struct rte_mbuf **rx_pkts,
 161         __rte_unused uint16_t nb_pkts)
 162 {
 163         return 0;
 164 }
 165
 166 uint16_t __attribute__((weak))
 167 fm10k_recv_scattered_pkts_vec(
 168                 __rte_unused void *rx_queue,
 169                 __rte_unused struct rte_mbuf **rx_pkts,
 170                 __rte_unused uint16_t nb_pkts)
 171 {
 172         return 0;
 173 }
 174
 175 int __attribute__((weak))
 176 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 177
 178 {
 179         return -1;
 180 }
 181
 182 void __attribute__((weak))
 183 fm10k_rx_queue_release_mbufs_vec(
 184                 __rte_unused struct fm10k_rx_queue *rxq)
 185 {
 186         return;
 187 }
 188
 189 void __attribute__((weak))
 190 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 191 {
 192         return;
 193 }
 194
 195 int __attribute__((weak))
 196 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 197 {
 198         return -1;
 199 }
 200
 201 uint16_t __attribute__((weak))
 202 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 203                            __rte_unused struct rte_mbuf **tx_pkts,
 204                            __rte_unused uint16_t nb_pkts)
 205 {
 206         return 0;
 207 }
 208
 209 /*
 210  * reset queue to initial state, allocate software buffers used when starting
 211  * device.
 212  * return 0 on success
 213  * return -ENOMEM if buffers cannot be allocated
 214  * return -EINVAL if buffers do not satisfy alignment condition
 215  */
 216 static inline int
 217 rx_queue_reset(struct fm10k_rx_queue *q)
 218 {
 219         static const union fm10k_rx_desc zero = {{0} };
 220         uint64_t dma_addr;
 221         int i, diag;
 222         PMD_INIT_FUNC_TRACE();
 223
 224         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 225         if (diag != 0)
 226                 return -ENOMEM;
 227
 228         for (i = 0; i < q->nb_desc; ++i) {
 229                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 230                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 231                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 232                                                 q->nb_desc);
 233                         return -EINVAL;
 234                 }
 235                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 236                 q->hw_ring[i].q.pkt_addr = dma_addr;
 237                 q->hw_ring[i].q.hdr_addr = dma_addr;
 238         }
 239
 240         /* initialize extra software ring entries. Space for these extra
 241          * entries is always allocated.
 242          */
 243         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 244         for (i = 0; i < q->nb_fake_desc; ++i) {
 245                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 246                 q->hw_ring[q->nb_desc + i] = zero;
 247         }
 248
 249         q->next_dd = 0;
 250         q->next_alloc = 0;
 251         q->next_trigger = q->alloc_thresh - 1;
 252         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 253         q->rxrearm_start = 0;
 254         q->rxrearm_nb = 0;
 255
 256         return 0;
 257 }
 258
 259 /*
 260  * clean queue, descriptor rings, free software buffers used when stopping
 261  * device.
 262  */
 263 static inline void
 264 rx_queue_clean(struct fm10k_rx_queue *q)
 265 {
 266         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 267         uint32_t i;
 268         PMD_INIT_FUNC_TRACE();
 269
 270         /* zero descriptor rings */
 271         for (i = 0; i < q->nb_desc; ++i)
 272                 q->hw_ring[i] = zero;
 273
 274         /* zero faked descriptors */
 275         for (i = 0; i < q->nb_fake_desc; ++i)
 276                 q->hw_ring[q->nb_desc + i] = zero;
 277
 278         /* vPMD driver has a different way of releasing mbufs. */
 279         if (q->rx_using_sse) {
 280                 fm10k_rx_queue_release_mbufs_vec(q);
 281                 return;
 282         }
 283
 284         /* free software buffers */
 285         for (i = 0; i < q->nb_desc; ++i) {
 286                 if (q->sw_ring[i]) {
 287                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 288                         q->sw_ring[i] = NULL;
 289                 }
 290         }
 291 }
 292
 293 /*
 294  * free all queue memory used when releasing the queue (i.e. configure)
 295  */
 296 static inline void
 297 rx_queue_free(struct fm10k_rx_queue *q)
 298 {
 299         PMD_INIT_FUNC_TRACE();
 300         if (q) {
 301                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 302                 rx_queue_clean(q);
 303                 if (q->sw_ring) {
 304                         rte_free(q->sw_ring);
 305                         q->sw_ring = NULL;
 306                 }
 307                 rte_free(q);
 308                 q = NULL;
 309         }
 310 }
 311
 312 /*
 313  * disable RX queue, wait unitl HW finished necessary flush operation
 314  */
 315 static inline int
 316 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 317 {
 318         uint32_t reg, i;
 319
 320         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 321         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 322                         reg & ~FM10K_RXQCTL_ENABLE);
 323
 324         /* Wait 100us at most */
 325         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 326                 rte_delay_us(1);
 327                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 328                 if (!(reg & FM10K_RXQCTL_ENABLE))
 329                         break;
 330         }
 331
 332         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 333                 return -1;
 334
 335         return 0;
 336 }
 337
 338 /*
 339  * reset queue to initial state, allocate software buffers used when starting
 340  * device
 341  */
 342 static inline void
 343 tx_queue_reset(struct fm10k_tx_queue *q)
 344 {
 345         PMD_INIT_FUNC_TRACE();
 346         q->last_free = 0;
 347         q->next_free = 0;
 348         q->nb_used = 0;
 349         q->nb_free = q->nb_desc - 1;
 350         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 351         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 352 }
 353
 354 /*
 355  * clean queue, descriptor rings, free software buffers used when stopping
 356  * device
 357  */
 358 static inline void
 359 tx_queue_clean(struct fm10k_tx_queue *q)
 360 {
 361         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 362         uint32_t i;
 363         PMD_INIT_FUNC_TRACE();
 364
 365         /* zero descriptor rings */
 366         for (i = 0; i < q->nb_desc; ++i)
 367                 q->hw_ring[i] = zero;
 368
 369         /* free software buffers */
 370         for (i = 0; i < q->nb_desc; ++i) {
 371                 if (q->sw_ring[i]) {
 372                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 373                         q->sw_ring[i] = NULL;
 374                 }
 375         }
 376 }
 377
 378 /*
 379  * free all queue memory used when releasing the queue (i.e. configure)
 380  */
 381 static inline void
 382 tx_queue_free(struct fm10k_tx_queue *q)
 383 {
 384         PMD_INIT_FUNC_TRACE();
 385         if (q) {
 386                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 387                 tx_queue_clean(q);
 388                 if (q->rs_tracker.list) {
 389                         rte_free(q->rs_tracker.list);
 390                         q->rs_tracker.list = NULL;
 391                 }
 392                 if (q->sw_ring) {
 393                         rte_free(q->sw_ring);
 394                         q->sw_ring = NULL;
 395                 }
 396                 rte_free(q);
 397                 q = NULL;
 398         }
 399 }
 400
 401 /*
 402  * disable TX queue, wait unitl HW finished necessary flush operation
 403  */
 404 static inline int
 405 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 406 {
 407         uint32_t reg, i;
 408
 409         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 410         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 411                         reg & ~FM10K_TXDCTL_ENABLE);
 412
 413         /* Wait 100us at most */
 414         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 415                 rte_delay_us(1);
 416                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 417                 if (!(reg & FM10K_TXDCTL_ENABLE))
 418                         break;
 419         }
 420
 421         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 422                 return -1;
 423
 424         return 0;
 425 }
 426
 427 static int
 428 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 429 {
 430         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 431         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 432         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 433         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 434
 435         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 436
 437         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 438                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 439                 return -EINVAL;
 440         }
 441
 442         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 443                 return 0;
 444
 445         if (hw->mac.type == fm10k_mac_vf) {
 446                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 447                 return -EINVAL;
 448         }
 449
 450         /* Check VMDQ queue pool number */
 451         if (vmdq_conf->nb_queue_pools >
 452                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 453                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 454                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 455                         vmdq_conf->nb_queue_pools);
 456                 return -EINVAL;
 457         }
 458
 459         return 0;
 460 }
 461
 462 static const struct fm10k_txq_ops def_txq_ops = {
 463         .reset = tx_queue_reset,
 464 };
 465
 466 static int
 467 fm10k_dev_configure(struct rte_eth_dev *dev)
 468 {
 469         int ret;
 470
 471         PMD_INIT_FUNC_TRACE();
 472
 473         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 474                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 475         /* multipe queue mode checking */
 476         ret  = fm10k_check_mq_mode(dev);
 477         if (ret != 0) {
 478                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 479                             ret);
 480                 return ret;
 481         }
 482
 483         return 0;
 484 }
 485
 486 /* fls = find last set bit = 32 minus the number of leading zeros */
 487 #ifndef fls
 488 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 489 #endif
 490
 491 static void
 492 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 493 {
 494         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 495         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 496         uint32_t i;
 497
 498         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 499
 500         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 501                 if (!vmdq_conf->pool_map[i].pools)
 502                         continue;
 503                 fm10k_mbx_lock(hw);
 504                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 505                 fm10k_mbx_unlock(hw);
 506         }
 507 }
 508
 509 static void
 510 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 511 {
 512         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 513
 514         /* Add default mac address */
 515         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 516                 MAIN_VSI_POOL_NUMBER);
 517 }
 518
 519 static void
 520 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 521 {
 522         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 523         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 524         uint32_t mrqc, *key, i, reta, j;
 525         uint64_t hf;
 526
 527 #define RSS_KEY_SIZE 40
 528         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 529                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 530                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 531                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 532                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 533                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 534         };
 535
 536         if (dev->data->nb_rx_queues == 1 ||
 537             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 538             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 539                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 540                 return;
 541         }
 542
 543         /* random key is rss_intel_key (default) or user provided (rss_key) */
 544         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 545                 key = (uint32_t *)rss_intel_key;
 546         else
 547                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 548
 549         /* Now fill our hash function seeds, 4 bytes at a time */
 550         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 551                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 552
 553         /*
 554          * Fill in redirection table
 555          * The byte-swap is needed because NIC registers are in
 556          * little-endian order.
 557          */
 558         reta = 0;
 559         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 560                 if (j == dev->data->nb_rx_queues)
 561                         j = 0;
 562                 reta = (reta << CHAR_BIT) | j;
 563                 if ((i & 3) == 3)
 564                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 565                                         rte_bswap32(reta));
 566         }
 567
 568         /*
 569          * Generate RSS hash based on packet types, TCP/UDP
 570          * port numbers and/or IPv4/v6 src and dst addresses
 571          */
 572         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 573         mrqc = 0;
 574         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 575         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 576         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 577         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 578         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 579         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 580         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 581         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 582         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 583
 584         if (mrqc == 0) {
 585                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 586                         "supported", hf);
 587                 return;
 588         }
 589
 590         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 591 }
 592
 593 static void
 594 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 595 {
 596         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 597         uint32_t i;
 598
 599         for (i = 0; i < nb_lport_new; i++) {
 600                 /* Set unicast mode by default. App can change
 601                  * to other mode in other API func.
 602                  */
 603                 fm10k_mbx_lock(hw);
 604                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 605                         FM10K_XCAST_MODE_NONE);
 606                 fm10k_mbx_unlock(hw);
 607         }
 608 }
 609
 610 static void
 611 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 612 {
 613         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 614         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 615         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 616         struct fm10k_macvlan_filter_info *macvlan;
 617         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 618         uint16_t nb_lport_new;
 619
 620         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 621         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 622
 623         fm10k_dev_rss_configure(dev);
 624
 625         /* only PF supports VMDQ */
 626         if (hw->mac.type != fm10k_mac_pf)
 627                 return;
 628
 629         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 630                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 631
 632         /* no pool number change, no need to update logic port and VLAN/MAC */
 633         if (macvlan->nb_queue_pools == nb_queue_pools)
 634                 return;
 635
 636         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 637         fm10k_dev_logic_port_update(dev, nb_lport_new);
 638
 639         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 640         memset(dev->data->mac_addrs, 0,
 641                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 642         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 643                 &dev->data->mac_addrs[0]);
 644         memset(macvlan, 0, sizeof(*macvlan));
 645         macvlan->nb_queue_pools = nb_queue_pools;
 646
 647         if (nb_queue_pools)
 648                 fm10k_dev_vmdq_rx_configure(dev);
 649         else
 650                 fm10k_dev_pf_main_vsi_reset(dev);
 651 }
 652
 653 static int
 654 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 655 {
 656         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 657         int i, ret;
 658         struct fm10k_tx_queue *txq;
 659         uint64_t base_addr;
 660         uint32_t size;
 661
 662         /* Disable TXINT to avoid possible interrupt */
 663         for (i = 0; i < hw->mac.max_queues; i++)
 664                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 665                                 3 << FM10K_TXINT_TIMER_SHIFT);
 666
 667         /* Setup TX queue */
 668         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 669                 txq = dev->data->tx_queues[i];
 670                 base_addr = txq->hw_ring_phys_addr;
 671                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 672
 673                 /* disable queue to avoid issues while updating state */
 674                 ret = tx_queue_disable(hw, i);
 675                 if (ret) {
 676                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 677                         return -1;
 678                 }
 679                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 680                  * register is read-only for VF.
 681                  */
 682                 if (fm10k_check_ftag(dev->device->devargs)) {
 683                         if (hw->mac.type == fm10k_mac_pf) {
 684                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 685                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 686                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 687                         } else {
 688                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 689                                 return -ENOTSUP;
 690                         }
 691                 }
 692
 693                 /* set location and size for descriptor ring */
 694                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 695                                 base_addr & UINT64_LOWER_32BITS_MASK);
 696                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 697                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 698                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 699
 700                 /* assign default SGLORT for each TX queue by PF */
 701                 if (hw->mac.type == fm10k_mac_pf)
 702                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 703         }
 704
 705         /* set up vector or scalar TX function as appropriate */
 706         fm10k_set_tx_function(dev);
 707
 708         return 0;
 709 }
 710
 711 static int
 712 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 713 {
 714         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 715         struct fm10k_macvlan_filter_info *macvlan;
 716         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 717         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 718         int i, ret;
 719         struct fm10k_rx_queue *rxq;
 720         uint64_t base_addr;
 721         uint32_t size;
 722         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 723         uint32_t logic_port = hw->mac.dglort_map;
 724         uint16_t buf_size;
 725         uint16_t queue_stride = 0;
 726
 727         /* enable RXINT for interrupt mode */
 728         i = 0;
 729         if (rte_intr_dp_is_en(intr_handle)) {
 730                 for (; i < dev->data->nb_rx_queues; i++) {
 731                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 732                         if (hw->mac.type == fm10k_mac_pf)
 733                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 734                                         FM10K_ITR_AUTOMASK |
 735                                         FM10K_ITR_MASK_CLEAR);
 736                         else
 737                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 738                                         FM10K_ITR_AUTOMASK |
 739                                         FM10K_ITR_MASK_CLEAR);
 740                 }
 741         }
 742         /* Disable other RXINT to avoid possible interrupt */
 743         for (; i < hw->mac.max_queues; i++)
 744                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 745                         3 << FM10K_RXINT_TIMER_SHIFT);
 746
 747         /* Setup RX queues */
 748         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 749                 rxq = dev->data->rx_queues[i];
 750                 base_addr = rxq->hw_ring_phys_addr;
 751                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 752
 753                 /* disable queue to avoid issues while updating state */
 754                 ret = rx_queue_disable(hw, i);
 755                 if (ret) {
 756                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 757                         return -1;
 758                 }
 759
 760                 /* Setup the Base and Length of the Rx Descriptor Ring */
 761                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 762                                 base_addr & UINT64_LOWER_32BITS_MASK);
 763                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 764                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 765                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 766
 767                 /* Configure the Rx buffer size for one buff without split */
 768                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 769                         RTE_PKTMBUF_HEADROOM);
 770                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 771                  * reserved for this purpose, and the worst case could be 511B.
 772                  * But SRR reg assumes all buffers have the same size. In order
 773                  * to fill the gap, we'll have to consider the worst case and
 774                  * assume 512B is reserved. If we don't do so, it's possible
 775                  * for HW to overwrite data to next mbuf.
 776                  */
 777                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 778
 779                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 780                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 781                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 782
 783                 /* It adds dual VLAN length for supporting dual VLAN */
 784                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 785                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 786                         dev->data->dev_conf.rxmode.enable_scatter) {
 787                         uint32_t reg;
 788                         dev->data->scattered_rx = 1;
 789                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 790                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 791                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 792                 }
 793
 794                 /* Enable drop on empty, it's RO for VF */
 795                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 796                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 797
 798                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 799                 FM10K_WRITE_FLUSH(hw);
 800         }
 801
 802         /* Configure VMDQ/RSS if applicable */
 803         fm10k_dev_mq_rx_configure(dev);
 804
 805         /* Decide the best RX function */
 806         fm10k_set_rx_function(dev);
 807
 808         /* update RX_SGLORT for loopback suppress*/
 809         if (hw->mac.type != fm10k_mac_pf)
 810                 return 0;
 811         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 812         if (macvlan->nb_queue_pools)
 813                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 814         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 815                 if (i && queue_stride && !(i % queue_stride))
 816                         logic_port++;
 817                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 818         }
 819
 820         return 0;
 821 }
 822
 823 static int
 824 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 825 {
 826         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 827         int err = -1;
 828         uint32_t reg;
 829         struct fm10k_rx_queue *rxq;
 830
 831         PMD_INIT_FUNC_TRACE();
 832
 833         if (rx_queue_id < dev->data->nb_rx_queues) {
 834                 rxq = dev->data->rx_queues[rx_queue_id];
 835                 err = rx_queue_reset(rxq);
 836                 if (err == -ENOMEM) {
 837                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 838                         return err;
 839                 } else if (err == -EINVAL) {
 840                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 841                                 " %d", err);
 842                         return err;
 843                 }
 844
 845                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 846                  * Note: this must be done AFTER the queue is enabled on real
 847                  * hardware, but BEFORE the queue is enabled when using the
 848                  * emulation platform. Do it in both places for now and remove
 849                  * this comment and the following two register writes when the
 850                  * emulation platform is no longer being used.
 851                  */
 852                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 853                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 854
 855                 /* Set PF ownership flag for PF devices */
 856                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 857                 if (hw->mac.type == fm10k_mac_pf)
 858                         reg |= FM10K_RXQCTL_PF;
 859                 reg |= FM10K_RXQCTL_ENABLE;
 860                 /* enable RX queue */
 861                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 862                 FM10K_WRITE_FLUSH(hw);
 863
 864                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 865                  * Note: this must be done AFTER the queue is enabled
 866                  */
 867                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 868                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 869                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 870         }
 871
 872         return err;
 873 }
 874
 875 static int
 876 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 877 {
 878         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 879
 880         PMD_INIT_FUNC_TRACE();
 881
 882         if (rx_queue_id < dev->data->nb_rx_queues) {
 883                 /* Disable RX queue */
 884                 rx_queue_disable(hw, rx_queue_id);
 885
 886                 /* Free mbuf and clean HW ring */
 887                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 888                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 889         }
 890
 891         return 0;
 892 }
 893
 894 static int
 895 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 896 {
 897         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 898         /** @todo - this should be defined in the shared code */
 899 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 900         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 901         int err = 0;
 902
 903         PMD_INIT_FUNC_TRACE();
 904
 905         if (tx_queue_id < dev->data->nb_tx_queues) {
 906                 struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 907
 908                 q->ops->reset(q);
 909
 910                 /* reset head and tail pointers */
 911                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 912                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 913
 914                 /* enable TX queue */
 915                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 916                                         FM10K_TXDCTL_ENABLE | txdctl);
 917                 FM10K_WRITE_FLUSH(hw);
 918                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 919         } else
 920                 err = -1;
 921
 922         return err;
 923 }
 924
 925 static int
 926 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 927 {
 928         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 929
 930         PMD_INIT_FUNC_TRACE();
 931
 932         if (tx_queue_id < dev->data->nb_tx_queues) {
 933                 tx_queue_disable(hw, tx_queue_id);
 934                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 935                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 936         }
 937
 938         return 0;
 939 }
 940
 941 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 942 {
 943         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 944                 != FM10K_DGLORTMAP_NONE);
 945 }
 946
 947 static void
 948 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 949 {
 950         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 951         int status;
 952
 953         PMD_INIT_FUNC_TRACE();
 954
 955         /* Return if it didn't acquire valid glort range */
 956         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 957                 return;
 958
 959         fm10k_mbx_lock(hw);
 960         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 961                                 FM10K_XCAST_MODE_PROMISC);
 962         fm10k_mbx_unlock(hw);
 963
 964         if (status != FM10K_SUCCESS)
 965                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 966 }
 967
 968 static void
 969 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 970 {
 971         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 972         uint8_t mode;
 973         int status;
 974
 975         PMD_INIT_FUNC_TRACE();
 976
 977         /* Return if it didn't acquire valid glort range */
 978         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 979                 return;
 980
 981         if (dev->data->all_multicast == 1)
 982                 mode = FM10K_XCAST_MODE_ALLMULTI;
 983         else
 984                 mode = FM10K_XCAST_MODE_NONE;
 985
 986         fm10k_mbx_lock(hw);
 987         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 988                                 mode);
 989         fm10k_mbx_unlock(hw);
 990
 991         if (status != FM10K_SUCCESS)
 992                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 993 }
 994
 995 static void
 996 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 997 {
 998         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 999         int status;
1000
1001         PMD_INIT_FUNC_TRACE();
1002
1003         /* Return if it didn't acquire valid glort range */
1004         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1005                 return;
1006
1007         /* If promiscuous mode is enabled, it doesn't make sense to enable
1008          * allmulticast and disable promiscuous since fm10k only can select
1009          * one of the modes.
1010          */
1011         if (dev->data->promiscuous) {
1012                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
1013                         "needn't enable allmulticast");
1014                 return;
1015         }
1016
1017         fm10k_mbx_lock(hw);
1018         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1019                                 FM10K_XCAST_MODE_ALLMULTI);
1020         fm10k_mbx_unlock(hw);
1021
1022         if (status != FM10K_SUCCESS)
1023                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
1024 }
1025
1026 static void
1027 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1028 {
1029         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1030         int status;
1031
1032         PMD_INIT_FUNC_TRACE();
1033
1034         /* Return if it didn't acquire valid glort range */
1035         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1036                 return;
1037
1038         if (dev->data->promiscuous) {
1039                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1040                         "since promisc mode is enabled");
1041                 return;
1042         }
1043
1044         fm10k_mbx_lock(hw);
1045         /* Change mode to unicast mode */
1046         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1047                                 FM10K_XCAST_MODE_NONE);
1048         fm10k_mbx_unlock(hw);
1049
1050         if (status != FM10K_SUCCESS)
1051                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1052 }
1053
1054 static void
1055 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1056 {
1057         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1058         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1059         uint16_t nb_queue_pools;
1060         struct fm10k_macvlan_filter_info *macvlan;
1061
1062         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1063         nb_queue_pools = macvlan->nb_queue_pools;
1064         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
1065         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
1066
1067         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1068         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1069         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1070                         hw->mac.dglort_map;
1071         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1072         /* Configure VMDQ/RSS DGlort Decoder */
1073         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1074
1075         /* Flow Director configurations, only queue number is valid. */
1076         dglortdec = fls(dev->data->nb_rx_queues - 1);
1077         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1078                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1079         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1080         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1081
1082         /* Invalidate all other GLORT entries */
1083         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1084                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1085                                 FM10K_DGLORTMAP_NONE);
1086 }
1087
1088 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1089 static int
1090 fm10k_dev_start(struct rte_eth_dev *dev)
1091 {
1092         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1093         int i, diag;
1094
1095         PMD_INIT_FUNC_TRACE();
1096
1097         /* stop, init, then start the hw */
1098         diag = fm10k_stop_hw(hw);
1099         if (diag != FM10K_SUCCESS) {
1100                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1101                 return -EIO;
1102         }
1103
1104         diag = fm10k_init_hw(hw);
1105         if (diag != FM10K_SUCCESS) {
1106                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1107                 return -EIO;
1108         }
1109
1110         diag = fm10k_start_hw(hw);
1111         if (diag != FM10K_SUCCESS) {
1112                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1113                 return -EIO;
1114         }
1115
1116         diag = fm10k_dev_tx_init(dev);
1117         if (diag) {
1118                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1119                 return diag;
1120         }
1121
1122         if (fm10k_dev_rxq_interrupt_setup(dev))
1123                 return -EIO;
1124
1125         diag = fm10k_dev_rx_init(dev);
1126         if (diag) {
1127                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1128                 return diag;
1129         }
1130
1131         if (hw->mac.type == fm10k_mac_pf)
1132                 fm10k_dev_dglort_map_configure(dev);
1133
1134         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1135                 struct fm10k_rx_queue *rxq;
1136                 rxq = dev->data->rx_queues[i];
1137
1138                 if (rxq->rx_deferred_start)
1139                         continue;
1140                 diag = fm10k_dev_rx_queue_start(dev, i);
1141                 if (diag != 0) {
1142                         int j;
1143                         for (j = 0; j < i; ++j)
1144                                 rx_queue_clean(dev->data->rx_queues[j]);
1145                         return diag;
1146                 }
1147         }
1148
1149         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1150                 struct fm10k_tx_queue *txq;
1151                 txq = dev->data->tx_queues[i];
1152
1153                 if (txq->tx_deferred_start)
1154                         continue;
1155                 diag = fm10k_dev_tx_queue_start(dev, i);
1156                 if (diag != 0) {
1157                         int j;
1158                         for (j = 0; j < i; ++j)
1159                                 tx_queue_clean(dev->data->tx_queues[j]);
1160                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1161                                 rx_queue_clean(dev->data->rx_queues[j]);
1162                         return diag;
1163                 }
1164         }
1165
1166         /* Update default vlan when not in VMDQ mode */
1167         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1168                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1169
1170         fm10k_link_update(dev, 0);
1171
1172         return 0;
1173 }
1174
1175 static void
1176 fm10k_dev_stop(struct rte_eth_dev *dev)
1177 {
1178         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1179         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1180         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1181         int i;
1182
1183         PMD_INIT_FUNC_TRACE();
1184
1185         if (dev->data->tx_queues)
1186                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1187                         fm10k_dev_tx_queue_stop(dev, i);
1188
1189         if (dev->data->rx_queues)
1190                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1191                         fm10k_dev_rx_queue_stop(dev, i);
1192
1193         /* Disable datapath event */
1194         if (rte_intr_dp_is_en(intr_handle)) {
1195                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1196                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1197                                 3 << FM10K_RXINT_TIMER_SHIFT);
1198                         if (hw->mac.type == fm10k_mac_pf)
1199                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1200                                         FM10K_ITR_MASK_SET);
1201                         else
1202                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1203                                         FM10K_ITR_MASK_SET);
1204                 }
1205         }
1206         /* Clean datapath event and queue/vec mapping */
1207         rte_intr_efd_disable(intr_handle);
1208         rte_free(intr_handle->intr_vec);
1209         intr_handle->intr_vec = NULL;
1210 }
1211
1212 static void
1213 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1214 {
1215         int i;
1216
1217         PMD_INIT_FUNC_TRACE();
1218
1219         if (dev->data->tx_queues) {
1220                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1221                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1222
1223                         tx_queue_free(txq);
1224                 }
1225         }
1226
1227         if (dev->data->rx_queues) {
1228                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1229                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1230         }
1231 }
1232
1233 static void
1234 fm10k_dev_close(struct rte_eth_dev *dev)
1235 {
1236         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1237
1238         PMD_INIT_FUNC_TRACE();
1239
1240         fm10k_mbx_lock(hw);
1241         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1242                 MAX_LPORT_NUM, false);
1243         fm10k_mbx_unlock(hw);
1244
1245         /* allow 10ms for device to quiesce */
1246         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
1247
1248         /* Stop mailbox service first */
1249         fm10k_close_mbx_service(hw);
1250         fm10k_dev_stop(dev);
1251         fm10k_dev_queue_release(dev);
1252         fm10k_stop_hw(hw);
1253 }
1254
1255 static int
1256 fm10k_link_update(struct rte_eth_dev *dev,
1257         __rte_unused int wait_to_complete)
1258 {
1259         struct fm10k_dev_info *dev_info =
1260                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1261         PMD_INIT_FUNC_TRACE();
1262
1263         /* The speed is ~50Gbps per Gen3 x8 PCIe interface. For now, we
1264          * leave the speed undefined since there is no 50Gbps Ethernet.
1265          */
1266         dev->data->dev_link.link_speed  = 0;
1267         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1268         dev->data->dev_link.link_status =
1269                 dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP;
1270
1271         return 0;
1272 }
1273
1274 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1275         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1276 {
1277         unsigned i, q;
1278         unsigned count = 0;
1279
1280         if (xstats_names != NULL) {
1281                 /* Note: limit checked in rte_eth_xstats_names() */
1282
1283                 /* Global stats */
1284                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1285                         snprintf(xstats_names[count].name,
1286                                 sizeof(xstats_names[count].name),
1287                                 "%s", fm10k_hw_stats_strings[count].name);
1288                         count++;
1289                 }
1290
1291                 /* PF queue stats */
1292                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1293                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1294                                 snprintf(xstats_names[count].name,
1295                                         sizeof(xstats_names[count].name),
1296                                         "rx_q%u_%s", q,
1297                                         fm10k_hw_stats_rx_q_strings[i].name);
1298                                 count++;
1299                         }
1300                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1301                                 snprintf(xstats_names[count].name,
1302                                         sizeof(xstats_names[count].name),
1303                                         "tx_q%u_%s", q,
1304                                         fm10k_hw_stats_tx_q_strings[i].name);
1305                                 count++;
1306                         }
1307                 }
1308         }
1309         return FM10K_NB_XSTATS;
1310 }
1311
1312 static int
1313 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1314                  unsigned n)
1315 {
1316         struct fm10k_hw_stats *hw_stats =
1317                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1318         unsigned i, q, count = 0;
1319
1320         if (n < FM10K_NB_XSTATS)
1321                 return FM10K_NB_XSTATS;
1322
1323         /* Global stats */
1324         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1325                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1326                         fm10k_hw_stats_strings[count].offset);
1327                 xstats[count].id = count;
1328                 count++;
1329         }
1330
1331         /* PF queue stats */
1332         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1333                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1334                         xstats[count].value =
1335                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1336                                 fm10k_hw_stats_rx_q_strings[i].offset);
1337                         xstats[count].id = count;
1338                         count++;
1339                 }
1340                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1341                         xstats[count].value =
1342                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1343                                 fm10k_hw_stats_tx_q_strings[i].offset);
1344                         xstats[count].id = count;
1345                         count++;
1346                 }
1347         }
1348
1349         return FM10K_NB_XSTATS;
1350 }
1351
1352 static int
1353 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1354 {
1355         uint64_t ipackets, opackets, ibytes, obytes;
1356         struct fm10k_hw *hw =
1357                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1358         struct fm10k_hw_stats *hw_stats =
1359                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1360         int i;
1361
1362         PMD_INIT_FUNC_TRACE();
1363
1364         fm10k_update_hw_stats(hw, hw_stats);
1365
1366         ipackets = opackets = ibytes = obytes = 0;
1367         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1368                 (i < hw->mac.max_queues); ++i) {
1369                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1370                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1371                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1372                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1373                 ipackets += stats->q_ipackets[i];
1374                 opackets += stats->q_opackets[i];
1375                 ibytes   += stats->q_ibytes[i];
1376                 obytes   += stats->q_obytes[i];
1377         }
1378         stats->ipackets = ipackets;
1379         stats->opackets = opackets;
1380         stats->ibytes = ibytes;
1381         stats->obytes = obytes;
1382         return 0;
1383 }
1384
1385 static void
1386 fm10k_stats_reset(struct rte_eth_dev *dev)
1387 {
1388         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1389         struct fm10k_hw_stats *hw_stats =
1390                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1391
1392         PMD_INIT_FUNC_TRACE();
1393
1394         memset(hw_stats, 0, sizeof(*hw_stats));
1395         fm10k_rebind_hw_stats(hw, hw_stats);
1396 }
1397
1398 static void
1399 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1400         struct rte_eth_dev_info *dev_info)
1401 {
1402         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1403         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1404
1405         PMD_INIT_FUNC_TRACE();
1406
1407         dev_info->pci_dev            = pdev;
1408         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1409         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1410         dev_info->max_rx_queues      = hw->mac.max_queues;
1411         dev_info->max_tx_queues      = hw->mac.max_queues;
1412         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1413         dev_info->max_hash_mac_addrs = 0;
1414         dev_info->max_vfs            = pdev->max_vfs;
1415         dev_info->vmdq_pool_base     = 0;
1416         dev_info->vmdq_queue_base    = 0;
1417         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1418         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1419         dev_info->rx_offload_capa =
1420                 DEV_RX_OFFLOAD_VLAN_STRIP |
1421                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1422                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1423                 DEV_RX_OFFLOAD_TCP_CKSUM;
1424         dev_info->tx_offload_capa =
1425                 DEV_TX_OFFLOAD_VLAN_INSERT |
1426                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1427                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1428                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1429                 DEV_TX_OFFLOAD_TCP_TSO;
1430
1431         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1432         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1433
1434         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1435                 .rx_thresh = {
1436                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1437                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1438                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1439                 },
1440                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1441                 .rx_drop_en = 0,
1442         };
1443
1444         dev_info->default_txconf = (struct rte_eth_txconf) {
1445                 .tx_thresh = {
1446                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1447                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1448                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1449                 },
1450                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1451                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1452                 .txq_flags = FM10K_SIMPLE_TX_FLAG,
1453         };
1454
1455         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1456                 .nb_max = FM10K_MAX_RX_DESC,
1457                 .nb_min = FM10K_MIN_RX_DESC,
1458                 .nb_align = FM10K_MULT_RX_DESC,
1459         };
1460
1461         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1462                 .nb_max = FM10K_MAX_TX_DESC,
1463                 .nb_min = FM10K_MIN_TX_DESC,
1464                 .nb_align = FM10K_MULT_TX_DESC,
1465                 .nb_seg_max = FM10K_TX_MAX_SEG,
1466                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1467         };
1468
1469         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1470                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1471                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1472 }
1473
1474 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1475 static const uint32_t *
1476 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1477 {
1478         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1479             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1480                 static uint32_t ptypes[] = {
1481                         /* refers to rx_desc_to_ol_flags() */
1482                         RTE_PTYPE_L2_ETHER,
1483                         RTE_PTYPE_L3_IPV4,
1484                         RTE_PTYPE_L3_IPV4_EXT,
1485                         RTE_PTYPE_L3_IPV6,
1486                         RTE_PTYPE_L3_IPV6_EXT,
1487                         RTE_PTYPE_L4_TCP,
1488                         RTE_PTYPE_L4_UDP,
1489                         RTE_PTYPE_UNKNOWN
1490                 };
1491
1492                 return ptypes;
1493         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1494                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1495                 static uint32_t ptypes_vec[] = {
1496                         /* refers to fm10k_desc_to_pktype_v() */
1497                         RTE_PTYPE_L3_IPV4,
1498                         RTE_PTYPE_L3_IPV4_EXT,
1499                         RTE_PTYPE_L3_IPV6,
1500                         RTE_PTYPE_L3_IPV6_EXT,
1501                         RTE_PTYPE_L4_TCP,
1502                         RTE_PTYPE_L4_UDP,
1503                         RTE_PTYPE_TUNNEL_GENEVE,
1504                         RTE_PTYPE_TUNNEL_NVGRE,
1505                         RTE_PTYPE_TUNNEL_VXLAN,
1506                         RTE_PTYPE_TUNNEL_GRE,
1507                         RTE_PTYPE_UNKNOWN
1508                 };
1509
1510                 return ptypes_vec;
1511         }
1512
1513         return NULL;
1514 }
1515 #else
1516 static const uint32_t *
1517 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1518 {
1519         return NULL;
1520 }
1521 #endif
1522
1523 static int
1524 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1525 {
1526         s32 result;
1527         uint16_t mac_num = 0;
1528         uint32_t vid_idx, vid_bit, mac_index;
1529         struct fm10k_hw *hw;
1530         struct fm10k_macvlan_filter_info *macvlan;
1531         struct rte_eth_dev_data *data = dev->data;
1532
1533         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1534         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1535
1536         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1537                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1538                 return -EINVAL;
1539         }
1540
1541         if (vlan_id > ETH_VLAN_ID_MAX) {
1542                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1543                 return -EINVAL;
1544         }
1545
1546         vid_idx = FM10K_VFTA_IDX(vlan_id);
1547         vid_bit = FM10K_VFTA_BIT(vlan_id);
1548         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1549         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1550                 return 0;
1551         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1552         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1553                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1554                         "in the VLAN filter table");
1555                 return -EINVAL;
1556         }
1557
1558         fm10k_mbx_lock(hw);
1559         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1560         fm10k_mbx_unlock(hw);
1561         if (result != FM10K_SUCCESS) {
1562                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1563                 return -EIO;
1564         }
1565
1566         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1567                         (result == FM10K_SUCCESS); mac_index++) {
1568                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1569                         continue;
1570                 if (mac_num > macvlan->mac_num - 1) {
1571                         PMD_INIT_LOG(ERR, "MAC address number "
1572                                         "not match");
1573                         break;
1574                 }
1575                 fm10k_mbx_lock(hw);
1576                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1577                         data->mac_addrs[mac_index].addr_bytes,
1578                         vlan_id, on, 0);
1579                 fm10k_mbx_unlock(hw);
1580                 mac_num++;
1581         }
1582         if (result != FM10K_SUCCESS) {
1583                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1584                 return -EIO;
1585         }
1586
1587         if (on) {
1588                 macvlan->vlan_num++;
1589                 macvlan->vfta[vid_idx] |= vid_bit;
1590         } else {
1591                 macvlan->vlan_num--;
1592                 macvlan->vfta[vid_idx] &= ~vid_bit;
1593         }
1594         return 0;
1595 }
1596
1597 static void
1598 fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1599 {
1600         if (mask & ETH_VLAN_STRIP_MASK) {
1601                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1602                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1603                                         "always on in fm10k");
1604         }
1605
1606         if (mask & ETH_VLAN_EXTEND_MASK) {
1607                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1608                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1609                                         "supported in fm10k");
1610         }
1611
1612         if (mask & ETH_VLAN_FILTER_MASK) {
1613                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1614                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1615         }
1616 }
1617
1618 /* Add/Remove a MAC address, and update filters to main VSI */
1619 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1620                 const u8 *mac, bool add, uint32_t pool)
1621 {
1622         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1623         struct fm10k_macvlan_filter_info *macvlan;
1624         uint32_t i, j, k;
1625
1626         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1627
1628         if (pool != MAIN_VSI_POOL_NUMBER) {
1629                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1630                         "mac to pool %u", pool);
1631                 return;
1632         }
1633         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1634                 if (!macvlan->vfta[j])
1635                         continue;
1636                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1637                         if (!(macvlan->vfta[j] & (1 << k)))
1638                                 continue;
1639                         if (i + 1 > macvlan->vlan_num) {
1640                                 PMD_INIT_LOG(ERR, "vlan number not match");
1641                                 return;
1642                         }
1643                         fm10k_mbx_lock(hw);
1644                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1645                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1646                         fm10k_mbx_unlock(hw);
1647                         i++;
1648                 }
1649         }
1650 }
1651
1652 /* Add/Remove a MAC address, and update filters to VMDQ */
1653 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1654                 const u8 *mac, bool add, uint32_t pool)
1655 {
1656         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1657         struct fm10k_macvlan_filter_info *macvlan;
1658         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1659         uint32_t i;
1660
1661         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1662         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1663
1664         if (pool > macvlan->nb_queue_pools) {
1665                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1666                         " Max pool is %u",
1667                         pool, macvlan->nb_queue_pools);
1668                 return;
1669         }
1670         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1671                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1672                         continue;
1673                 fm10k_mbx_lock(hw);
1674                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1675                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1676                 fm10k_mbx_unlock(hw);
1677         }
1678 }
1679
1680 /* Add/Remove a MAC address, and update filters */
1681 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1682                 const u8 *mac, bool add, uint32_t pool)
1683 {
1684         struct fm10k_macvlan_filter_info *macvlan;
1685
1686         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1687
1688         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1689                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1690         else
1691                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1692
1693         if (add)
1694                 macvlan->mac_num++;
1695         else
1696                 macvlan->mac_num--;
1697 }
1698
1699 /* Add a MAC address, and update filters */
1700 static int
1701 fm10k_macaddr_add(struct rte_eth_dev *dev,
1702                 struct ether_addr *mac_addr,
1703                 uint32_t index,
1704                 uint32_t pool)
1705 {
1706         struct fm10k_macvlan_filter_info *macvlan;
1707
1708         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1709         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1710         macvlan->mac_vmdq_id[index] = pool;
1711         return 0;
1712 }
1713
1714 /* Remove a MAC address, and update filters */
1715 static void
1716 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1717 {
1718         struct rte_eth_dev_data *data = dev->data;
1719         struct fm10k_macvlan_filter_info *macvlan;
1720
1721         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1722         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1723                         FALSE, macvlan->mac_vmdq_id[index]);
1724         macvlan->mac_vmdq_id[index] = 0;
1725 }
1726
1727 static inline int
1728 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1729 {
1730         if ((request < min) || (request > max) || ((request % mult) != 0))
1731                 return -1;
1732         else
1733                 return 0;
1734 }
1735
1736
1737 static inline int
1738 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1739 {
1740         if ((request < min) || (request > max) || ((div % request) != 0))
1741                 return -1;
1742         else
1743                 return 0;
1744 }
1745
1746 static inline int
1747 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1748 {
1749         uint16_t rx_free_thresh;
1750
1751         if (conf->rx_free_thresh == 0)
1752                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1753         else
1754                 rx_free_thresh = conf->rx_free_thresh;
1755
1756         /* make sure the requested threshold satisfies the constraints */
1757         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1758                         FM10K_RX_FREE_THRESH_MAX(q),
1759                         FM10K_RX_FREE_THRESH_DIV(q),
1760                         rx_free_thresh)) {
1761                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1762                         "less than or equal to %u, "
1763                         "greater than or equal to %u, "
1764                         "and a divisor of %u",
1765                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1766                         FM10K_RX_FREE_THRESH_MIN(q),
1767                         FM10K_RX_FREE_THRESH_DIV(q));
1768                 return -EINVAL;
1769         }
1770
1771         q->alloc_thresh = rx_free_thresh;
1772         q->drop_en = conf->rx_drop_en;
1773         q->rx_deferred_start = conf->rx_deferred_start;
1774
1775         return 0;
1776 }
1777
1778 /*
1779  * Hardware requires specific alignment for Rx packet buffers. At
1780  * least one of the following two conditions must be satisfied.
1781  *  1. Address is 512B aligned
1782  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1783  *
1784  * As such, the driver may need to adjust the DMA address within the
1785  * buffer by up to 512B.
1786  *
1787  * return 1 if the element size is valid, otherwise return 0.
1788  */
1789 static int
1790 mempool_element_size_valid(struct rte_mempool *mp)
1791 {
1792         uint32_t min_size;
1793
1794         /* elt_size includes mbuf header and headroom */
1795         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1796                         RTE_PKTMBUF_HEADROOM;
1797
1798         /* account for up to 512B of alignment */
1799         min_size -= FM10K_RX_DATABUF_ALIGN;
1800
1801         /* sanity check for overflow */
1802         if (min_size > mp->elt_size)
1803                 return 0;
1804
1805         /* size is valid */
1806         return 1;
1807 }
1808
1809 static int
1810 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1811         uint16_t nb_desc, unsigned int socket_id,
1812         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1813 {
1814         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1815         struct fm10k_dev_info *dev_info =
1816                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1817         struct fm10k_rx_queue *q;
1818         const struct rte_memzone *mz;
1819
1820         PMD_INIT_FUNC_TRACE();
1821
1822         /* make sure the mempool element size can account for alignment. */
1823         if (!mempool_element_size_valid(mp)) {
1824                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1825                 return -EINVAL;
1826         }
1827
1828         /* make sure a valid number of descriptors have been requested */
1829         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1830                                 FM10K_MULT_RX_DESC, nb_desc)) {
1831                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1832                         "less than or equal to %"PRIu32", "
1833                         "greater than or equal to %u, "
1834                         "and a multiple of %u",
1835                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1836                         FM10K_MULT_RX_DESC);
1837                 return -EINVAL;
1838         }
1839
1840         /*
1841          * if this queue existed already, free the associated memory. The
1842          * queue cannot be reused in case we need to allocate memory on
1843          * different socket than was previously used.
1844          */
1845         if (dev->data->rx_queues[queue_id] != NULL) {
1846                 rx_queue_free(dev->data->rx_queues[queue_id]);
1847                 dev->data->rx_queues[queue_id] = NULL;
1848         }
1849
1850         /* allocate memory for the queue structure */
1851         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1852                                 socket_id);
1853         if (q == NULL) {
1854                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1855                 return -ENOMEM;
1856         }
1857
1858         /* setup queue */
1859         q->mp = mp;
1860         q->nb_desc = nb_desc;
1861         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1862         q->port_id = dev->data->port_id;
1863         q->queue_id = queue_id;
1864         q->tail_ptr = (volatile uint32_t *)
1865                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1866         if (handle_rxconf(q, conf))
1867                 return -EINVAL;
1868
1869         /* allocate memory for the software ring */
1870         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1871                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1872                         RTE_CACHE_LINE_SIZE, socket_id);
1873         if (q->sw_ring == NULL) {
1874                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1875                 rte_free(q);
1876                 return -ENOMEM;
1877         }
1878
1879         /*
1880          * allocate memory for the hardware descriptor ring. A memzone large
1881          * enough to hold the maximum ring size is requested to allow for
1882          * resizing in later calls to the queue setup function.
1883          */
1884         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1885                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1886                                       socket_id);
1887         if (mz == NULL) {
1888                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1889                 rte_free(q->sw_ring);
1890                 rte_free(q);
1891                 return -ENOMEM;
1892         }
1893         q->hw_ring = mz->addr;
1894         q->hw_ring_phys_addr = mz->phys_addr;
1895
1896         /* Check if number of descs satisfied Vector requirement */
1897         if (!rte_is_power_of_2(nb_desc)) {
1898                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1899                                     "preconditions - canceling the feature for "
1900                                     "the whole port[%d]",
1901                              q->queue_id, q->port_id);
1902                 dev_info->rx_vec_allowed = false;
1903         } else
1904                 fm10k_rxq_vec_setup(q);
1905
1906         dev->data->rx_queues[queue_id] = q;
1907         return 0;
1908 }
1909
1910 static void
1911 fm10k_rx_queue_release(void *queue)
1912 {
1913         PMD_INIT_FUNC_TRACE();
1914
1915         rx_queue_free(queue);
1916 }
1917
1918 static inline int
1919 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1920 {
1921         uint16_t tx_free_thresh;
1922         uint16_t tx_rs_thresh;
1923
1924         /* constraint MACROs require that tx_free_thresh is configured
1925          * before tx_rs_thresh */
1926         if (conf->tx_free_thresh == 0)
1927                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1928         else
1929                 tx_free_thresh = conf->tx_free_thresh;
1930
1931         /* make sure the requested threshold satisfies the constraints */
1932         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1933                         FM10K_TX_FREE_THRESH_MAX(q),
1934                         FM10K_TX_FREE_THRESH_DIV(q),
1935                         tx_free_thresh)) {
1936                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1937                         "less than or equal to %u, "
1938                         "greater than or equal to %u, "
1939                         "and a divisor of %u",
1940                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1941                         FM10K_TX_FREE_THRESH_MIN(q),
1942                         FM10K_TX_FREE_THRESH_DIV(q));
1943                 return -EINVAL;
1944         }
1945
1946         q->free_thresh = tx_free_thresh;
1947
1948         if (conf->tx_rs_thresh == 0)
1949                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1950         else
1951                 tx_rs_thresh = conf->tx_rs_thresh;
1952
1953         q->tx_deferred_start = conf->tx_deferred_start;
1954
1955         /* make sure the requested threshold satisfies the constraints */
1956         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1957                         FM10K_TX_RS_THRESH_MAX(q),
1958                         FM10K_TX_RS_THRESH_DIV(q),
1959                         tx_rs_thresh)) {
1960                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1961                         "less than or equal to %u, "
1962                         "greater than or equal to %u, "
1963                         "and a divisor of %u",
1964                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1965                         FM10K_TX_RS_THRESH_MIN(q),
1966                         FM10K_TX_RS_THRESH_DIV(q));
1967                 return -EINVAL;
1968         }
1969
1970         q->rs_thresh = tx_rs_thresh;
1971
1972         return 0;
1973 }
1974
1975 static int
1976 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1977         uint16_t nb_desc, unsigned int socket_id,
1978         const struct rte_eth_txconf *conf)
1979 {
1980         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1981         struct fm10k_tx_queue *q;
1982         const struct rte_memzone *mz;
1983
1984         PMD_INIT_FUNC_TRACE();
1985
1986         /* make sure a valid number of descriptors have been requested */
1987         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1988                                 FM10K_MULT_TX_DESC, nb_desc)) {
1989                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1990                         "less than or equal to %"PRIu32", "
1991                         "greater than or equal to %u, "
1992                         "and a multiple of %u",
1993                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1994                         FM10K_MULT_TX_DESC);
1995                 return -EINVAL;
1996         }
1997
1998         /*
1999          * if this queue existed already, free the associated memory. The
2000          * queue cannot be reused in case we need to allocate memory on
2001          * different socket than was previously used.
2002          */
2003         if (dev->data->tx_queues[queue_id] != NULL) {
2004                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2005
2006                 tx_queue_free(txq);
2007                 dev->data->tx_queues[queue_id] = NULL;
2008         }
2009
2010         /* allocate memory for the queue structure */
2011         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2012                                 socket_id);
2013         if (q == NULL) {
2014                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2015                 return -ENOMEM;
2016         }
2017
2018         /* setup queue */
2019         q->nb_desc = nb_desc;
2020         q->port_id = dev->data->port_id;
2021         q->queue_id = queue_id;
2022         q->txq_flags = conf->txq_flags;
2023         q->ops = &def_txq_ops;
2024         q->tail_ptr = (volatile uint32_t *)
2025                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2026         if (handle_txconf(q, conf))
2027                 return -EINVAL;
2028
2029         /* allocate memory for the software ring */
2030         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2031                                         nb_desc * sizeof(struct rte_mbuf *),
2032                                         RTE_CACHE_LINE_SIZE, socket_id);
2033         if (q->sw_ring == NULL) {
2034                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2035                 rte_free(q);
2036                 return -ENOMEM;
2037         }
2038
2039         /*
2040          * allocate memory for the hardware descriptor ring. A memzone large
2041          * enough to hold the maximum ring size is requested to allow for
2042          * resizing in later calls to the queue setup function.
2043          */
2044         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2045                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2046                                       socket_id);
2047         if (mz == NULL) {
2048                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2049                 rte_free(q->sw_ring);
2050                 rte_free(q);
2051                 return -ENOMEM;
2052         }
2053         q->hw_ring = mz->addr;
2054         q->hw_ring_phys_addr = mz->phys_addr;
2055
2056         /*
2057          * allocate memory for the RS bit tracker. Enough slots to hold the
2058          * descriptor index for each RS bit needing to be set are required.
2059          */
2060         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2061                                 ((nb_desc + 1) / q->rs_thresh) *
2062                                 sizeof(uint16_t),
2063                                 RTE_CACHE_LINE_SIZE, socket_id);
2064         if (q->rs_tracker.list == NULL) {
2065                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2066                 rte_free(q->sw_ring);
2067                 rte_free(q);
2068                 return -ENOMEM;
2069         }
2070
2071         dev->data->tx_queues[queue_id] = q;
2072         return 0;
2073 }
2074
2075 static void
2076 fm10k_tx_queue_release(void *queue)
2077 {
2078         struct fm10k_tx_queue *q = queue;
2079         PMD_INIT_FUNC_TRACE();
2080
2081         tx_queue_free(q);
2082 }
2083
2084 static int
2085 fm10k_reta_update(struct rte_eth_dev *dev,
2086                         struct rte_eth_rss_reta_entry64 *reta_conf,
2087                         uint16_t reta_size)
2088 {
2089         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2090         uint16_t i, j, idx, shift;
2091         uint8_t mask;
2092         uint32_t reta;
2093
2094         PMD_INIT_FUNC_TRACE();
2095
2096         if (reta_size > FM10K_MAX_RSS_INDICES) {
2097                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2098                         "(%d) doesn't match the number hardware can supported "
2099                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2100                 return -EINVAL;
2101         }
2102
2103         /*
2104          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2105          * 128-entries in 32 registers
2106          */
2107         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2108                 idx = i / RTE_RETA_GROUP_SIZE;
2109                 shift = i % RTE_RETA_GROUP_SIZE;
2110                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2111                                 BIT_MASK_PER_UINT32);
2112                 if (mask == 0)
2113                         continue;
2114
2115                 reta = 0;
2116                 if (mask != BIT_MASK_PER_UINT32)
2117                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2118
2119                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2120                         if (mask & (0x1 << j)) {
2121                                 if (mask != 0xF)
2122                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2123                                 reta |= reta_conf[idx].reta[shift + j] <<
2124                                                 (CHAR_BIT * j);
2125                         }
2126                 }
2127                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2128         }
2129
2130         return 0;
2131 }
2132
2133 static int
2134 fm10k_reta_query(struct rte_eth_dev *dev,
2135                         struct rte_eth_rss_reta_entry64 *reta_conf,
2136                         uint16_t reta_size)
2137 {
2138         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2139         uint16_t i, j, idx, shift;
2140         uint8_t mask;
2141         uint32_t reta;
2142
2143         PMD_INIT_FUNC_TRACE();
2144
2145         if (reta_size < FM10K_MAX_RSS_INDICES) {
2146                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2147                         "(%d) doesn't match the number hardware can supported "
2148                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2149                 return -EINVAL;
2150         }
2151
2152         /*
2153          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2154          * 128-entries in 32 registers
2155          */
2156         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2157                 idx = i / RTE_RETA_GROUP_SIZE;
2158                 shift = i % RTE_RETA_GROUP_SIZE;
2159                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2160                                 BIT_MASK_PER_UINT32);
2161                 if (mask == 0)
2162                         continue;
2163
2164                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2165                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2166                         if (mask & (0x1 << j))
2167                                 reta_conf[idx].reta[shift + j] = ((reta >>
2168                                         CHAR_BIT * j) & UINT8_MAX);
2169                 }
2170         }
2171
2172         return 0;
2173 }
2174
2175 static int
2176 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2177         struct rte_eth_rss_conf *rss_conf)
2178 {
2179         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2180         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2181         uint32_t mrqc;
2182         uint64_t hf = rss_conf->rss_hf;
2183         int i;
2184
2185         PMD_INIT_FUNC_TRACE();
2186
2187         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2188                                 FM10K_RSSRK_ENTRIES_PER_REG))
2189                 return -EINVAL;
2190
2191         if (hf == 0)
2192                 return -EINVAL;
2193
2194         mrqc = 0;
2195         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2196         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2197         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2198         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2199         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2200         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2201         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2202         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2203         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2204
2205         /* If the mapping doesn't fit any supported, return */
2206         if (mrqc == 0)
2207                 return -EINVAL;
2208
2209         if (key != NULL)
2210                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2211                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2212
2213         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2214
2215         return 0;
2216 }
2217
2218 static int
2219 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2220         struct rte_eth_rss_conf *rss_conf)
2221 {
2222         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2223         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2224         uint32_t mrqc;
2225         uint64_t hf;
2226         int i;
2227
2228         PMD_INIT_FUNC_TRACE();
2229
2230         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2231                                 FM10K_RSSRK_ENTRIES_PER_REG))
2232                 return -EINVAL;
2233
2234         if (key != NULL)
2235                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2236                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2237
2238         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2239         hf = 0;
2240         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2241         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2242         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2243         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2244         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2245         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2246         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2247         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2248         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2249
2250         rss_conf->rss_hf = hf;
2251
2252         return 0;
2253 }
2254
2255 static void
2256 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2257 {
2258         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2259         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2260
2261         /* Bind all local non-queue interrupt to vector 0 */
2262         int_map |= FM10K_MISC_VEC_ID;
2263
2264         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2265         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2266         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2267         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2268         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2269         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2270
2271         /* Enable misc causes */
2272         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2273                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2274                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2275                                 FM10K_EIMR_ENABLE(MAILBOX) |
2276                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2277                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2278                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2279                                 FM10K_EIMR_ENABLE(VFLR));
2280
2281         /* Enable ITR 0 */
2282         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2283                                         FM10K_ITR_MASK_CLEAR);
2284         FM10K_WRITE_FLUSH(hw);
2285 }
2286
2287 static void
2288 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2289 {
2290         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2291         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2292
2293         int_map |= FM10K_MISC_VEC_ID;
2294
2295         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2296         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2297         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2298         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2299         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2300         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2301
2302         /* Disable misc causes */
2303         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2304                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2305                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2306                                 FM10K_EIMR_DISABLE(MAILBOX) |
2307                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2308                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2309                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2310                                 FM10K_EIMR_DISABLE(VFLR));
2311
2312         /* Disable ITR 0 */
2313         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2314         FM10K_WRITE_FLUSH(hw);
2315 }
2316
2317 static void
2318 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2319 {
2320         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2321         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2322
2323         /* Bind all local non-queue interrupt to vector 0 */
2324         int_map |= FM10K_MISC_VEC_ID;
2325
2326         /* Only INT 0 available, other 15 are reserved. */
2327         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2328
2329         /* Enable ITR 0 */
2330         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2331                                         FM10K_ITR_MASK_CLEAR);
2332         FM10K_WRITE_FLUSH(hw);
2333 }
2334
2335 static void
2336 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2337 {
2338         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2339         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2340
2341         int_map |= FM10K_MISC_VEC_ID;
2342
2343         /* Only INT 0 available, other 15 are reserved. */
2344         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2345
2346         /* Disable ITR 0 */
2347         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2348         FM10K_WRITE_FLUSH(hw);
2349 }
2350
2351 static int
2352 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2353 {
2354         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2355         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2356
2357         /* Enable ITR */
2358         if (hw->mac.type == fm10k_mac_pf)
2359                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2360                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2361         else
2362                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2363                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2364         rte_intr_enable(&pdev->intr_handle);
2365         return 0;
2366 }
2367
2368 static int
2369 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2370 {
2371         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2372         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2373
2374         /* Disable ITR */
2375         if (hw->mac.type == fm10k_mac_pf)
2376                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2377                         FM10K_ITR_MASK_SET);
2378         else
2379                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2380                         FM10K_ITR_MASK_SET);
2381         return 0;
2382 }
2383
2384 static int
2385 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2386 {
2387         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2388         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2389         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2390         uint32_t intr_vector, vec;
2391         uint16_t queue_id;
2392         int result = 0;
2393
2394         /* fm10k needs one separate interrupt for mailbox,
2395          * so only drivers which support multiple interrupt vectors
2396          * e.g. vfio-pci can work for fm10k interrupt mode
2397          */
2398         if (!rte_intr_cap_multiple(intr_handle) ||
2399                         dev->data->dev_conf.intr_conf.rxq == 0)
2400                 return result;
2401
2402         intr_vector = dev->data->nb_rx_queues;
2403
2404         /* disable interrupt first */
2405         rte_intr_disable(intr_handle);
2406         if (hw->mac.type == fm10k_mac_pf)
2407                 fm10k_dev_disable_intr_pf(dev);
2408         else
2409                 fm10k_dev_disable_intr_vf(dev);
2410
2411         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2412                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2413                 result = -EIO;
2414         }
2415
2416         if (rte_intr_dp_is_en(intr_handle) && !result) {
2417                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2418                         dev->data->nb_rx_queues * sizeof(int), 0);
2419                 if (intr_handle->intr_vec) {
2420                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2421                                         queue_id < dev->data->nb_rx_queues;
2422                                         queue_id++) {
2423                                 intr_handle->intr_vec[queue_id] = vec;
2424                                 if (vec < intr_handle->nb_efd - 1
2425                                                 + FM10K_RX_VEC_START)
2426                                         vec++;
2427                         }
2428                 } else {
2429                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2430                                 " intr_vec", dev->data->nb_rx_queues);
2431                         rte_intr_efd_disable(intr_handle);
2432                         result = -ENOMEM;
2433                 }
2434         }
2435
2436         if (hw->mac.type == fm10k_mac_pf)
2437                 fm10k_dev_enable_intr_pf(dev);
2438         else
2439                 fm10k_dev_enable_intr_vf(dev);
2440         rte_intr_enable(intr_handle);
2441         hw->mac.ops.update_int_moderator(hw);
2442         return result;
2443 }
2444
2445 static int
2446 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2447 {
2448         struct fm10k_fault fault;
2449         int err;
2450         const char *estr = "Unknown error";
2451
2452         /* Process PCA fault */
2453         if (eicr & FM10K_EICR_PCA_FAULT) {
2454                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2455                 if (err)
2456                         goto error;
2457                 switch (fault.type) {
2458                 case PCA_NO_FAULT:
2459                         estr = "PCA_NO_FAULT"; break;
2460                 case PCA_UNMAPPED_ADDR:
2461                         estr = "PCA_UNMAPPED_ADDR"; break;
2462                 case PCA_BAD_QACCESS_PF:
2463                         estr = "PCA_BAD_QACCESS_PF"; break;
2464                 case PCA_BAD_QACCESS_VF:
2465                         estr = "PCA_BAD_QACCESS_VF"; break;
2466                 case PCA_MALICIOUS_REQ:
2467                         estr = "PCA_MALICIOUS_REQ"; break;
2468                 case PCA_POISONED_TLP:
2469                         estr = "PCA_POISONED_TLP"; break;
2470                 case PCA_TLP_ABORT:
2471                         estr = "PCA_TLP_ABORT"; break;
2472                 default:
2473                         goto error;
2474                 }
2475                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2476                         estr, fault.func ? "VF" : "PF", fault.func,
2477                         fault.address, fault.specinfo);
2478         }
2479
2480         /* Process THI fault */
2481         if (eicr & FM10K_EICR_THI_FAULT) {
2482                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2483                 if (err)
2484                         goto error;
2485                 switch (fault.type) {
2486                 case THI_NO_FAULT:
2487                         estr = "THI_NO_FAULT"; break;
2488                 case THI_MAL_DIS_Q_FAULT:
2489                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2490                 default:
2491                         goto error;
2492                 }
2493                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2494                         estr, fault.func ? "VF" : "PF", fault.func,
2495                         fault.address, fault.specinfo);
2496         }
2497
2498         /* Process FUM fault */
2499         if (eicr & FM10K_EICR_FUM_FAULT) {
2500                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2501                 if (err)
2502                         goto error;
2503                 switch (fault.type) {
2504                 case FUM_NO_FAULT:
2505                         estr = "FUM_NO_FAULT"; break;
2506                 case FUM_UNMAPPED_ADDR:
2507                         estr = "FUM_UNMAPPED_ADDR"; break;
2508                 case FUM_POISONED_TLP:
2509                         estr = "FUM_POISONED_TLP"; break;
2510                 case FUM_BAD_VF_QACCESS:
2511                         estr = "FUM_BAD_VF_QACCESS"; break;
2512                 case FUM_ADD_DECODE_ERR:
2513                         estr = "FUM_ADD_DECODE_ERR"; break;
2514                 case FUM_RO_ERROR:
2515                         estr = "FUM_RO_ERROR"; break;
2516                 case FUM_QPRC_CRC_ERROR:
2517                         estr = "FUM_QPRC_CRC_ERROR"; break;
2518                 case FUM_CSR_TIMEOUT:
2519                         estr = "FUM_CSR_TIMEOUT"; break;
2520                 case FUM_INVALID_TYPE:
2521                         estr = "FUM_INVALID_TYPE"; break;
2522                 case FUM_INVALID_LENGTH:
2523                         estr = "FUM_INVALID_LENGTH"; break;
2524                 case FUM_INVALID_BE:
2525                         estr = "FUM_INVALID_BE"; break;
2526                 case FUM_INVALID_ALIGN:
2527                         estr = "FUM_INVALID_ALIGN"; break;
2528                 default:
2529                         goto error;
2530                 }
2531                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2532                         estr, fault.func ? "VF" : "PF", fault.func,
2533                         fault.address, fault.specinfo);
2534         }
2535
2536         return 0;
2537 error:
2538         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2539         return err;
2540 }
2541
2542 /**
2543  * PF interrupt handler triggered by NIC for handling specific interrupt.
2544  *
2545  * @param handle
2546  *  Pointer to interrupt handle.
2547  * @param param
2548  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2549  *
2550  * @return
2551  *  void
2552  */
2553 static void
2554 fm10k_dev_interrupt_handler_pf(void *param)
2555 {
2556         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2557         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2558         uint32_t cause, status;
2559         struct fm10k_dev_info *dev_info =
2560                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2561         int status_mbx;
2562         s32 err;
2563
2564         if (hw->mac.type != fm10k_mac_pf)
2565                 return;
2566
2567         cause = FM10K_READ_REG(hw, FM10K_EICR);
2568
2569         /* Handle PCI fault cases */
2570         if (cause & FM10K_EICR_FAULT_MASK) {
2571                 PMD_INIT_LOG(ERR, "INT: find fault!");
2572                 fm10k_dev_handle_fault(hw, cause);
2573         }
2574
2575         /* Handle switch up/down */
2576         if (cause & FM10K_EICR_SWITCHNOTREADY)
2577                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2578
2579         if (cause & FM10K_EICR_SWITCHREADY) {
2580                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2581                 if (dev_info->sm_down == 1) {
2582                         fm10k_mbx_lock(hw);
2583
2584                         /* For recreating logical ports */
2585                         status_mbx = hw->mac.ops.update_lport_state(hw,
2586                                         hw->mac.dglort_map, MAX_LPORT_NUM, 1);
2587                         if (status_mbx == FM10K_SUCCESS)
2588                                 PMD_INIT_LOG(INFO,
2589                                         "INT: Recreated Logical port");
2590                         else
2591                                 PMD_INIT_LOG(INFO,
2592                                         "INT: Logical ports weren't recreated");
2593
2594                         status_mbx = hw->mac.ops.update_xcast_mode(hw,
2595                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2596                         if (status_mbx != FM10K_SUCCESS)
2597                                 PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2598
2599                         fm10k_mbx_unlock(hw);
2600
2601                         /* first clear the internal SW recording structure */
2602                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2603                                                 ETH_MQ_RX_VMDQ_FLAG))
2604                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2605                                         false);
2606
2607                         fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2608                                         MAIN_VSI_POOL_NUMBER);
2609
2610                         /*
2611                          * Add default mac address and vlan for the logical
2612                          * ports that have been created, leave to the
2613                          * application to fully recover Rx filtering.
2614                          */
2615                         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2616                                         MAIN_VSI_POOL_NUMBER);
2617
2618                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2619                                                 ETH_MQ_RX_VMDQ_FLAG))
2620                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2621                                         true);
2622
2623                         dev_info->sm_down = 0;
2624                         _rte_eth_dev_callback_process(dev,
2625                                         RTE_ETH_EVENT_INTR_LSC,
2626                                         NULL, NULL);
2627                 }
2628         }
2629
2630         /* Handle mailbox message */
2631         fm10k_mbx_lock(hw);
2632         err = hw->mbx.ops.process(hw, &hw->mbx);
2633         fm10k_mbx_unlock(hw);
2634
2635         if (err == FM10K_ERR_RESET_REQUESTED) {
2636                 PMD_INIT_LOG(INFO, "INT: Switch is down");
2637                 dev_info->sm_down = 1;
2638                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2639                                 NULL, NULL);
2640         }
2641
2642         /* Handle SRAM error */
2643         if (cause & FM10K_EICR_SRAMERROR) {
2644                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2645
2646                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2647                 /* Write to clear pending bits */
2648                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2649
2650                 /* Todo: print out error message after shared code  updates */
2651         }
2652
2653         /* Clear these 3 events if having any */
2654         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2655                  FM10K_EICR_SWITCHREADY;
2656         if (cause)
2657                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2658
2659         /* Re-enable interrupt from device side */
2660         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2661                                         FM10K_ITR_MASK_CLEAR);
2662         /* Re-enable interrupt from host side */
2663         rte_intr_enable(dev->intr_handle);
2664 }
2665
2666 /**
2667  * VF interrupt handler triggered by NIC for handling specific interrupt.
2668  *
2669  * @param handle
2670  *  Pointer to interrupt handle.
2671  * @param param
2672  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2673  *
2674  * @return
2675  *  void
2676  */
2677 static void
2678 fm10k_dev_interrupt_handler_vf(void *param)
2679 {
2680         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2681         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2682         struct fm10k_mbx_info *mbx = &hw->mbx;
2683         struct fm10k_dev_info *dev_info =
2684                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2685         const enum fm10k_mbx_state state = mbx->state;
2686         int status_mbx;
2687
2688         if (hw->mac.type != fm10k_mac_vf)
2689                 return;
2690
2691         /* Handle mailbox message if lock is acquired */
2692         fm10k_mbx_lock(hw);
2693         hw->mbx.ops.process(hw, &hw->mbx);
2694         fm10k_mbx_unlock(hw);
2695
2696         if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) {
2697                 PMD_INIT_LOG(INFO, "INT: Switch has gone down");
2698
2699                 fm10k_mbx_lock(hw);
2700                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2701                                 MAX_LPORT_NUM, 1);
2702                 fm10k_mbx_unlock(hw);
2703
2704                 /* Setting reset flag */
2705                 dev_info->sm_down = 1;
2706                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2707                                 NULL, NULL);
2708         }
2709
2710         if (dev_info->sm_down == 1 &&
2711                         hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) {
2712                 PMD_INIT_LOG(INFO, "INT: Switch has gone up");
2713                 fm10k_mbx_lock(hw);
2714                 status_mbx = hw->mac.ops.update_xcast_mode(hw,
2715                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2716                 if (status_mbx != FM10K_SUCCESS)
2717                         PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2718                 fm10k_mbx_unlock(hw);
2719
2720                 /* first clear the internal SW recording structure */
2721                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false);
2722                 fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2723                                 MAIN_VSI_POOL_NUMBER);
2724
2725                 /*
2726                  * Add default mac address and vlan for the logical ports that
2727                  * have been created, leave to the application to fully recover
2728                  * Rx filtering.
2729                  */
2730                 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2731                                 MAIN_VSI_POOL_NUMBER);
2732                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
2733
2734                 dev_info->sm_down = 0;
2735                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2736                                 NULL, NULL);
2737         }
2738
2739         /* Re-enable interrupt from device side */
2740         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2741                                         FM10K_ITR_MASK_CLEAR);
2742         /* Re-enable interrupt from host side */
2743         rte_intr_enable(dev->intr_handle);
2744 }
2745
2746 /* Mailbox message handler in VF */
2747 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2748         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2749         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2750         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2751         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2752 };
2753
2754 static int
2755 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2756 {
2757         int err = 0;
2758
2759         /* Initialize mailbox lock */
2760         fm10k_mbx_initlock(hw);
2761
2762         /* Replace default message handler with new ones */
2763         if (hw->mac.type == fm10k_mac_vf)
2764                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2765
2766         if (err) {
2767                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2768                                 err);
2769                 return err;
2770         }
2771         /* Connect to SM for PF device or PF for VF device */
2772         return hw->mbx.ops.connect(hw, &hw->mbx);
2773 }
2774
2775 static void
2776 fm10k_close_mbx_service(struct fm10k_hw *hw)
2777 {
2778         /* Disconnect from SM for PF device or PF for VF device */
2779         hw->mbx.ops.disconnect(hw, &hw->mbx);
2780 }
2781
2782 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2783         .dev_configure          = fm10k_dev_configure,
2784         .dev_start              = fm10k_dev_start,
2785         .dev_stop               = fm10k_dev_stop,
2786         .dev_close              = fm10k_dev_close,
2787         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2788         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2789         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2790         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2791         .stats_get              = fm10k_stats_get,
2792         .xstats_get             = fm10k_xstats_get,
2793         .xstats_get_names       = fm10k_xstats_get_names,
2794         .stats_reset            = fm10k_stats_reset,
2795         .xstats_reset           = fm10k_stats_reset,
2796         .link_update            = fm10k_link_update,
2797         .dev_infos_get          = fm10k_dev_infos_get,
2798         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2799         .vlan_filter_set        = fm10k_vlan_filter_set,
2800         .vlan_offload_set       = fm10k_vlan_offload_set,
2801         .mac_addr_add           = fm10k_macaddr_add,
2802         .mac_addr_remove        = fm10k_macaddr_remove,
2803         .rx_queue_start         = fm10k_dev_rx_queue_start,
2804         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2805         .tx_queue_start         = fm10k_dev_tx_queue_start,
2806         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2807         .rx_queue_setup         = fm10k_rx_queue_setup,
2808         .rx_queue_release       = fm10k_rx_queue_release,
2809         .tx_queue_setup         = fm10k_tx_queue_setup,
2810         .tx_queue_release       = fm10k_tx_queue_release,
2811         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2812         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2813         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2814         .reta_update            = fm10k_reta_update,
2815         .reta_query             = fm10k_reta_query,
2816         .rss_hash_update        = fm10k_rss_hash_update,
2817         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2818 };
2819
2820 static int ftag_check_handler(__rte_unused const char *key,
2821                 const char *value, __rte_unused void *opaque)
2822 {
2823         if (strcmp(value, "1"))
2824                 return -1;
2825
2826         return 0;
2827 }
2828
2829 static int
2830 fm10k_check_ftag(struct rte_devargs *devargs)
2831 {
2832         struct rte_kvargs *kvlist;
2833         const char *ftag_key = "enable_ftag";
2834
2835         if (devargs == NULL)
2836                 return 0;
2837
2838         kvlist = rte_kvargs_parse(devargs->args, NULL);
2839         if (kvlist == NULL)
2840                 return 0;
2841
2842         if (!rte_kvargs_count(kvlist, ftag_key)) {
2843                 rte_kvargs_free(kvlist);
2844                 return 0;
2845         }
2846         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2847         if (rte_kvargs_process(kvlist, ftag_key,
2848                                 ftag_check_handler, NULL) < 0) {
2849                 rte_kvargs_free(kvlist);
2850                 return 0;
2851         }
2852         rte_kvargs_free(kvlist);
2853
2854         return 1;
2855 }
2856
2857 static uint16_t
2858 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2859                     uint16_t nb_pkts)
2860 {
2861         uint16_t nb_tx = 0;
2862         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2863
2864         while (nb_pkts) {
2865                 uint16_t ret, num;
2866
2867                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2868                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2869                                                  num);
2870                 nb_tx += ret;
2871                 nb_pkts -= ret;
2872                 if (ret < num)
2873                         break;
2874         }
2875
2876         return nb_tx;
2877 }
2878
2879 static void __attribute__((cold))
2880 fm10k_set_tx_function(struct rte_eth_dev *dev)
2881 {
2882         struct fm10k_tx_queue *txq;
2883         int i;
2884         int use_sse = 1;
2885         uint16_t tx_ftag_en = 0;
2886
2887         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2888                 /* primary process has set the ftag flag and txq_flags */
2889                 txq = dev->data->tx_queues[0];
2890                 if (fm10k_tx_vec_condition_check(txq)) {
2891                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2892                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2893                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2894                 } else {
2895                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2896                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2897                         dev->tx_pkt_prepare = NULL;
2898                 }
2899                 return;
2900         }
2901
2902         if (fm10k_check_ftag(dev->device->devargs))
2903                 tx_ftag_en = 1;
2904
2905         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2906                 txq = dev->data->tx_queues[i];
2907                 txq->tx_ftag_en = tx_ftag_en;
2908                 /* Check if Vector Tx is satisfied */
2909                 if (fm10k_tx_vec_condition_check(txq))
2910                         use_sse = 0;
2911         }
2912
2913         if (use_sse) {
2914                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2915                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2916                         txq = dev->data->tx_queues[i];
2917                         fm10k_txq_vec_setup(txq);
2918                 }
2919                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2920                 dev->tx_pkt_prepare = NULL;
2921         } else {
2922                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2923                 dev->tx_pkt_prepare = fm10k_prep_pkts;
2924                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2925         }
2926 }
2927
2928 static void __attribute__((cold))
2929 fm10k_set_rx_function(struct rte_eth_dev *dev)
2930 {
2931         struct fm10k_dev_info *dev_info =
2932                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2933         uint16_t i, rx_using_sse;
2934         uint16_t rx_ftag_en = 0;
2935
2936         if (fm10k_check_ftag(dev->device->devargs))
2937                 rx_ftag_en = 1;
2938
2939         /* In order to allow Vector Rx there are a few configuration
2940          * conditions to be met.
2941          */
2942         if (!fm10k_rx_vec_condition_check(dev) &&
2943                         dev_info->rx_vec_allowed && !rx_ftag_en) {
2944                 if (dev->data->scattered_rx)
2945                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2946                 else
2947                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2948         } else if (dev->data->scattered_rx)
2949                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2950         else
2951                 dev->rx_pkt_burst = fm10k_recv_pkts;
2952
2953         rx_using_sse =
2954                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2955                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2956
2957         if (rx_using_sse)
2958                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
2959         else
2960                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
2961
2962         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2963                 return;
2964
2965         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2966                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2967
2968                 rxq->rx_using_sse = rx_using_sse;
2969                 rxq->rx_ftag_en = rx_ftag_en;
2970         }
2971 }
2972
2973 static void
2974 fm10k_params_init(struct rte_eth_dev *dev)
2975 {
2976         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2977         struct fm10k_dev_info *info =
2978                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2979
2980         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2981          * there is no way to get link status without reading BAR4.  Until this
2982          * works, assume we have maximum bandwidth.
2983          * @todo - fix bus info
2984          */
2985         hw->bus_caps.speed = fm10k_bus_speed_8000;
2986         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2987         hw->bus_caps.payload = fm10k_bus_payload_512;
2988         hw->bus.speed = fm10k_bus_speed_8000;
2989         hw->bus.width = fm10k_bus_width_pcie_x8;
2990         hw->bus.payload = fm10k_bus_payload_256;
2991
2992         info->rx_vec_allowed = true;
2993 }
2994
2995 static int
2996 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2997 {
2998         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2999         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3000         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3001         int diag, i;
3002         struct fm10k_macvlan_filter_info *macvlan;
3003
3004         PMD_INIT_FUNC_TRACE();
3005
3006         dev->dev_ops = &fm10k_eth_dev_ops;
3007         dev->rx_pkt_burst = &fm10k_recv_pkts;
3008         dev->tx_pkt_burst = &fm10k_xmit_pkts;
3009         dev->tx_pkt_prepare = &fm10k_prep_pkts;
3010
3011         /*
3012          * Primary process does the whole initialization, for secondary
3013          * processes, we just select the same Rx and Tx function as primary.
3014          */
3015         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3016                 fm10k_set_rx_function(dev);
3017                 fm10k_set_tx_function(dev);
3018                 return 0;
3019         }
3020
3021         rte_eth_copy_pci_info(dev, pdev);
3022
3023         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
3024         memset(macvlan, 0, sizeof(*macvlan));
3025         /* Vendor and Device ID need to be set before init of shared code */
3026         memset(hw, 0, sizeof(*hw));
3027         hw->device_id = pdev->id.device_id;
3028         hw->vendor_id = pdev->id.vendor_id;
3029         hw->subsystem_device_id = pdev->id.subsystem_device_id;
3030         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
3031         hw->revision_id = 0;
3032         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
3033         if (hw->hw_addr == NULL) {
3034                 PMD_INIT_LOG(ERR, "Bad mem resource."
3035                         " Try to blacklist unused devices.");
3036                 return -EIO;
3037         }
3038
3039         /* Store fm10k_adapter pointer */
3040         hw->back = dev->data->dev_private;
3041
3042         /* Initialize the shared code */
3043         diag = fm10k_init_shared_code(hw);
3044         if (diag != FM10K_SUCCESS) {
3045                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
3046                 return -EIO;
3047         }
3048
3049         /* Initialize parameters */
3050         fm10k_params_init(dev);
3051
3052         /* Initialize the hw */
3053         diag = fm10k_init_hw(hw);
3054         if (diag != FM10K_SUCCESS) {
3055                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
3056                 return -EIO;
3057         }
3058
3059         /* Initialize MAC address(es) */
3060         dev->data->mac_addrs = rte_zmalloc("fm10k",
3061                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
3062         if (dev->data->mac_addrs == NULL) {
3063                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
3064                 return -ENOMEM;
3065         }
3066
3067         diag = fm10k_read_mac_addr(hw);
3068
3069         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
3070                         &dev->data->mac_addrs[0]);
3071
3072         if (diag != FM10K_SUCCESS ||
3073                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
3074
3075                 /* Generate a random addr */
3076                 eth_random_addr(hw->mac.addr);
3077                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
3078                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
3079                 &dev->data->mac_addrs[0]);
3080         }
3081
3082         /* Reset the hw statistics */
3083         fm10k_stats_reset(dev);
3084
3085         /* Reset the hw */
3086         diag = fm10k_reset_hw(hw);
3087         if (diag != FM10K_SUCCESS) {
3088                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
3089                 return -EIO;
3090         }
3091
3092         /* Setup mailbox service */
3093         diag = fm10k_setup_mbx_service(hw);
3094         if (diag != FM10K_SUCCESS) {
3095                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
3096                 return -EIO;
3097         }
3098
3099         /*PF/VF has different interrupt handling mechanism */
3100         if (hw->mac.type == fm10k_mac_pf) {
3101                 /* register callback func to eal lib */
3102                 rte_intr_callback_register(intr_handle,
3103                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3104
3105                 /* enable MISC interrupt */
3106                 fm10k_dev_enable_intr_pf(dev);
3107         } else { /* VF */
3108                 rte_intr_callback_register(intr_handle,
3109                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3110
3111                 fm10k_dev_enable_intr_vf(dev);
3112         }
3113
3114         /* Enable intr after callback registered */
3115         rte_intr_enable(intr_handle);
3116
3117         hw->mac.ops.update_int_moderator(hw);
3118
3119         /* Make sure Switch Manager is ready before going forward. */
3120         if (hw->mac.type == fm10k_mac_pf) {
3121                 int switch_ready = 0;
3122
3123                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3124                         fm10k_mbx_lock(hw);
3125                         hw->mac.ops.get_host_state(hw, &switch_ready);
3126                         fm10k_mbx_unlock(hw);
3127                         if (switch_ready)
3128                                 break;
3129                         /* Delay some time to acquire async LPORT_MAP info. */
3130                         rte_delay_us(WAIT_SWITCH_MSG_US);
3131                 }
3132
3133                 if (switch_ready == 0) {
3134                         PMD_INIT_LOG(ERR, "switch is not ready");
3135                         return -1;
3136                 }
3137         }
3138
3139         /*
3140          * Below function will trigger operations on mailbox, acquire lock to
3141          * avoid race condition from interrupt handler. Operations on mailbox
3142          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3143          * will handle and generate an interrupt to our side. Then,  FIFO in
3144          * mailbox will be touched.
3145          */
3146         fm10k_mbx_lock(hw);
3147         /* Enable port first */
3148         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3149                                         MAX_LPORT_NUM, 1);
3150
3151         /* Set unicast mode by default. App can change to other mode in other
3152          * API func.
3153          */
3154         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3155                                         FM10K_XCAST_MODE_NONE);
3156
3157         fm10k_mbx_unlock(hw);
3158
3159         /* Make sure default VID is ready before going forward. */
3160         if (hw->mac.type == fm10k_mac_pf) {
3161                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3162                         if (hw->mac.default_vid)
3163                                 break;
3164                         /* Delay some time to acquire async port VLAN info. */
3165                         rte_delay_us(WAIT_SWITCH_MSG_US);
3166                 }
3167
3168                 if (!hw->mac.default_vid) {
3169                         PMD_INIT_LOG(ERR, "default VID is not ready");
3170                         return -1;
3171                 }
3172         }
3173
3174         /* Add default mac address */
3175         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3176                 MAIN_VSI_POOL_NUMBER);
3177
3178         return 0;
3179 }
3180
3181 static int
3182 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3183 {
3184         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3185         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3186         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3187         PMD_INIT_FUNC_TRACE();
3188
3189         /* only uninitialize in the primary process */
3190         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3191                 return 0;
3192
3193         /* safe to close dev here */
3194         fm10k_dev_close(dev);
3195
3196         dev->dev_ops = NULL;
3197         dev->rx_pkt_burst = NULL;
3198         dev->tx_pkt_burst = NULL;
3199
3200         /* disable uio/vfio intr */
3201         rte_intr_disable(intr_handle);
3202
3203         /*PF/VF has different interrupt handling mechanism */
3204         if (hw->mac.type == fm10k_mac_pf) {
3205                 /* disable interrupt */
3206                 fm10k_dev_disable_intr_pf(dev);
3207
3208                 /* unregister callback func to eal lib */
3209                 rte_intr_callback_unregister(intr_handle,
3210                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3211         } else {
3212                 /* disable interrupt */
3213                 fm10k_dev_disable_intr_vf(dev);
3214
3215                 rte_intr_callback_unregister(intr_handle,
3216                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3217         }
3218
3219         /* free mac memory */
3220         if (dev->data->mac_addrs) {
3221                 rte_free(dev->data->mac_addrs);
3222                 dev->data->mac_addrs = NULL;
3223         }
3224
3225         memset(hw, 0, sizeof(*hw));
3226
3227         return 0;
3228 }
3229
3230 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3231         struct rte_pci_device *pci_dev)
3232 {
3233         return rte_eth_dev_pci_generic_probe(pci_dev,
3234                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3235 }
3236
3237 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3238 {
3239         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3240 }
3241
3242 /*
3243  * The set of PCI devices this driver supports. This driver will enable both PF
3244  * and SRIOV-VF devices.
3245  */
3246 static const struct rte_pci_id pci_id_fm10k_map[] = {
3247         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3248         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3249         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3250         { .vendor_id = 0, /* sentinel */ },
3251 };
3252
3253 static struct rte_pci_driver rte_pmd_fm10k = {
3254         .id_table = pci_id_fm10k_map,
3255         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
3256                      RTE_PCI_DRV_IOVA_AS_VA,
3257         .probe = eth_fm10k_pci_probe,
3258         .remove = eth_fm10k_pci_remove,
3259 };
3260
3261 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3262 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3263 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");