drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 /* Default delay to acquire mailbox lock */
  45 #define FM10K_MBXLOCK_DELAY_US 20
  46 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  47
  48 #define MAIN_VSI_POOL_NUMBER 0
  49
  50 /* Max try times to acquire switch status */
  51 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  52 /* Wait interval to get switch status */
  53 #define WAIT_SWITCH_MSG_US    100000
  54 /* Number of chars per uint32 type */
  55 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  56 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  57
  58 /* default 1:1 map from queue ID to interrupt vector ID */
  59 #define Q2V(dev, queue_id) (dev->pci_dev->intr_handle.intr_vec[queue_id])
  60
  61 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  62 #define MAX_LPORT_NUM    128
  63 #define GLORT_FD_Q_BASE  0x40
  64 #define GLORT_PF_MASK    0xFFC0
  65 #define GLORT_FD_MASK    GLORT_PF_MASK
  66 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  67
  68 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  69 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  70 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  71 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  72 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  73 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  74 static int
  75 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  76 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  77         const u8 *mac, bool add, uint32_t pool);
  78 static void fm10k_tx_queue_release(void *queue);
  79 static void fm10k_rx_queue_release(void *queue);
  80 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  81 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  82
  83 struct fm10k_xstats_name_off {
  84         char name[RTE_ETH_XSTATS_NAME_SIZE];
  85         unsigned offset;
  86 };
  87
  88 struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  89         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  90         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  91         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  92         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  93         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  94         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  95         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  96         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  97                 nodesc_drop)},
  98 };
  99
 100 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
 101                 sizeof(fm10k_hw_stats_strings[0]))
 102
 103 struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
 104         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
 105         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
 106         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
 107 };
 108
 109 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
 110                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
 111
 112 struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 113         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 114         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 115 };
 116
 117 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 118                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 119
 120 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 121                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 122 static int
 123 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 124
 125 static void
 126 fm10k_mbx_initlock(struct fm10k_hw *hw)
 127 {
 128         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 129 }
 130
 131 static void
 132 fm10k_mbx_lock(struct fm10k_hw *hw)
 133 {
 134         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 135                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 136 }
 137
 138 static void
 139 fm10k_mbx_unlock(struct fm10k_hw *hw)
 140 {
 141         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 142 }
 143
 144 /* Stubs needed for linkage when vPMD is disabled */
 145 int __attribute__((weak))
 146 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 147 {
 148         return -1;
 149 }
 150
 151 uint16_t __attribute__((weak))
 152 fm10k_recv_pkts_vec(
 153         __rte_unused void *rx_queue,
 154         __rte_unused struct rte_mbuf **rx_pkts,
 155         __rte_unused uint16_t nb_pkts)
 156 {
 157         return 0;
 158 }
 159
 160 uint16_t __attribute__((weak))
 161 fm10k_recv_scattered_pkts_vec(
 162                 __rte_unused void *rx_queue,
 163                 __rte_unused struct rte_mbuf **rx_pkts,
 164                 __rte_unused uint16_t nb_pkts)
 165 {
 166         return 0;
 167 }
 168
 169 int __attribute__((weak))
 170 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 171
 172 {
 173         return -1;
 174 }
 175
 176 void __attribute__((weak))
 177 fm10k_rx_queue_release_mbufs_vec(
 178                 __rte_unused struct fm10k_rx_queue *rxq)
 179 {
 180         return;
 181 }
 182
 183 void __attribute__((weak))
 184 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 185 {
 186         return;
 187 }
 188
 189 int __attribute__((weak))
 190 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 191 {
 192         return -1;
 193 }
 194
 195 uint16_t __attribute__((weak))
 196 fm10k_xmit_pkts_vec(__rte_unused void *tx_queue,
 197                 __rte_unused struct rte_mbuf **tx_pkts,
 198                 __rte_unused uint16_t nb_pkts)
 199 {
 200         return 0;
 201 }
 202
 203 /*
 204  * reset queue to initial state, allocate software buffers used when starting
 205  * device.
 206  * return 0 on success
 207  * return -ENOMEM if buffers cannot be allocated
 208  * return -EINVAL if buffers do not satisfy alignment condition
 209  */
 210 static inline int
 211 rx_queue_reset(struct fm10k_rx_queue *q)
 212 {
 213         static const union fm10k_rx_desc zero = {{0} };
 214         uint64_t dma_addr;
 215         int i, diag;
 216         PMD_INIT_FUNC_TRACE();
 217
 218         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 219         if (diag != 0)
 220                 return -ENOMEM;
 221
 222         for (i = 0; i < q->nb_desc; ++i) {
 223                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 224                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 225                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 226                                                 q->nb_desc);
 227                         return -EINVAL;
 228                 }
 229                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 230                 q->hw_ring[i].q.pkt_addr = dma_addr;
 231                 q->hw_ring[i].q.hdr_addr = dma_addr;
 232         }
 233
 234         /* initialize extra software ring entries. Space for these extra
 235          * entries is always allocated.
 236          */
 237         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 238         for (i = 0; i < q->nb_fake_desc; ++i) {
 239                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 240                 q->hw_ring[q->nb_desc + i] = zero;
 241         }
 242
 243         q->next_dd = 0;
 244         q->next_alloc = 0;
 245         q->next_trigger = q->alloc_thresh - 1;
 246         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 247         q->rxrearm_start = 0;
 248         q->rxrearm_nb = 0;
 249
 250         return 0;
 251 }
 252
 253 /*
 254  * clean queue, descriptor rings, free software buffers used when stopping
 255  * device.
 256  */
 257 static inline void
 258 rx_queue_clean(struct fm10k_rx_queue *q)
 259 {
 260         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 261         uint32_t i;
 262         PMD_INIT_FUNC_TRACE();
 263
 264         /* zero descriptor rings */
 265         for (i = 0; i < q->nb_desc; ++i)
 266                 q->hw_ring[i] = zero;
 267
 268         /* zero faked descriptors */
 269         for (i = 0; i < q->nb_fake_desc; ++i)
 270                 q->hw_ring[q->nb_desc + i] = zero;
 271
 272         /* vPMD driver has a different way of releasing mbufs. */
 273         if (q->rx_using_sse) {
 274                 fm10k_rx_queue_release_mbufs_vec(q);
 275                 return;
 276         }
 277
 278         /* free software buffers */
 279         for (i = 0; i < q->nb_desc; ++i) {
 280                 if (q->sw_ring[i]) {
 281                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 282                         q->sw_ring[i] = NULL;
 283                 }
 284         }
 285 }
 286
 287 /*
 288  * free all queue memory used when releasing the queue (i.e. configure)
 289  */
 290 static inline void
 291 rx_queue_free(struct fm10k_rx_queue *q)
 292 {
 293         PMD_INIT_FUNC_TRACE();
 294         if (q) {
 295                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 296                 rx_queue_clean(q);
 297                 if (q->sw_ring) {
 298                         rte_free(q->sw_ring);
 299                         q->sw_ring = NULL;
 300                 }
 301                 rte_free(q);
 302                 q = NULL;
 303         }
 304 }
 305
 306 /*
 307  * disable RX queue, wait unitl HW finished necessary flush operation
 308  */
 309 static inline int
 310 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 311 {
 312         uint32_t reg, i;
 313
 314         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 315         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 316                         reg & ~FM10K_RXQCTL_ENABLE);
 317
 318         /* Wait 100us at most */
 319         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 320                 rte_delay_us(1);
 321                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 322                 if (!(reg & FM10K_RXQCTL_ENABLE))
 323                         break;
 324         }
 325
 326         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 327                 return -1;
 328
 329         return 0;
 330 }
 331
 332 /*
 333  * reset queue to initial state, allocate software buffers used when starting
 334  * device
 335  */
 336 static inline void
 337 tx_queue_reset(struct fm10k_tx_queue *q)
 338 {
 339         PMD_INIT_FUNC_TRACE();
 340         q->last_free = 0;
 341         q->next_free = 0;
 342         q->nb_used = 0;
 343         q->nb_free = q->nb_desc - 1;
 344         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 345         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 346 }
 347
 348 /*
 349  * clean queue, descriptor rings, free software buffers used when stopping
 350  * device
 351  */
 352 static inline void
 353 tx_queue_clean(struct fm10k_tx_queue *q)
 354 {
 355         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 356         uint32_t i;
 357         PMD_INIT_FUNC_TRACE();
 358
 359         /* zero descriptor rings */
 360         for (i = 0; i < q->nb_desc; ++i)
 361                 q->hw_ring[i] = zero;
 362
 363         /* free software buffers */
 364         for (i = 0; i < q->nb_desc; ++i) {
 365                 if (q->sw_ring[i]) {
 366                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 367                         q->sw_ring[i] = NULL;
 368                 }
 369         }
 370 }
 371
 372 /*
 373  * free all queue memory used when releasing the queue (i.e. configure)
 374  */
 375 static inline void
 376 tx_queue_free(struct fm10k_tx_queue *q)
 377 {
 378         PMD_INIT_FUNC_TRACE();
 379         if (q) {
 380                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 381                 tx_queue_clean(q);
 382                 if (q->rs_tracker.list) {
 383                         rte_free(q->rs_tracker.list);
 384                         q->rs_tracker.list = NULL;
 385                 }
 386                 if (q->sw_ring) {
 387                         rte_free(q->sw_ring);
 388                         q->sw_ring = NULL;
 389                 }
 390                 rte_free(q);
 391                 q = NULL;
 392         }
 393 }
 394
 395 /*
 396  * disable TX queue, wait unitl HW finished necessary flush operation
 397  */
 398 static inline int
 399 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 400 {
 401         uint32_t reg, i;
 402
 403         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 404         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 405                         reg & ~FM10K_TXDCTL_ENABLE);
 406
 407         /* Wait 100us at most */
 408         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 409                 rte_delay_us(1);
 410                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 411                 if (!(reg & FM10K_TXDCTL_ENABLE))
 412                         break;
 413         }
 414
 415         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 416                 return -1;
 417
 418         return 0;
 419 }
 420
 421 static int
 422 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 423 {
 424         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 425         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 426         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 427         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 428
 429         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 430
 431         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 432                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 433                 return -EINVAL;
 434         }
 435
 436         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 437                 return 0;
 438
 439         if (hw->mac.type == fm10k_mac_vf) {
 440                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 441                 return -EINVAL;
 442         }
 443
 444         /* Check VMDQ queue pool number */
 445         if (vmdq_conf->nb_queue_pools >
 446                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 447                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 448                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 449                         vmdq_conf->nb_queue_pools);
 450                 return -EINVAL;
 451         }
 452
 453         return 0;
 454 }
 455
 456 static const struct fm10k_txq_ops def_txq_ops = {
 457         .reset = tx_queue_reset,
 458 };
 459
 460 static int
 461 fm10k_dev_configure(struct rte_eth_dev *dev)
 462 {
 463         int ret;
 464
 465         PMD_INIT_FUNC_TRACE();
 466
 467         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 468                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 469         /* multipe queue mode checking */
 470         ret  = fm10k_check_mq_mode(dev);
 471         if (ret != 0) {
 472                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 473                             ret);
 474                 return ret;
 475         }
 476
 477         return 0;
 478 }
 479
 480 /* fls = find last set bit = 32 minus the number of leading zeros */
 481 #ifndef fls
 482 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 483 #endif
 484
 485 static void
 486 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 487 {
 488         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 489         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 490         uint32_t i;
 491
 492         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 493
 494         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 495                 if (!vmdq_conf->pool_map[i].pools)
 496                         continue;
 497                 fm10k_mbx_lock(hw);
 498                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 499                 fm10k_mbx_unlock(hw);
 500         }
 501 }
 502
 503 static void
 504 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 505 {
 506         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 507
 508         /* Add default mac address */
 509         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 510                 MAIN_VSI_POOL_NUMBER);
 511 }
 512
 513 static void
 514 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 515 {
 516         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 517         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 518         uint32_t mrqc, *key, i, reta, j;
 519         uint64_t hf;
 520
 521 #define RSS_KEY_SIZE 40
 522         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 523                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 524                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 525                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 526                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 527                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 528         };
 529
 530         if (dev->data->nb_rx_queues == 1 ||
 531             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 532             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 533                 return;
 534
 535         /* random key is rss_intel_key (default) or user provided (rss_key) */
 536         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 537                 key = (uint32_t *)rss_intel_key;
 538         else
 539                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 540
 541         /* Now fill our hash function seeds, 4 bytes at a time */
 542         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 543                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 544
 545         /*
 546          * Fill in redirection table
 547          * The byte-swap is needed because NIC registers are in
 548          * little-endian order.
 549          */
 550         reta = 0;
 551         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 552                 if (j == dev->data->nb_rx_queues)
 553                         j = 0;
 554                 reta = (reta << CHAR_BIT) | j;
 555                 if ((i & 3) == 3)
 556                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 557                                         rte_bswap32(reta));
 558         }
 559
 560         /*
 561          * Generate RSS hash based on packet types, TCP/UDP
 562          * port numbers and/or IPv4/v6 src and dst addresses
 563          */
 564         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 565         mrqc = 0;
 566         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 567         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 568         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 569         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 570         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 571         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 572         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 573         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 574         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 575
 576         if (mrqc == 0) {
 577                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 578                         "supported", hf);
 579                 return;
 580         }
 581
 582         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 583 }
 584
 585 static void
 586 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 587 {
 588         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 589         uint32_t i;
 590
 591         for (i = 0; i < nb_lport_new; i++) {
 592                 /* Set unicast mode by default. App can change
 593                  * to other mode in other API func.
 594                  */
 595                 fm10k_mbx_lock(hw);
 596                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 597                         FM10K_XCAST_MODE_NONE);
 598                 fm10k_mbx_unlock(hw);
 599         }
 600 }
 601
 602 static void
 603 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 604 {
 605         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 606         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 607         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 608         struct fm10k_macvlan_filter_info *macvlan;
 609         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 610         uint16_t nb_lport_new;
 611
 612         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 613         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 614
 615         fm10k_dev_rss_configure(dev);
 616
 617         /* only PF supports VMDQ */
 618         if (hw->mac.type != fm10k_mac_pf)
 619                 return;
 620
 621         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 622                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 623
 624         /* no pool number change, no need to update logic port and VLAN/MAC */
 625         if (macvlan->nb_queue_pools == nb_queue_pools)
 626                 return;
 627
 628         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 629         fm10k_dev_logic_port_update(dev, nb_lport_new);
 630
 631         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 632         memset(dev->data->mac_addrs, 0,
 633                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 634         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 635                 &dev->data->mac_addrs[0]);
 636         memset(macvlan, 0, sizeof(*macvlan));
 637         macvlan->nb_queue_pools = nb_queue_pools;
 638
 639         if (nb_queue_pools)
 640                 fm10k_dev_vmdq_rx_configure(dev);
 641         else
 642                 fm10k_dev_pf_main_vsi_reset(dev);
 643 }
 644
 645 static int
 646 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 647 {
 648         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 649         int i, ret;
 650         struct fm10k_tx_queue *txq;
 651         uint64_t base_addr;
 652         uint32_t size;
 653
 654         /* Disable TXINT to avoid possible interrupt */
 655         for (i = 0; i < hw->mac.max_queues; i++)
 656                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 657                                 3 << FM10K_TXINT_TIMER_SHIFT);
 658
 659         /* Setup TX queue */
 660         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 661                 txq = dev->data->tx_queues[i];
 662                 base_addr = txq->hw_ring_phys_addr;
 663                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 664
 665                 /* disable queue to avoid issues while updating state */
 666                 ret = tx_queue_disable(hw, i);
 667                 if (ret) {
 668                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 669                         return -1;
 670                 }
 671
 672                 /* set location and size for descriptor ring */
 673                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 674                                 base_addr & UINT64_LOWER_32BITS_MASK);
 675                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 676                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 677                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 678         }
 679
 680         /* set up vector or scalar TX function as appropriate */
 681         fm10k_set_tx_function(dev);
 682
 683         return 0;
 684 }
 685
 686 static int
 687 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 688 {
 689         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 690         struct fm10k_macvlan_filter_info *macvlan;
 691         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
 692         int i, ret;
 693         struct fm10k_rx_queue *rxq;
 694         uint64_t base_addr;
 695         uint32_t size;
 696         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 697         uint32_t logic_port = hw->mac.dglort_map;
 698         uint16_t buf_size;
 699         uint16_t queue_stride = 0;
 700
 701         /* enable RXINT for interrupt mode */
 702         i = 0;
 703         if (rte_intr_dp_is_en(intr_handle)) {
 704                 for (; i < dev->data->nb_rx_queues; i++) {
 705                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(dev, i));
 706                         if (hw->mac.type == fm10k_mac_pf)
 707                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
 708                                         FM10K_ITR_AUTOMASK |
 709                                         FM10K_ITR_MASK_CLEAR);
 710                         else
 711                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
 712                                         FM10K_ITR_AUTOMASK |
 713                                         FM10K_ITR_MASK_CLEAR);
 714                 }
 715         }
 716         /* Disable other RXINT to avoid possible interrupt */
 717         for (; i < hw->mac.max_queues; i++)
 718                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 719                         3 << FM10K_RXINT_TIMER_SHIFT);
 720
 721         /* Setup RX queues */
 722         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 723                 rxq = dev->data->rx_queues[i];
 724                 base_addr = rxq->hw_ring_phys_addr;
 725                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 726
 727                 /* disable queue to avoid issues while updating state */
 728                 ret = rx_queue_disable(hw, i);
 729                 if (ret) {
 730                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 731                         return -1;
 732                 }
 733
 734                 /* Setup the Base and Length of the Rx Descriptor Ring */
 735                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 736                                 base_addr & UINT64_LOWER_32BITS_MASK);
 737                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 738                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 739                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 740
 741                 /* Configure the Rx buffer size for one buff without split */
 742                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 743                         RTE_PKTMBUF_HEADROOM);
 744                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 745                  * reserved for this purpose, and the worst case could be 511B.
 746                  * But SRR reg assumes all buffers have the same size. In order
 747                  * to fill the gap, we'll have to consider the worst case and
 748                  * assume 512B is reserved. If we don't do so, it's possible
 749                  * for HW to overwrite data to next mbuf.
 750                  */
 751                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 752
 753                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 754                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 755                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 756
 757                 /* It adds dual VLAN length for supporting dual VLAN */
 758                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 759                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 760                         dev->data->dev_conf.rxmode.enable_scatter) {
 761                         uint32_t reg;
 762                         dev->data->scattered_rx = 1;
 763                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 764                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 765                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 766                 }
 767
 768                 /* Enable drop on empty, it's RO for VF */
 769                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 770                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 771
 772                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 773                 FM10K_WRITE_FLUSH(hw);
 774         }
 775
 776         /* Configure VMDQ/RSS if applicable */
 777         fm10k_dev_mq_rx_configure(dev);
 778
 779         /* Decide the best RX function */
 780         fm10k_set_rx_function(dev);
 781
 782         /* update RX_SGLORT for loopback suppress*/
 783         if (hw->mac.type != fm10k_mac_pf)
 784                 return 0;
 785         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 786         if (macvlan->nb_queue_pools)
 787                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 788         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 789                 if (i && queue_stride && !(i % queue_stride))
 790                         logic_port++;
 791                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 792         }
 793
 794         return 0;
 795 }
 796
 797 static int
 798 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 799 {
 800         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 801         int err = -1;
 802         uint32_t reg;
 803         struct fm10k_rx_queue *rxq;
 804
 805         PMD_INIT_FUNC_TRACE();
 806
 807         if (rx_queue_id < dev->data->nb_rx_queues) {
 808                 rxq = dev->data->rx_queues[rx_queue_id];
 809                 err = rx_queue_reset(rxq);
 810                 if (err == -ENOMEM) {
 811                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 812                         return err;
 813                 } else if (err == -EINVAL) {
 814                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 815                                 " %d", err);
 816                         return err;
 817                 }
 818
 819                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 820                  * Note: this must be done AFTER the queue is enabled on real
 821                  * hardware, but BEFORE the queue is enabled when using the
 822                  * emulation platform. Do it in both places for now and remove
 823                  * this comment and the following two register writes when the
 824                  * emulation platform is no longer being used.
 825                  */
 826                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 827                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 828
 829                 /* Set PF ownership flag for PF devices */
 830                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 831                 if (hw->mac.type == fm10k_mac_pf)
 832                         reg |= FM10K_RXQCTL_PF;
 833                 reg |= FM10K_RXQCTL_ENABLE;
 834                 /* enable RX queue */
 835                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 836                 FM10K_WRITE_FLUSH(hw);
 837
 838                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 839                  * Note: this must be done AFTER the queue is enabled
 840                  */
 841                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 842                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 843                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 844         }
 845
 846         return err;
 847 }
 848
 849 static int
 850 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 851 {
 852         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 853
 854         PMD_INIT_FUNC_TRACE();
 855
 856         if (rx_queue_id < dev->data->nb_rx_queues) {
 857                 /* Disable RX queue */
 858                 rx_queue_disable(hw, rx_queue_id);
 859
 860                 /* Free mbuf and clean HW ring */
 861                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 862                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 863         }
 864
 865         return 0;
 866 }
 867
 868 static int
 869 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 870 {
 871         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 872         /** @todo - this should be defined in the shared code */
 873 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 874         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 875         int err = 0;
 876
 877         PMD_INIT_FUNC_TRACE();
 878
 879         if (tx_queue_id < dev->data->nb_tx_queues) {
 880                 struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 881
 882                 q->ops->reset(q);
 883
 884                 /* reset head and tail pointers */
 885                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 886                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 887
 888                 /* enable TX queue */
 889                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 890                                         FM10K_TXDCTL_ENABLE | txdctl);
 891                 FM10K_WRITE_FLUSH(hw);
 892                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 893         } else
 894                 err = -1;
 895
 896         return err;
 897 }
 898
 899 static int
 900 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 901 {
 902         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 903
 904         PMD_INIT_FUNC_TRACE();
 905
 906         if (tx_queue_id < dev->data->nb_tx_queues) {
 907                 tx_queue_disable(hw, tx_queue_id);
 908                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 909                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 910         }
 911
 912         return 0;
 913 }
 914
 915 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 916 {
 917         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 918                 != FM10K_DGLORTMAP_NONE);
 919 }
 920
 921 static void
 922 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 923 {
 924         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 925         int status;
 926
 927         PMD_INIT_FUNC_TRACE();
 928
 929         /* Return if it didn't acquire valid glort range */
 930         if (!fm10k_glort_valid(hw))
 931                 return;
 932
 933         fm10k_mbx_lock(hw);
 934         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 935                                 FM10K_XCAST_MODE_PROMISC);
 936         fm10k_mbx_unlock(hw);
 937
 938         if (status != FM10K_SUCCESS)
 939                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 940 }
 941
 942 static void
 943 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 944 {
 945         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 946         uint8_t mode;
 947         int status;
 948
 949         PMD_INIT_FUNC_TRACE();
 950
 951         /* Return if it didn't acquire valid glort range */
 952         if (!fm10k_glort_valid(hw))
 953                 return;
 954
 955         if (dev->data->all_multicast == 1)
 956                 mode = FM10K_XCAST_MODE_ALLMULTI;
 957         else
 958                 mode = FM10K_XCAST_MODE_NONE;
 959
 960         fm10k_mbx_lock(hw);
 961         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 962                                 mode);
 963         fm10k_mbx_unlock(hw);
 964
 965         if (status != FM10K_SUCCESS)
 966                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 967 }
 968
 969 static void
 970 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 971 {
 972         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 973         int status;
 974
 975         PMD_INIT_FUNC_TRACE();
 976
 977         /* Return if it didn't acquire valid glort range */
 978         if (!fm10k_glort_valid(hw))
 979                 return;
 980
 981         /* If promiscuous mode is enabled, it doesn't make sense to enable
 982          * allmulticast and disable promiscuous since fm10k only can select
 983          * one of the modes.
 984          */
 985         if (dev->data->promiscuous) {
 986                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 987                         "needn't enable allmulticast");
 988                 return;
 989         }
 990
 991         fm10k_mbx_lock(hw);
 992         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 993                                 FM10K_XCAST_MODE_ALLMULTI);
 994         fm10k_mbx_unlock(hw);
 995
 996         if (status != FM10K_SUCCESS)
 997                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 998 }
 999
1000 static void
1001 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1002 {
1003         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1004         int status;
1005
1006         PMD_INIT_FUNC_TRACE();
1007
1008         /* Return if it didn't acquire valid glort range */
1009         if (!fm10k_glort_valid(hw))
1010                 return;
1011
1012         if (dev->data->promiscuous) {
1013                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1014                         "since promisc mode is enabled");
1015                 return;
1016         }
1017
1018         fm10k_mbx_lock(hw);
1019         /* Change mode to unicast mode */
1020         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1021                                 FM10K_XCAST_MODE_NONE);
1022         fm10k_mbx_unlock(hw);
1023
1024         if (status != FM10K_SUCCESS)
1025                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1026 }
1027
1028 static void
1029 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1030 {
1031         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1032         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1033         uint16_t nb_queue_pools;
1034         struct fm10k_macvlan_filter_info *macvlan;
1035
1036         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1037         nb_queue_pools = macvlan->nb_queue_pools;
1038         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
1039         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
1040
1041         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1042         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1043         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1044                         hw->mac.dglort_map;
1045         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1046         /* Configure VMDQ/RSS DGlort Decoder */
1047         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1048
1049         /* Flow Director configurations, only queue number is valid. */
1050         dglortdec = fls(dev->data->nb_rx_queues - 1);
1051         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1052                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1053         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1054         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1055
1056         /* Invalidate all other GLORT entries */
1057         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1058                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1059                                 FM10K_DGLORTMAP_NONE);
1060 }
1061
1062 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1063 static int
1064 fm10k_dev_start(struct rte_eth_dev *dev)
1065 {
1066         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1067         int i, diag;
1068
1069         PMD_INIT_FUNC_TRACE();
1070
1071         /* stop, init, then start the hw */
1072         diag = fm10k_stop_hw(hw);
1073         if (diag != FM10K_SUCCESS) {
1074                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1075                 return -EIO;
1076         }
1077
1078         diag = fm10k_init_hw(hw);
1079         if (diag != FM10K_SUCCESS) {
1080                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1081                 return -EIO;
1082         }
1083
1084         diag = fm10k_start_hw(hw);
1085         if (diag != FM10K_SUCCESS) {
1086                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1087                 return -EIO;
1088         }
1089
1090         diag = fm10k_dev_tx_init(dev);
1091         if (diag) {
1092                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1093                 return diag;
1094         }
1095
1096         if (fm10k_dev_rxq_interrupt_setup(dev))
1097                 return -EIO;
1098
1099         diag = fm10k_dev_rx_init(dev);
1100         if (diag) {
1101                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1102                 return diag;
1103         }
1104
1105         if (hw->mac.type == fm10k_mac_pf)
1106                 fm10k_dev_dglort_map_configure(dev);
1107
1108         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1109                 struct fm10k_rx_queue *rxq;
1110                 rxq = dev->data->rx_queues[i];
1111
1112                 if (rxq->rx_deferred_start)
1113                         continue;
1114                 diag = fm10k_dev_rx_queue_start(dev, i);
1115                 if (diag != 0) {
1116                         int j;
1117                         for (j = 0; j < i; ++j)
1118                                 rx_queue_clean(dev->data->rx_queues[j]);
1119                         return diag;
1120                 }
1121         }
1122
1123         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1124                 struct fm10k_tx_queue *txq;
1125                 txq = dev->data->tx_queues[i];
1126
1127                 if (txq->tx_deferred_start)
1128                         continue;
1129                 diag = fm10k_dev_tx_queue_start(dev, i);
1130                 if (diag != 0) {
1131                         int j;
1132                         for (j = 0; j < i; ++j)
1133                                 tx_queue_clean(dev->data->tx_queues[j]);
1134                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1135                                 rx_queue_clean(dev->data->rx_queues[j]);
1136                         return diag;
1137                 }
1138         }
1139
1140         /* Update default vlan when not in VMDQ mode */
1141         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1142                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1143
1144         return 0;
1145 }
1146
1147 static void
1148 fm10k_dev_stop(struct rte_eth_dev *dev)
1149 {
1150         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1151         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1152         int i;
1153
1154         PMD_INIT_FUNC_TRACE();
1155
1156         if (dev->data->tx_queues)
1157                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1158                         fm10k_dev_tx_queue_stop(dev, i);
1159
1160         if (dev->data->rx_queues)
1161                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1162                         fm10k_dev_rx_queue_stop(dev, i);
1163
1164         /* Disable datapath event */
1165         if (rte_intr_dp_is_en(intr_handle)) {
1166                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1167                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1168                                 3 << FM10K_RXINT_TIMER_SHIFT);
1169                         if (hw->mac.type == fm10k_mac_pf)
1170                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
1171                                         FM10K_ITR_MASK_SET);
1172                         else
1173                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
1174                                         FM10K_ITR_MASK_SET);
1175                 }
1176         }
1177         /* Clean datapath event and queue/vec mapping */
1178         rte_intr_efd_disable(intr_handle);
1179         rte_free(intr_handle->intr_vec);
1180         intr_handle->intr_vec = NULL;
1181 }
1182
1183 static void
1184 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1185 {
1186         int i;
1187
1188         PMD_INIT_FUNC_TRACE();
1189
1190         if (dev->data->tx_queues) {
1191                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1192                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1193
1194                         tx_queue_free(txq);
1195                 }
1196         }
1197
1198         if (dev->data->rx_queues) {
1199                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1200                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1201         }
1202 }
1203
1204 static void
1205 fm10k_dev_close(struct rte_eth_dev *dev)
1206 {
1207         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1208
1209         PMD_INIT_FUNC_TRACE();
1210
1211         fm10k_mbx_lock(hw);
1212         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1213                 MAX_LPORT_NUM, false);
1214         fm10k_mbx_unlock(hw);
1215
1216         /* Stop mailbox service first */
1217         fm10k_close_mbx_service(hw);
1218         fm10k_dev_stop(dev);
1219         fm10k_dev_queue_release(dev);
1220         fm10k_stop_hw(hw);
1221 }
1222
1223 static int
1224 fm10k_link_update(struct rte_eth_dev *dev,
1225         __rte_unused int wait_to_complete)
1226 {
1227         PMD_INIT_FUNC_TRACE();
1228
1229         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
1230          * x8 PCIe interface. For now, we leave the speed undefined since there
1231          * is no 50Gbps Ethernet. */
1232         dev->data->dev_link.link_speed  = 0;
1233         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1234         dev->data->dev_link.link_status = 1;
1235
1236         return 0;
1237 }
1238
1239 static int
1240 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstats *xstats,
1241                  unsigned n)
1242 {
1243         struct fm10k_hw_stats *hw_stats =
1244                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1245         unsigned i, q, count = 0;
1246
1247         if (n < FM10K_NB_XSTATS)
1248                 return FM10K_NB_XSTATS;
1249
1250         /* Global stats */
1251         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1252                 snprintf(xstats[count].name, sizeof(xstats[count].name),
1253                          "%s", fm10k_hw_stats_strings[count].name);
1254                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1255                         fm10k_hw_stats_strings[count].offset);
1256                 count++;
1257         }
1258
1259         /* PF queue stats */
1260         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1261                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1262                         snprintf(xstats[count].name, sizeof(xstats[count].name),
1263                                  "rx_q%u_%s", q,
1264                                  fm10k_hw_stats_rx_q_strings[i].name);
1265                         xstats[count].value =
1266                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1267                                 fm10k_hw_stats_rx_q_strings[i].offset);
1268                         count++;
1269                 }
1270                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1271                         snprintf(xstats[count].name, sizeof(xstats[count].name),
1272                                  "tx_q%u_%s", q,
1273                                  fm10k_hw_stats_tx_q_strings[i].name);
1274                         xstats[count].value =
1275                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1276                                 fm10k_hw_stats_tx_q_strings[i].offset);
1277                         count++;
1278                 }
1279         }
1280
1281         return FM10K_NB_XSTATS;
1282 }
1283
1284 static void
1285 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1286 {
1287         uint64_t ipackets, opackets, ibytes, obytes;
1288         struct fm10k_hw *hw =
1289                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1290         struct fm10k_hw_stats *hw_stats =
1291                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1292         int i;
1293
1294         PMD_INIT_FUNC_TRACE();
1295
1296         fm10k_update_hw_stats(hw, hw_stats);
1297
1298         ipackets = opackets = ibytes = obytes = 0;
1299         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1300                 (i < hw->mac.max_queues); ++i) {
1301                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1302                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1303                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1304                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1305                 ipackets += stats->q_ipackets[i];
1306                 opackets += stats->q_opackets[i];
1307                 ibytes   += stats->q_ibytes[i];
1308                 obytes   += stats->q_obytes[i];
1309         }
1310         stats->ipackets = ipackets;
1311         stats->opackets = opackets;
1312         stats->ibytes = ibytes;
1313         stats->obytes = obytes;
1314 }
1315
1316 static void
1317 fm10k_stats_reset(struct rte_eth_dev *dev)
1318 {
1319         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1320         struct fm10k_hw_stats *hw_stats =
1321                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1322
1323         PMD_INIT_FUNC_TRACE();
1324
1325         memset(hw_stats, 0, sizeof(*hw_stats));
1326         fm10k_rebind_hw_stats(hw, hw_stats);
1327 }
1328
1329 static void
1330 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1331         struct rte_eth_dev_info *dev_info)
1332 {
1333         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1334
1335         PMD_INIT_FUNC_TRACE();
1336
1337         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1338         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1339         dev_info->max_rx_queues      = hw->mac.max_queues;
1340         dev_info->max_tx_queues      = hw->mac.max_queues;
1341         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1342         dev_info->max_hash_mac_addrs = 0;
1343         dev_info->max_vfs            = dev->pci_dev->max_vfs;
1344         dev_info->vmdq_pool_base     = 0;
1345         dev_info->vmdq_queue_base    = 0;
1346         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1347         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1348         dev_info->rx_offload_capa =
1349                 DEV_RX_OFFLOAD_VLAN_STRIP |
1350                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1351                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1352                 DEV_RX_OFFLOAD_TCP_CKSUM;
1353         dev_info->tx_offload_capa =
1354                 DEV_TX_OFFLOAD_VLAN_INSERT |
1355                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1356                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1357                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1358                 DEV_TX_OFFLOAD_TCP_TSO;
1359
1360         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1361         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1362
1363         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1364                 .rx_thresh = {
1365                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1366                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1367                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1368                 },
1369                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1370                 .rx_drop_en = 0,
1371         };
1372
1373         dev_info->default_txconf = (struct rte_eth_txconf) {
1374                 .tx_thresh = {
1375                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1376                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1377                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1378                 },
1379                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1380                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1381                 .txq_flags = FM10K_SIMPLE_TX_FLAG,
1382         };
1383
1384         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1385                 .nb_max = FM10K_MAX_RX_DESC,
1386                 .nb_min = FM10K_MIN_RX_DESC,
1387                 .nb_align = FM10K_MULT_RX_DESC,
1388         };
1389
1390         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1391                 .nb_max = FM10K_MAX_TX_DESC,
1392                 .nb_min = FM10K_MIN_TX_DESC,
1393                 .nb_align = FM10K_MULT_TX_DESC,
1394         };
1395 }
1396
1397 static int
1398 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1399 {
1400         s32 result;
1401         uint16_t mac_num = 0;
1402         uint32_t vid_idx, vid_bit, mac_index;
1403         struct fm10k_hw *hw;
1404         struct fm10k_macvlan_filter_info *macvlan;
1405         struct rte_eth_dev_data *data = dev->data;
1406
1407         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1408         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1409
1410         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1411                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1412                 return -EINVAL;
1413         }
1414
1415         if (vlan_id > ETH_VLAN_ID_MAX) {
1416                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1417                 return -EINVAL;
1418         }
1419
1420         vid_idx = FM10K_VFTA_IDX(vlan_id);
1421         vid_bit = FM10K_VFTA_BIT(vlan_id);
1422         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1423         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1424                 return 0;
1425         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1426         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1427                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1428                         "in the VLAN filter table");
1429                 return -EINVAL;
1430         }
1431
1432         fm10k_mbx_lock(hw);
1433         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1434         fm10k_mbx_unlock(hw);
1435         if (result != FM10K_SUCCESS) {
1436                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1437                 return -EIO;
1438         }
1439
1440         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1441                         (result == FM10K_SUCCESS); mac_index++) {
1442                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1443                         continue;
1444                 if (mac_num > macvlan->mac_num - 1) {
1445                         PMD_INIT_LOG(ERR, "MAC address number "
1446                                         "not match");
1447                         break;
1448                 }
1449                 fm10k_mbx_lock(hw);
1450                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1451                         data->mac_addrs[mac_index].addr_bytes,
1452                         vlan_id, on, 0);
1453                 fm10k_mbx_unlock(hw);
1454                 mac_num++;
1455         }
1456         if (result != FM10K_SUCCESS) {
1457                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1458                 return -EIO;
1459         }
1460
1461         if (on) {
1462                 macvlan->vlan_num++;
1463                 macvlan->vfta[vid_idx] |= vid_bit;
1464         } else {
1465                 macvlan->vlan_num--;
1466                 macvlan->vfta[vid_idx] &= ~vid_bit;
1467         }
1468         return 0;
1469 }
1470
1471 static void
1472 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1473 {
1474         if (mask & ETH_VLAN_STRIP_MASK) {
1475                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1476                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1477                                         "always on in fm10k");
1478         }
1479
1480         if (mask & ETH_VLAN_EXTEND_MASK) {
1481                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1482                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1483                                         "supported in fm10k");
1484         }
1485
1486         if (mask & ETH_VLAN_FILTER_MASK) {
1487                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1488                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1489         }
1490 }
1491
1492 /* Add/Remove a MAC address, and update filters to main VSI */
1493 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1494                 const u8 *mac, bool add, uint32_t pool)
1495 {
1496         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1497         struct fm10k_macvlan_filter_info *macvlan;
1498         uint32_t i, j, k;
1499
1500         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1501
1502         if (pool != MAIN_VSI_POOL_NUMBER) {
1503                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1504                         "mac to pool %u", pool);
1505                 return;
1506         }
1507         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1508                 if (!macvlan->vfta[j])
1509                         continue;
1510                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1511                         if (!(macvlan->vfta[j] & (1 << k)))
1512                                 continue;
1513                         if (i + 1 > macvlan->vlan_num) {
1514                                 PMD_INIT_LOG(ERR, "vlan number not match");
1515                                 return;
1516                         }
1517                         fm10k_mbx_lock(hw);
1518                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1519                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1520                         fm10k_mbx_unlock(hw);
1521                         i++;
1522                 }
1523         }
1524 }
1525
1526 /* Add/Remove a MAC address, and update filters to VMDQ */
1527 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1528                 const u8 *mac, bool add, uint32_t pool)
1529 {
1530         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1531         struct fm10k_macvlan_filter_info *macvlan;
1532         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1533         uint32_t i;
1534
1535         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1536         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1537
1538         if (pool > macvlan->nb_queue_pools) {
1539                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1540                         " Max pool is %u",
1541                         pool, macvlan->nb_queue_pools);
1542                 return;
1543         }
1544         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1545                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1546                         continue;
1547                 fm10k_mbx_lock(hw);
1548                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1549                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1550                 fm10k_mbx_unlock(hw);
1551         }
1552 }
1553
1554 /* Add/Remove a MAC address, and update filters */
1555 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1556                 const u8 *mac, bool add, uint32_t pool)
1557 {
1558         struct fm10k_macvlan_filter_info *macvlan;
1559
1560         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1561
1562         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1563                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1564         else
1565                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1566
1567         if (add)
1568                 macvlan->mac_num++;
1569         else
1570                 macvlan->mac_num--;
1571 }
1572
1573 /* Add a MAC address, and update filters */
1574 static void
1575 fm10k_macaddr_add(struct rte_eth_dev *dev,
1576                 struct ether_addr *mac_addr,
1577                 uint32_t index,
1578                 uint32_t pool)
1579 {
1580         struct fm10k_macvlan_filter_info *macvlan;
1581
1582         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1583         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1584         macvlan->mac_vmdq_id[index] = pool;
1585 }
1586
1587 /* Remove a MAC address, and update filters */
1588 static void
1589 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1590 {
1591         struct rte_eth_dev_data *data = dev->data;
1592         struct fm10k_macvlan_filter_info *macvlan;
1593
1594         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1595         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1596                         FALSE, macvlan->mac_vmdq_id[index]);
1597         macvlan->mac_vmdq_id[index] = 0;
1598 }
1599
1600 static inline int
1601 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1602 {
1603         if ((request < min) || (request > max) || ((request % mult) != 0))
1604                 return -1;
1605         else
1606                 return 0;
1607 }
1608
1609
1610 static inline int
1611 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1612 {
1613         if ((request < min) || (request > max) || ((div % request) != 0))
1614                 return -1;
1615         else
1616                 return 0;
1617 }
1618
1619 static inline int
1620 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1621 {
1622         uint16_t rx_free_thresh;
1623
1624         if (conf->rx_free_thresh == 0)
1625                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1626         else
1627                 rx_free_thresh = conf->rx_free_thresh;
1628
1629         /* make sure the requested threshold satisfies the constraints */
1630         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1631                         FM10K_RX_FREE_THRESH_MAX(q),
1632                         FM10K_RX_FREE_THRESH_DIV(q),
1633                         rx_free_thresh)) {
1634                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1635                         "less than or equal to %u, "
1636                         "greater than or equal to %u, "
1637                         "and a divisor of %u",
1638                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1639                         FM10K_RX_FREE_THRESH_MIN(q),
1640                         FM10K_RX_FREE_THRESH_DIV(q));
1641                 return -EINVAL;
1642         }
1643
1644         q->alloc_thresh = rx_free_thresh;
1645         q->drop_en = conf->rx_drop_en;
1646         q->rx_deferred_start = conf->rx_deferred_start;
1647
1648         return 0;
1649 }
1650
1651 /*
1652  * Hardware requires specific alignment for Rx packet buffers. At
1653  * least one of the following two conditions must be satisfied.
1654  *  1. Address is 512B aligned
1655  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1656  *
1657  * As such, the driver may need to adjust the DMA address within the
1658  * buffer by up to 512B.
1659  *
1660  * return 1 if the element size is valid, otherwise return 0.
1661  */
1662 static int
1663 mempool_element_size_valid(struct rte_mempool *mp)
1664 {
1665         uint32_t min_size;
1666
1667         /* elt_size includes mbuf header and headroom */
1668         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1669                         RTE_PKTMBUF_HEADROOM;
1670
1671         /* account for up to 512B of alignment */
1672         min_size -= FM10K_RX_DATABUF_ALIGN;
1673
1674         /* sanity check for overflow */
1675         if (min_size > mp->elt_size)
1676                 return 0;
1677
1678         /* size is valid */
1679         return 1;
1680 }
1681
1682 static int
1683 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1684         uint16_t nb_desc, unsigned int socket_id,
1685         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1686 {
1687         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1688         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
1689         struct fm10k_rx_queue *q;
1690         const struct rte_memzone *mz;
1691
1692         PMD_INIT_FUNC_TRACE();
1693
1694         /* make sure the mempool element size can account for alignment. */
1695         if (!mempool_element_size_valid(mp)) {
1696                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1697                 return -EINVAL;
1698         }
1699
1700         /* make sure a valid number of descriptors have been requested */
1701         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1702                                 FM10K_MULT_RX_DESC, nb_desc)) {
1703                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1704                         "less than or equal to %"PRIu32", "
1705                         "greater than or equal to %u, "
1706                         "and a multiple of %u",
1707                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1708                         FM10K_MULT_RX_DESC);
1709                 return -EINVAL;
1710         }
1711
1712         /*
1713          * if this queue existed already, free the associated memory. The
1714          * queue cannot be reused in case we need to allocate memory on
1715          * different socket than was previously used.
1716          */
1717         if (dev->data->rx_queues[queue_id] != NULL) {
1718                 rx_queue_free(dev->data->rx_queues[queue_id]);
1719                 dev->data->rx_queues[queue_id] = NULL;
1720         }
1721
1722         /* allocate memory for the queue structure */
1723         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1724                                 socket_id);
1725         if (q == NULL) {
1726                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1727                 return -ENOMEM;
1728         }
1729
1730         /* setup queue */
1731         q->mp = mp;
1732         q->nb_desc = nb_desc;
1733         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1734         q->port_id = dev->data->port_id;
1735         q->queue_id = queue_id;
1736         q->tail_ptr = (volatile uint32_t *)
1737                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1738         if (handle_rxconf(q, conf))
1739                 return -EINVAL;
1740
1741         /* allocate memory for the software ring */
1742         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1743                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1744                         RTE_CACHE_LINE_SIZE, socket_id);
1745         if (q->sw_ring == NULL) {
1746                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1747                 rte_free(q);
1748                 return -ENOMEM;
1749         }
1750
1751         /*
1752          * allocate memory for the hardware descriptor ring. A memzone large
1753          * enough to hold the maximum ring size is requested to allow for
1754          * resizing in later calls to the queue setup function.
1755          */
1756         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1757                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1758                                       socket_id);
1759         if (mz == NULL) {
1760                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1761                 rte_free(q->sw_ring);
1762                 rte_free(q);
1763                 return -ENOMEM;
1764         }
1765         q->hw_ring = mz->addr;
1766         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1767
1768         /* Check if number of descs satisfied Vector requirement */
1769         if (!rte_is_power_of_2(nb_desc)) {
1770                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1771                                     "preconditions - canceling the feature for "
1772                                     "the whole port[%d]",
1773                              q->queue_id, q->port_id);
1774                 dev_info->rx_vec_allowed = false;
1775         } else
1776                 fm10k_rxq_vec_setup(q);
1777
1778         dev->data->rx_queues[queue_id] = q;
1779         return 0;
1780 }
1781
1782 static void
1783 fm10k_rx_queue_release(void *queue)
1784 {
1785         PMD_INIT_FUNC_TRACE();
1786
1787         rx_queue_free(queue);
1788 }
1789
1790 static inline int
1791 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1792 {
1793         uint16_t tx_free_thresh;
1794         uint16_t tx_rs_thresh;
1795
1796         /* constraint MACROs require that tx_free_thresh is configured
1797          * before tx_rs_thresh */
1798         if (conf->tx_free_thresh == 0)
1799                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1800         else
1801                 tx_free_thresh = conf->tx_free_thresh;
1802
1803         /* make sure the requested threshold satisfies the constraints */
1804         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1805                         FM10K_TX_FREE_THRESH_MAX(q),
1806                         FM10K_TX_FREE_THRESH_DIV(q),
1807                         tx_free_thresh)) {
1808                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1809                         "less than or equal to %u, "
1810                         "greater than or equal to %u, "
1811                         "and a divisor of %u",
1812                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1813                         FM10K_TX_FREE_THRESH_MIN(q),
1814                         FM10K_TX_FREE_THRESH_DIV(q));
1815                 return -EINVAL;
1816         }
1817
1818         q->free_thresh = tx_free_thresh;
1819
1820         if (conf->tx_rs_thresh == 0)
1821                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1822         else
1823                 tx_rs_thresh = conf->tx_rs_thresh;
1824
1825         q->tx_deferred_start = conf->tx_deferred_start;
1826
1827         /* make sure the requested threshold satisfies the constraints */
1828         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1829                         FM10K_TX_RS_THRESH_MAX(q),
1830                         FM10K_TX_RS_THRESH_DIV(q),
1831                         tx_rs_thresh)) {
1832                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1833                         "less than or equal to %u, "
1834                         "greater than or equal to %u, "
1835                         "and a divisor of %u",
1836                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1837                         FM10K_TX_RS_THRESH_MIN(q),
1838                         FM10K_TX_RS_THRESH_DIV(q));
1839                 return -EINVAL;
1840         }
1841
1842         q->rs_thresh = tx_rs_thresh;
1843
1844         return 0;
1845 }
1846
1847 static int
1848 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1849         uint16_t nb_desc, unsigned int socket_id,
1850         const struct rte_eth_txconf *conf)
1851 {
1852         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1853         struct fm10k_tx_queue *q;
1854         const struct rte_memzone *mz;
1855
1856         PMD_INIT_FUNC_TRACE();
1857
1858         /* make sure a valid number of descriptors have been requested */
1859         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1860                                 FM10K_MULT_TX_DESC, nb_desc)) {
1861                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1862                         "less than or equal to %"PRIu32", "
1863                         "greater than or equal to %u, "
1864                         "and a multiple of %u",
1865                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1866                         FM10K_MULT_TX_DESC);
1867                 return -EINVAL;
1868         }
1869
1870         /*
1871          * if this queue existed already, free the associated memory. The
1872          * queue cannot be reused in case we need to allocate memory on
1873          * different socket than was previously used.
1874          */
1875         if (dev->data->tx_queues[queue_id] != NULL) {
1876                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
1877
1878                 tx_queue_free(txq);
1879                 dev->data->tx_queues[queue_id] = NULL;
1880         }
1881
1882         /* allocate memory for the queue structure */
1883         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1884                                 socket_id);
1885         if (q == NULL) {
1886                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1887                 return -ENOMEM;
1888         }
1889
1890         /* setup queue */
1891         q->nb_desc = nb_desc;
1892         q->port_id = dev->data->port_id;
1893         q->queue_id = queue_id;
1894         q->txq_flags = conf->txq_flags;
1895         q->ops = &def_txq_ops;
1896         q->tail_ptr = (volatile uint32_t *)
1897                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1898         if (handle_txconf(q, conf))
1899                 return -EINVAL;
1900
1901         /* allocate memory for the software ring */
1902         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1903                                         nb_desc * sizeof(struct rte_mbuf *),
1904                                         RTE_CACHE_LINE_SIZE, socket_id);
1905         if (q->sw_ring == NULL) {
1906                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1907                 rte_free(q);
1908                 return -ENOMEM;
1909         }
1910
1911         /*
1912          * allocate memory for the hardware descriptor ring. A memzone large
1913          * enough to hold the maximum ring size is requested to allow for
1914          * resizing in later calls to the queue setup function.
1915          */
1916         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
1917                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
1918                                       socket_id);
1919         if (mz == NULL) {
1920                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1921                 rte_free(q->sw_ring);
1922                 rte_free(q);
1923                 return -ENOMEM;
1924         }
1925         q->hw_ring = mz->addr;
1926         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1927
1928         /*
1929          * allocate memory for the RS bit tracker. Enough slots to hold the
1930          * descriptor index for each RS bit needing to be set are required.
1931          */
1932         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1933                                 ((nb_desc + 1) / q->rs_thresh) *
1934                                 sizeof(uint16_t),
1935                                 RTE_CACHE_LINE_SIZE, socket_id);
1936         if (q->rs_tracker.list == NULL) {
1937                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1938                 rte_free(q->sw_ring);
1939                 rte_free(q);
1940                 return -ENOMEM;
1941         }
1942
1943         dev->data->tx_queues[queue_id] = q;
1944         return 0;
1945 }
1946
1947 static void
1948 fm10k_tx_queue_release(void *queue)
1949 {
1950         struct fm10k_tx_queue *q = queue;
1951         PMD_INIT_FUNC_TRACE();
1952
1953         tx_queue_free(q);
1954 }
1955
1956 static int
1957 fm10k_reta_update(struct rte_eth_dev *dev,
1958                         struct rte_eth_rss_reta_entry64 *reta_conf,
1959                         uint16_t reta_size)
1960 {
1961         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1962         uint16_t i, j, idx, shift;
1963         uint8_t mask;
1964         uint32_t reta;
1965
1966         PMD_INIT_FUNC_TRACE();
1967
1968         if (reta_size > FM10K_MAX_RSS_INDICES) {
1969                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1970                         "(%d) doesn't match the number hardware can supported "
1971                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1972                 return -EINVAL;
1973         }
1974
1975         /*
1976          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1977          * 128-entries in 32 registers
1978          */
1979         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1980                 idx = i / RTE_RETA_GROUP_SIZE;
1981                 shift = i % RTE_RETA_GROUP_SIZE;
1982                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1983                                 BIT_MASK_PER_UINT32);
1984                 if (mask == 0)
1985                         continue;
1986
1987                 reta = 0;
1988                 if (mask != BIT_MASK_PER_UINT32)
1989                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1990
1991                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1992                         if (mask & (0x1 << j)) {
1993                                 if (mask != 0xF)
1994                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1995                                 reta |= reta_conf[idx].reta[shift + j] <<
1996                                                 (CHAR_BIT * j);
1997                         }
1998                 }
1999                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2000         }
2001
2002         return 0;
2003 }
2004
2005 static int
2006 fm10k_reta_query(struct rte_eth_dev *dev,
2007                         struct rte_eth_rss_reta_entry64 *reta_conf,
2008                         uint16_t reta_size)
2009 {
2010         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2011         uint16_t i, j, idx, shift;
2012         uint8_t mask;
2013         uint32_t reta;
2014
2015         PMD_INIT_FUNC_TRACE();
2016
2017         if (reta_size < FM10K_MAX_RSS_INDICES) {
2018                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2019                         "(%d) doesn't match the number hardware can supported "
2020                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2021                 return -EINVAL;
2022         }
2023
2024         /*
2025          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2026          * 128-entries in 32 registers
2027          */
2028         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2029                 idx = i / RTE_RETA_GROUP_SIZE;
2030                 shift = i % RTE_RETA_GROUP_SIZE;
2031                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2032                                 BIT_MASK_PER_UINT32);
2033                 if (mask == 0)
2034                         continue;
2035
2036                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2037                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2038                         if (mask & (0x1 << j))
2039                                 reta_conf[idx].reta[shift + j] = ((reta >>
2040                                         CHAR_BIT * j) & UINT8_MAX);
2041                 }
2042         }
2043
2044         return 0;
2045 }
2046
2047 static int
2048 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2049         struct rte_eth_rss_conf *rss_conf)
2050 {
2051         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2052         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2053         uint32_t mrqc;
2054         uint64_t hf = rss_conf->rss_hf;
2055         int i;
2056
2057         PMD_INIT_FUNC_TRACE();
2058
2059         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2060                 FM10K_RSSRK_ENTRIES_PER_REG)
2061                 return -EINVAL;
2062
2063         if (hf == 0)
2064                 return -EINVAL;
2065
2066         mrqc = 0;
2067         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2068         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2069         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2070         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2071         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2072         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2073         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2074         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2075         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2076
2077         /* If the mapping doesn't fit any supported, return */
2078         if (mrqc == 0)
2079                 return -EINVAL;
2080
2081         if (key != NULL)
2082                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2083                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2084
2085         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2086
2087         return 0;
2088 }
2089
2090 static int
2091 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2092         struct rte_eth_rss_conf *rss_conf)
2093 {
2094         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2095         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2096         uint32_t mrqc;
2097         uint64_t hf;
2098         int i;
2099
2100         PMD_INIT_FUNC_TRACE();
2101
2102         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2103                                 FM10K_RSSRK_ENTRIES_PER_REG)
2104                 return -EINVAL;
2105
2106         if (key != NULL)
2107                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2108                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2109
2110         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2111         hf = 0;
2112         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2113         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2114         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2115         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2116         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2117         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2118         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2119         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2120         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2121
2122         rss_conf->rss_hf = hf;
2123
2124         return 0;
2125 }
2126
2127 static void
2128 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2129 {
2130         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2131         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2132
2133         /* Bind all local non-queue interrupt to vector 0 */
2134         int_map |= FM10K_MISC_VEC_ID;
2135
2136         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
2137         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
2138         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
2139         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
2140         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
2141         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
2142
2143         /* Enable misc causes */
2144         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2145                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2146                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2147                                 FM10K_EIMR_ENABLE(MAILBOX) |
2148                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2149                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2150                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2151                                 FM10K_EIMR_ENABLE(VFLR));
2152
2153         /* Enable ITR 0 */
2154         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2155                                         FM10K_ITR_MASK_CLEAR);
2156         FM10K_WRITE_FLUSH(hw);
2157 }
2158
2159 static void
2160 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2161 {
2162         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2163         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2164
2165         int_map |= FM10K_MISC_VEC_ID;
2166
2167         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
2168         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
2169         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
2170         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
2171         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
2172         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
2173
2174         /* Disable misc causes */
2175         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2176                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2177                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2178                                 FM10K_EIMR_DISABLE(MAILBOX) |
2179                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2180                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2181                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2182                                 FM10K_EIMR_DISABLE(VFLR));
2183
2184         /* Disable ITR 0 */
2185         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2186         FM10K_WRITE_FLUSH(hw);
2187 }
2188
2189 static void
2190 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2191 {
2192         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2193         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2194
2195         /* Bind all local non-queue interrupt to vector 0 */
2196         int_map |= FM10K_MISC_VEC_ID;
2197
2198         /* Only INT 0 available, other 15 are reserved. */
2199         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2200
2201         /* Enable ITR 0 */
2202         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2203                                         FM10K_ITR_MASK_CLEAR);
2204         FM10K_WRITE_FLUSH(hw);
2205 }
2206
2207 static void
2208 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2209 {
2210         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2211         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2212
2213         int_map |= FM10K_MISC_VEC_ID;
2214
2215         /* Only INT 0 available, other 15 are reserved. */
2216         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2217
2218         /* Disable ITR 0 */
2219         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2220         FM10K_WRITE_FLUSH(hw);
2221 }
2222
2223 static int
2224 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2225 {
2226         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2227
2228         /* Enable ITR */
2229         if (hw->mac.type == fm10k_mac_pf)
2230                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
2231                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2232         else
2233                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
2234                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2235         rte_intr_enable(&dev->pci_dev->intr_handle);
2236         return 0;
2237 }
2238
2239 static int
2240 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2241 {
2242         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2243
2244         /* Disable ITR */
2245         if (hw->mac.type == fm10k_mac_pf)
2246                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
2247                         FM10K_ITR_MASK_SET);
2248         else
2249                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
2250                         FM10K_ITR_MASK_SET);
2251         return 0;
2252 }
2253
2254 static int
2255 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2256 {
2257         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2258         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
2259         uint32_t intr_vector, vec;
2260         uint16_t queue_id;
2261         int result = 0;
2262
2263         /* fm10k needs one separate interrupt for mailbox,
2264          * so only drivers which support multiple interrupt vectors
2265          * e.g. vfio-pci can work for fm10k interrupt mode
2266          */
2267         if (!rte_intr_cap_multiple(intr_handle) ||
2268                         dev->data->dev_conf.intr_conf.rxq == 0)
2269                 return result;
2270
2271         intr_vector = dev->data->nb_rx_queues;
2272
2273         /* disable interrupt first */
2274         rte_intr_disable(&dev->pci_dev->intr_handle);
2275         if (hw->mac.type == fm10k_mac_pf)
2276                 fm10k_dev_disable_intr_pf(dev);
2277         else
2278                 fm10k_dev_disable_intr_vf(dev);
2279
2280         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2281                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2282                 result = -EIO;
2283         }
2284
2285         if (rte_intr_dp_is_en(intr_handle) && !result) {
2286                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2287                         dev->data->nb_rx_queues * sizeof(int), 0);
2288                 if (intr_handle->intr_vec) {
2289                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2290                                         queue_id < dev->data->nb_rx_queues;
2291                                         queue_id++) {
2292                                 intr_handle->intr_vec[queue_id] = vec;
2293                                 if (vec < intr_handle->nb_efd - 1
2294                                                 + FM10K_RX_VEC_START)
2295                                         vec++;
2296                         }
2297                 } else {
2298                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2299                                 " intr_vec", dev->data->nb_rx_queues);
2300                         rte_intr_efd_disable(intr_handle);
2301                         result = -ENOMEM;
2302                 }
2303         }
2304
2305         if (hw->mac.type == fm10k_mac_pf)
2306                 fm10k_dev_enable_intr_pf(dev);
2307         else
2308                 fm10k_dev_enable_intr_vf(dev);
2309         rte_intr_enable(&dev->pci_dev->intr_handle);
2310         hw->mac.ops.update_int_moderator(hw);
2311         return result;
2312 }
2313
2314 static int
2315 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2316 {
2317         struct fm10k_fault fault;
2318         int err;
2319         const char *estr = "Unknown error";
2320
2321         /* Process PCA fault */
2322         if (eicr & FM10K_EICR_PCA_FAULT) {
2323                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2324                 if (err)
2325                         goto error;
2326                 switch (fault.type) {
2327                 case PCA_NO_FAULT:
2328                         estr = "PCA_NO_FAULT"; break;
2329                 case PCA_UNMAPPED_ADDR:
2330                         estr = "PCA_UNMAPPED_ADDR"; break;
2331                 case PCA_BAD_QACCESS_PF:
2332                         estr = "PCA_BAD_QACCESS_PF"; break;
2333                 case PCA_BAD_QACCESS_VF:
2334                         estr = "PCA_BAD_QACCESS_VF"; break;
2335                 case PCA_MALICIOUS_REQ:
2336                         estr = "PCA_MALICIOUS_REQ"; break;
2337                 case PCA_POISONED_TLP:
2338                         estr = "PCA_POISONED_TLP"; break;
2339                 case PCA_TLP_ABORT:
2340                         estr = "PCA_TLP_ABORT"; break;
2341                 default:
2342                         goto error;
2343                 }
2344                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2345                         estr, fault.func ? "VF" : "PF", fault.func,
2346                         fault.address, fault.specinfo);
2347         }
2348
2349         /* Process THI fault */
2350         if (eicr & FM10K_EICR_THI_FAULT) {
2351                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2352                 if (err)
2353                         goto error;
2354                 switch (fault.type) {
2355                 case THI_NO_FAULT:
2356                         estr = "THI_NO_FAULT"; break;
2357                 case THI_MAL_DIS_Q_FAULT:
2358                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2359                 default:
2360                         goto error;
2361                 }
2362                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2363                         estr, fault.func ? "VF" : "PF", fault.func,
2364                         fault.address, fault.specinfo);
2365         }
2366
2367         /* Process FUM fault */
2368         if (eicr & FM10K_EICR_FUM_FAULT) {
2369                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2370                 if (err)
2371                         goto error;
2372                 switch (fault.type) {
2373                 case FUM_NO_FAULT:
2374                         estr = "FUM_NO_FAULT"; break;
2375                 case FUM_UNMAPPED_ADDR:
2376                         estr = "FUM_UNMAPPED_ADDR"; break;
2377                 case FUM_POISONED_TLP:
2378                         estr = "FUM_POISONED_TLP"; break;
2379                 case FUM_BAD_VF_QACCESS:
2380                         estr = "FUM_BAD_VF_QACCESS"; break;
2381                 case FUM_ADD_DECODE_ERR:
2382                         estr = "FUM_ADD_DECODE_ERR"; break;
2383                 case FUM_RO_ERROR:
2384                         estr = "FUM_RO_ERROR"; break;
2385                 case FUM_QPRC_CRC_ERROR:
2386                         estr = "FUM_QPRC_CRC_ERROR"; break;
2387                 case FUM_CSR_TIMEOUT:
2388                         estr = "FUM_CSR_TIMEOUT"; break;
2389                 case FUM_INVALID_TYPE:
2390                         estr = "FUM_INVALID_TYPE"; break;
2391                 case FUM_INVALID_LENGTH:
2392                         estr = "FUM_INVALID_LENGTH"; break;
2393                 case FUM_INVALID_BE:
2394                         estr = "FUM_INVALID_BE"; break;
2395                 case FUM_INVALID_ALIGN:
2396                         estr = "FUM_INVALID_ALIGN"; break;
2397                 default:
2398                         goto error;
2399                 }
2400                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2401                         estr, fault.func ? "VF" : "PF", fault.func,
2402                         fault.address, fault.specinfo);
2403         }
2404
2405         return 0;
2406 error:
2407         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2408         return err;
2409 }
2410
2411 /**
2412  * PF interrupt handler triggered by NIC for handling specific interrupt.
2413  *
2414  * @param handle
2415  *  Pointer to interrupt handle.
2416  * @param param
2417  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2418  *
2419  * @return
2420  *  void
2421  */
2422 static void
2423 fm10k_dev_interrupt_handler_pf(
2424                         __rte_unused struct rte_intr_handle *handle,
2425                         void *param)
2426 {
2427         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2428         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2429         uint32_t cause, status;
2430
2431         if (hw->mac.type != fm10k_mac_pf)
2432                 return;
2433
2434         cause = FM10K_READ_REG(hw, FM10K_EICR);
2435
2436         /* Handle PCI fault cases */
2437         if (cause & FM10K_EICR_FAULT_MASK) {
2438                 PMD_INIT_LOG(ERR, "INT: find fault!");
2439                 fm10k_dev_handle_fault(hw, cause);
2440         }
2441
2442         /* Handle switch up/down */
2443         if (cause & FM10K_EICR_SWITCHNOTREADY)
2444                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2445
2446         if (cause & FM10K_EICR_SWITCHREADY)
2447                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2448
2449         /* Handle mailbox message */
2450         fm10k_mbx_lock(hw);
2451         hw->mbx.ops.process(hw, &hw->mbx);
2452         fm10k_mbx_unlock(hw);
2453
2454         /* Handle SRAM error */
2455         if (cause & FM10K_EICR_SRAMERROR) {
2456                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2457
2458                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2459                 /* Write to clear pending bits */
2460                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2461
2462                 /* Todo: print out error message after shared code  updates */
2463         }
2464
2465         /* Clear these 3 events if having any */
2466         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2467                  FM10K_EICR_SWITCHREADY;
2468         if (cause)
2469                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2470
2471         /* Re-enable interrupt from device side */
2472         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2473                                         FM10K_ITR_MASK_CLEAR);
2474         /* Re-enable interrupt from host side */
2475         rte_intr_enable(&(dev->pci_dev->intr_handle));
2476 }
2477
2478 /**
2479  * VF interrupt handler triggered by NIC for handling specific interrupt.
2480  *
2481  * @param handle
2482  *  Pointer to interrupt handle.
2483  * @param param
2484  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2485  *
2486  * @return
2487  *  void
2488  */
2489 static void
2490 fm10k_dev_interrupt_handler_vf(
2491                         __rte_unused struct rte_intr_handle *handle,
2492                         void *param)
2493 {
2494         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2495         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2496
2497         if (hw->mac.type != fm10k_mac_vf)
2498                 return;
2499
2500         /* Handle mailbox message if lock is acquired */
2501         fm10k_mbx_lock(hw);
2502         hw->mbx.ops.process(hw, &hw->mbx);
2503         fm10k_mbx_unlock(hw);
2504
2505         /* Re-enable interrupt from device side */
2506         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2507                                         FM10K_ITR_MASK_CLEAR);
2508         /* Re-enable interrupt from host side */
2509         rte_intr_enable(&(dev->pci_dev->intr_handle));
2510 }
2511
2512 /* Mailbox message handler in VF */
2513 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2514         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2515         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2516         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2517         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2518 };
2519
2520 /* Mailbox message handler in PF */
2521 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
2522         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
2523         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
2524         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
2525         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
2526         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
2527         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
2528         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2529 };
2530
2531 static int
2532 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2533 {
2534         int err;
2535
2536         /* Initialize mailbox lock */
2537         fm10k_mbx_initlock(hw);
2538
2539         /* Replace default message handler with new ones */
2540         if (hw->mac.type == fm10k_mac_pf)
2541                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
2542         else
2543                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2544
2545         if (err) {
2546                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2547                                 err);
2548                 return err;
2549         }
2550         /* Connect to SM for PF device or PF for VF device */
2551         return hw->mbx.ops.connect(hw, &hw->mbx);
2552 }
2553
2554 static void
2555 fm10k_close_mbx_service(struct fm10k_hw *hw)
2556 {
2557         /* Disconnect from SM for PF device or PF for VF device */
2558         hw->mbx.ops.disconnect(hw, &hw->mbx);
2559 }
2560
2561 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2562         .dev_configure          = fm10k_dev_configure,
2563         .dev_start              = fm10k_dev_start,
2564         .dev_stop               = fm10k_dev_stop,
2565         .dev_close              = fm10k_dev_close,
2566         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2567         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2568         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2569         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2570         .stats_get              = fm10k_stats_get,
2571         .xstats_get             = fm10k_xstats_get,
2572         .stats_reset            = fm10k_stats_reset,
2573         .xstats_reset           = fm10k_stats_reset,
2574         .link_update            = fm10k_link_update,
2575         .dev_infos_get          = fm10k_dev_infos_get,
2576         .vlan_filter_set        = fm10k_vlan_filter_set,
2577         .vlan_offload_set       = fm10k_vlan_offload_set,
2578         .mac_addr_add           = fm10k_macaddr_add,
2579         .mac_addr_remove        = fm10k_macaddr_remove,
2580         .rx_queue_start         = fm10k_dev_rx_queue_start,
2581         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2582         .tx_queue_start         = fm10k_dev_tx_queue_start,
2583         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2584         .rx_queue_setup         = fm10k_rx_queue_setup,
2585         .rx_queue_release       = fm10k_rx_queue_release,
2586         .tx_queue_setup         = fm10k_tx_queue_setup,
2587         .tx_queue_release       = fm10k_tx_queue_release,
2588         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2589         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2590         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2591         .reta_update            = fm10k_reta_update,
2592         .reta_query             = fm10k_reta_query,
2593         .rss_hash_update        = fm10k_rss_hash_update,
2594         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2595 };
2596
2597 static void __attribute__((cold))
2598 fm10k_set_tx_function(struct rte_eth_dev *dev)
2599 {
2600         struct fm10k_tx_queue *txq;
2601         int i;
2602         int use_sse = 1;
2603
2604         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2605                 txq = dev->data->tx_queues[i];
2606                 /* Check if Vector Tx is satisfied */
2607                 if (fm10k_tx_vec_condition_check(txq)) {
2608                         use_sse = 0;
2609                         break;
2610                 }
2611         }
2612
2613         if (use_sse) {
2614                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2615                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2616                         txq = dev->data->tx_queues[i];
2617                         fm10k_txq_vec_setup(txq);
2618                 }
2619                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2620         } else {
2621                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2622                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2623         }
2624 }
2625
2626 static void __attribute__((cold))
2627 fm10k_set_rx_function(struct rte_eth_dev *dev)
2628 {
2629         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2630         uint16_t i, rx_using_sse;
2631
2632         /* In order to allow Vector Rx there are a few configuration
2633          * conditions to be met.
2634          */
2635         if (!fm10k_rx_vec_condition_check(dev) && dev_info->rx_vec_allowed) {
2636                 if (dev->data->scattered_rx)
2637                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2638                 else
2639                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2640         } else if (dev->data->scattered_rx)
2641                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2642         else
2643                 dev->rx_pkt_burst = fm10k_recv_pkts;
2644
2645         rx_using_sse =
2646                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2647                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2648
2649         if (rx_using_sse)
2650                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
2651         else
2652                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
2653
2654         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2655                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2656
2657                 rxq->rx_using_sse = rx_using_sse;
2658         }
2659 }
2660
2661 static void
2662 fm10k_params_init(struct rte_eth_dev *dev)
2663 {
2664         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2665         struct fm10k_dev_info *info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2666
2667         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2668          * there is no way to get link status without reading BAR4.  Until this
2669          * works, assume we have maximum bandwidth.
2670          * @todo - fix bus info
2671          */
2672         hw->bus_caps.speed = fm10k_bus_speed_8000;
2673         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2674         hw->bus_caps.payload = fm10k_bus_payload_512;
2675         hw->bus.speed = fm10k_bus_speed_8000;
2676         hw->bus.width = fm10k_bus_width_pcie_x8;
2677         hw->bus.payload = fm10k_bus_payload_256;
2678
2679         info->rx_vec_allowed = true;
2680 }
2681
2682 static int
2683 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2684 {
2685         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2686         int diag, i;
2687         struct fm10k_macvlan_filter_info *macvlan;
2688
2689         PMD_INIT_FUNC_TRACE();
2690
2691         dev->dev_ops = &fm10k_eth_dev_ops;
2692         dev->rx_pkt_burst = &fm10k_recv_pkts;
2693         dev->tx_pkt_burst = &fm10k_xmit_pkts;
2694
2695         /* only initialize in the primary process */
2696         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2697                 return 0;
2698
2699         rte_eth_copy_pci_info(dev, dev->pci_dev);
2700
2701         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2702         memset(macvlan, 0, sizeof(*macvlan));
2703         /* Vendor and Device ID need to be set before init of shared code */
2704         memset(hw, 0, sizeof(*hw));
2705         hw->device_id = dev->pci_dev->id.device_id;
2706         hw->vendor_id = dev->pci_dev->id.vendor_id;
2707         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
2708         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
2709         hw->revision_id = 0;
2710         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2711         if (hw->hw_addr == NULL) {
2712                 PMD_INIT_LOG(ERR, "Bad mem resource."
2713                         " Try to blacklist unused devices.");
2714                 return -EIO;
2715         }
2716
2717         /* Store fm10k_adapter pointer */
2718         hw->back = dev->data->dev_private;
2719
2720         /* Initialize the shared code */
2721         diag = fm10k_init_shared_code(hw);
2722         if (diag != FM10K_SUCCESS) {
2723                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2724                 return -EIO;
2725         }
2726
2727         /* Initialize parameters */
2728         fm10k_params_init(dev);
2729
2730         /* Initialize the hw */
2731         diag = fm10k_init_hw(hw);
2732         if (diag != FM10K_SUCCESS) {
2733                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2734                 return -EIO;
2735         }
2736
2737         /* Initialize MAC address(es) */
2738         dev->data->mac_addrs = rte_zmalloc("fm10k",
2739                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2740         if (dev->data->mac_addrs == NULL) {
2741                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2742                 return -ENOMEM;
2743         }
2744
2745         diag = fm10k_read_mac_addr(hw);
2746
2747         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2748                         &dev->data->mac_addrs[0]);
2749
2750         if (diag != FM10K_SUCCESS ||
2751                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2752
2753                 /* Generate a random addr */
2754                 eth_random_addr(hw->mac.addr);
2755                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2756                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2757                 &dev->data->mac_addrs[0]);
2758         }
2759
2760         /* Reset the hw statistics */
2761         fm10k_stats_reset(dev);
2762
2763         /* Reset the hw */
2764         diag = fm10k_reset_hw(hw);
2765         if (diag != FM10K_SUCCESS) {
2766                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2767                 return -EIO;
2768         }
2769
2770         /* Setup mailbox service */
2771         diag = fm10k_setup_mbx_service(hw);
2772         if (diag != FM10K_SUCCESS) {
2773                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2774                 return -EIO;
2775         }
2776
2777         /*PF/VF has different interrupt handling mechanism */
2778         if (hw->mac.type == fm10k_mac_pf) {
2779                 /* register callback func to eal lib */
2780                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2781                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2782
2783                 /* enable MISC interrupt */
2784                 fm10k_dev_enable_intr_pf(dev);
2785         } else { /* VF */
2786                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2787                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2788
2789                 fm10k_dev_enable_intr_vf(dev);
2790         }
2791
2792         /* Enable intr after callback registered */
2793         rte_intr_enable(&(dev->pci_dev->intr_handle));
2794
2795         hw->mac.ops.update_int_moderator(hw);
2796
2797         /* Make sure Switch Manager is ready before going forward. */
2798         if (hw->mac.type == fm10k_mac_pf) {
2799                 int switch_ready = 0;
2800
2801                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2802                         fm10k_mbx_lock(hw);
2803                         hw->mac.ops.get_host_state(hw, &switch_ready);
2804                         fm10k_mbx_unlock(hw);
2805                         if (switch_ready)
2806                                 break;
2807                         /* Delay some time to acquire async LPORT_MAP info. */
2808                         rte_delay_us(WAIT_SWITCH_MSG_US);
2809                 }
2810
2811                 if (switch_ready == 0) {
2812                         PMD_INIT_LOG(ERR, "switch is not ready");
2813                         return -1;
2814                 }
2815         }
2816
2817         /*
2818          * Below function will trigger operations on mailbox, acquire lock to
2819          * avoid race condition from interrupt handler. Operations on mailbox
2820          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2821          * will handle and generate an interrupt to our side. Then,  FIFO in
2822          * mailbox will be touched.
2823          */
2824         fm10k_mbx_lock(hw);
2825         /* Enable port first */
2826         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2827                                         MAX_LPORT_NUM, 1);
2828
2829         /* Set unicast mode by default. App can change to other mode in other
2830          * API func.
2831          */
2832         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2833                                         FM10K_XCAST_MODE_NONE);
2834
2835         fm10k_mbx_unlock(hw);
2836
2837         /* Make sure default VID is ready before going forward. */
2838         if (hw->mac.type == fm10k_mac_pf) {
2839                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2840                         if (hw->mac.default_vid)
2841                                 break;
2842                         /* Delay some time to acquire async port VLAN info. */
2843                         rte_delay_us(WAIT_SWITCH_MSG_US);
2844                 }
2845
2846                 if (!hw->mac.default_vid) {
2847                         PMD_INIT_LOG(ERR, "default VID is not ready");
2848                         return -1;
2849                 }
2850         }
2851
2852         /* Add default mac address */
2853         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2854                 MAIN_VSI_POOL_NUMBER);
2855
2856         return 0;
2857 }
2858
2859 static int
2860 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
2861 {
2862         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2863
2864         PMD_INIT_FUNC_TRACE();
2865
2866         /* only uninitialize in the primary process */
2867         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2868                 return 0;
2869
2870         /* safe to close dev here */
2871         fm10k_dev_close(dev);
2872
2873         dev->dev_ops = NULL;
2874         dev->rx_pkt_burst = NULL;
2875         dev->tx_pkt_burst = NULL;
2876
2877         /* disable uio/vfio intr */
2878         rte_intr_disable(&(dev->pci_dev->intr_handle));
2879
2880         /*PF/VF has different interrupt handling mechanism */
2881         if (hw->mac.type == fm10k_mac_pf) {
2882                 /* disable interrupt */
2883                 fm10k_dev_disable_intr_pf(dev);
2884
2885                 /* unregister callback func to eal lib */
2886                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2887                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2888         } else {
2889                 /* disable interrupt */
2890                 fm10k_dev_disable_intr_vf(dev);
2891
2892                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2893                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2894         }
2895
2896         /* free mac memory */
2897         if (dev->data->mac_addrs) {
2898                 rte_free(dev->data->mac_addrs);
2899                 dev->data->mac_addrs = NULL;
2900         }
2901
2902         memset(hw, 0, sizeof(*hw));
2903
2904         return 0;
2905 }
2906
2907 /*
2908  * The set of PCI devices this driver supports. This driver will enable both PF
2909  * and SRIOV-VF devices.
2910  */
2911 static const struct rte_pci_id pci_id_fm10k_map[] = {
2912 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2913 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2914 #include "rte_pci_dev_ids.h"
2915         { .vendor_id = 0, /* sentinel */ },
2916 };
2917
2918 static struct eth_driver rte_pmd_fm10k = {
2919         .pci_drv = {
2920                 .name = "rte_pmd_fm10k",
2921                 .id_table = pci_id_fm10k_map,
2922                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
2923                         RTE_PCI_DRV_DETACHABLE,
2924         },
2925         .eth_dev_init = eth_fm10k_dev_init,
2926         .eth_dev_uninit = eth_fm10k_dev_uninit,
2927         .dev_private_size = sizeof(struct fm10k_adapter),
2928 };
2929
2930 /*
2931  * Driver initialization routine.
2932  * Invoked once at EAL init time.
2933  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2934  */
2935 static int
2936 rte_pmd_fm10k_init(__rte_unused const char *name,
2937         __rte_unused const char *params)
2938 {
2939         PMD_INIT_FUNC_TRACE();
2940         rte_eth_driver_register(&rte_pmd_fm10k);
2941         return 0;
2942 }
2943
2944 static struct rte_driver rte_fm10k_driver = {
2945         .type = PMD_PDEV,
2946         .init = rte_pmd_fm10k_init,
2947 };
2948
2949 PMD_REGISTER_DRIVER(rte_fm10k_driver);