drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 /* Default delay to acquire mailbox lock */
  45 #define FM10K_MBXLOCK_DELAY_US 20
  46 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  47
  48 #define MAIN_VSI_POOL_NUMBER 0
  49
  50 /* Max try times to acquire switch status */
  51 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  52 /* Wait interval to get switch status */
  53 #define WAIT_SWITCH_MSG_US    100000
  54 /* Number of chars per uint32 type */
  55 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  56 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  57
  58 /* default 1:1 map from queue ID to interrupt vector ID */
  59 #define Q2V(dev, queue_id) (dev->pci_dev->intr_handle.intr_vec[queue_id])
  60
  61 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  62 #define MAX_LPORT_NUM    128
  63 #define GLORT_FD_Q_BASE  0x40
  64 #define GLORT_PF_MASK    0xFFC0
  65 #define GLORT_FD_MASK    GLORT_PF_MASK
  66 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  67
  68 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  69 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  70 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  71 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  72 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  73 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  74 static int
  75 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  76 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  77         const u8 *mac, bool add, uint32_t pool);
  78 static void fm10k_tx_queue_release(void *queue);
  79 static void fm10k_rx_queue_release(void *queue);
  80 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  81 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  82
  83 struct fm10k_xstats_name_off {
  84         char name[RTE_ETH_XSTATS_NAME_SIZE];
  85         unsigned offset;
  86 };
  87
  88 struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  89         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  90         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  91         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  92         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  93         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  94         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  95         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  96         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  97                 nodesc_drop)},
  98 };
  99
 100 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
 101                 sizeof(fm10k_hw_stats_strings[0]))
 102
 103 struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
 104         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
 105         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
 106         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
 107 };
 108
 109 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
 110                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
 111
 112 struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 113         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 114         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 115 };
 116
 117 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 118                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 119
 120 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 121                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 122 static int
 123 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 124
 125 static void
 126 fm10k_mbx_initlock(struct fm10k_hw *hw)
 127 {
 128         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 129 }
 130
 131 static void
 132 fm10k_mbx_lock(struct fm10k_hw *hw)
 133 {
 134         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 135                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 136 }
 137
 138 static void
 139 fm10k_mbx_unlock(struct fm10k_hw *hw)
 140 {
 141         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 142 }
 143
 144 /* Stubs needed for linkage when vPMD is disabled */
 145 int __attribute__((weak))
 146 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 147 {
 148         return -1;
 149 }
 150
 151 uint16_t __attribute__((weak))
 152 fm10k_recv_pkts_vec(
 153         __rte_unused void *rx_queue,
 154         __rte_unused struct rte_mbuf **rx_pkts,
 155         __rte_unused uint16_t nb_pkts)
 156 {
 157         return 0;
 158 }
 159
 160 uint16_t __attribute__((weak))
 161 fm10k_recv_scattered_pkts_vec(
 162                 __rte_unused void *rx_queue,
 163                 __rte_unused struct rte_mbuf **rx_pkts,
 164                 __rte_unused uint16_t nb_pkts)
 165 {
 166         return 0;
 167 }
 168
 169 int __attribute__((weak))
 170 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 171
 172 {
 173         return -1;
 174 }
 175
 176 void __attribute__((weak))
 177 fm10k_rx_queue_release_mbufs_vec(
 178                 __rte_unused struct fm10k_rx_queue *rxq)
 179 {
 180         return;
 181 }
 182
 183 void __attribute__((weak))
 184 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 185 {
 186         return;
 187 }
 188
 189 int __attribute__((weak))
 190 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 191 {
 192         return -1;
 193 }
 194
 195 uint16_t __attribute__((weak))
 196 fm10k_xmit_pkts_vec(__rte_unused void *tx_queue,
 197                 __rte_unused struct rte_mbuf **tx_pkts,
 198                 __rte_unused uint16_t nb_pkts)
 199 {
 200         return 0;
 201 }
 202
 203 /*
 204  * reset queue to initial state, allocate software buffers used when starting
 205  * device.
 206  * return 0 on success
 207  * return -ENOMEM if buffers cannot be allocated
 208  * return -EINVAL if buffers do not satisfy alignment condition
 209  */
 210 static inline int
 211 rx_queue_reset(struct fm10k_rx_queue *q)
 212 {
 213         static const union fm10k_rx_desc zero = {{0} };
 214         uint64_t dma_addr;
 215         int i, diag;
 216         PMD_INIT_FUNC_TRACE();
 217
 218         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 219         if (diag != 0)
 220                 return -ENOMEM;
 221
 222         for (i = 0; i < q->nb_desc; ++i) {
 223                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 224                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 225                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 226                                                 q->nb_desc);
 227                         return -EINVAL;
 228                 }
 229                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 230                 q->hw_ring[i].q.pkt_addr = dma_addr;
 231                 q->hw_ring[i].q.hdr_addr = dma_addr;
 232         }
 233
 234         /* initialize extra software ring entries. Space for these extra
 235          * entries is always allocated.
 236          */
 237         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 238         for (i = 0; i < q->nb_fake_desc; ++i) {
 239                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 240                 q->hw_ring[q->nb_desc + i] = zero;
 241         }
 242
 243         q->next_dd = 0;
 244         q->next_alloc = 0;
 245         q->next_trigger = q->alloc_thresh - 1;
 246         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 247         q->rxrearm_start = 0;
 248         q->rxrearm_nb = 0;
 249
 250         return 0;
 251 }
 252
 253 /*
 254  * clean queue, descriptor rings, free software buffers used when stopping
 255  * device.
 256  */
 257 static inline void
 258 rx_queue_clean(struct fm10k_rx_queue *q)
 259 {
 260         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 261         uint32_t i;
 262         PMD_INIT_FUNC_TRACE();
 263
 264         /* zero descriptor rings */
 265         for (i = 0; i < q->nb_desc; ++i)
 266                 q->hw_ring[i] = zero;
 267
 268         /* zero faked descriptors */
 269         for (i = 0; i < q->nb_fake_desc; ++i)
 270                 q->hw_ring[q->nb_desc + i] = zero;
 271
 272         /* vPMD driver has a different way of releasing mbufs. */
 273         if (q->rx_using_sse) {
 274                 fm10k_rx_queue_release_mbufs_vec(q);
 275                 return;
 276         }
 277
 278         /* free software buffers */
 279         for (i = 0; i < q->nb_desc; ++i) {
 280                 if (q->sw_ring[i]) {
 281                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 282                         q->sw_ring[i] = NULL;
 283                 }
 284         }
 285 }
 286
 287 /*
 288  * free all queue memory used when releasing the queue (i.e. configure)
 289  */
 290 static inline void
 291 rx_queue_free(struct fm10k_rx_queue *q)
 292 {
 293         PMD_INIT_FUNC_TRACE();
 294         if (q) {
 295                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 296                 rx_queue_clean(q);
 297                 if (q->sw_ring) {
 298                         rte_free(q->sw_ring);
 299                         q->sw_ring = NULL;
 300                 }
 301                 rte_free(q);
 302                 q = NULL;
 303         }
 304 }
 305
 306 /*
 307  * disable RX queue, wait unitl HW finished necessary flush operation
 308  */
 309 static inline int
 310 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 311 {
 312         uint32_t reg, i;
 313
 314         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 315         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 316                         reg & ~FM10K_RXQCTL_ENABLE);
 317
 318         /* Wait 100us at most */
 319         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 320                 rte_delay_us(1);
 321                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 322                 if (!(reg & FM10K_RXQCTL_ENABLE))
 323                         break;
 324         }
 325
 326         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 327                 return -1;
 328
 329         return 0;
 330 }
 331
 332 /*
 333  * reset queue to initial state, allocate software buffers used when starting
 334  * device
 335  */
 336 static inline void
 337 tx_queue_reset(struct fm10k_tx_queue *q)
 338 {
 339         PMD_INIT_FUNC_TRACE();
 340         q->last_free = 0;
 341         q->next_free = 0;
 342         q->nb_used = 0;
 343         q->nb_free = q->nb_desc - 1;
 344         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 345         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 346 }
 347
 348 /*
 349  * clean queue, descriptor rings, free software buffers used when stopping
 350  * device
 351  */
 352 static inline void
 353 tx_queue_clean(struct fm10k_tx_queue *q)
 354 {
 355         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 356         uint32_t i;
 357         PMD_INIT_FUNC_TRACE();
 358
 359         /* zero descriptor rings */
 360         for (i = 0; i < q->nb_desc; ++i)
 361                 q->hw_ring[i] = zero;
 362
 363         /* free software buffers */
 364         for (i = 0; i < q->nb_desc; ++i) {
 365                 if (q->sw_ring[i]) {
 366                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 367                         q->sw_ring[i] = NULL;
 368                 }
 369         }
 370 }
 371
 372 /*
 373  * free all queue memory used when releasing the queue (i.e. configure)
 374  */
 375 static inline void
 376 tx_queue_free(struct fm10k_tx_queue *q)
 377 {
 378         PMD_INIT_FUNC_TRACE();
 379         if (q) {
 380                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 381                 tx_queue_clean(q);
 382                 if (q->rs_tracker.list) {
 383                         rte_free(q->rs_tracker.list);
 384                         q->rs_tracker.list = NULL;
 385                 }
 386                 if (q->sw_ring) {
 387                         rte_free(q->sw_ring);
 388                         q->sw_ring = NULL;
 389                 }
 390                 rte_free(q);
 391                 q = NULL;
 392         }
 393 }
 394
 395 /*
 396  * disable TX queue, wait unitl HW finished necessary flush operation
 397  */
 398 static inline int
 399 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 400 {
 401         uint32_t reg, i;
 402
 403         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 404         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 405                         reg & ~FM10K_TXDCTL_ENABLE);
 406
 407         /* Wait 100us at most */
 408         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 409                 rte_delay_us(1);
 410                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 411                 if (!(reg & FM10K_TXDCTL_ENABLE))
 412                         break;
 413         }
 414
 415         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 416                 return -1;
 417
 418         return 0;
 419 }
 420
 421 static int
 422 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 423 {
 424         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 425         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 426         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 427         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 428
 429         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 430
 431         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 432                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 433                 return -EINVAL;
 434         }
 435
 436         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 437                 return 0;
 438
 439         if (hw->mac.type == fm10k_mac_vf) {
 440                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 441                 return -EINVAL;
 442         }
 443
 444         /* Check VMDQ queue pool number */
 445         if (vmdq_conf->nb_queue_pools >
 446                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 447                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 448                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 449                         vmdq_conf->nb_queue_pools);
 450                 return -EINVAL;
 451         }
 452
 453         return 0;
 454 }
 455
 456 static const struct fm10k_txq_ops def_txq_ops = {
 457         .reset = tx_queue_reset,
 458 };
 459
 460 static int
 461 fm10k_dev_configure(struct rte_eth_dev *dev)
 462 {
 463         int ret;
 464
 465         PMD_INIT_FUNC_TRACE();
 466
 467         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 468                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 469         /* multipe queue mode checking */
 470         ret  = fm10k_check_mq_mode(dev);
 471         if (ret != 0) {
 472                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 473                             ret);
 474                 return ret;
 475         }
 476
 477         return 0;
 478 }
 479
 480 /* fls = find last set bit = 32 minus the number of leading zeros */
 481 #ifndef fls
 482 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 483 #endif
 484
 485 static void
 486 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 487 {
 488         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 489         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 490         uint32_t i;
 491
 492         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 493
 494         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 495                 if (!vmdq_conf->pool_map[i].pools)
 496                         continue;
 497                 fm10k_mbx_lock(hw);
 498                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 499                 fm10k_mbx_unlock(hw);
 500         }
 501 }
 502
 503 static void
 504 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 505 {
 506         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 507
 508         /* Add default mac address */
 509         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 510                 MAIN_VSI_POOL_NUMBER);
 511 }
 512
 513 static void
 514 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 515 {
 516         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 517         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 518         uint32_t mrqc, *key, i, reta, j;
 519         uint64_t hf;
 520
 521 #define RSS_KEY_SIZE 40
 522         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 523                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 524                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 525                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 526                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 527                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 528         };
 529
 530         if (dev->data->nb_rx_queues == 1 ||
 531             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 532             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 533                 return;
 534
 535         /* random key is rss_intel_key (default) or user provided (rss_key) */
 536         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 537                 key = (uint32_t *)rss_intel_key;
 538         else
 539                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 540
 541         /* Now fill our hash function seeds, 4 bytes at a time */
 542         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 543                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 544
 545         /*
 546          * Fill in redirection table
 547          * The byte-swap is needed because NIC registers are in
 548          * little-endian order.
 549          */
 550         reta = 0;
 551         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 552                 if (j == dev->data->nb_rx_queues)
 553                         j = 0;
 554                 reta = (reta << CHAR_BIT) | j;
 555                 if ((i & 3) == 3)
 556                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 557                                         rte_bswap32(reta));
 558         }
 559
 560         /*
 561          * Generate RSS hash based on packet types, TCP/UDP
 562          * port numbers and/or IPv4/v6 src and dst addresses
 563          */
 564         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 565         mrqc = 0;
 566         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 567         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 568         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 569         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 570         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 571         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 572         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 573         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 574         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 575
 576         if (mrqc == 0) {
 577                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 578                         "supported", hf);
 579                 return;
 580         }
 581
 582         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 583 }
 584
 585 static void
 586 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 587 {
 588         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 589         uint32_t i;
 590
 591         for (i = 0; i < nb_lport_new; i++) {
 592                 /* Set unicast mode by default. App can change
 593                  * to other mode in other API func.
 594                  */
 595                 fm10k_mbx_lock(hw);
 596                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 597                         FM10K_XCAST_MODE_NONE);
 598                 fm10k_mbx_unlock(hw);
 599         }
 600 }
 601
 602 static void
 603 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 604 {
 605         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 606         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 607         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 608         struct fm10k_macvlan_filter_info *macvlan;
 609         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 610         uint16_t nb_lport_new;
 611
 612         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 613         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 614
 615         fm10k_dev_rss_configure(dev);
 616
 617         /* only PF supports VMDQ */
 618         if (hw->mac.type != fm10k_mac_pf)
 619                 return;
 620
 621         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 622                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 623
 624         /* no pool number change, no need to update logic port and VLAN/MAC */
 625         if (macvlan->nb_queue_pools == nb_queue_pools)
 626                 return;
 627
 628         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 629         fm10k_dev_logic_port_update(dev, nb_lport_new);
 630
 631         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 632         memset(dev->data->mac_addrs, 0,
 633                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 634         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 635                 &dev->data->mac_addrs[0]);
 636         memset(macvlan, 0, sizeof(*macvlan));
 637         macvlan->nb_queue_pools = nb_queue_pools;
 638
 639         if (nb_queue_pools)
 640                 fm10k_dev_vmdq_rx_configure(dev);
 641         else
 642                 fm10k_dev_pf_main_vsi_reset(dev);
 643 }
 644
 645 static int
 646 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 647 {
 648         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 649         int i, ret;
 650         struct fm10k_tx_queue *txq;
 651         uint64_t base_addr;
 652         uint32_t size;
 653
 654         /* Disable TXINT to avoid possible interrupt */
 655         for (i = 0; i < hw->mac.max_queues; i++)
 656                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 657                                 3 << FM10K_TXINT_TIMER_SHIFT);
 658
 659         /* Setup TX queue */
 660         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 661                 txq = dev->data->tx_queues[i];
 662                 base_addr = txq->hw_ring_phys_addr;
 663                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 664
 665                 /* disable queue to avoid issues while updating state */
 666                 ret = tx_queue_disable(hw, i);
 667                 if (ret) {
 668                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 669                         return -1;
 670                 }
 671
 672                 /* set location and size for descriptor ring */
 673                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 674                                 base_addr & UINT64_LOWER_32BITS_MASK);
 675                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 676                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 677                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 678
 679                 /* assign default SGLORT for each TX queue */
 680                 FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 681         }
 682
 683         /* set up vector or scalar TX function as appropriate */
 684         fm10k_set_tx_function(dev);
 685
 686         return 0;
 687 }
 688
 689 static int
 690 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 691 {
 692         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 693         struct fm10k_macvlan_filter_info *macvlan;
 694         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
 695         int i, ret;
 696         struct fm10k_rx_queue *rxq;
 697         uint64_t base_addr;
 698         uint32_t size;
 699         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 700         uint32_t logic_port = hw->mac.dglort_map;
 701         uint16_t buf_size;
 702         uint16_t queue_stride = 0;
 703
 704         /* enable RXINT for interrupt mode */
 705         i = 0;
 706         if (rte_intr_dp_is_en(intr_handle)) {
 707                 for (; i < dev->data->nb_rx_queues; i++) {
 708                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(dev, i));
 709                         if (hw->mac.type == fm10k_mac_pf)
 710                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
 711                                         FM10K_ITR_AUTOMASK |
 712                                         FM10K_ITR_MASK_CLEAR);
 713                         else
 714                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
 715                                         FM10K_ITR_AUTOMASK |
 716                                         FM10K_ITR_MASK_CLEAR);
 717                 }
 718         }
 719         /* Disable other RXINT to avoid possible interrupt */
 720         for (; i < hw->mac.max_queues; i++)
 721                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 722                         3 << FM10K_RXINT_TIMER_SHIFT);
 723
 724         /* Setup RX queues */
 725         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 726                 rxq = dev->data->rx_queues[i];
 727                 base_addr = rxq->hw_ring_phys_addr;
 728                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 729
 730                 /* disable queue to avoid issues while updating state */
 731                 ret = rx_queue_disable(hw, i);
 732                 if (ret) {
 733                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 734                         return -1;
 735                 }
 736
 737                 /* Setup the Base and Length of the Rx Descriptor Ring */
 738                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 739                                 base_addr & UINT64_LOWER_32BITS_MASK);
 740                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 741                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 742                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 743
 744                 /* Configure the Rx buffer size for one buff without split */
 745                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 746                         RTE_PKTMBUF_HEADROOM);
 747                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 748                  * reserved for this purpose, and the worst case could be 511B.
 749                  * But SRR reg assumes all buffers have the same size. In order
 750                  * to fill the gap, we'll have to consider the worst case and
 751                  * assume 512B is reserved. If we don't do so, it's possible
 752                  * for HW to overwrite data to next mbuf.
 753                  */
 754                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 755
 756                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 757                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 758                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 759
 760                 /* It adds dual VLAN length for supporting dual VLAN */
 761                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 762                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 763                         dev->data->dev_conf.rxmode.enable_scatter) {
 764                         uint32_t reg;
 765                         dev->data->scattered_rx = 1;
 766                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 767                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 768                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 769                 }
 770
 771                 /* Enable drop on empty, it's RO for VF */
 772                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 773                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 774
 775                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 776                 FM10K_WRITE_FLUSH(hw);
 777         }
 778
 779         /* Configure VMDQ/RSS if applicable */
 780         fm10k_dev_mq_rx_configure(dev);
 781
 782         /* Decide the best RX function */
 783         fm10k_set_rx_function(dev);
 784
 785         /* update RX_SGLORT for loopback suppress*/
 786         if (hw->mac.type != fm10k_mac_pf)
 787                 return 0;
 788         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 789         if (macvlan->nb_queue_pools)
 790                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 791         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 792                 if (i && queue_stride && !(i % queue_stride))
 793                         logic_port++;
 794                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 795         }
 796
 797         return 0;
 798 }
 799
 800 static int
 801 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 802 {
 803         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 804         int err = -1;
 805         uint32_t reg;
 806         struct fm10k_rx_queue *rxq;
 807
 808         PMD_INIT_FUNC_TRACE();
 809
 810         if (rx_queue_id < dev->data->nb_rx_queues) {
 811                 rxq = dev->data->rx_queues[rx_queue_id];
 812                 err = rx_queue_reset(rxq);
 813                 if (err == -ENOMEM) {
 814                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 815                         return err;
 816                 } else if (err == -EINVAL) {
 817                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 818                                 " %d", err);
 819                         return err;
 820                 }
 821
 822                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 823                  * Note: this must be done AFTER the queue is enabled on real
 824                  * hardware, but BEFORE the queue is enabled when using the
 825                  * emulation platform. Do it in both places for now and remove
 826                  * this comment and the following two register writes when the
 827                  * emulation platform is no longer being used.
 828                  */
 829                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 830                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 831
 832                 /* Set PF ownership flag for PF devices */
 833                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 834                 if (hw->mac.type == fm10k_mac_pf)
 835                         reg |= FM10K_RXQCTL_PF;
 836                 reg |= FM10K_RXQCTL_ENABLE;
 837                 /* enable RX queue */
 838                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 839                 FM10K_WRITE_FLUSH(hw);
 840
 841                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 842                  * Note: this must be done AFTER the queue is enabled
 843                  */
 844                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 845                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 846                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 847         }
 848
 849         return err;
 850 }
 851
 852 static int
 853 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 854 {
 855         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 856
 857         PMD_INIT_FUNC_TRACE();
 858
 859         if (rx_queue_id < dev->data->nb_rx_queues) {
 860                 /* Disable RX queue */
 861                 rx_queue_disable(hw, rx_queue_id);
 862
 863                 /* Free mbuf and clean HW ring */
 864                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 865                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 866         }
 867
 868         return 0;
 869 }
 870
 871 static int
 872 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 873 {
 874         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 875         /** @todo - this should be defined in the shared code */
 876 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 877         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 878         int err = 0;
 879
 880         PMD_INIT_FUNC_TRACE();
 881
 882         if (tx_queue_id < dev->data->nb_tx_queues) {
 883                 struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 884
 885                 q->ops->reset(q);
 886
 887                 /* reset head and tail pointers */
 888                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 889                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 890
 891                 /* enable TX queue */
 892                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 893                                         FM10K_TXDCTL_ENABLE | txdctl);
 894                 FM10K_WRITE_FLUSH(hw);
 895                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 896         } else
 897                 err = -1;
 898
 899         return err;
 900 }
 901
 902 static int
 903 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 904 {
 905         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 906
 907         PMD_INIT_FUNC_TRACE();
 908
 909         if (tx_queue_id < dev->data->nb_tx_queues) {
 910                 tx_queue_disable(hw, tx_queue_id);
 911                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 912                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 913         }
 914
 915         return 0;
 916 }
 917
 918 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 919 {
 920         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 921                 != FM10K_DGLORTMAP_NONE);
 922 }
 923
 924 static void
 925 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 926 {
 927         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 928         int status;
 929
 930         PMD_INIT_FUNC_TRACE();
 931
 932         /* Return if it didn't acquire valid glort range */
 933         if (!fm10k_glort_valid(hw))
 934                 return;
 935
 936         fm10k_mbx_lock(hw);
 937         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 938                                 FM10K_XCAST_MODE_PROMISC);
 939         fm10k_mbx_unlock(hw);
 940
 941         if (status != FM10K_SUCCESS)
 942                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 943 }
 944
 945 static void
 946 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 947 {
 948         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 949         uint8_t mode;
 950         int status;
 951
 952         PMD_INIT_FUNC_TRACE();
 953
 954         /* Return if it didn't acquire valid glort range */
 955         if (!fm10k_glort_valid(hw))
 956                 return;
 957
 958         if (dev->data->all_multicast == 1)
 959                 mode = FM10K_XCAST_MODE_ALLMULTI;
 960         else
 961                 mode = FM10K_XCAST_MODE_NONE;
 962
 963         fm10k_mbx_lock(hw);
 964         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 965                                 mode);
 966         fm10k_mbx_unlock(hw);
 967
 968         if (status != FM10K_SUCCESS)
 969                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 970 }
 971
 972 static void
 973 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 974 {
 975         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 976         int status;
 977
 978         PMD_INIT_FUNC_TRACE();
 979
 980         /* Return if it didn't acquire valid glort range */
 981         if (!fm10k_glort_valid(hw))
 982                 return;
 983
 984         /* If promiscuous mode is enabled, it doesn't make sense to enable
 985          * allmulticast and disable promiscuous since fm10k only can select
 986          * one of the modes.
 987          */
 988         if (dev->data->promiscuous) {
 989                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 990                         "needn't enable allmulticast");
 991                 return;
 992         }
 993
 994         fm10k_mbx_lock(hw);
 995         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 996                                 FM10K_XCAST_MODE_ALLMULTI);
 997         fm10k_mbx_unlock(hw);
 998
 999         if (status != FM10K_SUCCESS)
1000                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
1001 }
1002
1003 static void
1004 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1005 {
1006         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1007         int status;
1008
1009         PMD_INIT_FUNC_TRACE();
1010
1011         /* Return if it didn't acquire valid glort range */
1012         if (!fm10k_glort_valid(hw))
1013                 return;
1014
1015         if (dev->data->promiscuous) {
1016                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1017                         "since promisc mode is enabled");
1018                 return;
1019         }
1020
1021         fm10k_mbx_lock(hw);
1022         /* Change mode to unicast mode */
1023         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1024                                 FM10K_XCAST_MODE_NONE);
1025         fm10k_mbx_unlock(hw);
1026
1027         if (status != FM10K_SUCCESS)
1028                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1029 }
1030
1031 static void
1032 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1033 {
1034         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1035         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1036         uint16_t nb_queue_pools;
1037         struct fm10k_macvlan_filter_info *macvlan;
1038
1039         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1040         nb_queue_pools = macvlan->nb_queue_pools;
1041         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
1042         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
1043
1044         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1045         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1046         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1047                         hw->mac.dglort_map;
1048         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1049         /* Configure VMDQ/RSS DGlort Decoder */
1050         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1051
1052         /* Flow Director configurations, only queue number is valid. */
1053         dglortdec = fls(dev->data->nb_rx_queues - 1);
1054         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1055                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1056         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1057         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1058
1059         /* Invalidate all other GLORT entries */
1060         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1061                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1062                                 FM10K_DGLORTMAP_NONE);
1063 }
1064
1065 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1066 static int
1067 fm10k_dev_start(struct rte_eth_dev *dev)
1068 {
1069         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1070         int i, diag;
1071
1072         PMD_INIT_FUNC_TRACE();
1073
1074         /* stop, init, then start the hw */
1075         diag = fm10k_stop_hw(hw);
1076         if (diag != FM10K_SUCCESS) {
1077                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1078                 return -EIO;
1079         }
1080
1081         diag = fm10k_init_hw(hw);
1082         if (diag != FM10K_SUCCESS) {
1083                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1084                 return -EIO;
1085         }
1086
1087         diag = fm10k_start_hw(hw);
1088         if (diag != FM10K_SUCCESS) {
1089                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1090                 return -EIO;
1091         }
1092
1093         diag = fm10k_dev_tx_init(dev);
1094         if (diag) {
1095                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1096                 return diag;
1097         }
1098
1099         if (fm10k_dev_rxq_interrupt_setup(dev))
1100                 return -EIO;
1101
1102         diag = fm10k_dev_rx_init(dev);
1103         if (diag) {
1104                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1105                 return diag;
1106         }
1107
1108         if (hw->mac.type == fm10k_mac_pf)
1109                 fm10k_dev_dglort_map_configure(dev);
1110
1111         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1112                 struct fm10k_rx_queue *rxq;
1113                 rxq = dev->data->rx_queues[i];
1114
1115                 if (rxq->rx_deferred_start)
1116                         continue;
1117                 diag = fm10k_dev_rx_queue_start(dev, i);
1118                 if (diag != 0) {
1119                         int j;
1120                         for (j = 0; j < i; ++j)
1121                                 rx_queue_clean(dev->data->rx_queues[j]);
1122                         return diag;
1123                 }
1124         }
1125
1126         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1127                 struct fm10k_tx_queue *txq;
1128                 txq = dev->data->tx_queues[i];
1129
1130                 if (txq->tx_deferred_start)
1131                         continue;
1132                 diag = fm10k_dev_tx_queue_start(dev, i);
1133                 if (diag != 0) {
1134                         int j;
1135                         for (j = 0; j < i; ++j)
1136                                 tx_queue_clean(dev->data->tx_queues[j]);
1137                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1138                                 rx_queue_clean(dev->data->rx_queues[j]);
1139                         return diag;
1140                 }
1141         }
1142
1143         /* Update default vlan when not in VMDQ mode */
1144         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1145                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1146
1147         return 0;
1148 }
1149
1150 static void
1151 fm10k_dev_stop(struct rte_eth_dev *dev)
1152 {
1153         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1154         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1155         int i;
1156
1157         PMD_INIT_FUNC_TRACE();
1158
1159         if (dev->data->tx_queues)
1160                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1161                         fm10k_dev_tx_queue_stop(dev, i);
1162
1163         if (dev->data->rx_queues)
1164                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1165                         fm10k_dev_rx_queue_stop(dev, i);
1166
1167         /* Disable datapath event */
1168         if (rte_intr_dp_is_en(intr_handle)) {
1169                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1170                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1171                                 3 << FM10K_RXINT_TIMER_SHIFT);
1172                         if (hw->mac.type == fm10k_mac_pf)
1173                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
1174                                         FM10K_ITR_MASK_SET);
1175                         else
1176                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
1177                                         FM10K_ITR_MASK_SET);
1178                 }
1179         }
1180         /* Clean datapath event and queue/vec mapping */
1181         rte_intr_efd_disable(intr_handle);
1182         rte_free(intr_handle->intr_vec);
1183         intr_handle->intr_vec = NULL;
1184 }
1185
1186 static void
1187 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1188 {
1189         int i;
1190
1191         PMD_INIT_FUNC_TRACE();
1192
1193         if (dev->data->tx_queues) {
1194                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1195                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1196
1197                         tx_queue_free(txq);
1198                 }
1199         }
1200
1201         if (dev->data->rx_queues) {
1202                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1203                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1204         }
1205 }
1206
1207 static void
1208 fm10k_dev_close(struct rte_eth_dev *dev)
1209 {
1210         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1211
1212         PMD_INIT_FUNC_TRACE();
1213
1214         fm10k_mbx_lock(hw);
1215         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1216                 MAX_LPORT_NUM, false);
1217         fm10k_mbx_unlock(hw);
1218
1219         /* Stop mailbox service first */
1220         fm10k_close_mbx_service(hw);
1221         fm10k_dev_stop(dev);
1222         fm10k_dev_queue_release(dev);
1223         fm10k_stop_hw(hw);
1224 }
1225
1226 static int
1227 fm10k_link_update(struct rte_eth_dev *dev,
1228         __rte_unused int wait_to_complete)
1229 {
1230         PMD_INIT_FUNC_TRACE();
1231
1232         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
1233          * x8 PCIe interface. For now, we leave the speed undefined since there
1234          * is no 50Gbps Ethernet. */
1235         dev->data->dev_link.link_speed  = 0;
1236         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1237         dev->data->dev_link.link_status = 1;
1238
1239         return 0;
1240 }
1241
1242 static int
1243 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstats *xstats,
1244                  unsigned n)
1245 {
1246         struct fm10k_hw_stats *hw_stats =
1247                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1248         unsigned i, q, count = 0;
1249
1250         if (n < FM10K_NB_XSTATS)
1251                 return FM10K_NB_XSTATS;
1252
1253         /* Global stats */
1254         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1255                 snprintf(xstats[count].name, sizeof(xstats[count].name),
1256                          "%s", fm10k_hw_stats_strings[count].name);
1257                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1258                         fm10k_hw_stats_strings[count].offset);
1259                 count++;
1260         }
1261
1262         /* PF queue stats */
1263         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1264                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1265                         snprintf(xstats[count].name, sizeof(xstats[count].name),
1266                                  "rx_q%u_%s", q,
1267                                  fm10k_hw_stats_rx_q_strings[i].name);
1268                         xstats[count].value =
1269                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1270                                 fm10k_hw_stats_rx_q_strings[i].offset);
1271                         count++;
1272                 }
1273                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1274                         snprintf(xstats[count].name, sizeof(xstats[count].name),
1275                                  "tx_q%u_%s", q,
1276                                  fm10k_hw_stats_tx_q_strings[i].name);
1277                         xstats[count].value =
1278                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1279                                 fm10k_hw_stats_tx_q_strings[i].offset);
1280                         count++;
1281                 }
1282         }
1283
1284         return FM10K_NB_XSTATS;
1285 }
1286
1287 static void
1288 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1289 {
1290         uint64_t ipackets, opackets, ibytes, obytes;
1291         struct fm10k_hw *hw =
1292                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1293         struct fm10k_hw_stats *hw_stats =
1294                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1295         int i;
1296
1297         PMD_INIT_FUNC_TRACE();
1298
1299         fm10k_update_hw_stats(hw, hw_stats);
1300
1301         ipackets = opackets = ibytes = obytes = 0;
1302         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1303                 (i < hw->mac.max_queues); ++i) {
1304                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1305                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1306                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1307                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1308                 ipackets += stats->q_ipackets[i];
1309                 opackets += stats->q_opackets[i];
1310                 ibytes   += stats->q_ibytes[i];
1311                 obytes   += stats->q_obytes[i];
1312         }
1313         stats->ipackets = ipackets;
1314         stats->opackets = opackets;
1315         stats->ibytes = ibytes;
1316         stats->obytes = obytes;
1317 }
1318
1319 static void
1320 fm10k_stats_reset(struct rte_eth_dev *dev)
1321 {
1322         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1323         struct fm10k_hw_stats *hw_stats =
1324                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1325
1326         PMD_INIT_FUNC_TRACE();
1327
1328         memset(hw_stats, 0, sizeof(*hw_stats));
1329         fm10k_rebind_hw_stats(hw, hw_stats);
1330 }
1331
1332 static void
1333 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1334         struct rte_eth_dev_info *dev_info)
1335 {
1336         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1337
1338         PMD_INIT_FUNC_TRACE();
1339
1340         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1341         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1342         dev_info->max_rx_queues      = hw->mac.max_queues;
1343         dev_info->max_tx_queues      = hw->mac.max_queues;
1344         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1345         dev_info->max_hash_mac_addrs = 0;
1346         dev_info->max_vfs            = dev->pci_dev->max_vfs;
1347         dev_info->vmdq_pool_base     = 0;
1348         dev_info->vmdq_queue_base    = 0;
1349         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1350         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1351         dev_info->rx_offload_capa =
1352                 DEV_RX_OFFLOAD_VLAN_STRIP |
1353                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1354                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1355                 DEV_RX_OFFLOAD_TCP_CKSUM;
1356         dev_info->tx_offload_capa =
1357                 DEV_TX_OFFLOAD_VLAN_INSERT |
1358                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1359                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1360                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1361                 DEV_TX_OFFLOAD_TCP_TSO;
1362
1363         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1364         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1365
1366         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1367                 .rx_thresh = {
1368                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1369                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1370                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1371                 },
1372                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1373                 .rx_drop_en = 0,
1374         };
1375
1376         dev_info->default_txconf = (struct rte_eth_txconf) {
1377                 .tx_thresh = {
1378                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1379                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1380                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1381                 },
1382                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1383                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1384                 .txq_flags = FM10K_SIMPLE_TX_FLAG,
1385         };
1386
1387         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1388                 .nb_max = FM10K_MAX_RX_DESC,
1389                 .nb_min = FM10K_MIN_RX_DESC,
1390                 .nb_align = FM10K_MULT_RX_DESC,
1391         };
1392
1393         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1394                 .nb_max = FM10K_MAX_TX_DESC,
1395                 .nb_min = FM10K_MIN_TX_DESC,
1396                 .nb_align = FM10K_MULT_TX_DESC,
1397         };
1398 }
1399
1400 static int
1401 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1402 {
1403         s32 result;
1404         uint16_t mac_num = 0;
1405         uint32_t vid_idx, vid_bit, mac_index;
1406         struct fm10k_hw *hw;
1407         struct fm10k_macvlan_filter_info *macvlan;
1408         struct rte_eth_dev_data *data = dev->data;
1409
1410         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1411         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1412
1413         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1414                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1415                 return -EINVAL;
1416         }
1417
1418         if (vlan_id > ETH_VLAN_ID_MAX) {
1419                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1420                 return -EINVAL;
1421         }
1422
1423         vid_idx = FM10K_VFTA_IDX(vlan_id);
1424         vid_bit = FM10K_VFTA_BIT(vlan_id);
1425         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1426         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1427                 return 0;
1428         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1429         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1430                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1431                         "in the VLAN filter table");
1432                 return -EINVAL;
1433         }
1434
1435         fm10k_mbx_lock(hw);
1436         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1437         fm10k_mbx_unlock(hw);
1438         if (result != FM10K_SUCCESS) {
1439                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1440                 return -EIO;
1441         }
1442
1443         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1444                         (result == FM10K_SUCCESS); mac_index++) {
1445                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1446                         continue;
1447                 if (mac_num > macvlan->mac_num - 1) {
1448                         PMD_INIT_LOG(ERR, "MAC address number "
1449                                         "not match");
1450                         break;
1451                 }
1452                 fm10k_mbx_lock(hw);
1453                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1454                         data->mac_addrs[mac_index].addr_bytes,
1455                         vlan_id, on, 0);
1456                 fm10k_mbx_unlock(hw);
1457                 mac_num++;
1458         }
1459         if (result != FM10K_SUCCESS) {
1460                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1461                 return -EIO;
1462         }
1463
1464         if (on) {
1465                 macvlan->vlan_num++;
1466                 macvlan->vfta[vid_idx] |= vid_bit;
1467         } else {
1468                 macvlan->vlan_num--;
1469                 macvlan->vfta[vid_idx] &= ~vid_bit;
1470         }
1471         return 0;
1472 }
1473
1474 static void
1475 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1476 {
1477         if (mask & ETH_VLAN_STRIP_MASK) {
1478                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1479                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1480                                         "always on in fm10k");
1481         }
1482
1483         if (mask & ETH_VLAN_EXTEND_MASK) {
1484                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1485                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1486                                         "supported in fm10k");
1487         }
1488
1489         if (mask & ETH_VLAN_FILTER_MASK) {
1490                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1491                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1492         }
1493 }
1494
1495 /* Add/Remove a MAC address, and update filters to main VSI */
1496 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1497                 const u8 *mac, bool add, uint32_t pool)
1498 {
1499         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1500         struct fm10k_macvlan_filter_info *macvlan;
1501         uint32_t i, j, k;
1502
1503         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1504
1505         if (pool != MAIN_VSI_POOL_NUMBER) {
1506                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1507                         "mac to pool %u", pool);
1508                 return;
1509         }
1510         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1511                 if (!macvlan->vfta[j])
1512                         continue;
1513                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1514                         if (!(macvlan->vfta[j] & (1 << k)))
1515                                 continue;
1516                         if (i + 1 > macvlan->vlan_num) {
1517                                 PMD_INIT_LOG(ERR, "vlan number not match");
1518                                 return;
1519                         }
1520                         fm10k_mbx_lock(hw);
1521                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1522                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1523                         fm10k_mbx_unlock(hw);
1524                         i++;
1525                 }
1526         }
1527 }
1528
1529 /* Add/Remove a MAC address, and update filters to VMDQ */
1530 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1531                 const u8 *mac, bool add, uint32_t pool)
1532 {
1533         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1534         struct fm10k_macvlan_filter_info *macvlan;
1535         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1536         uint32_t i;
1537
1538         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1539         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1540
1541         if (pool > macvlan->nb_queue_pools) {
1542                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1543                         " Max pool is %u",
1544                         pool, macvlan->nb_queue_pools);
1545                 return;
1546         }
1547         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1548                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1549                         continue;
1550                 fm10k_mbx_lock(hw);
1551                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1552                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1553                 fm10k_mbx_unlock(hw);
1554         }
1555 }
1556
1557 /* Add/Remove a MAC address, and update filters */
1558 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1559                 const u8 *mac, bool add, uint32_t pool)
1560 {
1561         struct fm10k_macvlan_filter_info *macvlan;
1562
1563         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1564
1565         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1566                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1567         else
1568                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1569
1570         if (add)
1571                 macvlan->mac_num++;
1572         else
1573                 macvlan->mac_num--;
1574 }
1575
1576 /* Add a MAC address, and update filters */
1577 static void
1578 fm10k_macaddr_add(struct rte_eth_dev *dev,
1579                 struct ether_addr *mac_addr,
1580                 uint32_t index,
1581                 uint32_t pool)
1582 {
1583         struct fm10k_macvlan_filter_info *macvlan;
1584
1585         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1586         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1587         macvlan->mac_vmdq_id[index] = pool;
1588 }
1589
1590 /* Remove a MAC address, and update filters */
1591 static void
1592 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1593 {
1594         struct rte_eth_dev_data *data = dev->data;
1595         struct fm10k_macvlan_filter_info *macvlan;
1596
1597         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1598         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1599                         FALSE, macvlan->mac_vmdq_id[index]);
1600         macvlan->mac_vmdq_id[index] = 0;
1601 }
1602
1603 static inline int
1604 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1605 {
1606         if ((request < min) || (request > max) || ((request % mult) != 0))
1607                 return -1;
1608         else
1609                 return 0;
1610 }
1611
1612
1613 static inline int
1614 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1615 {
1616         if ((request < min) || (request > max) || ((div % request) != 0))
1617                 return -1;
1618         else
1619                 return 0;
1620 }
1621
1622 static inline int
1623 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1624 {
1625         uint16_t rx_free_thresh;
1626
1627         if (conf->rx_free_thresh == 0)
1628                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1629         else
1630                 rx_free_thresh = conf->rx_free_thresh;
1631
1632         /* make sure the requested threshold satisfies the constraints */
1633         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1634                         FM10K_RX_FREE_THRESH_MAX(q),
1635                         FM10K_RX_FREE_THRESH_DIV(q),
1636                         rx_free_thresh)) {
1637                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1638                         "less than or equal to %u, "
1639                         "greater than or equal to %u, "
1640                         "and a divisor of %u",
1641                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1642                         FM10K_RX_FREE_THRESH_MIN(q),
1643                         FM10K_RX_FREE_THRESH_DIV(q));
1644                 return -EINVAL;
1645         }
1646
1647         q->alloc_thresh = rx_free_thresh;
1648         q->drop_en = conf->rx_drop_en;
1649         q->rx_deferred_start = conf->rx_deferred_start;
1650
1651         return 0;
1652 }
1653
1654 /*
1655  * Hardware requires specific alignment for Rx packet buffers. At
1656  * least one of the following two conditions must be satisfied.
1657  *  1. Address is 512B aligned
1658  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1659  *
1660  * As such, the driver may need to adjust the DMA address within the
1661  * buffer by up to 512B.
1662  *
1663  * return 1 if the element size is valid, otherwise return 0.
1664  */
1665 static int
1666 mempool_element_size_valid(struct rte_mempool *mp)
1667 {
1668         uint32_t min_size;
1669
1670         /* elt_size includes mbuf header and headroom */
1671         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1672                         RTE_PKTMBUF_HEADROOM;
1673
1674         /* account for up to 512B of alignment */
1675         min_size -= FM10K_RX_DATABUF_ALIGN;
1676
1677         /* sanity check for overflow */
1678         if (min_size > mp->elt_size)
1679                 return 0;
1680
1681         /* size is valid */
1682         return 1;
1683 }
1684
1685 static int
1686 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1687         uint16_t nb_desc, unsigned int socket_id,
1688         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1689 {
1690         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1691         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
1692         struct fm10k_rx_queue *q;
1693         const struct rte_memzone *mz;
1694
1695         PMD_INIT_FUNC_TRACE();
1696
1697         /* make sure the mempool element size can account for alignment. */
1698         if (!mempool_element_size_valid(mp)) {
1699                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1700                 return -EINVAL;
1701         }
1702
1703         /* make sure a valid number of descriptors have been requested */
1704         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1705                                 FM10K_MULT_RX_DESC, nb_desc)) {
1706                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1707                         "less than or equal to %"PRIu32", "
1708                         "greater than or equal to %u, "
1709                         "and a multiple of %u",
1710                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1711                         FM10K_MULT_RX_DESC);
1712                 return -EINVAL;
1713         }
1714
1715         /*
1716          * if this queue existed already, free the associated memory. The
1717          * queue cannot be reused in case we need to allocate memory on
1718          * different socket than was previously used.
1719          */
1720         if (dev->data->rx_queues[queue_id] != NULL) {
1721                 rx_queue_free(dev->data->rx_queues[queue_id]);
1722                 dev->data->rx_queues[queue_id] = NULL;
1723         }
1724
1725         /* allocate memory for the queue structure */
1726         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1727                                 socket_id);
1728         if (q == NULL) {
1729                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1730                 return -ENOMEM;
1731         }
1732
1733         /* setup queue */
1734         q->mp = mp;
1735         q->nb_desc = nb_desc;
1736         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1737         q->port_id = dev->data->port_id;
1738         q->queue_id = queue_id;
1739         q->tail_ptr = (volatile uint32_t *)
1740                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1741         if (handle_rxconf(q, conf))
1742                 return -EINVAL;
1743
1744         /* allocate memory for the software ring */
1745         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1746                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1747                         RTE_CACHE_LINE_SIZE, socket_id);
1748         if (q->sw_ring == NULL) {
1749                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1750                 rte_free(q);
1751                 return -ENOMEM;
1752         }
1753
1754         /*
1755          * allocate memory for the hardware descriptor ring. A memzone large
1756          * enough to hold the maximum ring size is requested to allow for
1757          * resizing in later calls to the queue setup function.
1758          */
1759         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1760                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1761                                       socket_id);
1762         if (mz == NULL) {
1763                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1764                 rte_free(q->sw_ring);
1765                 rte_free(q);
1766                 return -ENOMEM;
1767         }
1768         q->hw_ring = mz->addr;
1769         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1770
1771         /* Check if number of descs satisfied Vector requirement */
1772         if (!rte_is_power_of_2(nb_desc)) {
1773                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1774                                     "preconditions - canceling the feature for "
1775                                     "the whole port[%d]",
1776                              q->queue_id, q->port_id);
1777                 dev_info->rx_vec_allowed = false;
1778         } else
1779                 fm10k_rxq_vec_setup(q);
1780
1781         dev->data->rx_queues[queue_id] = q;
1782         return 0;
1783 }
1784
1785 static void
1786 fm10k_rx_queue_release(void *queue)
1787 {
1788         PMD_INIT_FUNC_TRACE();
1789
1790         rx_queue_free(queue);
1791 }
1792
1793 static inline int
1794 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1795 {
1796         uint16_t tx_free_thresh;
1797         uint16_t tx_rs_thresh;
1798
1799         /* constraint MACROs require that tx_free_thresh is configured
1800          * before tx_rs_thresh */
1801         if (conf->tx_free_thresh == 0)
1802                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1803         else
1804                 tx_free_thresh = conf->tx_free_thresh;
1805
1806         /* make sure the requested threshold satisfies the constraints */
1807         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1808                         FM10K_TX_FREE_THRESH_MAX(q),
1809                         FM10K_TX_FREE_THRESH_DIV(q),
1810                         tx_free_thresh)) {
1811                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1812                         "less than or equal to %u, "
1813                         "greater than or equal to %u, "
1814                         "and a divisor of %u",
1815                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1816                         FM10K_TX_FREE_THRESH_MIN(q),
1817                         FM10K_TX_FREE_THRESH_DIV(q));
1818                 return -EINVAL;
1819         }
1820
1821         q->free_thresh = tx_free_thresh;
1822
1823         if (conf->tx_rs_thresh == 0)
1824                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1825         else
1826                 tx_rs_thresh = conf->tx_rs_thresh;
1827
1828         q->tx_deferred_start = conf->tx_deferred_start;
1829
1830         /* make sure the requested threshold satisfies the constraints */
1831         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1832                         FM10K_TX_RS_THRESH_MAX(q),
1833                         FM10K_TX_RS_THRESH_DIV(q),
1834                         tx_rs_thresh)) {
1835                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1836                         "less than or equal to %u, "
1837                         "greater than or equal to %u, "
1838                         "and a divisor of %u",
1839                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1840                         FM10K_TX_RS_THRESH_MIN(q),
1841                         FM10K_TX_RS_THRESH_DIV(q));
1842                 return -EINVAL;
1843         }
1844
1845         q->rs_thresh = tx_rs_thresh;
1846
1847         return 0;
1848 }
1849
1850 static int
1851 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1852         uint16_t nb_desc, unsigned int socket_id,
1853         const struct rte_eth_txconf *conf)
1854 {
1855         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1856         struct fm10k_tx_queue *q;
1857         const struct rte_memzone *mz;
1858
1859         PMD_INIT_FUNC_TRACE();
1860
1861         /* make sure a valid number of descriptors have been requested */
1862         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1863                                 FM10K_MULT_TX_DESC, nb_desc)) {
1864                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1865                         "less than or equal to %"PRIu32", "
1866                         "greater than or equal to %u, "
1867                         "and a multiple of %u",
1868                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1869                         FM10K_MULT_TX_DESC);
1870                 return -EINVAL;
1871         }
1872
1873         /*
1874          * if this queue existed already, free the associated memory. The
1875          * queue cannot be reused in case we need to allocate memory on
1876          * different socket than was previously used.
1877          */
1878         if (dev->data->tx_queues[queue_id] != NULL) {
1879                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
1880
1881                 tx_queue_free(txq);
1882                 dev->data->tx_queues[queue_id] = NULL;
1883         }
1884
1885         /* allocate memory for the queue structure */
1886         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1887                                 socket_id);
1888         if (q == NULL) {
1889                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1890                 return -ENOMEM;
1891         }
1892
1893         /* setup queue */
1894         q->nb_desc = nb_desc;
1895         q->port_id = dev->data->port_id;
1896         q->queue_id = queue_id;
1897         q->txq_flags = conf->txq_flags;
1898         q->ops = &def_txq_ops;
1899         q->tail_ptr = (volatile uint32_t *)
1900                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1901         if (handle_txconf(q, conf))
1902                 return -EINVAL;
1903
1904         /* allocate memory for the software ring */
1905         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1906                                         nb_desc * sizeof(struct rte_mbuf *),
1907                                         RTE_CACHE_LINE_SIZE, socket_id);
1908         if (q->sw_ring == NULL) {
1909                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1910                 rte_free(q);
1911                 return -ENOMEM;
1912         }
1913
1914         /*
1915          * allocate memory for the hardware descriptor ring. A memzone large
1916          * enough to hold the maximum ring size is requested to allow for
1917          * resizing in later calls to the queue setup function.
1918          */
1919         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
1920                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
1921                                       socket_id);
1922         if (mz == NULL) {
1923                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1924                 rte_free(q->sw_ring);
1925                 rte_free(q);
1926                 return -ENOMEM;
1927         }
1928         q->hw_ring = mz->addr;
1929         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1930
1931         /*
1932          * allocate memory for the RS bit tracker. Enough slots to hold the
1933          * descriptor index for each RS bit needing to be set are required.
1934          */
1935         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1936                                 ((nb_desc + 1) / q->rs_thresh) *
1937                                 sizeof(uint16_t),
1938                                 RTE_CACHE_LINE_SIZE, socket_id);
1939         if (q->rs_tracker.list == NULL) {
1940                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1941                 rte_free(q->sw_ring);
1942                 rte_free(q);
1943                 return -ENOMEM;
1944         }
1945
1946         dev->data->tx_queues[queue_id] = q;
1947         return 0;
1948 }
1949
1950 static void
1951 fm10k_tx_queue_release(void *queue)
1952 {
1953         struct fm10k_tx_queue *q = queue;
1954         PMD_INIT_FUNC_TRACE();
1955
1956         tx_queue_free(q);
1957 }
1958
1959 static int
1960 fm10k_reta_update(struct rte_eth_dev *dev,
1961                         struct rte_eth_rss_reta_entry64 *reta_conf,
1962                         uint16_t reta_size)
1963 {
1964         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1965         uint16_t i, j, idx, shift;
1966         uint8_t mask;
1967         uint32_t reta;
1968
1969         PMD_INIT_FUNC_TRACE();
1970
1971         if (reta_size > FM10K_MAX_RSS_INDICES) {
1972                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1973                         "(%d) doesn't match the number hardware can supported "
1974                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1975                 return -EINVAL;
1976         }
1977
1978         /*
1979          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1980          * 128-entries in 32 registers
1981          */
1982         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1983                 idx = i / RTE_RETA_GROUP_SIZE;
1984                 shift = i % RTE_RETA_GROUP_SIZE;
1985                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1986                                 BIT_MASK_PER_UINT32);
1987                 if (mask == 0)
1988                         continue;
1989
1990                 reta = 0;
1991                 if (mask != BIT_MASK_PER_UINT32)
1992                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1993
1994                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1995                         if (mask & (0x1 << j)) {
1996                                 if (mask != 0xF)
1997                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1998                                 reta |= reta_conf[idx].reta[shift + j] <<
1999                                                 (CHAR_BIT * j);
2000                         }
2001                 }
2002                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2003         }
2004
2005         return 0;
2006 }
2007
2008 static int
2009 fm10k_reta_query(struct rte_eth_dev *dev,
2010                         struct rte_eth_rss_reta_entry64 *reta_conf,
2011                         uint16_t reta_size)
2012 {
2013         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2014         uint16_t i, j, idx, shift;
2015         uint8_t mask;
2016         uint32_t reta;
2017
2018         PMD_INIT_FUNC_TRACE();
2019
2020         if (reta_size < FM10K_MAX_RSS_INDICES) {
2021                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2022                         "(%d) doesn't match the number hardware can supported "
2023                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2024                 return -EINVAL;
2025         }
2026
2027         /*
2028          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2029          * 128-entries in 32 registers
2030          */
2031         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2032                 idx = i / RTE_RETA_GROUP_SIZE;
2033                 shift = i % RTE_RETA_GROUP_SIZE;
2034                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2035                                 BIT_MASK_PER_UINT32);
2036                 if (mask == 0)
2037                         continue;
2038
2039                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2040                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2041                         if (mask & (0x1 << j))
2042                                 reta_conf[idx].reta[shift + j] = ((reta >>
2043                                         CHAR_BIT * j) & UINT8_MAX);
2044                 }
2045         }
2046
2047         return 0;
2048 }
2049
2050 static int
2051 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2052         struct rte_eth_rss_conf *rss_conf)
2053 {
2054         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2055         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2056         uint32_t mrqc;
2057         uint64_t hf = rss_conf->rss_hf;
2058         int i;
2059
2060         PMD_INIT_FUNC_TRACE();
2061
2062         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2063                 FM10K_RSSRK_ENTRIES_PER_REG)
2064                 return -EINVAL;
2065
2066         if (hf == 0)
2067                 return -EINVAL;
2068
2069         mrqc = 0;
2070         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2071         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2072         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2073         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2074         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2075         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2076         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2077         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2078         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2079
2080         /* If the mapping doesn't fit any supported, return */
2081         if (mrqc == 0)
2082                 return -EINVAL;
2083
2084         if (key != NULL)
2085                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2086                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2087
2088         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2089
2090         return 0;
2091 }
2092
2093 static int
2094 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2095         struct rte_eth_rss_conf *rss_conf)
2096 {
2097         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2098         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2099         uint32_t mrqc;
2100         uint64_t hf;
2101         int i;
2102
2103         PMD_INIT_FUNC_TRACE();
2104
2105         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2106                                 FM10K_RSSRK_ENTRIES_PER_REG)
2107                 return -EINVAL;
2108
2109         if (key != NULL)
2110                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2111                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2112
2113         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2114         hf = 0;
2115         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2116         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2117         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2118         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2119         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2120         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2121         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2122         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2123         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2124
2125         rss_conf->rss_hf = hf;
2126
2127         return 0;
2128 }
2129
2130 static void
2131 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2132 {
2133         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2134         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2135
2136         /* Bind all local non-queue interrupt to vector 0 */
2137         int_map |= FM10K_MISC_VEC_ID;
2138
2139         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2140         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2141         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2142         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2143         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2144         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2145
2146         /* Enable misc causes */
2147         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2148                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2149                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2150                                 FM10K_EIMR_ENABLE(MAILBOX) |
2151                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2152                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2153                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2154                                 FM10K_EIMR_ENABLE(VFLR));
2155
2156         /* Enable ITR 0 */
2157         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2158                                         FM10K_ITR_MASK_CLEAR);
2159         FM10K_WRITE_FLUSH(hw);
2160 }
2161
2162 static void
2163 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2164 {
2165         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2166         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2167
2168         int_map |= FM10K_MISC_VEC_ID;
2169
2170         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2171         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2172         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2173         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2174         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2175         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2176
2177         /* Disable misc causes */
2178         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2179                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2180                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2181                                 FM10K_EIMR_DISABLE(MAILBOX) |
2182                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2183                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2184                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2185                                 FM10K_EIMR_DISABLE(VFLR));
2186
2187         /* Disable ITR 0 */
2188         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2189         FM10K_WRITE_FLUSH(hw);
2190 }
2191
2192 static void
2193 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2194 {
2195         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2196         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2197
2198         /* Bind all local non-queue interrupt to vector 0 */
2199         int_map |= FM10K_MISC_VEC_ID;
2200
2201         /* Only INT 0 available, other 15 are reserved. */
2202         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2203
2204         /* Enable ITR 0 */
2205         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2206                                         FM10K_ITR_MASK_CLEAR);
2207         FM10K_WRITE_FLUSH(hw);
2208 }
2209
2210 static void
2211 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2212 {
2213         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2214         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2215
2216         int_map |= FM10K_MISC_VEC_ID;
2217
2218         /* Only INT 0 available, other 15 are reserved. */
2219         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2220
2221         /* Disable ITR 0 */
2222         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2223         FM10K_WRITE_FLUSH(hw);
2224 }
2225
2226 static int
2227 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2228 {
2229         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2230
2231         /* Enable ITR */
2232         if (hw->mac.type == fm10k_mac_pf)
2233                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
2234                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2235         else
2236                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
2237                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2238         rte_intr_enable(&dev->pci_dev->intr_handle);
2239         return 0;
2240 }
2241
2242 static int
2243 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2244 {
2245         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2246
2247         /* Disable ITR */
2248         if (hw->mac.type == fm10k_mac_pf)
2249                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
2250                         FM10K_ITR_MASK_SET);
2251         else
2252                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
2253                         FM10K_ITR_MASK_SET);
2254         return 0;
2255 }
2256
2257 static int
2258 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2259 {
2260         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2261         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
2262         uint32_t intr_vector, vec;
2263         uint16_t queue_id;
2264         int result = 0;
2265
2266         /* fm10k needs one separate interrupt for mailbox,
2267          * so only drivers which support multiple interrupt vectors
2268          * e.g. vfio-pci can work for fm10k interrupt mode
2269          */
2270         if (!rte_intr_cap_multiple(intr_handle) ||
2271                         dev->data->dev_conf.intr_conf.rxq == 0)
2272                 return result;
2273
2274         intr_vector = dev->data->nb_rx_queues;
2275
2276         /* disable interrupt first */
2277         rte_intr_disable(&dev->pci_dev->intr_handle);
2278         if (hw->mac.type == fm10k_mac_pf)
2279                 fm10k_dev_disable_intr_pf(dev);
2280         else
2281                 fm10k_dev_disable_intr_vf(dev);
2282
2283         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2284                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2285                 result = -EIO;
2286         }
2287
2288         if (rte_intr_dp_is_en(intr_handle) && !result) {
2289                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2290                         dev->data->nb_rx_queues * sizeof(int), 0);
2291                 if (intr_handle->intr_vec) {
2292                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2293                                         queue_id < dev->data->nb_rx_queues;
2294                                         queue_id++) {
2295                                 intr_handle->intr_vec[queue_id] = vec;
2296                                 if (vec < intr_handle->nb_efd - 1
2297                                                 + FM10K_RX_VEC_START)
2298                                         vec++;
2299                         }
2300                 } else {
2301                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2302                                 " intr_vec", dev->data->nb_rx_queues);
2303                         rte_intr_efd_disable(intr_handle);
2304                         result = -ENOMEM;
2305                 }
2306         }
2307
2308         if (hw->mac.type == fm10k_mac_pf)
2309                 fm10k_dev_enable_intr_pf(dev);
2310         else
2311                 fm10k_dev_enable_intr_vf(dev);
2312         rte_intr_enable(&dev->pci_dev->intr_handle);
2313         hw->mac.ops.update_int_moderator(hw);
2314         return result;
2315 }
2316
2317 static int
2318 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2319 {
2320         struct fm10k_fault fault;
2321         int err;
2322         const char *estr = "Unknown error";
2323
2324         /* Process PCA fault */
2325         if (eicr & FM10K_EICR_PCA_FAULT) {
2326                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2327                 if (err)
2328                         goto error;
2329                 switch (fault.type) {
2330                 case PCA_NO_FAULT:
2331                         estr = "PCA_NO_FAULT"; break;
2332                 case PCA_UNMAPPED_ADDR:
2333                         estr = "PCA_UNMAPPED_ADDR"; break;
2334                 case PCA_BAD_QACCESS_PF:
2335                         estr = "PCA_BAD_QACCESS_PF"; break;
2336                 case PCA_BAD_QACCESS_VF:
2337                         estr = "PCA_BAD_QACCESS_VF"; break;
2338                 case PCA_MALICIOUS_REQ:
2339                         estr = "PCA_MALICIOUS_REQ"; break;
2340                 case PCA_POISONED_TLP:
2341                         estr = "PCA_POISONED_TLP"; break;
2342                 case PCA_TLP_ABORT:
2343                         estr = "PCA_TLP_ABORT"; break;
2344                 default:
2345                         goto error;
2346                 }
2347                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2348                         estr, fault.func ? "VF" : "PF", fault.func,
2349                         fault.address, fault.specinfo);
2350         }
2351
2352         /* Process THI fault */
2353         if (eicr & FM10K_EICR_THI_FAULT) {
2354                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2355                 if (err)
2356                         goto error;
2357                 switch (fault.type) {
2358                 case THI_NO_FAULT:
2359                         estr = "THI_NO_FAULT"; break;
2360                 case THI_MAL_DIS_Q_FAULT:
2361                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2362                 default:
2363                         goto error;
2364                 }
2365                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2366                         estr, fault.func ? "VF" : "PF", fault.func,
2367                         fault.address, fault.specinfo);
2368         }
2369
2370         /* Process FUM fault */
2371         if (eicr & FM10K_EICR_FUM_FAULT) {
2372                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2373                 if (err)
2374                         goto error;
2375                 switch (fault.type) {
2376                 case FUM_NO_FAULT:
2377                         estr = "FUM_NO_FAULT"; break;
2378                 case FUM_UNMAPPED_ADDR:
2379                         estr = "FUM_UNMAPPED_ADDR"; break;
2380                 case FUM_POISONED_TLP:
2381                         estr = "FUM_POISONED_TLP"; break;
2382                 case FUM_BAD_VF_QACCESS:
2383                         estr = "FUM_BAD_VF_QACCESS"; break;
2384                 case FUM_ADD_DECODE_ERR:
2385                         estr = "FUM_ADD_DECODE_ERR"; break;
2386                 case FUM_RO_ERROR:
2387                         estr = "FUM_RO_ERROR"; break;
2388                 case FUM_QPRC_CRC_ERROR:
2389                         estr = "FUM_QPRC_CRC_ERROR"; break;
2390                 case FUM_CSR_TIMEOUT:
2391                         estr = "FUM_CSR_TIMEOUT"; break;
2392                 case FUM_INVALID_TYPE:
2393                         estr = "FUM_INVALID_TYPE"; break;
2394                 case FUM_INVALID_LENGTH:
2395                         estr = "FUM_INVALID_LENGTH"; break;
2396                 case FUM_INVALID_BE:
2397                         estr = "FUM_INVALID_BE"; break;
2398                 case FUM_INVALID_ALIGN:
2399                         estr = "FUM_INVALID_ALIGN"; break;
2400                 default:
2401                         goto error;
2402                 }
2403                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2404                         estr, fault.func ? "VF" : "PF", fault.func,
2405                         fault.address, fault.specinfo);
2406         }
2407
2408         return 0;
2409 error:
2410         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2411         return err;
2412 }
2413
2414 /**
2415  * PF interrupt handler triggered by NIC for handling specific interrupt.
2416  *
2417  * @param handle
2418  *  Pointer to interrupt handle.
2419  * @param param
2420  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2421  *
2422  * @return
2423  *  void
2424  */
2425 static void
2426 fm10k_dev_interrupt_handler_pf(
2427                         __rte_unused struct rte_intr_handle *handle,
2428                         void *param)
2429 {
2430         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2431         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2432         uint32_t cause, status;
2433
2434         if (hw->mac.type != fm10k_mac_pf)
2435                 return;
2436
2437         cause = FM10K_READ_REG(hw, FM10K_EICR);
2438
2439         /* Handle PCI fault cases */
2440         if (cause & FM10K_EICR_FAULT_MASK) {
2441                 PMD_INIT_LOG(ERR, "INT: find fault!");
2442                 fm10k_dev_handle_fault(hw, cause);
2443         }
2444
2445         /* Handle switch up/down */
2446         if (cause & FM10K_EICR_SWITCHNOTREADY)
2447                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2448
2449         if (cause & FM10K_EICR_SWITCHREADY)
2450                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2451
2452         /* Handle mailbox message */
2453         fm10k_mbx_lock(hw);
2454         hw->mbx.ops.process(hw, &hw->mbx);
2455         fm10k_mbx_unlock(hw);
2456
2457         /* Handle SRAM error */
2458         if (cause & FM10K_EICR_SRAMERROR) {
2459                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2460
2461                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2462                 /* Write to clear pending bits */
2463                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2464
2465                 /* Todo: print out error message after shared code  updates */
2466         }
2467
2468         /* Clear these 3 events if having any */
2469         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2470                  FM10K_EICR_SWITCHREADY;
2471         if (cause)
2472                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2473
2474         /* Re-enable interrupt from device side */
2475         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2476                                         FM10K_ITR_MASK_CLEAR);
2477         /* Re-enable interrupt from host side */
2478         rte_intr_enable(&(dev->pci_dev->intr_handle));
2479 }
2480
2481 /**
2482  * VF interrupt handler triggered by NIC for handling specific interrupt.
2483  *
2484  * @param handle
2485  *  Pointer to interrupt handle.
2486  * @param param
2487  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2488  *
2489  * @return
2490  *  void
2491  */
2492 static void
2493 fm10k_dev_interrupt_handler_vf(
2494                         __rte_unused struct rte_intr_handle *handle,
2495                         void *param)
2496 {
2497         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2498         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2499
2500         if (hw->mac.type != fm10k_mac_vf)
2501                 return;
2502
2503         /* Handle mailbox message if lock is acquired */
2504         fm10k_mbx_lock(hw);
2505         hw->mbx.ops.process(hw, &hw->mbx);
2506         fm10k_mbx_unlock(hw);
2507
2508         /* Re-enable interrupt from device side */
2509         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2510                                         FM10K_ITR_MASK_CLEAR);
2511         /* Re-enable interrupt from host side */
2512         rte_intr_enable(&(dev->pci_dev->intr_handle));
2513 }
2514
2515 /* Mailbox message handler in VF */
2516 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2517         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2518         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2519         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2520         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2521 };
2522
2523 static int
2524 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2525 {
2526         int err = 0;
2527
2528         /* Initialize mailbox lock */
2529         fm10k_mbx_initlock(hw);
2530
2531         /* Replace default message handler with new ones */
2532         if (hw->mac.type == fm10k_mac_vf)
2533                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2534
2535         if (err) {
2536                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2537                                 err);
2538                 return err;
2539         }
2540         /* Connect to SM for PF device or PF for VF device */
2541         return hw->mbx.ops.connect(hw, &hw->mbx);
2542 }
2543
2544 static void
2545 fm10k_close_mbx_service(struct fm10k_hw *hw)
2546 {
2547         /* Disconnect from SM for PF device or PF for VF device */
2548         hw->mbx.ops.disconnect(hw, &hw->mbx);
2549 }
2550
2551 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2552         .dev_configure          = fm10k_dev_configure,
2553         .dev_start              = fm10k_dev_start,
2554         .dev_stop               = fm10k_dev_stop,
2555         .dev_close              = fm10k_dev_close,
2556         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2557         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2558         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2559         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2560         .stats_get              = fm10k_stats_get,
2561         .xstats_get             = fm10k_xstats_get,
2562         .stats_reset            = fm10k_stats_reset,
2563         .xstats_reset           = fm10k_stats_reset,
2564         .link_update            = fm10k_link_update,
2565         .dev_infos_get          = fm10k_dev_infos_get,
2566         .vlan_filter_set        = fm10k_vlan_filter_set,
2567         .vlan_offload_set       = fm10k_vlan_offload_set,
2568         .mac_addr_add           = fm10k_macaddr_add,
2569         .mac_addr_remove        = fm10k_macaddr_remove,
2570         .rx_queue_start         = fm10k_dev_rx_queue_start,
2571         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2572         .tx_queue_start         = fm10k_dev_tx_queue_start,
2573         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2574         .rx_queue_setup         = fm10k_rx_queue_setup,
2575         .rx_queue_release       = fm10k_rx_queue_release,
2576         .tx_queue_setup         = fm10k_tx_queue_setup,
2577         .tx_queue_release       = fm10k_tx_queue_release,
2578         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2579         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2580         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2581         .reta_update            = fm10k_reta_update,
2582         .reta_query             = fm10k_reta_query,
2583         .rss_hash_update        = fm10k_rss_hash_update,
2584         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2585 };
2586
2587 static void __attribute__((cold))
2588 fm10k_set_tx_function(struct rte_eth_dev *dev)
2589 {
2590         struct fm10k_tx_queue *txq;
2591         int i;
2592         int use_sse = 1;
2593
2594         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2595                 txq = dev->data->tx_queues[i];
2596                 /* Check if Vector Tx is satisfied */
2597                 if (fm10k_tx_vec_condition_check(txq)) {
2598                         use_sse = 0;
2599                         break;
2600                 }
2601         }
2602
2603         if (use_sse) {
2604                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2605                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2606                         txq = dev->data->tx_queues[i];
2607                         fm10k_txq_vec_setup(txq);
2608                 }
2609                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2610         } else {
2611                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2612                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2613         }
2614 }
2615
2616 static void __attribute__((cold))
2617 fm10k_set_rx_function(struct rte_eth_dev *dev)
2618 {
2619         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2620         uint16_t i, rx_using_sse;
2621
2622         /* In order to allow Vector Rx there are a few configuration
2623          * conditions to be met.
2624          */
2625         if (!fm10k_rx_vec_condition_check(dev) && dev_info->rx_vec_allowed) {
2626                 if (dev->data->scattered_rx)
2627                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2628                 else
2629                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2630         } else if (dev->data->scattered_rx)
2631                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2632         else
2633                 dev->rx_pkt_burst = fm10k_recv_pkts;
2634
2635         rx_using_sse =
2636                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2637                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2638
2639         if (rx_using_sse)
2640                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
2641         else
2642                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
2643
2644         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2645                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2646
2647                 rxq->rx_using_sse = rx_using_sse;
2648         }
2649 }
2650
2651 static void
2652 fm10k_params_init(struct rte_eth_dev *dev)
2653 {
2654         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2655         struct fm10k_dev_info *info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2656
2657         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2658          * there is no way to get link status without reading BAR4.  Until this
2659          * works, assume we have maximum bandwidth.
2660          * @todo - fix bus info
2661          */
2662         hw->bus_caps.speed = fm10k_bus_speed_8000;
2663         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2664         hw->bus_caps.payload = fm10k_bus_payload_512;
2665         hw->bus.speed = fm10k_bus_speed_8000;
2666         hw->bus.width = fm10k_bus_width_pcie_x8;
2667         hw->bus.payload = fm10k_bus_payload_256;
2668
2669         info->rx_vec_allowed = true;
2670 }
2671
2672 static int
2673 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2674 {
2675         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2676         int diag, i;
2677         struct fm10k_macvlan_filter_info *macvlan;
2678
2679         PMD_INIT_FUNC_TRACE();
2680
2681         dev->dev_ops = &fm10k_eth_dev_ops;
2682         dev->rx_pkt_burst = &fm10k_recv_pkts;
2683         dev->tx_pkt_burst = &fm10k_xmit_pkts;
2684
2685         /* only initialize in the primary process */
2686         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2687                 return 0;
2688
2689         rte_eth_copy_pci_info(dev, dev->pci_dev);
2690
2691         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2692         memset(macvlan, 0, sizeof(*macvlan));
2693         /* Vendor and Device ID need to be set before init of shared code */
2694         memset(hw, 0, sizeof(*hw));
2695         hw->device_id = dev->pci_dev->id.device_id;
2696         hw->vendor_id = dev->pci_dev->id.vendor_id;
2697         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
2698         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
2699         hw->revision_id = 0;
2700         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2701         if (hw->hw_addr == NULL) {
2702                 PMD_INIT_LOG(ERR, "Bad mem resource."
2703                         " Try to blacklist unused devices.");
2704                 return -EIO;
2705         }
2706
2707         /* Store fm10k_adapter pointer */
2708         hw->back = dev->data->dev_private;
2709
2710         /* Initialize the shared code */
2711         diag = fm10k_init_shared_code(hw);
2712         if (diag != FM10K_SUCCESS) {
2713                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2714                 return -EIO;
2715         }
2716
2717         /* Initialize parameters */
2718         fm10k_params_init(dev);
2719
2720         /* Initialize the hw */
2721         diag = fm10k_init_hw(hw);
2722         if (diag != FM10K_SUCCESS) {
2723                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2724                 return -EIO;
2725         }
2726
2727         /* Initialize MAC address(es) */
2728         dev->data->mac_addrs = rte_zmalloc("fm10k",
2729                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2730         if (dev->data->mac_addrs == NULL) {
2731                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2732                 return -ENOMEM;
2733         }
2734
2735         diag = fm10k_read_mac_addr(hw);
2736
2737         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2738                         &dev->data->mac_addrs[0]);
2739
2740         if (diag != FM10K_SUCCESS ||
2741                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2742
2743                 /* Generate a random addr */
2744                 eth_random_addr(hw->mac.addr);
2745                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2746                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2747                 &dev->data->mac_addrs[0]);
2748         }
2749
2750         /* Reset the hw statistics */
2751         fm10k_stats_reset(dev);
2752
2753         /* Reset the hw */
2754         diag = fm10k_reset_hw(hw);
2755         if (diag != FM10K_SUCCESS) {
2756                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2757                 return -EIO;
2758         }
2759
2760         /* Setup mailbox service */
2761         diag = fm10k_setup_mbx_service(hw);
2762         if (diag != FM10K_SUCCESS) {
2763                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2764                 return -EIO;
2765         }
2766
2767         /*PF/VF has different interrupt handling mechanism */
2768         if (hw->mac.type == fm10k_mac_pf) {
2769                 /* register callback func to eal lib */
2770                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2771                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2772
2773                 /* enable MISC interrupt */
2774                 fm10k_dev_enable_intr_pf(dev);
2775         } else { /* VF */
2776                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2777                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2778
2779                 fm10k_dev_enable_intr_vf(dev);
2780         }
2781
2782         /* Enable intr after callback registered */
2783         rte_intr_enable(&(dev->pci_dev->intr_handle));
2784
2785         hw->mac.ops.update_int_moderator(hw);
2786
2787         /* Make sure Switch Manager is ready before going forward. */
2788         if (hw->mac.type == fm10k_mac_pf) {
2789                 int switch_ready = 0;
2790
2791                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2792                         fm10k_mbx_lock(hw);
2793                         hw->mac.ops.get_host_state(hw, &switch_ready);
2794                         fm10k_mbx_unlock(hw);
2795                         if (switch_ready)
2796                                 break;
2797                         /* Delay some time to acquire async LPORT_MAP info. */
2798                         rte_delay_us(WAIT_SWITCH_MSG_US);
2799                 }
2800
2801                 if (switch_ready == 0) {
2802                         PMD_INIT_LOG(ERR, "switch is not ready");
2803                         return -1;
2804                 }
2805         }
2806
2807         /*
2808          * Below function will trigger operations on mailbox, acquire lock to
2809          * avoid race condition from interrupt handler. Operations on mailbox
2810          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2811          * will handle and generate an interrupt to our side. Then,  FIFO in
2812          * mailbox will be touched.
2813          */
2814         fm10k_mbx_lock(hw);
2815         /* Enable port first */
2816         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2817                                         MAX_LPORT_NUM, 1);
2818
2819         /* Set unicast mode by default. App can change to other mode in other
2820          * API func.
2821          */
2822         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2823                                         FM10K_XCAST_MODE_NONE);
2824
2825         fm10k_mbx_unlock(hw);
2826
2827         /* Make sure default VID is ready before going forward. */
2828         if (hw->mac.type == fm10k_mac_pf) {
2829                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2830                         if (hw->mac.default_vid)
2831                                 break;
2832                         /* Delay some time to acquire async port VLAN info. */
2833                         rte_delay_us(WAIT_SWITCH_MSG_US);
2834                 }
2835
2836                 if (!hw->mac.default_vid) {
2837                         PMD_INIT_LOG(ERR, "default VID is not ready");
2838                         return -1;
2839                 }
2840         }
2841
2842         /* Add default mac address */
2843         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2844                 MAIN_VSI_POOL_NUMBER);
2845
2846         return 0;
2847 }
2848
2849 static int
2850 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
2851 {
2852         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2853
2854         PMD_INIT_FUNC_TRACE();
2855
2856         /* only uninitialize in the primary process */
2857         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2858                 return 0;
2859
2860         /* safe to close dev here */
2861         fm10k_dev_close(dev);
2862
2863         dev->dev_ops = NULL;
2864         dev->rx_pkt_burst = NULL;
2865         dev->tx_pkt_burst = NULL;
2866
2867         /* disable uio/vfio intr */
2868         rte_intr_disable(&(dev->pci_dev->intr_handle));
2869
2870         /*PF/VF has different interrupt handling mechanism */
2871         if (hw->mac.type == fm10k_mac_pf) {
2872                 /* disable interrupt */
2873                 fm10k_dev_disable_intr_pf(dev);
2874
2875                 /* unregister callback func to eal lib */
2876                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2877                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2878         } else {
2879                 /* disable interrupt */
2880                 fm10k_dev_disable_intr_vf(dev);
2881
2882                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2883                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2884         }
2885
2886         /* free mac memory */
2887         if (dev->data->mac_addrs) {
2888                 rte_free(dev->data->mac_addrs);
2889                 dev->data->mac_addrs = NULL;
2890         }
2891
2892         memset(hw, 0, sizeof(*hw));
2893
2894         return 0;
2895 }
2896
2897 /*
2898  * The set of PCI devices this driver supports. This driver will enable both PF
2899  * and SRIOV-VF devices.
2900  */
2901 static const struct rte_pci_id pci_id_fm10k_map[] = {
2902 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2903 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2904 #include "rte_pci_dev_ids.h"
2905         { .vendor_id = 0, /* sentinel */ },
2906 };
2907
2908 static struct eth_driver rte_pmd_fm10k = {
2909         .pci_drv = {
2910                 .name = "rte_pmd_fm10k",
2911                 .id_table = pci_id_fm10k_map,
2912                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
2913                         RTE_PCI_DRV_DETACHABLE,
2914         },
2915         .eth_dev_init = eth_fm10k_dev_init,
2916         .eth_dev_uninit = eth_fm10k_dev_uninit,
2917         .dev_private_size = sizeof(struct fm10k_adapter),
2918 };
2919
2920 /*
2921  * Driver initialization routine.
2922  * Invoked once at EAL init time.
2923  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2924  */
2925 static int
2926 rte_pmd_fm10k_init(__rte_unused const char *name,
2927         __rte_unused const char *params)
2928 {
2929         PMD_INIT_FUNC_TRACE();
2930         rte_eth_driver_register(&rte_pmd_fm10k);
2931         return 0;
2932 }
2933
2934 static struct rte_driver rte_fm10k_driver = {
2935         .type = PMD_PDEV,
2936         .init = rte_pmd_fm10k_init,
2937 };
2938
2939 PMD_REGISTER_DRIVER(rte_fm10k_driver);