drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_ethdev_pci.h>
  36 #include <rte_malloc.h>
  37 #include <rte_memzone.h>
  38 #include <rte_string_fns.h>
  39 #include <rte_dev.h>
  40 #include <rte_spinlock.h>
  41 #include <rte_kvargs.h>
  42
  43 #include "fm10k.h"
  44 #include "base/fm10k_api.h"
  45
  46 /* Default delay to acquire mailbox lock */
  47 #define FM10K_MBXLOCK_DELAY_US 20
  48 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  49
  50 #define MAIN_VSI_POOL_NUMBER 0
  51
  52 /* Max try times to acquire switch status */
  53 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  54 /* Wait interval to get switch status */
  55 #define WAIT_SWITCH_MSG_US    100000
  56 /* A period of quiescence for switch */
  57 #define FM10K_SWITCH_QUIESCE_US 10000
  58 /* Number of chars per uint32 type */
  59 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  60 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  61
  62 /* default 1:1 map from queue ID to interrupt vector ID */
  63 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  64
  65 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  66 #define MAX_LPORT_NUM    128
  67 #define GLORT_FD_Q_BASE  0x40
  68 #define GLORT_PF_MASK    0xFFC0
  69 #define GLORT_FD_MASK    GLORT_PF_MASK
  70 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  71
  72 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  73 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  74 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  75 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  76 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  77 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  78 static int
  79 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  80 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  81         const u8 *mac, bool add, uint32_t pool);
  82 static void fm10k_tx_queue_release(void *queue);
  83 static void fm10k_rx_queue_release(void *queue);
  84 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  85 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  86 static int fm10k_check_ftag(struct rte_devargs *devargs);
  87 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  88
  89 struct fm10k_xstats_name_off {
  90         char name[RTE_ETH_XSTATS_NAME_SIZE];
  91         unsigned offset;
  92 };
  93
  94 struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  95         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  96         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  97         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  98         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  99         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
 100         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
 101         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
 102         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
 103                 nodesc_drop)},
 104 };
 105
 106 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
 107                 sizeof(fm10k_hw_stats_strings[0]))
 108
 109 struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
 110         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
 111         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
 112         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
 113 };
 114
 115 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
 116                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
 117
 118 struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 119         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 120         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 121 };
 122
 123 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 124                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 125
 126 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 127                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 128 static int
 129 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 130
 131 static void
 132 fm10k_mbx_initlock(struct fm10k_hw *hw)
 133 {
 134         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 135 }
 136
 137 static void
 138 fm10k_mbx_lock(struct fm10k_hw *hw)
 139 {
 140         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 141                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 142 }
 143
 144 static void
 145 fm10k_mbx_unlock(struct fm10k_hw *hw)
 146 {
 147         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 148 }
 149
 150 /* Stubs needed for linkage when vPMD is disabled */
 151 int __attribute__((weak))
 152 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 153 {
 154         return -1;
 155 }
 156
 157 uint16_t __attribute__((weak))
 158 fm10k_recv_pkts_vec(
 159         __rte_unused void *rx_queue,
 160         __rte_unused struct rte_mbuf **rx_pkts,
 161         __rte_unused uint16_t nb_pkts)
 162 {
 163         return 0;
 164 }
 165
 166 uint16_t __attribute__((weak))
 167 fm10k_recv_scattered_pkts_vec(
 168                 __rte_unused void *rx_queue,
 169                 __rte_unused struct rte_mbuf **rx_pkts,
 170                 __rte_unused uint16_t nb_pkts)
 171 {
 172         return 0;
 173 }
 174
 175 int __attribute__((weak))
 176 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 177
 178 {
 179         return -1;
 180 }
 181
 182 void __attribute__((weak))
 183 fm10k_rx_queue_release_mbufs_vec(
 184                 __rte_unused struct fm10k_rx_queue *rxq)
 185 {
 186         return;
 187 }
 188
 189 void __attribute__((weak))
 190 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 191 {
 192         return;
 193 }
 194
 195 int __attribute__((weak))
 196 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 197 {
 198         return -1;
 199 }
 200
 201 uint16_t __attribute__((weak))
 202 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 203                            __rte_unused struct rte_mbuf **tx_pkts,
 204                            __rte_unused uint16_t nb_pkts)
 205 {
 206         return 0;
 207 }
 208
 209 /*
 210  * reset queue to initial state, allocate software buffers used when starting
 211  * device.
 212  * return 0 on success
 213  * return -ENOMEM if buffers cannot be allocated
 214  * return -EINVAL if buffers do not satisfy alignment condition
 215  */
 216 static inline int
 217 rx_queue_reset(struct fm10k_rx_queue *q)
 218 {
 219         static const union fm10k_rx_desc zero = {{0} };
 220         uint64_t dma_addr;
 221         int i, diag;
 222         PMD_INIT_FUNC_TRACE();
 223
 224         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 225         if (diag != 0)
 226                 return -ENOMEM;
 227
 228         for (i = 0; i < q->nb_desc; ++i) {
 229                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 230                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 231                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 232                                                 q->nb_desc);
 233                         return -EINVAL;
 234                 }
 235                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 236                 q->hw_ring[i].q.pkt_addr = dma_addr;
 237                 q->hw_ring[i].q.hdr_addr = dma_addr;
 238         }
 239
 240         /* initialize extra software ring entries. Space for these extra
 241          * entries is always allocated.
 242          */
 243         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 244         for (i = 0; i < q->nb_fake_desc; ++i) {
 245                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 246                 q->hw_ring[q->nb_desc + i] = zero;
 247         }
 248
 249         q->next_dd = 0;
 250         q->next_alloc = 0;
 251         q->next_trigger = q->alloc_thresh - 1;
 252         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 253         q->rxrearm_start = 0;
 254         q->rxrearm_nb = 0;
 255
 256         return 0;
 257 }
 258
 259 /*
 260  * clean queue, descriptor rings, free software buffers used when stopping
 261  * device.
 262  */
 263 static inline void
 264 rx_queue_clean(struct fm10k_rx_queue *q)
 265 {
 266         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 267         uint32_t i;
 268         PMD_INIT_FUNC_TRACE();
 269
 270         /* zero descriptor rings */
 271         for (i = 0; i < q->nb_desc; ++i)
 272                 q->hw_ring[i] = zero;
 273
 274         /* zero faked descriptors */
 275         for (i = 0; i < q->nb_fake_desc; ++i)
 276                 q->hw_ring[q->nb_desc + i] = zero;
 277
 278         /* vPMD driver has a different way of releasing mbufs. */
 279         if (q->rx_using_sse) {
 280                 fm10k_rx_queue_release_mbufs_vec(q);
 281                 return;
 282         }
 283
 284         /* free software buffers */
 285         for (i = 0; i < q->nb_desc; ++i) {
 286                 if (q->sw_ring[i]) {
 287                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 288                         q->sw_ring[i] = NULL;
 289                 }
 290         }
 291 }
 292
 293 /*
 294  * free all queue memory used when releasing the queue (i.e. configure)
 295  */
 296 static inline void
 297 rx_queue_free(struct fm10k_rx_queue *q)
 298 {
 299         PMD_INIT_FUNC_TRACE();
 300         if (q) {
 301                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 302                 rx_queue_clean(q);
 303                 if (q->sw_ring) {
 304                         rte_free(q->sw_ring);
 305                         q->sw_ring = NULL;
 306                 }
 307                 rte_free(q);
 308                 q = NULL;
 309         }
 310 }
 311
 312 /*
 313  * disable RX queue, wait unitl HW finished necessary flush operation
 314  */
 315 static inline int
 316 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 317 {
 318         uint32_t reg, i;
 319
 320         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 321         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 322                         reg & ~FM10K_RXQCTL_ENABLE);
 323
 324         /* Wait 100us at most */
 325         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 326                 rte_delay_us(1);
 327                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 328                 if (!(reg & FM10K_RXQCTL_ENABLE))
 329                         break;
 330         }
 331
 332         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 333                 return -1;
 334
 335         return 0;
 336 }
 337
 338 /*
 339  * reset queue to initial state, allocate software buffers used when starting
 340  * device
 341  */
 342 static inline void
 343 tx_queue_reset(struct fm10k_tx_queue *q)
 344 {
 345         PMD_INIT_FUNC_TRACE();
 346         q->last_free = 0;
 347         q->next_free = 0;
 348         q->nb_used = 0;
 349         q->nb_free = q->nb_desc - 1;
 350         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 351         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 352 }
 353
 354 /*
 355  * clean queue, descriptor rings, free software buffers used when stopping
 356  * device
 357  */
 358 static inline void
 359 tx_queue_clean(struct fm10k_tx_queue *q)
 360 {
 361         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 362         uint32_t i;
 363         PMD_INIT_FUNC_TRACE();
 364
 365         /* zero descriptor rings */
 366         for (i = 0; i < q->nb_desc; ++i)
 367                 q->hw_ring[i] = zero;
 368
 369         /* free software buffers */
 370         for (i = 0; i < q->nb_desc; ++i) {
 371                 if (q->sw_ring[i]) {
 372                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 373                         q->sw_ring[i] = NULL;
 374                 }
 375         }
 376 }
 377
 378 /*
 379  * free all queue memory used when releasing the queue (i.e. configure)
 380  */
 381 static inline void
 382 tx_queue_free(struct fm10k_tx_queue *q)
 383 {
 384         PMD_INIT_FUNC_TRACE();
 385         if (q) {
 386                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 387                 tx_queue_clean(q);
 388                 if (q->rs_tracker.list) {
 389                         rte_free(q->rs_tracker.list);
 390                         q->rs_tracker.list = NULL;
 391                 }
 392                 if (q->sw_ring) {
 393                         rte_free(q->sw_ring);
 394                         q->sw_ring = NULL;
 395                 }
 396                 rte_free(q);
 397                 q = NULL;
 398         }
 399 }
 400
 401 /*
 402  * disable TX queue, wait unitl HW finished necessary flush operation
 403  */
 404 static inline int
 405 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 406 {
 407         uint32_t reg, i;
 408
 409         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 410         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 411                         reg & ~FM10K_TXDCTL_ENABLE);
 412
 413         /* Wait 100us at most */
 414         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 415                 rte_delay_us(1);
 416                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 417                 if (!(reg & FM10K_TXDCTL_ENABLE))
 418                         break;
 419         }
 420
 421         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 422                 return -1;
 423
 424         return 0;
 425 }
 426
 427 static int
 428 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 429 {
 430         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 431         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 432         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 433         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 434
 435         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 436
 437         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 438                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 439                 return -EINVAL;
 440         }
 441
 442         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 443                 return 0;
 444
 445         if (hw->mac.type == fm10k_mac_vf) {
 446                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 447                 return -EINVAL;
 448         }
 449
 450         /* Check VMDQ queue pool number */
 451         if (vmdq_conf->nb_queue_pools >
 452                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 453                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 454                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 455                         vmdq_conf->nb_queue_pools);
 456                 return -EINVAL;
 457         }
 458
 459         return 0;
 460 }
 461
 462 static const struct fm10k_txq_ops def_txq_ops = {
 463         .reset = tx_queue_reset,
 464 };
 465
 466 static int
 467 fm10k_dev_configure(struct rte_eth_dev *dev)
 468 {
 469         int ret;
 470
 471         PMD_INIT_FUNC_TRACE();
 472
 473         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 474                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 475         /* multipe queue mode checking */
 476         ret  = fm10k_check_mq_mode(dev);
 477         if (ret != 0) {
 478                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 479                             ret);
 480                 return ret;
 481         }
 482
 483         return 0;
 484 }
 485
 486 /* fls = find last set bit = 32 minus the number of leading zeros */
 487 #ifndef fls
 488 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 489 #endif
 490
 491 static void
 492 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 493 {
 494         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 495         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 496         uint32_t i;
 497
 498         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 499
 500         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 501                 if (!vmdq_conf->pool_map[i].pools)
 502                         continue;
 503                 fm10k_mbx_lock(hw);
 504                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 505                 fm10k_mbx_unlock(hw);
 506         }
 507 }
 508
 509 static void
 510 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 511 {
 512         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 513
 514         /* Add default mac address */
 515         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 516                 MAIN_VSI_POOL_NUMBER);
 517 }
 518
 519 static void
 520 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 521 {
 522         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 523         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 524         uint32_t mrqc, *key, i, reta, j;
 525         uint64_t hf;
 526
 527 #define RSS_KEY_SIZE 40
 528         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 529                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 530                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 531                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 532                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 533                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 534         };
 535
 536         if (dev->data->nb_rx_queues == 1 ||
 537             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 538             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 539                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 540                 return;
 541         }
 542
 543         /* random key is rss_intel_key (default) or user provided (rss_key) */
 544         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 545                 key = (uint32_t *)rss_intel_key;
 546         else
 547                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 548
 549         /* Now fill our hash function seeds, 4 bytes at a time */
 550         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 551                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 552
 553         /*
 554          * Fill in redirection table
 555          * The byte-swap is needed because NIC registers are in
 556          * little-endian order.
 557          */
 558         reta = 0;
 559         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 560                 if (j == dev->data->nb_rx_queues)
 561                         j = 0;
 562                 reta = (reta << CHAR_BIT) | j;
 563                 if ((i & 3) == 3)
 564                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 565                                         rte_bswap32(reta));
 566         }
 567
 568         /*
 569          * Generate RSS hash based on packet types, TCP/UDP
 570          * port numbers and/or IPv4/v6 src and dst addresses
 571          */
 572         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 573         mrqc = 0;
 574         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 575         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 576         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 577         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 578         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 579         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 580         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 581         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 582         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 583
 584         if (mrqc == 0) {
 585                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 586                         "supported", hf);
 587                 return;
 588         }
 589
 590         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 591 }
 592
 593 static void
 594 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 595 {
 596         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 597         uint32_t i;
 598
 599         for (i = 0; i < nb_lport_new; i++) {
 600                 /* Set unicast mode by default. App can change
 601                  * to other mode in other API func.
 602                  */
 603                 fm10k_mbx_lock(hw);
 604                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 605                         FM10K_XCAST_MODE_NONE);
 606                 fm10k_mbx_unlock(hw);
 607         }
 608 }
 609
 610 static void
 611 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 612 {
 613         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 614         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 615         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 616         struct fm10k_macvlan_filter_info *macvlan;
 617         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 618         uint16_t nb_lport_new;
 619
 620         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 621         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 622
 623         fm10k_dev_rss_configure(dev);
 624
 625         /* only PF supports VMDQ */
 626         if (hw->mac.type != fm10k_mac_pf)
 627                 return;
 628
 629         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 630                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 631
 632         /* no pool number change, no need to update logic port and VLAN/MAC */
 633         if (macvlan->nb_queue_pools == nb_queue_pools)
 634                 return;
 635
 636         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 637         fm10k_dev_logic_port_update(dev, nb_lport_new);
 638
 639         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 640         memset(dev->data->mac_addrs, 0,
 641                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 642         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 643                 &dev->data->mac_addrs[0]);
 644         memset(macvlan, 0, sizeof(*macvlan));
 645         macvlan->nb_queue_pools = nb_queue_pools;
 646
 647         if (nb_queue_pools)
 648                 fm10k_dev_vmdq_rx_configure(dev);
 649         else
 650                 fm10k_dev_pf_main_vsi_reset(dev);
 651 }
 652
 653 static int
 654 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 655 {
 656         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 657         int i, ret;
 658         struct fm10k_tx_queue *txq;
 659         uint64_t base_addr;
 660         uint32_t size;
 661
 662         /* Disable TXINT to avoid possible interrupt */
 663         for (i = 0; i < hw->mac.max_queues; i++)
 664                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 665                                 3 << FM10K_TXINT_TIMER_SHIFT);
 666
 667         /* Setup TX queue */
 668         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 669                 txq = dev->data->tx_queues[i];
 670                 base_addr = txq->hw_ring_phys_addr;
 671                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 672
 673                 /* disable queue to avoid issues while updating state */
 674                 ret = tx_queue_disable(hw, i);
 675                 if (ret) {
 676                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 677                         return -1;
 678                 }
 679                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 680                  * register is read-only for VF.
 681                  */
 682                 if (fm10k_check_ftag(dev->device->devargs)) {
 683                         if (hw->mac.type == fm10k_mac_pf) {
 684                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 685                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 686                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 687                         } else {
 688                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 689                                 return -ENOTSUP;
 690                         }
 691                 }
 692
 693                 /* set location and size for descriptor ring */
 694                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 695                                 base_addr & UINT64_LOWER_32BITS_MASK);
 696                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 697                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 698                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 699
 700                 /* assign default SGLORT for each TX queue by PF */
 701                 if (hw->mac.type == fm10k_mac_pf)
 702                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 703         }
 704
 705         /* set up vector or scalar TX function as appropriate */
 706         fm10k_set_tx_function(dev);
 707
 708         return 0;
 709 }
 710
 711 static int
 712 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 713 {
 714         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 715         struct fm10k_macvlan_filter_info *macvlan;
 716         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 717         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 718         int i, ret;
 719         struct fm10k_rx_queue *rxq;
 720         uint64_t base_addr;
 721         uint32_t size;
 722         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 723         uint32_t logic_port = hw->mac.dglort_map;
 724         uint16_t buf_size;
 725         uint16_t queue_stride = 0;
 726
 727         /* enable RXINT for interrupt mode */
 728         i = 0;
 729         if (rte_intr_dp_is_en(intr_handle)) {
 730                 for (; i < dev->data->nb_rx_queues; i++) {
 731                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 732                         if (hw->mac.type == fm10k_mac_pf)
 733                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 734                                         FM10K_ITR_AUTOMASK |
 735                                         FM10K_ITR_MASK_CLEAR);
 736                         else
 737                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 738                                         FM10K_ITR_AUTOMASK |
 739                                         FM10K_ITR_MASK_CLEAR);
 740                 }
 741         }
 742         /* Disable other RXINT to avoid possible interrupt */
 743         for (; i < hw->mac.max_queues; i++)
 744                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 745                         3 << FM10K_RXINT_TIMER_SHIFT);
 746
 747         /* Setup RX queues */
 748         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 749                 rxq = dev->data->rx_queues[i];
 750                 base_addr = rxq->hw_ring_phys_addr;
 751                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 752
 753                 /* disable queue to avoid issues while updating state */
 754                 ret = rx_queue_disable(hw, i);
 755                 if (ret) {
 756                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 757                         return -1;
 758                 }
 759
 760                 /* Setup the Base and Length of the Rx Descriptor Ring */
 761                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 762                                 base_addr & UINT64_LOWER_32BITS_MASK);
 763                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 764                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 765                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 766
 767                 /* Configure the Rx buffer size for one buff without split */
 768                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 769                         RTE_PKTMBUF_HEADROOM);
 770                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 771                  * reserved for this purpose, and the worst case could be 511B.
 772                  * But SRR reg assumes all buffers have the same size. In order
 773                  * to fill the gap, we'll have to consider the worst case and
 774                  * assume 512B is reserved. If we don't do so, it's possible
 775                  * for HW to overwrite data to next mbuf.
 776                  */
 777                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 778
 779                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 780                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 781                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 782
 783                 /* It adds dual VLAN length for supporting dual VLAN */
 784                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 785                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 786                         dev->data->dev_conf.rxmode.enable_scatter) {
 787                         uint32_t reg;
 788                         dev->data->scattered_rx = 1;
 789                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 790                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 791                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 792                 }
 793
 794                 /* Enable drop on empty, it's RO for VF */
 795                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 796                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 797
 798                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 799                 FM10K_WRITE_FLUSH(hw);
 800         }
 801
 802         /* Configure VMDQ/RSS if applicable */
 803         fm10k_dev_mq_rx_configure(dev);
 804
 805         /* Decide the best RX function */
 806         fm10k_set_rx_function(dev);
 807
 808         /* update RX_SGLORT for loopback suppress*/
 809         if (hw->mac.type != fm10k_mac_pf)
 810                 return 0;
 811         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 812         if (macvlan->nb_queue_pools)
 813                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 814         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 815                 if (i && queue_stride && !(i % queue_stride))
 816                         logic_port++;
 817                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 818         }
 819
 820         return 0;
 821 }
 822
 823 static int
 824 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 825 {
 826         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 827         int err = -1;
 828         uint32_t reg;
 829         struct fm10k_rx_queue *rxq;
 830
 831         PMD_INIT_FUNC_TRACE();
 832
 833         if (rx_queue_id < dev->data->nb_rx_queues) {
 834                 rxq = dev->data->rx_queues[rx_queue_id];
 835                 err = rx_queue_reset(rxq);
 836                 if (err == -ENOMEM) {
 837                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 838                         return err;
 839                 } else if (err == -EINVAL) {
 840                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 841                                 " %d", err);
 842                         return err;
 843                 }
 844
 845                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 846                  * Note: this must be done AFTER the queue is enabled on real
 847                  * hardware, but BEFORE the queue is enabled when using the
 848                  * emulation platform. Do it in both places for now and remove
 849                  * this comment and the following two register writes when the
 850                  * emulation platform is no longer being used.
 851                  */
 852                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 853                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 854
 855                 /* Set PF ownership flag for PF devices */
 856                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 857                 if (hw->mac.type == fm10k_mac_pf)
 858                         reg |= FM10K_RXQCTL_PF;
 859                 reg |= FM10K_RXQCTL_ENABLE;
 860                 /* enable RX queue */
 861                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 862                 FM10K_WRITE_FLUSH(hw);
 863
 864                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 865                  * Note: this must be done AFTER the queue is enabled
 866                  */
 867                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 868                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 869                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 870         }
 871
 872         return err;
 873 }
 874
 875 static int
 876 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 877 {
 878         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 879
 880         PMD_INIT_FUNC_TRACE();
 881
 882         if (rx_queue_id < dev->data->nb_rx_queues) {
 883                 /* Disable RX queue */
 884                 rx_queue_disable(hw, rx_queue_id);
 885
 886                 /* Free mbuf and clean HW ring */
 887                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 888                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 889         }
 890
 891         return 0;
 892 }
 893
 894 static int
 895 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 896 {
 897         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 898         /** @todo - this should be defined in the shared code */
 899 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 900         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 901         int err = 0;
 902
 903         PMD_INIT_FUNC_TRACE();
 904
 905         if (tx_queue_id < dev->data->nb_tx_queues) {
 906                 struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 907
 908                 q->ops->reset(q);
 909
 910                 /* reset head and tail pointers */
 911                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 912                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 913
 914                 /* enable TX queue */
 915                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 916                                         FM10K_TXDCTL_ENABLE | txdctl);
 917                 FM10K_WRITE_FLUSH(hw);
 918                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 919         } else
 920                 err = -1;
 921
 922         return err;
 923 }
 924
 925 static int
 926 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 927 {
 928         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 929
 930         PMD_INIT_FUNC_TRACE();
 931
 932         if (tx_queue_id < dev->data->nb_tx_queues) {
 933                 tx_queue_disable(hw, tx_queue_id);
 934                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 935                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 936         }
 937
 938         return 0;
 939 }
 940
 941 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 942 {
 943         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 944                 != FM10K_DGLORTMAP_NONE);
 945 }
 946
 947 static void
 948 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 949 {
 950         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 951         int status;
 952
 953         PMD_INIT_FUNC_TRACE();
 954
 955         /* Return if it didn't acquire valid glort range */
 956         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 957                 return;
 958
 959         fm10k_mbx_lock(hw);
 960         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 961                                 FM10K_XCAST_MODE_PROMISC);
 962         fm10k_mbx_unlock(hw);
 963
 964         if (status != FM10K_SUCCESS)
 965                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 966 }
 967
 968 static void
 969 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 970 {
 971         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 972         uint8_t mode;
 973         int status;
 974
 975         PMD_INIT_FUNC_TRACE();
 976
 977         /* Return if it didn't acquire valid glort range */
 978         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 979                 return;
 980
 981         if (dev->data->all_multicast == 1)
 982                 mode = FM10K_XCAST_MODE_ALLMULTI;
 983         else
 984                 mode = FM10K_XCAST_MODE_NONE;
 985
 986         fm10k_mbx_lock(hw);
 987         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 988                                 mode);
 989         fm10k_mbx_unlock(hw);
 990
 991         if (status != FM10K_SUCCESS)
 992                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 993 }
 994
 995 static void
 996 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 997 {
 998         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 999         int status;
1000
1001         PMD_INIT_FUNC_TRACE();
1002
1003         /* Return if it didn't acquire valid glort range */
1004         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1005                 return;
1006
1007         /* If promiscuous mode is enabled, it doesn't make sense to enable
1008          * allmulticast and disable promiscuous since fm10k only can select
1009          * one of the modes.
1010          */
1011         if (dev->data->promiscuous) {
1012                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
1013                         "needn't enable allmulticast");
1014                 return;
1015         }
1016
1017         fm10k_mbx_lock(hw);
1018         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1019                                 FM10K_XCAST_MODE_ALLMULTI);
1020         fm10k_mbx_unlock(hw);
1021
1022         if (status != FM10K_SUCCESS)
1023                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
1024 }
1025
1026 static void
1027 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1028 {
1029         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1030         int status;
1031
1032         PMD_INIT_FUNC_TRACE();
1033
1034         /* Return if it didn't acquire valid glort range */
1035         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1036                 return;
1037
1038         if (dev->data->promiscuous) {
1039                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1040                         "since promisc mode is enabled");
1041                 return;
1042         }
1043
1044         fm10k_mbx_lock(hw);
1045         /* Change mode to unicast mode */
1046         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1047                                 FM10K_XCAST_MODE_NONE);
1048         fm10k_mbx_unlock(hw);
1049
1050         if (status != FM10K_SUCCESS)
1051                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1052 }
1053
1054 static void
1055 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1056 {
1057         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1058         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1059         uint16_t nb_queue_pools;
1060         struct fm10k_macvlan_filter_info *macvlan;
1061
1062         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1063         nb_queue_pools = macvlan->nb_queue_pools;
1064         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
1065         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
1066
1067         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1068         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1069         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1070                         hw->mac.dglort_map;
1071         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1072         /* Configure VMDQ/RSS DGlort Decoder */
1073         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1074
1075         /* Flow Director configurations, only queue number is valid. */
1076         dglortdec = fls(dev->data->nb_rx_queues - 1);
1077         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1078                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1079         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1080         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1081
1082         /* Invalidate all other GLORT entries */
1083         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1084                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1085                                 FM10K_DGLORTMAP_NONE);
1086 }
1087
1088 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1089 static int
1090 fm10k_dev_start(struct rte_eth_dev *dev)
1091 {
1092         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1093         int i, diag;
1094
1095         PMD_INIT_FUNC_TRACE();
1096
1097         /* stop, init, then start the hw */
1098         diag = fm10k_stop_hw(hw);
1099         if (diag != FM10K_SUCCESS) {
1100                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1101                 return -EIO;
1102         }
1103
1104         diag = fm10k_init_hw(hw);
1105         if (diag != FM10K_SUCCESS) {
1106                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1107                 return -EIO;
1108         }
1109
1110         diag = fm10k_start_hw(hw);
1111         if (diag != FM10K_SUCCESS) {
1112                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1113                 return -EIO;
1114         }
1115
1116         diag = fm10k_dev_tx_init(dev);
1117         if (diag) {
1118                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1119                 return diag;
1120         }
1121
1122         if (fm10k_dev_rxq_interrupt_setup(dev))
1123                 return -EIO;
1124
1125         diag = fm10k_dev_rx_init(dev);
1126         if (diag) {
1127                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1128                 return diag;
1129         }
1130
1131         if (hw->mac.type == fm10k_mac_pf)
1132                 fm10k_dev_dglort_map_configure(dev);
1133
1134         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1135                 struct fm10k_rx_queue *rxq;
1136                 rxq = dev->data->rx_queues[i];
1137
1138                 if (rxq->rx_deferred_start)
1139                         continue;
1140                 diag = fm10k_dev_rx_queue_start(dev, i);
1141                 if (diag != 0) {
1142                         int j;
1143                         for (j = 0; j < i; ++j)
1144                                 rx_queue_clean(dev->data->rx_queues[j]);
1145                         return diag;
1146                 }
1147         }
1148
1149         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1150                 struct fm10k_tx_queue *txq;
1151                 txq = dev->data->tx_queues[i];
1152
1153                 if (txq->tx_deferred_start)
1154                         continue;
1155                 diag = fm10k_dev_tx_queue_start(dev, i);
1156                 if (diag != 0) {
1157                         int j;
1158                         for (j = 0; j < i; ++j)
1159                                 tx_queue_clean(dev->data->tx_queues[j]);
1160                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1161                                 rx_queue_clean(dev->data->rx_queues[j]);
1162                         return diag;
1163                 }
1164         }
1165
1166         /* Update default vlan when not in VMDQ mode */
1167         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1168                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1169
1170         fm10k_link_update(dev, 0);
1171
1172         return 0;
1173 }
1174
1175 static void
1176 fm10k_dev_stop(struct rte_eth_dev *dev)
1177 {
1178         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1179         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1180         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1181         int i;
1182
1183         PMD_INIT_FUNC_TRACE();
1184
1185         if (dev->data->tx_queues)
1186                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1187                         fm10k_dev_tx_queue_stop(dev, i);
1188
1189         if (dev->data->rx_queues)
1190                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1191                         fm10k_dev_rx_queue_stop(dev, i);
1192
1193         /* Disable datapath event */
1194         if (rte_intr_dp_is_en(intr_handle)) {
1195                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1196                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1197                                 3 << FM10K_RXINT_TIMER_SHIFT);
1198                         if (hw->mac.type == fm10k_mac_pf)
1199                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1200                                         FM10K_ITR_MASK_SET);
1201                         else
1202                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1203                                         FM10K_ITR_MASK_SET);
1204                 }
1205         }
1206         /* Clean datapath event and queue/vec mapping */
1207         rte_intr_efd_disable(intr_handle);
1208         rte_free(intr_handle->intr_vec);
1209         intr_handle->intr_vec = NULL;
1210 }
1211
1212 static void
1213 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1214 {
1215         int i;
1216
1217         PMD_INIT_FUNC_TRACE();
1218
1219         if (dev->data->tx_queues) {
1220                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1221                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1222
1223                         tx_queue_free(txq);
1224                 }
1225         }
1226
1227         if (dev->data->rx_queues) {
1228                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1229                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1230         }
1231 }
1232
1233 static void
1234 fm10k_dev_close(struct rte_eth_dev *dev)
1235 {
1236         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1237
1238         PMD_INIT_FUNC_TRACE();
1239
1240         fm10k_mbx_lock(hw);
1241         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1242                 MAX_LPORT_NUM, false);
1243         fm10k_mbx_unlock(hw);
1244
1245         /* allow 10ms for device to quiesce */
1246         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
1247
1248         /* Stop mailbox service first */
1249         fm10k_close_mbx_service(hw);
1250         fm10k_dev_stop(dev);
1251         fm10k_dev_queue_release(dev);
1252         fm10k_stop_hw(hw);
1253 }
1254
1255 static int
1256 fm10k_link_update(struct rte_eth_dev *dev,
1257         __rte_unused int wait_to_complete)
1258 {
1259         PMD_INIT_FUNC_TRACE();
1260
1261         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
1262          * x8 PCIe interface. For now, we leave the speed undefined since there
1263          * is no 50Gbps Ethernet. */
1264         dev->data->dev_link.link_speed  = 0;
1265         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1266         dev->data->dev_link.link_status = ETH_LINK_UP;
1267
1268         return 0;
1269 }
1270
1271 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1272         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1273 {
1274         unsigned i, q;
1275         unsigned count = 0;
1276
1277         if (xstats_names != NULL) {
1278                 /* Note: limit checked in rte_eth_xstats_names() */
1279
1280                 /* Global stats */
1281                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1282                         snprintf(xstats_names[count].name,
1283                                 sizeof(xstats_names[count].name),
1284                                 "%s", fm10k_hw_stats_strings[count].name);
1285                         count++;
1286                 }
1287
1288                 /* PF queue stats */
1289                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1290                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1291                                 snprintf(xstats_names[count].name,
1292                                         sizeof(xstats_names[count].name),
1293                                         "rx_q%u_%s", q,
1294                                         fm10k_hw_stats_rx_q_strings[i].name);
1295                                 count++;
1296                         }
1297                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1298                                 snprintf(xstats_names[count].name,
1299                                         sizeof(xstats_names[count].name),
1300                                         "tx_q%u_%s", q,
1301                                         fm10k_hw_stats_tx_q_strings[i].name);
1302                                 count++;
1303                         }
1304                 }
1305         }
1306         return FM10K_NB_XSTATS;
1307 }
1308
1309 static int
1310 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1311                  unsigned n)
1312 {
1313         struct fm10k_hw_stats *hw_stats =
1314                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1315         unsigned i, q, count = 0;
1316
1317         if (n < FM10K_NB_XSTATS)
1318                 return FM10K_NB_XSTATS;
1319
1320         /* Global stats */
1321         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1322                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1323                         fm10k_hw_stats_strings[count].offset);
1324                 xstats[count].id = count;
1325                 count++;
1326         }
1327
1328         /* PF queue stats */
1329         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1330                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1331                         xstats[count].value =
1332                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1333                                 fm10k_hw_stats_rx_q_strings[i].offset);
1334                         xstats[count].id = count;
1335                         count++;
1336                 }
1337                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1338                         xstats[count].value =
1339                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1340                                 fm10k_hw_stats_tx_q_strings[i].offset);
1341                         xstats[count].id = count;
1342                         count++;
1343                 }
1344         }
1345
1346         return FM10K_NB_XSTATS;
1347 }
1348
1349 static int
1350 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1351 {
1352         uint64_t ipackets, opackets, ibytes, obytes;
1353         struct fm10k_hw *hw =
1354                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1355         struct fm10k_hw_stats *hw_stats =
1356                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1357         int i;
1358
1359         PMD_INIT_FUNC_TRACE();
1360
1361         fm10k_update_hw_stats(hw, hw_stats);
1362
1363         ipackets = opackets = ibytes = obytes = 0;
1364         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1365                 (i < hw->mac.max_queues); ++i) {
1366                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1367                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1368                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1369                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1370                 ipackets += stats->q_ipackets[i];
1371                 opackets += stats->q_opackets[i];
1372                 ibytes   += stats->q_ibytes[i];
1373                 obytes   += stats->q_obytes[i];
1374         }
1375         stats->ipackets = ipackets;
1376         stats->opackets = opackets;
1377         stats->ibytes = ibytes;
1378         stats->obytes = obytes;
1379         return 0;
1380 }
1381
1382 static void
1383 fm10k_stats_reset(struct rte_eth_dev *dev)
1384 {
1385         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1386         struct fm10k_hw_stats *hw_stats =
1387                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1388
1389         PMD_INIT_FUNC_TRACE();
1390
1391         memset(hw_stats, 0, sizeof(*hw_stats));
1392         fm10k_rebind_hw_stats(hw, hw_stats);
1393 }
1394
1395 static void
1396 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1397         struct rte_eth_dev_info *dev_info)
1398 {
1399         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1400         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1401
1402         PMD_INIT_FUNC_TRACE();
1403
1404         dev_info->pci_dev            = pdev;
1405         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1406         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1407         dev_info->max_rx_queues      = hw->mac.max_queues;
1408         dev_info->max_tx_queues      = hw->mac.max_queues;
1409         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1410         dev_info->max_hash_mac_addrs = 0;
1411         dev_info->max_vfs            = pdev->max_vfs;
1412         dev_info->vmdq_pool_base     = 0;
1413         dev_info->vmdq_queue_base    = 0;
1414         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1415         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1416         dev_info->rx_offload_capa =
1417                 DEV_RX_OFFLOAD_VLAN_STRIP |
1418                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1419                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1420                 DEV_RX_OFFLOAD_TCP_CKSUM;
1421         dev_info->tx_offload_capa =
1422                 DEV_TX_OFFLOAD_VLAN_INSERT |
1423                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1424                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1425                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1426                 DEV_TX_OFFLOAD_TCP_TSO;
1427
1428         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1429         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1430
1431         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1432                 .rx_thresh = {
1433                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1434                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1435                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1436                 },
1437                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1438                 .rx_drop_en = 0,
1439         };
1440
1441         dev_info->default_txconf = (struct rte_eth_txconf) {
1442                 .tx_thresh = {
1443                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1444                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1445                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1446                 },
1447                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1448                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1449                 .txq_flags = FM10K_SIMPLE_TX_FLAG,
1450         };
1451
1452         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1453                 .nb_max = FM10K_MAX_RX_DESC,
1454                 .nb_min = FM10K_MIN_RX_DESC,
1455                 .nb_align = FM10K_MULT_RX_DESC,
1456         };
1457
1458         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1459                 .nb_max = FM10K_MAX_TX_DESC,
1460                 .nb_min = FM10K_MIN_TX_DESC,
1461                 .nb_align = FM10K_MULT_TX_DESC,
1462                 .nb_seg_max = FM10K_TX_MAX_SEG,
1463                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1464         };
1465
1466         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1467                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1468                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1469 }
1470
1471 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1472 static const uint32_t *
1473 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1474 {
1475         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1476             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1477                 static uint32_t ptypes[] = {
1478                         /* refers to rx_desc_to_ol_flags() */
1479                         RTE_PTYPE_L2_ETHER,
1480                         RTE_PTYPE_L3_IPV4,
1481                         RTE_PTYPE_L3_IPV4_EXT,
1482                         RTE_PTYPE_L3_IPV6,
1483                         RTE_PTYPE_L3_IPV6_EXT,
1484                         RTE_PTYPE_L4_TCP,
1485                         RTE_PTYPE_L4_UDP,
1486                         RTE_PTYPE_UNKNOWN
1487                 };
1488
1489                 return ptypes;
1490         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1491                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1492                 static uint32_t ptypes_vec[] = {
1493                         /* refers to fm10k_desc_to_pktype_v() */
1494                         RTE_PTYPE_L3_IPV4,
1495                         RTE_PTYPE_L3_IPV4_EXT,
1496                         RTE_PTYPE_L3_IPV6,
1497                         RTE_PTYPE_L3_IPV6_EXT,
1498                         RTE_PTYPE_L4_TCP,
1499                         RTE_PTYPE_L4_UDP,
1500                         RTE_PTYPE_TUNNEL_GENEVE,
1501                         RTE_PTYPE_TUNNEL_NVGRE,
1502                         RTE_PTYPE_TUNNEL_VXLAN,
1503                         RTE_PTYPE_TUNNEL_GRE,
1504                         RTE_PTYPE_UNKNOWN
1505                 };
1506
1507                 return ptypes_vec;
1508         }
1509
1510         return NULL;
1511 }
1512 #else
1513 static const uint32_t *
1514 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1515 {
1516         return NULL;
1517 }
1518 #endif
1519
1520 static int
1521 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1522 {
1523         s32 result;
1524         uint16_t mac_num = 0;
1525         uint32_t vid_idx, vid_bit, mac_index;
1526         struct fm10k_hw *hw;
1527         struct fm10k_macvlan_filter_info *macvlan;
1528         struct rte_eth_dev_data *data = dev->data;
1529
1530         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1531         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1532
1533         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1534                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1535                 return -EINVAL;
1536         }
1537
1538         if (vlan_id > ETH_VLAN_ID_MAX) {
1539                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1540                 return -EINVAL;
1541         }
1542
1543         vid_idx = FM10K_VFTA_IDX(vlan_id);
1544         vid_bit = FM10K_VFTA_BIT(vlan_id);
1545         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1546         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1547                 return 0;
1548         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1549         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1550                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1551                         "in the VLAN filter table");
1552                 return -EINVAL;
1553         }
1554
1555         fm10k_mbx_lock(hw);
1556         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1557         fm10k_mbx_unlock(hw);
1558         if (result != FM10K_SUCCESS) {
1559                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1560                 return -EIO;
1561         }
1562
1563         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1564                         (result == FM10K_SUCCESS); mac_index++) {
1565                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1566                         continue;
1567                 if (mac_num > macvlan->mac_num - 1) {
1568                         PMD_INIT_LOG(ERR, "MAC address number "
1569                                         "not match");
1570                         break;
1571                 }
1572                 fm10k_mbx_lock(hw);
1573                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1574                         data->mac_addrs[mac_index].addr_bytes,
1575                         vlan_id, on, 0);
1576                 fm10k_mbx_unlock(hw);
1577                 mac_num++;
1578         }
1579         if (result != FM10K_SUCCESS) {
1580                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1581                 return -EIO;
1582         }
1583
1584         if (on) {
1585                 macvlan->vlan_num++;
1586                 macvlan->vfta[vid_idx] |= vid_bit;
1587         } else {
1588                 macvlan->vlan_num--;
1589                 macvlan->vfta[vid_idx] &= ~vid_bit;
1590         }
1591         return 0;
1592 }
1593
1594 static void
1595 fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1596 {
1597         if (mask & ETH_VLAN_STRIP_MASK) {
1598                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1599                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1600                                         "always on in fm10k");
1601         }
1602
1603         if (mask & ETH_VLAN_EXTEND_MASK) {
1604                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1605                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1606                                         "supported in fm10k");
1607         }
1608
1609         if (mask & ETH_VLAN_FILTER_MASK) {
1610                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1611                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1612         }
1613 }
1614
1615 /* Add/Remove a MAC address, and update filters to main VSI */
1616 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1617                 const u8 *mac, bool add, uint32_t pool)
1618 {
1619         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1620         struct fm10k_macvlan_filter_info *macvlan;
1621         uint32_t i, j, k;
1622
1623         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1624
1625         if (pool != MAIN_VSI_POOL_NUMBER) {
1626                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1627                         "mac to pool %u", pool);
1628                 return;
1629         }
1630         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1631                 if (!macvlan->vfta[j])
1632                         continue;
1633                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1634                         if (!(macvlan->vfta[j] & (1 << k)))
1635                                 continue;
1636                         if (i + 1 > macvlan->vlan_num) {
1637                                 PMD_INIT_LOG(ERR, "vlan number not match");
1638                                 return;
1639                         }
1640                         fm10k_mbx_lock(hw);
1641                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1642                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1643                         fm10k_mbx_unlock(hw);
1644                         i++;
1645                 }
1646         }
1647 }
1648
1649 /* Add/Remove a MAC address, and update filters to VMDQ */
1650 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1651                 const u8 *mac, bool add, uint32_t pool)
1652 {
1653         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1654         struct fm10k_macvlan_filter_info *macvlan;
1655         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1656         uint32_t i;
1657
1658         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1659         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1660
1661         if (pool > macvlan->nb_queue_pools) {
1662                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1663                         " Max pool is %u",
1664                         pool, macvlan->nb_queue_pools);
1665                 return;
1666         }
1667         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1668                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1669                         continue;
1670                 fm10k_mbx_lock(hw);
1671                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1672                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1673                 fm10k_mbx_unlock(hw);
1674         }
1675 }
1676
1677 /* Add/Remove a MAC address, and update filters */
1678 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1679                 const u8 *mac, bool add, uint32_t pool)
1680 {
1681         struct fm10k_macvlan_filter_info *macvlan;
1682
1683         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1684
1685         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1686                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1687         else
1688                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1689
1690         if (add)
1691                 macvlan->mac_num++;
1692         else
1693                 macvlan->mac_num--;
1694 }
1695
1696 /* Add a MAC address, and update filters */
1697 static int
1698 fm10k_macaddr_add(struct rte_eth_dev *dev,
1699                 struct ether_addr *mac_addr,
1700                 uint32_t index,
1701                 uint32_t pool)
1702 {
1703         struct fm10k_macvlan_filter_info *macvlan;
1704
1705         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1706         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1707         macvlan->mac_vmdq_id[index] = pool;
1708         return 0;
1709 }
1710
1711 /* Remove a MAC address, and update filters */
1712 static void
1713 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1714 {
1715         struct rte_eth_dev_data *data = dev->data;
1716         struct fm10k_macvlan_filter_info *macvlan;
1717
1718         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1719         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1720                         FALSE, macvlan->mac_vmdq_id[index]);
1721         macvlan->mac_vmdq_id[index] = 0;
1722 }
1723
1724 static inline int
1725 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1726 {
1727         if ((request < min) || (request > max) || ((request % mult) != 0))
1728                 return -1;
1729         else
1730                 return 0;
1731 }
1732
1733
1734 static inline int
1735 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1736 {
1737         if ((request < min) || (request > max) || ((div % request) != 0))
1738                 return -1;
1739         else
1740                 return 0;
1741 }
1742
1743 static inline int
1744 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1745 {
1746         uint16_t rx_free_thresh;
1747
1748         if (conf->rx_free_thresh == 0)
1749                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1750         else
1751                 rx_free_thresh = conf->rx_free_thresh;
1752
1753         /* make sure the requested threshold satisfies the constraints */
1754         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1755                         FM10K_RX_FREE_THRESH_MAX(q),
1756                         FM10K_RX_FREE_THRESH_DIV(q),
1757                         rx_free_thresh)) {
1758                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1759                         "less than or equal to %u, "
1760                         "greater than or equal to %u, "
1761                         "and a divisor of %u",
1762                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1763                         FM10K_RX_FREE_THRESH_MIN(q),
1764                         FM10K_RX_FREE_THRESH_DIV(q));
1765                 return -EINVAL;
1766         }
1767
1768         q->alloc_thresh = rx_free_thresh;
1769         q->drop_en = conf->rx_drop_en;
1770         q->rx_deferred_start = conf->rx_deferred_start;
1771
1772         return 0;
1773 }
1774
1775 /*
1776  * Hardware requires specific alignment for Rx packet buffers. At
1777  * least one of the following two conditions must be satisfied.
1778  *  1. Address is 512B aligned
1779  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1780  *
1781  * As such, the driver may need to adjust the DMA address within the
1782  * buffer by up to 512B.
1783  *
1784  * return 1 if the element size is valid, otherwise return 0.
1785  */
1786 static int
1787 mempool_element_size_valid(struct rte_mempool *mp)
1788 {
1789         uint32_t min_size;
1790
1791         /* elt_size includes mbuf header and headroom */
1792         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1793                         RTE_PKTMBUF_HEADROOM;
1794
1795         /* account for up to 512B of alignment */
1796         min_size -= FM10K_RX_DATABUF_ALIGN;
1797
1798         /* sanity check for overflow */
1799         if (min_size > mp->elt_size)
1800                 return 0;
1801
1802         /* size is valid */
1803         return 1;
1804 }
1805
1806 static int
1807 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1808         uint16_t nb_desc, unsigned int socket_id,
1809         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1810 {
1811         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1812         struct fm10k_dev_info *dev_info =
1813                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1814         struct fm10k_rx_queue *q;
1815         const struct rte_memzone *mz;
1816
1817         PMD_INIT_FUNC_TRACE();
1818
1819         /* make sure the mempool element size can account for alignment. */
1820         if (!mempool_element_size_valid(mp)) {
1821                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1822                 return -EINVAL;
1823         }
1824
1825         /* make sure a valid number of descriptors have been requested */
1826         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1827                                 FM10K_MULT_RX_DESC, nb_desc)) {
1828                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1829                         "less than or equal to %"PRIu32", "
1830                         "greater than or equal to %u, "
1831                         "and a multiple of %u",
1832                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1833                         FM10K_MULT_RX_DESC);
1834                 return -EINVAL;
1835         }
1836
1837         /*
1838          * if this queue existed already, free the associated memory. The
1839          * queue cannot be reused in case we need to allocate memory on
1840          * different socket than was previously used.
1841          */
1842         if (dev->data->rx_queues[queue_id] != NULL) {
1843                 rx_queue_free(dev->data->rx_queues[queue_id]);
1844                 dev->data->rx_queues[queue_id] = NULL;
1845         }
1846
1847         /* allocate memory for the queue structure */
1848         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1849                                 socket_id);
1850         if (q == NULL) {
1851                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1852                 return -ENOMEM;
1853         }
1854
1855         /* setup queue */
1856         q->mp = mp;
1857         q->nb_desc = nb_desc;
1858         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1859         q->port_id = dev->data->port_id;
1860         q->queue_id = queue_id;
1861         q->tail_ptr = (volatile uint32_t *)
1862                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1863         if (handle_rxconf(q, conf))
1864                 return -EINVAL;
1865
1866         /* allocate memory for the software ring */
1867         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1868                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1869                         RTE_CACHE_LINE_SIZE, socket_id);
1870         if (q->sw_ring == NULL) {
1871                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1872                 rte_free(q);
1873                 return -ENOMEM;
1874         }
1875
1876         /*
1877          * allocate memory for the hardware descriptor ring. A memzone large
1878          * enough to hold the maximum ring size is requested to allow for
1879          * resizing in later calls to the queue setup function.
1880          */
1881         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1882                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1883                                       socket_id);
1884         if (mz == NULL) {
1885                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1886                 rte_free(q->sw_ring);
1887                 rte_free(q);
1888                 return -ENOMEM;
1889         }
1890         q->hw_ring = mz->addr;
1891         q->hw_ring_phys_addr = mz->phys_addr;
1892
1893         /* Check if number of descs satisfied Vector requirement */
1894         if (!rte_is_power_of_2(nb_desc)) {
1895                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1896                                     "preconditions - canceling the feature for "
1897                                     "the whole port[%d]",
1898                              q->queue_id, q->port_id);
1899                 dev_info->rx_vec_allowed = false;
1900         } else
1901                 fm10k_rxq_vec_setup(q);
1902
1903         dev->data->rx_queues[queue_id] = q;
1904         return 0;
1905 }
1906
1907 static void
1908 fm10k_rx_queue_release(void *queue)
1909 {
1910         PMD_INIT_FUNC_TRACE();
1911
1912         rx_queue_free(queue);
1913 }
1914
1915 static inline int
1916 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1917 {
1918         uint16_t tx_free_thresh;
1919         uint16_t tx_rs_thresh;
1920
1921         /* constraint MACROs require that tx_free_thresh is configured
1922          * before tx_rs_thresh */
1923         if (conf->tx_free_thresh == 0)
1924                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1925         else
1926                 tx_free_thresh = conf->tx_free_thresh;
1927
1928         /* make sure the requested threshold satisfies the constraints */
1929         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1930                         FM10K_TX_FREE_THRESH_MAX(q),
1931                         FM10K_TX_FREE_THRESH_DIV(q),
1932                         tx_free_thresh)) {
1933                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1934                         "less than or equal to %u, "
1935                         "greater than or equal to %u, "
1936                         "and a divisor of %u",
1937                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1938                         FM10K_TX_FREE_THRESH_MIN(q),
1939                         FM10K_TX_FREE_THRESH_DIV(q));
1940                 return -EINVAL;
1941         }
1942
1943         q->free_thresh = tx_free_thresh;
1944
1945         if (conf->tx_rs_thresh == 0)
1946                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1947         else
1948                 tx_rs_thresh = conf->tx_rs_thresh;
1949
1950         q->tx_deferred_start = conf->tx_deferred_start;
1951
1952         /* make sure the requested threshold satisfies the constraints */
1953         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1954                         FM10K_TX_RS_THRESH_MAX(q),
1955                         FM10K_TX_RS_THRESH_DIV(q),
1956                         tx_rs_thresh)) {
1957                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1958                         "less than or equal to %u, "
1959                         "greater than or equal to %u, "
1960                         "and a divisor of %u",
1961                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1962                         FM10K_TX_RS_THRESH_MIN(q),
1963                         FM10K_TX_RS_THRESH_DIV(q));
1964                 return -EINVAL;
1965         }
1966
1967         q->rs_thresh = tx_rs_thresh;
1968
1969         return 0;
1970 }
1971
1972 static int
1973 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1974         uint16_t nb_desc, unsigned int socket_id,
1975         const struct rte_eth_txconf *conf)
1976 {
1977         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1978         struct fm10k_tx_queue *q;
1979         const struct rte_memzone *mz;
1980
1981         PMD_INIT_FUNC_TRACE();
1982
1983         /* make sure a valid number of descriptors have been requested */
1984         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1985                                 FM10K_MULT_TX_DESC, nb_desc)) {
1986                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1987                         "less than or equal to %"PRIu32", "
1988                         "greater than or equal to %u, "
1989                         "and a multiple of %u",
1990                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1991                         FM10K_MULT_TX_DESC);
1992                 return -EINVAL;
1993         }
1994
1995         /*
1996          * if this queue existed already, free the associated memory. The
1997          * queue cannot be reused in case we need to allocate memory on
1998          * different socket than was previously used.
1999          */
2000         if (dev->data->tx_queues[queue_id] != NULL) {
2001                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2002
2003                 tx_queue_free(txq);
2004                 dev->data->tx_queues[queue_id] = NULL;
2005         }
2006
2007         /* allocate memory for the queue structure */
2008         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2009                                 socket_id);
2010         if (q == NULL) {
2011                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2012                 return -ENOMEM;
2013         }
2014
2015         /* setup queue */
2016         q->nb_desc = nb_desc;
2017         q->port_id = dev->data->port_id;
2018         q->queue_id = queue_id;
2019         q->txq_flags = conf->txq_flags;
2020         q->ops = &def_txq_ops;
2021         q->tail_ptr = (volatile uint32_t *)
2022                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2023         if (handle_txconf(q, conf))
2024                 return -EINVAL;
2025
2026         /* allocate memory for the software ring */
2027         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2028                                         nb_desc * sizeof(struct rte_mbuf *),
2029                                         RTE_CACHE_LINE_SIZE, socket_id);
2030         if (q->sw_ring == NULL) {
2031                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2032                 rte_free(q);
2033                 return -ENOMEM;
2034         }
2035
2036         /*
2037          * allocate memory for the hardware descriptor ring. A memzone large
2038          * enough to hold the maximum ring size is requested to allow for
2039          * resizing in later calls to the queue setup function.
2040          */
2041         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2042                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2043                                       socket_id);
2044         if (mz == NULL) {
2045                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2046                 rte_free(q->sw_ring);
2047                 rte_free(q);
2048                 return -ENOMEM;
2049         }
2050         q->hw_ring = mz->addr;
2051         q->hw_ring_phys_addr = mz->phys_addr;
2052
2053         /*
2054          * allocate memory for the RS bit tracker. Enough slots to hold the
2055          * descriptor index for each RS bit needing to be set are required.
2056          */
2057         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2058                                 ((nb_desc + 1) / q->rs_thresh) *
2059                                 sizeof(uint16_t),
2060                                 RTE_CACHE_LINE_SIZE, socket_id);
2061         if (q->rs_tracker.list == NULL) {
2062                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2063                 rte_free(q->sw_ring);
2064                 rte_free(q);
2065                 return -ENOMEM;
2066         }
2067
2068         dev->data->tx_queues[queue_id] = q;
2069         return 0;
2070 }
2071
2072 static void
2073 fm10k_tx_queue_release(void *queue)
2074 {
2075         struct fm10k_tx_queue *q = queue;
2076         PMD_INIT_FUNC_TRACE();
2077
2078         tx_queue_free(q);
2079 }
2080
2081 static int
2082 fm10k_reta_update(struct rte_eth_dev *dev,
2083                         struct rte_eth_rss_reta_entry64 *reta_conf,
2084                         uint16_t reta_size)
2085 {
2086         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2087         uint16_t i, j, idx, shift;
2088         uint8_t mask;
2089         uint32_t reta;
2090
2091         PMD_INIT_FUNC_TRACE();
2092
2093         if (reta_size > FM10K_MAX_RSS_INDICES) {
2094                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2095                         "(%d) doesn't match the number hardware can supported "
2096                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2097                 return -EINVAL;
2098         }
2099
2100         /*
2101          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2102          * 128-entries in 32 registers
2103          */
2104         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2105                 idx = i / RTE_RETA_GROUP_SIZE;
2106                 shift = i % RTE_RETA_GROUP_SIZE;
2107                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2108                                 BIT_MASK_PER_UINT32);
2109                 if (mask == 0)
2110                         continue;
2111
2112                 reta = 0;
2113                 if (mask != BIT_MASK_PER_UINT32)
2114                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2115
2116                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2117                         if (mask & (0x1 << j)) {
2118                                 if (mask != 0xF)
2119                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2120                                 reta |= reta_conf[idx].reta[shift + j] <<
2121                                                 (CHAR_BIT * j);
2122                         }
2123                 }
2124                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2125         }
2126
2127         return 0;
2128 }
2129
2130 static int
2131 fm10k_reta_query(struct rte_eth_dev *dev,
2132                         struct rte_eth_rss_reta_entry64 *reta_conf,
2133                         uint16_t reta_size)
2134 {
2135         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2136         uint16_t i, j, idx, shift;
2137         uint8_t mask;
2138         uint32_t reta;
2139
2140         PMD_INIT_FUNC_TRACE();
2141
2142         if (reta_size < FM10K_MAX_RSS_INDICES) {
2143                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2144                         "(%d) doesn't match the number hardware can supported "
2145                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2146                 return -EINVAL;
2147         }
2148
2149         /*
2150          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2151          * 128-entries in 32 registers
2152          */
2153         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2154                 idx = i / RTE_RETA_GROUP_SIZE;
2155                 shift = i % RTE_RETA_GROUP_SIZE;
2156                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2157                                 BIT_MASK_PER_UINT32);
2158                 if (mask == 0)
2159                         continue;
2160
2161                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2162                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2163                         if (mask & (0x1 << j))
2164                                 reta_conf[idx].reta[shift + j] = ((reta >>
2165                                         CHAR_BIT * j) & UINT8_MAX);
2166                 }
2167         }
2168
2169         return 0;
2170 }
2171
2172 static int
2173 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2174         struct rte_eth_rss_conf *rss_conf)
2175 {
2176         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2177         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2178         uint32_t mrqc;
2179         uint64_t hf = rss_conf->rss_hf;
2180         int i;
2181
2182         PMD_INIT_FUNC_TRACE();
2183
2184         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2185                                 FM10K_RSSRK_ENTRIES_PER_REG))
2186                 return -EINVAL;
2187
2188         if (hf == 0)
2189                 return -EINVAL;
2190
2191         mrqc = 0;
2192         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2193         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2194         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2195         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2196         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2197         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2198         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2199         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2200         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2201
2202         /* If the mapping doesn't fit any supported, return */
2203         if (mrqc == 0)
2204                 return -EINVAL;
2205
2206         if (key != NULL)
2207                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2208                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2209
2210         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2211
2212         return 0;
2213 }
2214
2215 static int
2216 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2217         struct rte_eth_rss_conf *rss_conf)
2218 {
2219         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2220         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2221         uint32_t mrqc;
2222         uint64_t hf;
2223         int i;
2224
2225         PMD_INIT_FUNC_TRACE();
2226
2227         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2228                                 FM10K_RSSRK_ENTRIES_PER_REG))
2229                 return -EINVAL;
2230
2231         if (key != NULL)
2232                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2233                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2234
2235         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2236         hf = 0;
2237         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2238         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2239         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2240         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2241         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2242         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2243         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2244         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2245         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2246
2247         rss_conf->rss_hf = hf;
2248
2249         return 0;
2250 }
2251
2252 static void
2253 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2254 {
2255         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2256         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2257
2258         /* Bind all local non-queue interrupt to vector 0 */
2259         int_map |= FM10K_MISC_VEC_ID;
2260
2261         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2262         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2263         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2264         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2265         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2266         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2267
2268         /* Enable misc causes */
2269         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2270                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2271                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2272                                 FM10K_EIMR_ENABLE(MAILBOX) |
2273                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2274                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2275                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2276                                 FM10K_EIMR_ENABLE(VFLR));
2277
2278         /* Enable ITR 0 */
2279         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2280                                         FM10K_ITR_MASK_CLEAR);
2281         FM10K_WRITE_FLUSH(hw);
2282 }
2283
2284 static void
2285 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2286 {
2287         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2288         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2289
2290         int_map |= FM10K_MISC_VEC_ID;
2291
2292         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2293         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2294         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2295         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2296         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2297         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2298
2299         /* Disable misc causes */
2300         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2301                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2302                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2303                                 FM10K_EIMR_DISABLE(MAILBOX) |
2304                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2305                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2306                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2307                                 FM10K_EIMR_DISABLE(VFLR));
2308
2309         /* Disable ITR 0 */
2310         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2311         FM10K_WRITE_FLUSH(hw);
2312 }
2313
2314 static void
2315 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2316 {
2317         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2318         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2319
2320         /* Bind all local non-queue interrupt to vector 0 */
2321         int_map |= FM10K_MISC_VEC_ID;
2322
2323         /* Only INT 0 available, other 15 are reserved. */
2324         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2325
2326         /* Enable ITR 0 */
2327         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2328                                         FM10K_ITR_MASK_CLEAR);
2329         FM10K_WRITE_FLUSH(hw);
2330 }
2331
2332 static void
2333 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2334 {
2335         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2336         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2337
2338         int_map |= FM10K_MISC_VEC_ID;
2339
2340         /* Only INT 0 available, other 15 are reserved. */
2341         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2342
2343         /* Disable ITR 0 */
2344         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2345         FM10K_WRITE_FLUSH(hw);
2346 }
2347
2348 static int
2349 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2350 {
2351         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2352         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2353
2354         /* Enable ITR */
2355         if (hw->mac.type == fm10k_mac_pf)
2356                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2357                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2358         else
2359                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2360                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2361         rte_intr_enable(&pdev->intr_handle);
2362         return 0;
2363 }
2364
2365 static int
2366 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2367 {
2368         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2369         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2370
2371         /* Disable ITR */
2372         if (hw->mac.type == fm10k_mac_pf)
2373                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2374                         FM10K_ITR_MASK_SET);
2375         else
2376                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2377                         FM10K_ITR_MASK_SET);
2378         return 0;
2379 }
2380
2381 static int
2382 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2383 {
2384         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2385         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2386         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2387         uint32_t intr_vector, vec;
2388         uint16_t queue_id;
2389         int result = 0;
2390
2391         /* fm10k needs one separate interrupt for mailbox,
2392          * so only drivers which support multiple interrupt vectors
2393          * e.g. vfio-pci can work for fm10k interrupt mode
2394          */
2395         if (!rte_intr_cap_multiple(intr_handle) ||
2396                         dev->data->dev_conf.intr_conf.rxq == 0)
2397                 return result;
2398
2399         intr_vector = dev->data->nb_rx_queues;
2400
2401         /* disable interrupt first */
2402         rte_intr_disable(intr_handle);
2403         if (hw->mac.type == fm10k_mac_pf)
2404                 fm10k_dev_disable_intr_pf(dev);
2405         else
2406                 fm10k_dev_disable_intr_vf(dev);
2407
2408         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2409                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2410                 result = -EIO;
2411         }
2412
2413         if (rte_intr_dp_is_en(intr_handle) && !result) {
2414                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2415                         dev->data->nb_rx_queues * sizeof(int), 0);
2416                 if (intr_handle->intr_vec) {
2417                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2418                                         queue_id < dev->data->nb_rx_queues;
2419                                         queue_id++) {
2420                                 intr_handle->intr_vec[queue_id] = vec;
2421                                 if (vec < intr_handle->nb_efd - 1
2422                                                 + FM10K_RX_VEC_START)
2423                                         vec++;
2424                         }
2425                 } else {
2426                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2427                                 " intr_vec", dev->data->nb_rx_queues);
2428                         rte_intr_efd_disable(intr_handle);
2429                         result = -ENOMEM;
2430                 }
2431         }
2432
2433         if (hw->mac.type == fm10k_mac_pf)
2434                 fm10k_dev_enable_intr_pf(dev);
2435         else
2436                 fm10k_dev_enable_intr_vf(dev);
2437         rte_intr_enable(intr_handle);
2438         hw->mac.ops.update_int_moderator(hw);
2439         return result;
2440 }
2441
2442 static int
2443 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2444 {
2445         struct fm10k_fault fault;
2446         int err;
2447         const char *estr = "Unknown error";
2448
2449         /* Process PCA fault */
2450         if (eicr & FM10K_EICR_PCA_FAULT) {
2451                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2452                 if (err)
2453                         goto error;
2454                 switch (fault.type) {
2455                 case PCA_NO_FAULT:
2456                         estr = "PCA_NO_FAULT"; break;
2457                 case PCA_UNMAPPED_ADDR:
2458                         estr = "PCA_UNMAPPED_ADDR"; break;
2459                 case PCA_BAD_QACCESS_PF:
2460                         estr = "PCA_BAD_QACCESS_PF"; break;
2461                 case PCA_BAD_QACCESS_VF:
2462                         estr = "PCA_BAD_QACCESS_VF"; break;
2463                 case PCA_MALICIOUS_REQ:
2464                         estr = "PCA_MALICIOUS_REQ"; break;
2465                 case PCA_POISONED_TLP:
2466                         estr = "PCA_POISONED_TLP"; break;
2467                 case PCA_TLP_ABORT:
2468                         estr = "PCA_TLP_ABORT"; break;
2469                 default:
2470                         goto error;
2471                 }
2472                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2473                         estr, fault.func ? "VF" : "PF", fault.func,
2474                         fault.address, fault.specinfo);
2475         }
2476
2477         /* Process THI fault */
2478         if (eicr & FM10K_EICR_THI_FAULT) {
2479                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2480                 if (err)
2481                         goto error;
2482                 switch (fault.type) {
2483                 case THI_NO_FAULT:
2484                         estr = "THI_NO_FAULT"; break;
2485                 case THI_MAL_DIS_Q_FAULT:
2486                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2487                 default:
2488                         goto error;
2489                 }
2490                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2491                         estr, fault.func ? "VF" : "PF", fault.func,
2492                         fault.address, fault.specinfo);
2493         }
2494
2495         /* Process FUM fault */
2496         if (eicr & FM10K_EICR_FUM_FAULT) {
2497                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2498                 if (err)
2499                         goto error;
2500                 switch (fault.type) {
2501                 case FUM_NO_FAULT:
2502                         estr = "FUM_NO_FAULT"; break;
2503                 case FUM_UNMAPPED_ADDR:
2504                         estr = "FUM_UNMAPPED_ADDR"; break;
2505                 case FUM_POISONED_TLP:
2506                         estr = "FUM_POISONED_TLP"; break;
2507                 case FUM_BAD_VF_QACCESS:
2508                         estr = "FUM_BAD_VF_QACCESS"; break;
2509                 case FUM_ADD_DECODE_ERR:
2510                         estr = "FUM_ADD_DECODE_ERR"; break;
2511                 case FUM_RO_ERROR:
2512                         estr = "FUM_RO_ERROR"; break;
2513                 case FUM_QPRC_CRC_ERROR:
2514                         estr = "FUM_QPRC_CRC_ERROR"; break;
2515                 case FUM_CSR_TIMEOUT:
2516                         estr = "FUM_CSR_TIMEOUT"; break;
2517                 case FUM_INVALID_TYPE:
2518                         estr = "FUM_INVALID_TYPE"; break;
2519                 case FUM_INVALID_LENGTH:
2520                         estr = "FUM_INVALID_LENGTH"; break;
2521                 case FUM_INVALID_BE:
2522                         estr = "FUM_INVALID_BE"; break;
2523                 case FUM_INVALID_ALIGN:
2524                         estr = "FUM_INVALID_ALIGN"; break;
2525                 default:
2526                         goto error;
2527                 }
2528                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2529                         estr, fault.func ? "VF" : "PF", fault.func,
2530                         fault.address, fault.specinfo);
2531         }
2532
2533         return 0;
2534 error:
2535         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2536         return err;
2537 }
2538
2539 /**
2540  * PF interrupt handler triggered by NIC for handling specific interrupt.
2541  *
2542  * @param handle
2543  *  Pointer to interrupt handle.
2544  * @param param
2545  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2546  *
2547  * @return
2548  *  void
2549  */
2550 static void
2551 fm10k_dev_interrupt_handler_pf(void *param)
2552 {
2553         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2554         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2555         uint32_t cause, status;
2556
2557         if (hw->mac.type != fm10k_mac_pf)
2558                 return;
2559
2560         cause = FM10K_READ_REG(hw, FM10K_EICR);
2561
2562         /* Handle PCI fault cases */
2563         if (cause & FM10K_EICR_FAULT_MASK) {
2564                 PMD_INIT_LOG(ERR, "INT: find fault!");
2565                 fm10k_dev_handle_fault(hw, cause);
2566         }
2567
2568         /* Handle switch up/down */
2569         if (cause & FM10K_EICR_SWITCHNOTREADY)
2570                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2571
2572         if (cause & FM10K_EICR_SWITCHREADY)
2573                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2574
2575         /* Handle mailbox message */
2576         fm10k_mbx_lock(hw);
2577         hw->mbx.ops.process(hw, &hw->mbx);
2578         fm10k_mbx_unlock(hw);
2579
2580         /* Handle SRAM error */
2581         if (cause & FM10K_EICR_SRAMERROR) {
2582                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2583
2584                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2585                 /* Write to clear pending bits */
2586                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2587
2588                 /* Todo: print out error message after shared code  updates */
2589         }
2590
2591         /* Clear these 3 events if having any */
2592         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2593                  FM10K_EICR_SWITCHREADY;
2594         if (cause)
2595                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2596
2597         /* Re-enable interrupt from device side */
2598         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2599                                         FM10K_ITR_MASK_CLEAR);
2600         /* Re-enable interrupt from host side */
2601         rte_intr_enable(dev->intr_handle);
2602 }
2603
2604 /**
2605  * VF interrupt handler triggered by NIC for handling specific interrupt.
2606  *
2607  * @param handle
2608  *  Pointer to interrupt handle.
2609  * @param param
2610  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2611  *
2612  * @return
2613  *  void
2614  */
2615 static void
2616 fm10k_dev_interrupt_handler_vf(void *param)
2617 {
2618         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2619         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2620
2621         if (hw->mac.type != fm10k_mac_vf)
2622                 return;
2623
2624         /* Handle mailbox message if lock is acquired */
2625         fm10k_mbx_lock(hw);
2626         hw->mbx.ops.process(hw, &hw->mbx);
2627         fm10k_mbx_unlock(hw);
2628
2629         /* Re-enable interrupt from device side */
2630         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2631                                         FM10K_ITR_MASK_CLEAR);
2632         /* Re-enable interrupt from host side */
2633         rte_intr_enable(dev->intr_handle);
2634 }
2635
2636 /* Mailbox message handler in VF */
2637 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2638         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2639         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2640         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2641         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2642 };
2643
2644 static int
2645 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2646 {
2647         int err = 0;
2648
2649         /* Initialize mailbox lock */
2650         fm10k_mbx_initlock(hw);
2651
2652         /* Replace default message handler with new ones */
2653         if (hw->mac.type == fm10k_mac_vf)
2654                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2655
2656         if (err) {
2657                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2658                                 err);
2659                 return err;
2660         }
2661         /* Connect to SM for PF device or PF for VF device */
2662         return hw->mbx.ops.connect(hw, &hw->mbx);
2663 }
2664
2665 static void
2666 fm10k_close_mbx_service(struct fm10k_hw *hw)
2667 {
2668         /* Disconnect from SM for PF device or PF for VF device */
2669         hw->mbx.ops.disconnect(hw, &hw->mbx);
2670 }
2671
2672 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2673         .dev_configure          = fm10k_dev_configure,
2674         .dev_start              = fm10k_dev_start,
2675         .dev_stop               = fm10k_dev_stop,
2676         .dev_close              = fm10k_dev_close,
2677         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2678         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2679         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2680         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2681         .stats_get              = fm10k_stats_get,
2682         .xstats_get             = fm10k_xstats_get,
2683         .xstats_get_names       = fm10k_xstats_get_names,
2684         .stats_reset            = fm10k_stats_reset,
2685         .xstats_reset           = fm10k_stats_reset,
2686         .link_update            = fm10k_link_update,
2687         .dev_infos_get          = fm10k_dev_infos_get,
2688         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2689         .vlan_filter_set        = fm10k_vlan_filter_set,
2690         .vlan_offload_set       = fm10k_vlan_offload_set,
2691         .mac_addr_add           = fm10k_macaddr_add,
2692         .mac_addr_remove        = fm10k_macaddr_remove,
2693         .rx_queue_start         = fm10k_dev_rx_queue_start,
2694         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2695         .tx_queue_start         = fm10k_dev_tx_queue_start,
2696         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2697         .rx_queue_setup         = fm10k_rx_queue_setup,
2698         .rx_queue_release       = fm10k_rx_queue_release,
2699         .tx_queue_setup         = fm10k_tx_queue_setup,
2700         .tx_queue_release       = fm10k_tx_queue_release,
2701         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2702         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2703         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2704         .reta_update            = fm10k_reta_update,
2705         .reta_query             = fm10k_reta_query,
2706         .rss_hash_update        = fm10k_rss_hash_update,
2707         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2708 };
2709
2710 static int ftag_check_handler(__rte_unused const char *key,
2711                 const char *value, __rte_unused void *opaque)
2712 {
2713         if (strcmp(value, "1"))
2714                 return -1;
2715
2716         return 0;
2717 }
2718
2719 static int
2720 fm10k_check_ftag(struct rte_devargs *devargs)
2721 {
2722         struct rte_kvargs *kvlist;
2723         const char *ftag_key = "enable_ftag";
2724
2725         if (devargs == NULL)
2726                 return 0;
2727
2728         kvlist = rte_kvargs_parse(devargs->args, NULL);
2729         if (kvlist == NULL)
2730                 return 0;
2731
2732         if (!rte_kvargs_count(kvlist, ftag_key)) {
2733                 rte_kvargs_free(kvlist);
2734                 return 0;
2735         }
2736         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2737         if (rte_kvargs_process(kvlist, ftag_key,
2738                                 ftag_check_handler, NULL) < 0) {
2739                 rte_kvargs_free(kvlist);
2740                 return 0;
2741         }
2742         rte_kvargs_free(kvlist);
2743
2744         return 1;
2745 }
2746
2747 static uint16_t
2748 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2749                     uint16_t nb_pkts)
2750 {
2751         uint16_t nb_tx = 0;
2752         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2753
2754         while (nb_pkts) {
2755                 uint16_t ret, num;
2756
2757                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2758                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2759                                                  num);
2760                 nb_tx += ret;
2761                 nb_pkts -= ret;
2762                 if (ret < num)
2763                         break;
2764         }
2765
2766         return nb_tx;
2767 }
2768
2769 static void __attribute__((cold))
2770 fm10k_set_tx_function(struct rte_eth_dev *dev)
2771 {
2772         struct fm10k_tx_queue *txq;
2773         int i;
2774         int use_sse = 1;
2775         uint16_t tx_ftag_en = 0;
2776
2777         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2778                 /* primary process has set the ftag flag and txq_flags */
2779                 txq = dev->data->tx_queues[0];
2780                 if (fm10k_tx_vec_condition_check(txq)) {
2781                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2782                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2783                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2784                 } else {
2785                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2786                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2787                         dev->tx_pkt_prepare = NULL;
2788                 }
2789                 return;
2790         }
2791
2792         if (fm10k_check_ftag(dev->device->devargs))
2793                 tx_ftag_en = 1;
2794
2795         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2796                 txq = dev->data->tx_queues[i];
2797                 txq->tx_ftag_en = tx_ftag_en;
2798                 /* Check if Vector Tx is satisfied */
2799                 if (fm10k_tx_vec_condition_check(txq))
2800                         use_sse = 0;
2801         }
2802
2803         if (use_sse) {
2804                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2805                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2806                         txq = dev->data->tx_queues[i];
2807                         fm10k_txq_vec_setup(txq);
2808                 }
2809                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2810                 dev->tx_pkt_prepare = NULL;
2811         } else {
2812                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2813                 dev->tx_pkt_prepare = fm10k_prep_pkts;
2814                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2815         }
2816 }
2817
2818 static void __attribute__((cold))
2819 fm10k_set_rx_function(struct rte_eth_dev *dev)
2820 {
2821         struct fm10k_dev_info *dev_info =
2822                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2823         uint16_t i, rx_using_sse;
2824         uint16_t rx_ftag_en = 0;
2825
2826         if (fm10k_check_ftag(dev->device->devargs))
2827                 rx_ftag_en = 1;
2828
2829         /* In order to allow Vector Rx there are a few configuration
2830          * conditions to be met.
2831          */
2832         if (!fm10k_rx_vec_condition_check(dev) &&
2833                         dev_info->rx_vec_allowed && !rx_ftag_en) {
2834                 if (dev->data->scattered_rx)
2835                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2836                 else
2837                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2838         } else if (dev->data->scattered_rx)
2839                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2840         else
2841                 dev->rx_pkt_burst = fm10k_recv_pkts;
2842
2843         rx_using_sse =
2844                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2845                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2846
2847         if (rx_using_sse)
2848                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
2849         else
2850                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
2851
2852         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2853                 return;
2854
2855         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2856                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2857
2858                 rxq->rx_using_sse = rx_using_sse;
2859                 rxq->rx_ftag_en = rx_ftag_en;
2860         }
2861 }
2862
2863 static void
2864 fm10k_params_init(struct rte_eth_dev *dev)
2865 {
2866         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2867         struct fm10k_dev_info *info =
2868                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2869
2870         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2871          * there is no way to get link status without reading BAR4.  Until this
2872          * works, assume we have maximum bandwidth.
2873          * @todo - fix bus info
2874          */
2875         hw->bus_caps.speed = fm10k_bus_speed_8000;
2876         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2877         hw->bus_caps.payload = fm10k_bus_payload_512;
2878         hw->bus.speed = fm10k_bus_speed_8000;
2879         hw->bus.width = fm10k_bus_width_pcie_x8;
2880         hw->bus.payload = fm10k_bus_payload_256;
2881
2882         info->rx_vec_allowed = true;
2883 }
2884
2885 static int
2886 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2887 {
2888         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2889         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2890         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2891         int diag, i;
2892         struct fm10k_macvlan_filter_info *macvlan;
2893
2894         PMD_INIT_FUNC_TRACE();
2895
2896         dev->dev_ops = &fm10k_eth_dev_ops;
2897         dev->rx_pkt_burst = &fm10k_recv_pkts;
2898         dev->tx_pkt_burst = &fm10k_xmit_pkts;
2899         dev->tx_pkt_prepare = &fm10k_prep_pkts;
2900
2901         /*
2902          * Primary process does the whole initialization, for secondary
2903          * processes, we just select the same Rx and Tx function as primary.
2904          */
2905         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2906                 fm10k_set_rx_function(dev);
2907                 fm10k_set_tx_function(dev);
2908                 return 0;
2909         }
2910
2911         rte_eth_copy_pci_info(dev, pdev);
2912         dev->data->dev_flags |= RTE_ETH_DEV_DETACHABLE;
2913
2914         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2915         memset(macvlan, 0, sizeof(*macvlan));
2916         /* Vendor and Device ID need to be set before init of shared code */
2917         memset(hw, 0, sizeof(*hw));
2918         hw->device_id = pdev->id.device_id;
2919         hw->vendor_id = pdev->id.vendor_id;
2920         hw->subsystem_device_id = pdev->id.subsystem_device_id;
2921         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
2922         hw->revision_id = 0;
2923         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
2924         if (hw->hw_addr == NULL) {
2925                 PMD_INIT_LOG(ERR, "Bad mem resource."
2926                         " Try to blacklist unused devices.");
2927                 return -EIO;
2928         }
2929
2930         /* Store fm10k_adapter pointer */
2931         hw->back = dev->data->dev_private;
2932
2933         /* Initialize the shared code */
2934         diag = fm10k_init_shared_code(hw);
2935         if (diag != FM10K_SUCCESS) {
2936                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2937                 return -EIO;
2938         }
2939
2940         /* Initialize parameters */
2941         fm10k_params_init(dev);
2942
2943         /* Initialize the hw */
2944         diag = fm10k_init_hw(hw);
2945         if (diag != FM10K_SUCCESS) {
2946                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2947                 return -EIO;
2948         }
2949
2950         /* Initialize MAC address(es) */
2951         dev->data->mac_addrs = rte_zmalloc("fm10k",
2952                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2953         if (dev->data->mac_addrs == NULL) {
2954                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2955                 return -ENOMEM;
2956         }
2957
2958         diag = fm10k_read_mac_addr(hw);
2959
2960         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2961                         &dev->data->mac_addrs[0]);
2962
2963         if (diag != FM10K_SUCCESS ||
2964                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2965
2966                 /* Generate a random addr */
2967                 eth_random_addr(hw->mac.addr);
2968                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2969                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2970                 &dev->data->mac_addrs[0]);
2971         }
2972
2973         /* Reset the hw statistics */
2974         fm10k_stats_reset(dev);
2975
2976         /* Reset the hw */
2977         diag = fm10k_reset_hw(hw);
2978         if (diag != FM10K_SUCCESS) {
2979                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2980                 return -EIO;
2981         }
2982
2983         /* Setup mailbox service */
2984         diag = fm10k_setup_mbx_service(hw);
2985         if (diag != FM10K_SUCCESS) {
2986                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2987                 return -EIO;
2988         }
2989
2990         /*PF/VF has different interrupt handling mechanism */
2991         if (hw->mac.type == fm10k_mac_pf) {
2992                 /* register callback func to eal lib */
2993                 rte_intr_callback_register(intr_handle,
2994                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2995
2996                 /* enable MISC interrupt */
2997                 fm10k_dev_enable_intr_pf(dev);
2998         } else { /* VF */
2999                 rte_intr_callback_register(intr_handle,
3000                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3001
3002                 fm10k_dev_enable_intr_vf(dev);
3003         }
3004
3005         /* Enable intr after callback registered */
3006         rte_intr_enable(intr_handle);
3007
3008         hw->mac.ops.update_int_moderator(hw);
3009
3010         /* Make sure Switch Manager is ready before going forward. */
3011         if (hw->mac.type == fm10k_mac_pf) {
3012                 int switch_ready = 0;
3013
3014                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3015                         fm10k_mbx_lock(hw);
3016                         hw->mac.ops.get_host_state(hw, &switch_ready);
3017                         fm10k_mbx_unlock(hw);
3018                         if (switch_ready)
3019                                 break;
3020                         /* Delay some time to acquire async LPORT_MAP info. */
3021                         rte_delay_us(WAIT_SWITCH_MSG_US);
3022                 }
3023
3024                 if (switch_ready == 0) {
3025                         PMD_INIT_LOG(ERR, "switch is not ready");
3026                         return -1;
3027                 }
3028         }
3029
3030         /*
3031          * Below function will trigger operations on mailbox, acquire lock to
3032          * avoid race condition from interrupt handler. Operations on mailbox
3033          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3034          * will handle and generate an interrupt to our side. Then,  FIFO in
3035          * mailbox will be touched.
3036          */
3037         fm10k_mbx_lock(hw);
3038         /* Enable port first */
3039         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3040                                         MAX_LPORT_NUM, 1);
3041
3042         /* Set unicast mode by default. App can change to other mode in other
3043          * API func.
3044          */
3045         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3046                                         FM10K_XCAST_MODE_NONE);
3047
3048         fm10k_mbx_unlock(hw);
3049
3050         /* Make sure default VID is ready before going forward. */
3051         if (hw->mac.type == fm10k_mac_pf) {
3052                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3053                         if (hw->mac.default_vid)
3054                                 break;
3055                         /* Delay some time to acquire async port VLAN info. */
3056                         rte_delay_us(WAIT_SWITCH_MSG_US);
3057                 }
3058
3059                 if (!hw->mac.default_vid) {
3060                         PMD_INIT_LOG(ERR, "default VID is not ready");
3061                         return -1;
3062                 }
3063         }
3064
3065         /* Add default mac address */
3066         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3067                 MAIN_VSI_POOL_NUMBER);
3068
3069         return 0;
3070 }
3071
3072 static int
3073 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3074 {
3075         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3076         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3077         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3078         PMD_INIT_FUNC_TRACE();
3079
3080         /* only uninitialize in the primary process */
3081         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3082                 return 0;
3083
3084         /* safe to close dev here */
3085         fm10k_dev_close(dev);
3086
3087         dev->dev_ops = NULL;
3088         dev->rx_pkt_burst = NULL;
3089         dev->tx_pkt_burst = NULL;
3090
3091         /* disable uio/vfio intr */
3092         rte_intr_disable(intr_handle);
3093
3094         /*PF/VF has different interrupt handling mechanism */
3095         if (hw->mac.type == fm10k_mac_pf) {
3096                 /* disable interrupt */
3097                 fm10k_dev_disable_intr_pf(dev);
3098
3099                 /* unregister callback func to eal lib */
3100                 rte_intr_callback_unregister(intr_handle,
3101                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3102         } else {
3103                 /* disable interrupt */
3104                 fm10k_dev_disable_intr_vf(dev);
3105
3106                 rte_intr_callback_unregister(intr_handle,
3107                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3108         }
3109
3110         /* free mac memory */
3111         if (dev->data->mac_addrs) {
3112                 rte_free(dev->data->mac_addrs);
3113                 dev->data->mac_addrs = NULL;
3114         }
3115
3116         memset(hw, 0, sizeof(*hw));
3117
3118         return 0;
3119 }
3120
3121 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3122         struct rte_pci_device *pci_dev)
3123 {
3124         return rte_eth_dev_pci_generic_probe(pci_dev,
3125                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3126 }
3127
3128 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3129 {
3130         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3131 }
3132
3133 /*
3134  * The set of PCI devices this driver supports. This driver will enable both PF
3135  * and SRIOV-VF devices.
3136  */
3137 static const struct rte_pci_id pci_id_fm10k_map[] = {
3138         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3139         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3140         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3141         { .vendor_id = 0, /* sentinel */ },
3142 };
3143
3144 static struct rte_pci_driver rte_pmd_fm10k = {
3145         .id_table = pci_id_fm10k_map,
3146         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
3147         .probe = eth_fm10k_pci_probe,
3148         .remove = eth_fm10k_pci_remove,
3149 };
3150
3151 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3152 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3153 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");