drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_ethdev_pci.h>
  36 #include <rte_malloc.h>
  37 #include <rte_memzone.h>
  38 #include <rte_string_fns.h>
  39 #include <rte_dev.h>
  40 #include <rte_spinlock.h>
  41 #include <rte_kvargs.h>
  42
  43 #include "fm10k.h"
  44 #include "base/fm10k_api.h"
  45
  46 /* Default delay to acquire mailbox lock */
  47 #define FM10K_MBXLOCK_DELAY_US 20
  48 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  49
  50 #define MAIN_VSI_POOL_NUMBER 0
  51
  52 /* Max try times to acquire switch status */
  53 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  54 /* Wait interval to get switch status */
  55 #define WAIT_SWITCH_MSG_US    100000
  56 /* A period of quiescence for switch */
  57 #define FM10K_SWITCH_QUIESCE_US 10000
  58 /* Number of chars per uint32 type */
  59 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  60 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  61
  62 /* default 1:1 map from queue ID to interrupt vector ID */
  63 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  64
  65 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  66 #define MAX_LPORT_NUM    128
  67 #define GLORT_FD_Q_BASE  0x40
  68 #define GLORT_PF_MASK    0xFFC0
  69 #define GLORT_FD_MASK    GLORT_PF_MASK
  70 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  71
  72 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  73 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  74 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  75 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  76 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  77 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  78 static int
  79 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  80 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  81         const u8 *mac, bool add, uint32_t pool);
  82 static void fm10k_tx_queue_release(void *queue);
  83 static void fm10k_rx_queue_release(void *queue);
  84 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  85 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  86 static int fm10k_check_ftag(struct rte_devargs *devargs);
  87
  88 struct fm10k_xstats_name_off {
  89         char name[RTE_ETH_XSTATS_NAME_SIZE];
  90         unsigned offset;
  91 };
  92
  93 struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  94         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  95         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  96         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  97         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  98         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  99         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
 100         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
 101         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
 102                 nodesc_drop)},
 103 };
 104
 105 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
 106                 sizeof(fm10k_hw_stats_strings[0]))
 107
 108 struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
 109         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
 110         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
 111         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
 112 };
 113
 114 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
 115                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
 116
 117 struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 118         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 119         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 120 };
 121
 122 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 123                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 124
 125 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 126                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 127 static int
 128 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 129
 130 static void
 131 fm10k_mbx_initlock(struct fm10k_hw *hw)
 132 {
 133         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 134 }
 135
 136 static void
 137 fm10k_mbx_lock(struct fm10k_hw *hw)
 138 {
 139         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 140                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 141 }
 142
 143 static void
 144 fm10k_mbx_unlock(struct fm10k_hw *hw)
 145 {
 146         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 147 }
 148
 149 /* Stubs needed for linkage when vPMD is disabled */
 150 int __attribute__((weak))
 151 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 152 {
 153         return -1;
 154 }
 155
 156 uint16_t __attribute__((weak))
 157 fm10k_recv_pkts_vec(
 158         __rte_unused void *rx_queue,
 159         __rte_unused struct rte_mbuf **rx_pkts,
 160         __rte_unused uint16_t nb_pkts)
 161 {
 162         return 0;
 163 }
 164
 165 uint16_t __attribute__((weak))
 166 fm10k_recv_scattered_pkts_vec(
 167                 __rte_unused void *rx_queue,
 168                 __rte_unused struct rte_mbuf **rx_pkts,
 169                 __rte_unused uint16_t nb_pkts)
 170 {
 171         return 0;
 172 }
 173
 174 int __attribute__((weak))
 175 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 176
 177 {
 178         return -1;
 179 }
 180
 181 void __attribute__((weak))
 182 fm10k_rx_queue_release_mbufs_vec(
 183                 __rte_unused struct fm10k_rx_queue *rxq)
 184 {
 185         return;
 186 }
 187
 188 void __attribute__((weak))
 189 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 190 {
 191         return;
 192 }
 193
 194 int __attribute__((weak))
 195 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 196 {
 197         return -1;
 198 }
 199
 200 uint16_t __attribute__((weak))
 201 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 202                            __rte_unused struct rte_mbuf **tx_pkts,
 203                            __rte_unused uint16_t nb_pkts)
 204 {
 205         return 0;
 206 }
 207
 208 /*
 209  * reset queue to initial state, allocate software buffers used when starting
 210  * device.
 211  * return 0 on success
 212  * return -ENOMEM if buffers cannot be allocated
 213  * return -EINVAL if buffers do not satisfy alignment condition
 214  */
 215 static inline int
 216 rx_queue_reset(struct fm10k_rx_queue *q)
 217 {
 218         static const union fm10k_rx_desc zero = {{0} };
 219         uint64_t dma_addr;
 220         int i, diag;
 221         PMD_INIT_FUNC_TRACE();
 222
 223         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 224         if (diag != 0)
 225                 return -ENOMEM;
 226
 227         for (i = 0; i < q->nb_desc; ++i) {
 228                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 229                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 230                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 231                                                 q->nb_desc);
 232                         return -EINVAL;
 233                 }
 234                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 235                 q->hw_ring[i].q.pkt_addr = dma_addr;
 236                 q->hw_ring[i].q.hdr_addr = dma_addr;
 237         }
 238
 239         /* initialize extra software ring entries. Space for these extra
 240          * entries is always allocated.
 241          */
 242         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 243         for (i = 0; i < q->nb_fake_desc; ++i) {
 244                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 245                 q->hw_ring[q->nb_desc + i] = zero;
 246         }
 247
 248         q->next_dd = 0;
 249         q->next_alloc = 0;
 250         q->next_trigger = q->alloc_thresh - 1;
 251         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 252         q->rxrearm_start = 0;
 253         q->rxrearm_nb = 0;
 254
 255         return 0;
 256 }
 257
 258 /*
 259  * clean queue, descriptor rings, free software buffers used when stopping
 260  * device.
 261  */
 262 static inline void
 263 rx_queue_clean(struct fm10k_rx_queue *q)
 264 {
 265         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 266         uint32_t i;
 267         PMD_INIT_FUNC_TRACE();
 268
 269         /* zero descriptor rings */
 270         for (i = 0; i < q->nb_desc; ++i)
 271                 q->hw_ring[i] = zero;
 272
 273         /* zero faked descriptors */
 274         for (i = 0; i < q->nb_fake_desc; ++i)
 275                 q->hw_ring[q->nb_desc + i] = zero;
 276
 277         /* vPMD driver has a different way of releasing mbufs. */
 278         if (q->rx_using_sse) {
 279                 fm10k_rx_queue_release_mbufs_vec(q);
 280                 return;
 281         }
 282
 283         /* free software buffers */
 284         for (i = 0; i < q->nb_desc; ++i) {
 285                 if (q->sw_ring[i]) {
 286                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 287                         q->sw_ring[i] = NULL;
 288                 }
 289         }
 290 }
 291
 292 /*
 293  * free all queue memory used when releasing the queue (i.e. configure)
 294  */
 295 static inline void
 296 rx_queue_free(struct fm10k_rx_queue *q)
 297 {
 298         PMD_INIT_FUNC_TRACE();
 299         if (q) {
 300                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 301                 rx_queue_clean(q);
 302                 if (q->sw_ring) {
 303                         rte_free(q->sw_ring);
 304                         q->sw_ring = NULL;
 305                 }
 306                 rte_free(q);
 307                 q = NULL;
 308         }
 309 }
 310
 311 /*
 312  * disable RX queue, wait unitl HW finished necessary flush operation
 313  */
 314 static inline int
 315 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 316 {
 317         uint32_t reg, i;
 318
 319         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 320         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 321                         reg & ~FM10K_RXQCTL_ENABLE);
 322
 323         /* Wait 100us at most */
 324         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 325                 rte_delay_us(1);
 326                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 327                 if (!(reg & FM10K_RXQCTL_ENABLE))
 328                         break;
 329         }
 330
 331         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 332                 return -1;
 333
 334         return 0;
 335 }
 336
 337 /*
 338  * reset queue to initial state, allocate software buffers used when starting
 339  * device
 340  */
 341 static inline void
 342 tx_queue_reset(struct fm10k_tx_queue *q)
 343 {
 344         PMD_INIT_FUNC_TRACE();
 345         q->last_free = 0;
 346         q->next_free = 0;
 347         q->nb_used = 0;
 348         q->nb_free = q->nb_desc - 1;
 349         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 350         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 351 }
 352
 353 /*
 354  * clean queue, descriptor rings, free software buffers used when stopping
 355  * device
 356  */
 357 static inline void
 358 tx_queue_clean(struct fm10k_tx_queue *q)
 359 {
 360         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 361         uint32_t i;
 362         PMD_INIT_FUNC_TRACE();
 363
 364         /* zero descriptor rings */
 365         for (i = 0; i < q->nb_desc; ++i)
 366                 q->hw_ring[i] = zero;
 367
 368         /* free software buffers */
 369         for (i = 0; i < q->nb_desc; ++i) {
 370                 if (q->sw_ring[i]) {
 371                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 372                         q->sw_ring[i] = NULL;
 373                 }
 374         }
 375 }
 376
 377 /*
 378  * free all queue memory used when releasing the queue (i.e. configure)
 379  */
 380 static inline void
 381 tx_queue_free(struct fm10k_tx_queue *q)
 382 {
 383         PMD_INIT_FUNC_TRACE();
 384         if (q) {
 385                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 386                 tx_queue_clean(q);
 387                 if (q->rs_tracker.list) {
 388                         rte_free(q->rs_tracker.list);
 389                         q->rs_tracker.list = NULL;
 390                 }
 391                 if (q->sw_ring) {
 392                         rte_free(q->sw_ring);
 393                         q->sw_ring = NULL;
 394                 }
 395                 rte_free(q);
 396                 q = NULL;
 397         }
 398 }
 399
 400 /*
 401  * disable TX queue, wait unitl HW finished necessary flush operation
 402  */
 403 static inline int
 404 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 405 {
 406         uint32_t reg, i;
 407
 408         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 409         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 410                         reg & ~FM10K_TXDCTL_ENABLE);
 411
 412         /* Wait 100us at most */
 413         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 414                 rte_delay_us(1);
 415                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 416                 if (!(reg & FM10K_TXDCTL_ENABLE))
 417                         break;
 418         }
 419
 420         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 421                 return -1;
 422
 423         return 0;
 424 }
 425
 426 static int
 427 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 428 {
 429         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 430         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 431         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 432         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 433
 434         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 435
 436         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 437                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 438                 return -EINVAL;
 439         }
 440
 441         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 442                 return 0;
 443
 444         if (hw->mac.type == fm10k_mac_vf) {
 445                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 446                 return -EINVAL;
 447         }
 448
 449         /* Check VMDQ queue pool number */
 450         if (vmdq_conf->nb_queue_pools >
 451                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 452                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 453                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 454                         vmdq_conf->nb_queue_pools);
 455                 return -EINVAL;
 456         }
 457
 458         return 0;
 459 }
 460
 461 static const struct fm10k_txq_ops def_txq_ops = {
 462         .reset = tx_queue_reset,
 463 };
 464
 465 static int
 466 fm10k_dev_configure(struct rte_eth_dev *dev)
 467 {
 468         int ret;
 469
 470         PMD_INIT_FUNC_TRACE();
 471
 472         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 473                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 474         /* multipe queue mode checking */
 475         ret  = fm10k_check_mq_mode(dev);
 476         if (ret != 0) {
 477                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 478                             ret);
 479                 return ret;
 480         }
 481
 482         return 0;
 483 }
 484
 485 /* fls = find last set bit = 32 minus the number of leading zeros */
 486 #ifndef fls
 487 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 488 #endif
 489
 490 static void
 491 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 492 {
 493         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 494         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 495         uint32_t i;
 496
 497         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 498
 499         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 500                 if (!vmdq_conf->pool_map[i].pools)
 501                         continue;
 502                 fm10k_mbx_lock(hw);
 503                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 504                 fm10k_mbx_unlock(hw);
 505         }
 506 }
 507
 508 static void
 509 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 510 {
 511         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 512
 513         /* Add default mac address */
 514         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 515                 MAIN_VSI_POOL_NUMBER);
 516 }
 517
 518 static void
 519 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 520 {
 521         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 522         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 523         uint32_t mrqc, *key, i, reta, j;
 524         uint64_t hf;
 525
 526 #define RSS_KEY_SIZE 40
 527         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 528                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 529                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 530                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 531                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 532                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 533         };
 534
 535         if (dev->data->nb_rx_queues == 1 ||
 536             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 537             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 538                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 539                 return;
 540         }
 541
 542         /* random key is rss_intel_key (default) or user provided (rss_key) */
 543         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 544                 key = (uint32_t *)rss_intel_key;
 545         else
 546                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 547
 548         /* Now fill our hash function seeds, 4 bytes at a time */
 549         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 550                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 551
 552         /*
 553          * Fill in redirection table
 554          * The byte-swap is needed because NIC registers are in
 555          * little-endian order.
 556          */
 557         reta = 0;
 558         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 559                 if (j == dev->data->nb_rx_queues)
 560                         j = 0;
 561                 reta = (reta << CHAR_BIT) | j;
 562                 if ((i & 3) == 3)
 563                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 564                                         rte_bswap32(reta));
 565         }
 566
 567         /*
 568          * Generate RSS hash based on packet types, TCP/UDP
 569          * port numbers and/or IPv4/v6 src and dst addresses
 570          */
 571         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 572         mrqc = 0;
 573         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 574         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 575         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 576         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 577         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 578         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 579         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 580         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 581         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 582
 583         if (mrqc == 0) {
 584                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 585                         "supported", hf);
 586                 return;
 587         }
 588
 589         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 590 }
 591
 592 static void
 593 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 594 {
 595         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 596         uint32_t i;
 597
 598         for (i = 0; i < nb_lport_new; i++) {
 599                 /* Set unicast mode by default. App can change
 600                  * to other mode in other API func.
 601                  */
 602                 fm10k_mbx_lock(hw);
 603                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 604                         FM10K_XCAST_MODE_NONE);
 605                 fm10k_mbx_unlock(hw);
 606         }
 607 }
 608
 609 static void
 610 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 611 {
 612         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 613         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 614         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 615         struct fm10k_macvlan_filter_info *macvlan;
 616         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 617         uint16_t nb_lport_new;
 618
 619         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 620         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 621
 622         fm10k_dev_rss_configure(dev);
 623
 624         /* only PF supports VMDQ */
 625         if (hw->mac.type != fm10k_mac_pf)
 626                 return;
 627
 628         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 629                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 630
 631         /* no pool number change, no need to update logic port and VLAN/MAC */
 632         if (macvlan->nb_queue_pools == nb_queue_pools)
 633                 return;
 634
 635         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 636         fm10k_dev_logic_port_update(dev, nb_lport_new);
 637
 638         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 639         memset(dev->data->mac_addrs, 0,
 640                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 641         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 642                 &dev->data->mac_addrs[0]);
 643         memset(macvlan, 0, sizeof(*macvlan));
 644         macvlan->nb_queue_pools = nb_queue_pools;
 645
 646         if (nb_queue_pools)
 647                 fm10k_dev_vmdq_rx_configure(dev);
 648         else
 649                 fm10k_dev_pf_main_vsi_reset(dev);
 650 }
 651
 652 static int
 653 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 654 {
 655         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 656         int i, ret;
 657         struct fm10k_tx_queue *txq;
 658         uint64_t base_addr;
 659         uint32_t size;
 660
 661         /* Disable TXINT to avoid possible interrupt */
 662         for (i = 0; i < hw->mac.max_queues; i++)
 663                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 664                                 3 << FM10K_TXINT_TIMER_SHIFT);
 665
 666         /* Setup TX queue */
 667         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 668                 txq = dev->data->tx_queues[i];
 669                 base_addr = txq->hw_ring_phys_addr;
 670                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 671
 672                 /* disable queue to avoid issues while updating state */
 673                 ret = tx_queue_disable(hw, i);
 674                 if (ret) {
 675                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 676                         return -1;
 677                 }
 678                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 679                  * register is read-only for VF.
 680                  */
 681                 if (fm10k_check_ftag(dev->device->devargs)) {
 682                         if (hw->mac.type == fm10k_mac_pf) {
 683                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 684                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 685                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 686                         } else {
 687                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 688                                 return -ENOTSUP;
 689                         }
 690                 }
 691
 692                 /* set location and size for descriptor ring */
 693                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 694                                 base_addr & UINT64_LOWER_32BITS_MASK);
 695                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 696                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 697                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 698
 699                 /* assign default SGLORT for each TX queue by PF */
 700                 if (hw->mac.type == fm10k_mac_pf)
 701                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 702         }
 703
 704         /* set up vector or scalar TX function as appropriate */
 705         fm10k_set_tx_function(dev);
 706
 707         return 0;
 708 }
 709
 710 static int
 711 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 712 {
 713         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 714         struct fm10k_macvlan_filter_info *macvlan;
 715         struct rte_pci_device *pdev = RTE_DEV_TO_PCI(dev->device);
 716         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 717         int i, ret;
 718         struct fm10k_rx_queue *rxq;
 719         uint64_t base_addr;
 720         uint32_t size;
 721         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 722         uint32_t logic_port = hw->mac.dglort_map;
 723         uint16_t buf_size;
 724         uint16_t queue_stride = 0;
 725
 726         /* enable RXINT for interrupt mode */
 727         i = 0;
 728         if (rte_intr_dp_is_en(intr_handle)) {
 729                 for (; i < dev->data->nb_rx_queues; i++) {
 730                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 731                         if (hw->mac.type == fm10k_mac_pf)
 732                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 733                                         FM10K_ITR_AUTOMASK |
 734                                         FM10K_ITR_MASK_CLEAR);
 735                         else
 736                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 737                                         FM10K_ITR_AUTOMASK |
 738                                         FM10K_ITR_MASK_CLEAR);
 739                 }
 740         }
 741         /* Disable other RXINT to avoid possible interrupt */
 742         for (; i < hw->mac.max_queues; i++)
 743                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 744                         3 << FM10K_RXINT_TIMER_SHIFT);
 745
 746         /* Setup RX queues */
 747         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 748                 rxq = dev->data->rx_queues[i];
 749                 base_addr = rxq->hw_ring_phys_addr;
 750                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 751
 752                 /* disable queue to avoid issues while updating state */
 753                 ret = rx_queue_disable(hw, i);
 754                 if (ret) {
 755                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 756                         return -1;
 757                 }
 758
 759                 /* Setup the Base and Length of the Rx Descriptor Ring */
 760                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 761                                 base_addr & UINT64_LOWER_32BITS_MASK);
 762                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 763                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 764                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 765
 766                 /* Configure the Rx buffer size for one buff without split */
 767                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 768                         RTE_PKTMBUF_HEADROOM);
 769                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 770                  * reserved for this purpose, and the worst case could be 511B.
 771                  * But SRR reg assumes all buffers have the same size. In order
 772                  * to fill the gap, we'll have to consider the worst case and
 773                  * assume 512B is reserved. If we don't do so, it's possible
 774                  * for HW to overwrite data to next mbuf.
 775                  */
 776                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 777
 778                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 779                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 780                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 781
 782                 /* It adds dual VLAN length for supporting dual VLAN */
 783                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 784                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 785                         dev->data->dev_conf.rxmode.enable_scatter) {
 786                         uint32_t reg;
 787                         dev->data->scattered_rx = 1;
 788                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 789                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 790                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 791                 }
 792
 793                 /* Enable drop on empty, it's RO for VF */
 794                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 795                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 796
 797                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 798                 FM10K_WRITE_FLUSH(hw);
 799         }
 800
 801         /* Configure VMDQ/RSS if applicable */
 802         fm10k_dev_mq_rx_configure(dev);
 803
 804         /* Decide the best RX function */
 805         fm10k_set_rx_function(dev);
 806
 807         /* update RX_SGLORT for loopback suppress*/
 808         if (hw->mac.type != fm10k_mac_pf)
 809                 return 0;
 810         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 811         if (macvlan->nb_queue_pools)
 812                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 813         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 814                 if (i && queue_stride && !(i % queue_stride))
 815                         logic_port++;
 816                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 817         }
 818
 819         return 0;
 820 }
 821
 822 static int
 823 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 824 {
 825         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 826         int err = -1;
 827         uint32_t reg;
 828         struct fm10k_rx_queue *rxq;
 829
 830         PMD_INIT_FUNC_TRACE();
 831
 832         if (rx_queue_id < dev->data->nb_rx_queues) {
 833                 rxq = dev->data->rx_queues[rx_queue_id];
 834                 err = rx_queue_reset(rxq);
 835                 if (err == -ENOMEM) {
 836                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 837                         return err;
 838                 } else if (err == -EINVAL) {
 839                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 840                                 " %d", err);
 841                         return err;
 842                 }
 843
 844                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 845                  * Note: this must be done AFTER the queue is enabled on real
 846                  * hardware, but BEFORE the queue is enabled when using the
 847                  * emulation platform. Do it in both places for now and remove
 848                  * this comment and the following two register writes when the
 849                  * emulation platform is no longer being used.
 850                  */
 851                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 852                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 853
 854                 /* Set PF ownership flag for PF devices */
 855                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 856                 if (hw->mac.type == fm10k_mac_pf)
 857                         reg |= FM10K_RXQCTL_PF;
 858                 reg |= FM10K_RXQCTL_ENABLE;
 859                 /* enable RX queue */
 860                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 861                 FM10K_WRITE_FLUSH(hw);
 862
 863                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 864                  * Note: this must be done AFTER the queue is enabled
 865                  */
 866                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 867                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 868                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 869         }
 870
 871         return err;
 872 }
 873
 874 static int
 875 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 876 {
 877         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 878
 879         PMD_INIT_FUNC_TRACE();
 880
 881         if (rx_queue_id < dev->data->nb_rx_queues) {
 882                 /* Disable RX queue */
 883                 rx_queue_disable(hw, rx_queue_id);
 884
 885                 /* Free mbuf and clean HW ring */
 886                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 887                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 888         }
 889
 890         return 0;
 891 }
 892
 893 static int
 894 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 895 {
 896         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 897         /** @todo - this should be defined in the shared code */
 898 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 899         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 900         int err = 0;
 901
 902         PMD_INIT_FUNC_TRACE();
 903
 904         if (tx_queue_id < dev->data->nb_tx_queues) {
 905                 struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 906
 907                 q->ops->reset(q);
 908
 909                 /* reset head and tail pointers */
 910                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 911                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 912
 913                 /* enable TX queue */
 914                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 915                                         FM10K_TXDCTL_ENABLE | txdctl);
 916                 FM10K_WRITE_FLUSH(hw);
 917                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 918         } else
 919                 err = -1;
 920
 921         return err;
 922 }
 923
 924 static int
 925 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 926 {
 927         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 928
 929         PMD_INIT_FUNC_TRACE();
 930
 931         if (tx_queue_id < dev->data->nb_tx_queues) {
 932                 tx_queue_disable(hw, tx_queue_id);
 933                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 934                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 935         }
 936
 937         return 0;
 938 }
 939
 940 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 941 {
 942         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 943                 != FM10K_DGLORTMAP_NONE);
 944 }
 945
 946 static void
 947 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 948 {
 949         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 950         int status;
 951
 952         PMD_INIT_FUNC_TRACE();
 953
 954         /* Return if it didn't acquire valid glort range */
 955         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 956                 return;
 957
 958         fm10k_mbx_lock(hw);
 959         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 960                                 FM10K_XCAST_MODE_PROMISC);
 961         fm10k_mbx_unlock(hw);
 962
 963         if (status != FM10K_SUCCESS)
 964                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 965 }
 966
 967 static void
 968 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 969 {
 970         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 971         uint8_t mode;
 972         int status;
 973
 974         PMD_INIT_FUNC_TRACE();
 975
 976         /* Return if it didn't acquire valid glort range */
 977         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 978                 return;
 979
 980         if (dev->data->all_multicast == 1)
 981                 mode = FM10K_XCAST_MODE_ALLMULTI;
 982         else
 983                 mode = FM10K_XCAST_MODE_NONE;
 984
 985         fm10k_mbx_lock(hw);
 986         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 987                                 mode);
 988         fm10k_mbx_unlock(hw);
 989
 990         if (status != FM10K_SUCCESS)
 991                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 992 }
 993
 994 static void
 995 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 996 {
 997         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 998         int status;
 999
1000         PMD_INIT_FUNC_TRACE();
1001
1002         /* Return if it didn't acquire valid glort range */
1003         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1004                 return;
1005
1006         /* If promiscuous mode is enabled, it doesn't make sense to enable
1007          * allmulticast and disable promiscuous since fm10k only can select
1008          * one of the modes.
1009          */
1010         if (dev->data->promiscuous) {
1011                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
1012                         "needn't enable allmulticast");
1013                 return;
1014         }
1015
1016         fm10k_mbx_lock(hw);
1017         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1018                                 FM10K_XCAST_MODE_ALLMULTI);
1019         fm10k_mbx_unlock(hw);
1020
1021         if (status != FM10K_SUCCESS)
1022                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
1023 }
1024
1025 static void
1026 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1027 {
1028         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1029         int status;
1030
1031         PMD_INIT_FUNC_TRACE();
1032
1033         /* Return if it didn't acquire valid glort range */
1034         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1035                 return;
1036
1037         if (dev->data->promiscuous) {
1038                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1039                         "since promisc mode is enabled");
1040                 return;
1041         }
1042
1043         fm10k_mbx_lock(hw);
1044         /* Change mode to unicast mode */
1045         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1046                                 FM10K_XCAST_MODE_NONE);
1047         fm10k_mbx_unlock(hw);
1048
1049         if (status != FM10K_SUCCESS)
1050                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1051 }
1052
1053 static void
1054 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1055 {
1056         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1057         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1058         uint16_t nb_queue_pools;
1059         struct fm10k_macvlan_filter_info *macvlan;
1060
1061         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1062         nb_queue_pools = macvlan->nb_queue_pools;
1063         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
1064         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
1065
1066         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1067         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1068         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1069                         hw->mac.dglort_map;
1070         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1071         /* Configure VMDQ/RSS DGlort Decoder */
1072         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1073
1074         /* Flow Director configurations, only queue number is valid. */
1075         dglortdec = fls(dev->data->nb_rx_queues - 1);
1076         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1077                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1078         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1079         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1080
1081         /* Invalidate all other GLORT entries */
1082         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1083                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1084                                 FM10K_DGLORTMAP_NONE);
1085 }
1086
1087 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1088 static int
1089 fm10k_dev_start(struct rte_eth_dev *dev)
1090 {
1091         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1092         int i, diag;
1093
1094         PMD_INIT_FUNC_TRACE();
1095
1096         /* stop, init, then start the hw */
1097         diag = fm10k_stop_hw(hw);
1098         if (diag != FM10K_SUCCESS) {
1099                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1100                 return -EIO;
1101         }
1102
1103         diag = fm10k_init_hw(hw);
1104         if (diag != FM10K_SUCCESS) {
1105                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1106                 return -EIO;
1107         }
1108
1109         diag = fm10k_start_hw(hw);
1110         if (diag != FM10K_SUCCESS) {
1111                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1112                 return -EIO;
1113         }
1114
1115         diag = fm10k_dev_tx_init(dev);
1116         if (diag) {
1117                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1118                 return diag;
1119         }
1120
1121         if (fm10k_dev_rxq_interrupt_setup(dev))
1122                 return -EIO;
1123
1124         diag = fm10k_dev_rx_init(dev);
1125         if (diag) {
1126                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1127                 return diag;
1128         }
1129
1130         if (hw->mac.type == fm10k_mac_pf)
1131                 fm10k_dev_dglort_map_configure(dev);
1132
1133         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1134                 struct fm10k_rx_queue *rxq;
1135                 rxq = dev->data->rx_queues[i];
1136
1137                 if (rxq->rx_deferred_start)
1138                         continue;
1139                 diag = fm10k_dev_rx_queue_start(dev, i);
1140                 if (diag != 0) {
1141                         int j;
1142                         for (j = 0; j < i; ++j)
1143                                 rx_queue_clean(dev->data->rx_queues[j]);
1144                         return diag;
1145                 }
1146         }
1147
1148         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1149                 struct fm10k_tx_queue *txq;
1150                 txq = dev->data->tx_queues[i];
1151
1152                 if (txq->tx_deferred_start)
1153                         continue;
1154                 diag = fm10k_dev_tx_queue_start(dev, i);
1155                 if (diag != 0) {
1156                         int j;
1157                         for (j = 0; j < i; ++j)
1158                                 tx_queue_clean(dev->data->tx_queues[j]);
1159                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1160                                 rx_queue_clean(dev->data->rx_queues[j]);
1161                         return diag;
1162                 }
1163         }
1164
1165         /* Update default vlan when not in VMDQ mode */
1166         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1167                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1168
1169         return 0;
1170 }
1171
1172 static void
1173 fm10k_dev_stop(struct rte_eth_dev *dev)
1174 {
1175         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1176         struct rte_pci_device *pdev = RTE_DEV_TO_PCI(dev->device);
1177         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1178         int i;
1179
1180         PMD_INIT_FUNC_TRACE();
1181
1182         if (dev->data->tx_queues)
1183                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1184                         fm10k_dev_tx_queue_stop(dev, i);
1185
1186         if (dev->data->rx_queues)
1187                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1188                         fm10k_dev_rx_queue_stop(dev, i);
1189
1190         /* Disable datapath event */
1191         if (rte_intr_dp_is_en(intr_handle)) {
1192                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1193                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1194                                 3 << FM10K_RXINT_TIMER_SHIFT);
1195                         if (hw->mac.type == fm10k_mac_pf)
1196                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1197                                         FM10K_ITR_MASK_SET);
1198                         else
1199                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1200                                         FM10K_ITR_MASK_SET);
1201                 }
1202         }
1203         /* Clean datapath event and queue/vec mapping */
1204         rte_intr_efd_disable(intr_handle);
1205         rte_free(intr_handle->intr_vec);
1206         intr_handle->intr_vec = NULL;
1207 }
1208
1209 static void
1210 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1211 {
1212         int i;
1213
1214         PMD_INIT_FUNC_TRACE();
1215
1216         if (dev->data->tx_queues) {
1217                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1218                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1219
1220                         tx_queue_free(txq);
1221                 }
1222         }
1223
1224         if (dev->data->rx_queues) {
1225                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1226                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1227         }
1228 }
1229
1230 static void
1231 fm10k_dev_close(struct rte_eth_dev *dev)
1232 {
1233         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1234
1235         PMD_INIT_FUNC_TRACE();
1236
1237         fm10k_mbx_lock(hw);
1238         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1239                 MAX_LPORT_NUM, false);
1240         fm10k_mbx_unlock(hw);
1241
1242         /* allow 10ms for device to quiesce */
1243         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
1244
1245         /* Stop mailbox service first */
1246         fm10k_close_mbx_service(hw);
1247         fm10k_dev_stop(dev);
1248         fm10k_dev_queue_release(dev);
1249         fm10k_stop_hw(hw);
1250 }
1251
1252 static int
1253 fm10k_link_update(struct rte_eth_dev *dev,
1254         __rte_unused int wait_to_complete)
1255 {
1256         PMD_INIT_FUNC_TRACE();
1257
1258         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
1259          * x8 PCIe interface. For now, we leave the speed undefined since there
1260          * is no 50Gbps Ethernet. */
1261         dev->data->dev_link.link_speed  = 0;
1262         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1263         dev->data->dev_link.link_status = ETH_LINK_UP;
1264
1265         return 0;
1266 }
1267
1268 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1269         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1270 {
1271         unsigned i, q;
1272         unsigned count = 0;
1273
1274         if (xstats_names != NULL) {
1275                 /* Note: limit checked in rte_eth_xstats_names() */
1276
1277                 /* Global stats */
1278                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1279                         snprintf(xstats_names[count].name,
1280                                 sizeof(xstats_names[count].name),
1281                                 "%s", fm10k_hw_stats_strings[count].name);
1282                         count++;
1283                 }
1284
1285                 /* PF queue stats */
1286                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1287                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1288                                 snprintf(xstats_names[count].name,
1289                                         sizeof(xstats_names[count].name),
1290                                         "rx_q%u_%s", q,
1291                                         fm10k_hw_stats_rx_q_strings[i].name);
1292                                 count++;
1293                         }
1294                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1295                                 snprintf(xstats_names[count].name,
1296                                         sizeof(xstats_names[count].name),
1297                                         "tx_q%u_%s", q,
1298                                         fm10k_hw_stats_tx_q_strings[i].name);
1299                                 count++;
1300                         }
1301                 }
1302         }
1303         return FM10K_NB_XSTATS;
1304 }
1305
1306 static int
1307 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1308                  unsigned n)
1309 {
1310         struct fm10k_hw_stats *hw_stats =
1311                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1312         unsigned i, q, count = 0;
1313
1314         if (n < FM10K_NB_XSTATS)
1315                 return FM10K_NB_XSTATS;
1316
1317         /* Global stats */
1318         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1319                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1320                         fm10k_hw_stats_strings[count].offset);
1321                 xstats[count].id = count;
1322                 count++;
1323         }
1324
1325         /* PF queue stats */
1326         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1327                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1328                         xstats[count].value =
1329                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1330                                 fm10k_hw_stats_rx_q_strings[i].offset);
1331                         xstats[count].id = count;
1332                         count++;
1333                 }
1334                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1335                         xstats[count].value =
1336                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1337                                 fm10k_hw_stats_tx_q_strings[i].offset);
1338                         xstats[count].id = count;
1339                         count++;
1340                 }
1341         }
1342
1343         return FM10K_NB_XSTATS;
1344 }
1345
1346 static void
1347 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1348 {
1349         uint64_t ipackets, opackets, ibytes, obytes;
1350         struct fm10k_hw *hw =
1351                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1352         struct fm10k_hw_stats *hw_stats =
1353                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1354         int i;
1355
1356         PMD_INIT_FUNC_TRACE();
1357
1358         fm10k_update_hw_stats(hw, hw_stats);
1359
1360         ipackets = opackets = ibytes = obytes = 0;
1361         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1362                 (i < hw->mac.max_queues); ++i) {
1363                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1364                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1365                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1366                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1367                 ipackets += stats->q_ipackets[i];
1368                 opackets += stats->q_opackets[i];
1369                 ibytes   += stats->q_ibytes[i];
1370                 obytes   += stats->q_obytes[i];
1371         }
1372         stats->ipackets = ipackets;
1373         stats->opackets = opackets;
1374         stats->ibytes = ibytes;
1375         stats->obytes = obytes;
1376 }
1377
1378 static void
1379 fm10k_stats_reset(struct rte_eth_dev *dev)
1380 {
1381         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1382         struct fm10k_hw_stats *hw_stats =
1383                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1384
1385         PMD_INIT_FUNC_TRACE();
1386
1387         memset(hw_stats, 0, sizeof(*hw_stats));
1388         fm10k_rebind_hw_stats(hw, hw_stats);
1389 }
1390
1391 static void
1392 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1393         struct rte_eth_dev_info *dev_info)
1394 {
1395         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1396         struct rte_pci_device *pdev = RTE_DEV_TO_PCI(dev->device);
1397
1398         PMD_INIT_FUNC_TRACE();
1399
1400         dev_info->pci_dev            = pdev;
1401         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1402         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1403         dev_info->max_rx_queues      = hw->mac.max_queues;
1404         dev_info->max_tx_queues      = hw->mac.max_queues;
1405         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1406         dev_info->max_hash_mac_addrs = 0;
1407         dev_info->max_vfs            = pdev->max_vfs;
1408         dev_info->vmdq_pool_base     = 0;
1409         dev_info->vmdq_queue_base    = 0;
1410         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1411         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1412         dev_info->rx_offload_capa =
1413                 DEV_RX_OFFLOAD_VLAN_STRIP |
1414                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1415                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1416                 DEV_RX_OFFLOAD_TCP_CKSUM;
1417         dev_info->tx_offload_capa =
1418                 DEV_TX_OFFLOAD_VLAN_INSERT |
1419                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1420                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1421                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1422                 DEV_TX_OFFLOAD_TCP_TSO;
1423
1424         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1425         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1426
1427         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1428                 .rx_thresh = {
1429                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1430                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1431                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1432                 },
1433                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1434                 .rx_drop_en = 0,
1435         };
1436
1437         dev_info->default_txconf = (struct rte_eth_txconf) {
1438                 .tx_thresh = {
1439                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1440                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1441                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1442                 },
1443                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1444                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1445                 .txq_flags = FM10K_SIMPLE_TX_FLAG,
1446         };
1447
1448         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1449                 .nb_max = FM10K_MAX_RX_DESC,
1450                 .nb_min = FM10K_MIN_RX_DESC,
1451                 .nb_align = FM10K_MULT_RX_DESC,
1452         };
1453
1454         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1455                 .nb_max = FM10K_MAX_TX_DESC,
1456                 .nb_min = FM10K_MIN_TX_DESC,
1457                 .nb_align = FM10K_MULT_TX_DESC,
1458                 .nb_seg_max = FM10K_TX_MAX_SEG,
1459                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1460         };
1461
1462         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1463                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1464                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1465 }
1466
1467 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1468 static const uint32_t *
1469 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1470 {
1471         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1472             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1473                 static uint32_t ptypes[] = {
1474                         /* refers to rx_desc_to_ol_flags() */
1475                         RTE_PTYPE_L2_ETHER,
1476                         RTE_PTYPE_L3_IPV4,
1477                         RTE_PTYPE_L3_IPV4_EXT,
1478                         RTE_PTYPE_L3_IPV6,
1479                         RTE_PTYPE_L3_IPV6_EXT,
1480                         RTE_PTYPE_L4_TCP,
1481                         RTE_PTYPE_L4_UDP,
1482                         RTE_PTYPE_UNKNOWN
1483                 };
1484
1485                 return ptypes;
1486         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1487                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1488                 static uint32_t ptypes_vec[] = {
1489                         /* refers to fm10k_desc_to_pktype_v() */
1490                         RTE_PTYPE_L3_IPV4,
1491                         RTE_PTYPE_L3_IPV4_EXT,
1492                         RTE_PTYPE_L3_IPV6,
1493                         RTE_PTYPE_L3_IPV6_EXT,
1494                         RTE_PTYPE_L4_TCP,
1495                         RTE_PTYPE_L4_UDP,
1496                         RTE_PTYPE_TUNNEL_GENEVE,
1497                         RTE_PTYPE_TUNNEL_NVGRE,
1498                         RTE_PTYPE_TUNNEL_VXLAN,
1499                         RTE_PTYPE_TUNNEL_GRE,
1500                         RTE_PTYPE_UNKNOWN
1501                 };
1502
1503                 return ptypes_vec;
1504         }
1505
1506         return NULL;
1507 }
1508 #else
1509 static const uint32_t *
1510 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1511 {
1512         return NULL;
1513 }
1514 #endif
1515
1516 static int
1517 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1518 {
1519         s32 result;
1520         uint16_t mac_num = 0;
1521         uint32_t vid_idx, vid_bit, mac_index;
1522         struct fm10k_hw *hw;
1523         struct fm10k_macvlan_filter_info *macvlan;
1524         struct rte_eth_dev_data *data = dev->data;
1525
1526         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1527         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1528
1529         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1530                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1531                 return -EINVAL;
1532         }
1533
1534         if (vlan_id > ETH_VLAN_ID_MAX) {
1535                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1536                 return -EINVAL;
1537         }
1538
1539         vid_idx = FM10K_VFTA_IDX(vlan_id);
1540         vid_bit = FM10K_VFTA_BIT(vlan_id);
1541         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1542         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1543                 return 0;
1544         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1545         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1546                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1547                         "in the VLAN filter table");
1548                 return -EINVAL;
1549         }
1550
1551         fm10k_mbx_lock(hw);
1552         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1553         fm10k_mbx_unlock(hw);
1554         if (result != FM10K_SUCCESS) {
1555                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1556                 return -EIO;
1557         }
1558
1559         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1560                         (result == FM10K_SUCCESS); mac_index++) {
1561                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1562                         continue;
1563                 if (mac_num > macvlan->mac_num - 1) {
1564                         PMD_INIT_LOG(ERR, "MAC address number "
1565                                         "not match");
1566                         break;
1567                 }
1568                 fm10k_mbx_lock(hw);
1569                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1570                         data->mac_addrs[mac_index].addr_bytes,
1571                         vlan_id, on, 0);
1572                 fm10k_mbx_unlock(hw);
1573                 mac_num++;
1574         }
1575         if (result != FM10K_SUCCESS) {
1576                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1577                 return -EIO;
1578         }
1579
1580         if (on) {
1581                 macvlan->vlan_num++;
1582                 macvlan->vfta[vid_idx] |= vid_bit;
1583         } else {
1584                 macvlan->vlan_num--;
1585                 macvlan->vfta[vid_idx] &= ~vid_bit;
1586         }
1587         return 0;
1588 }
1589
1590 static void
1591 fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1592 {
1593         if (mask & ETH_VLAN_STRIP_MASK) {
1594                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1595                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1596                                         "always on in fm10k");
1597         }
1598
1599         if (mask & ETH_VLAN_EXTEND_MASK) {
1600                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1601                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1602                                         "supported in fm10k");
1603         }
1604
1605         if (mask & ETH_VLAN_FILTER_MASK) {
1606                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1607                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1608         }
1609 }
1610
1611 /* Add/Remove a MAC address, and update filters to main VSI */
1612 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1613                 const u8 *mac, bool add, uint32_t pool)
1614 {
1615         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1616         struct fm10k_macvlan_filter_info *macvlan;
1617         uint32_t i, j, k;
1618
1619         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1620
1621         if (pool != MAIN_VSI_POOL_NUMBER) {
1622                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1623                         "mac to pool %u", pool);
1624                 return;
1625         }
1626         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1627                 if (!macvlan->vfta[j])
1628                         continue;
1629                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1630                         if (!(macvlan->vfta[j] & (1 << k)))
1631                                 continue;
1632                         if (i + 1 > macvlan->vlan_num) {
1633                                 PMD_INIT_LOG(ERR, "vlan number not match");
1634                                 return;
1635                         }
1636                         fm10k_mbx_lock(hw);
1637                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1638                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1639                         fm10k_mbx_unlock(hw);
1640                         i++;
1641                 }
1642         }
1643 }
1644
1645 /* Add/Remove a MAC address, and update filters to VMDQ */
1646 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1647                 const u8 *mac, bool add, uint32_t pool)
1648 {
1649         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1650         struct fm10k_macvlan_filter_info *macvlan;
1651         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1652         uint32_t i;
1653
1654         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1655         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1656
1657         if (pool > macvlan->nb_queue_pools) {
1658                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1659                         " Max pool is %u",
1660                         pool, macvlan->nb_queue_pools);
1661                 return;
1662         }
1663         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1664                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1665                         continue;
1666                 fm10k_mbx_lock(hw);
1667                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1668                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1669                 fm10k_mbx_unlock(hw);
1670         }
1671 }
1672
1673 /* Add/Remove a MAC address, and update filters */
1674 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1675                 const u8 *mac, bool add, uint32_t pool)
1676 {
1677         struct fm10k_macvlan_filter_info *macvlan;
1678
1679         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1680
1681         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1682                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1683         else
1684                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1685
1686         if (add)
1687                 macvlan->mac_num++;
1688         else
1689                 macvlan->mac_num--;
1690 }
1691
1692 /* Add a MAC address, and update filters */
1693 static int
1694 fm10k_macaddr_add(struct rte_eth_dev *dev,
1695                 struct ether_addr *mac_addr,
1696                 uint32_t index,
1697                 uint32_t pool)
1698 {
1699         struct fm10k_macvlan_filter_info *macvlan;
1700
1701         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1702         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1703         macvlan->mac_vmdq_id[index] = pool;
1704         return 0;
1705 }
1706
1707 /* Remove a MAC address, and update filters */
1708 static void
1709 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1710 {
1711         struct rte_eth_dev_data *data = dev->data;
1712         struct fm10k_macvlan_filter_info *macvlan;
1713
1714         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1715         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1716                         FALSE, macvlan->mac_vmdq_id[index]);
1717         macvlan->mac_vmdq_id[index] = 0;
1718 }
1719
1720 static inline int
1721 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1722 {
1723         if ((request < min) || (request > max) || ((request % mult) != 0))
1724                 return -1;
1725         else
1726                 return 0;
1727 }
1728
1729
1730 static inline int
1731 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1732 {
1733         if ((request < min) || (request > max) || ((div % request) != 0))
1734                 return -1;
1735         else
1736                 return 0;
1737 }
1738
1739 static inline int
1740 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1741 {
1742         uint16_t rx_free_thresh;
1743
1744         if (conf->rx_free_thresh == 0)
1745                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1746         else
1747                 rx_free_thresh = conf->rx_free_thresh;
1748
1749         /* make sure the requested threshold satisfies the constraints */
1750         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1751                         FM10K_RX_FREE_THRESH_MAX(q),
1752                         FM10K_RX_FREE_THRESH_DIV(q),
1753                         rx_free_thresh)) {
1754                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1755                         "less than or equal to %u, "
1756                         "greater than or equal to %u, "
1757                         "and a divisor of %u",
1758                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1759                         FM10K_RX_FREE_THRESH_MIN(q),
1760                         FM10K_RX_FREE_THRESH_DIV(q));
1761                 return -EINVAL;
1762         }
1763
1764         q->alloc_thresh = rx_free_thresh;
1765         q->drop_en = conf->rx_drop_en;
1766         q->rx_deferred_start = conf->rx_deferred_start;
1767
1768         return 0;
1769 }
1770
1771 /*
1772  * Hardware requires specific alignment for Rx packet buffers. At
1773  * least one of the following two conditions must be satisfied.
1774  *  1. Address is 512B aligned
1775  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1776  *
1777  * As such, the driver may need to adjust the DMA address within the
1778  * buffer by up to 512B.
1779  *
1780  * return 1 if the element size is valid, otherwise return 0.
1781  */
1782 static int
1783 mempool_element_size_valid(struct rte_mempool *mp)
1784 {
1785         uint32_t min_size;
1786
1787         /* elt_size includes mbuf header and headroom */
1788         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1789                         RTE_PKTMBUF_HEADROOM;
1790
1791         /* account for up to 512B of alignment */
1792         min_size -= FM10K_RX_DATABUF_ALIGN;
1793
1794         /* sanity check for overflow */
1795         if (min_size > mp->elt_size)
1796                 return 0;
1797
1798         /* size is valid */
1799         return 1;
1800 }
1801
1802 static int
1803 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1804         uint16_t nb_desc, unsigned int socket_id,
1805         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1806 {
1807         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1808         struct fm10k_dev_info *dev_info =
1809                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1810         struct fm10k_rx_queue *q;
1811         const struct rte_memzone *mz;
1812
1813         PMD_INIT_FUNC_TRACE();
1814
1815         /* make sure the mempool element size can account for alignment. */
1816         if (!mempool_element_size_valid(mp)) {
1817                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1818                 return -EINVAL;
1819         }
1820
1821         /* make sure a valid number of descriptors have been requested */
1822         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1823                                 FM10K_MULT_RX_DESC, nb_desc)) {
1824                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1825                         "less than or equal to %"PRIu32", "
1826                         "greater than or equal to %u, "
1827                         "and a multiple of %u",
1828                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1829                         FM10K_MULT_RX_DESC);
1830                 return -EINVAL;
1831         }
1832
1833         /*
1834          * if this queue existed already, free the associated memory. The
1835          * queue cannot be reused in case we need to allocate memory on
1836          * different socket than was previously used.
1837          */
1838         if (dev->data->rx_queues[queue_id] != NULL) {
1839                 rx_queue_free(dev->data->rx_queues[queue_id]);
1840                 dev->data->rx_queues[queue_id] = NULL;
1841         }
1842
1843         /* allocate memory for the queue structure */
1844         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1845                                 socket_id);
1846         if (q == NULL) {
1847                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1848                 return -ENOMEM;
1849         }
1850
1851         /* setup queue */
1852         q->mp = mp;
1853         q->nb_desc = nb_desc;
1854         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1855         q->port_id = dev->data->port_id;
1856         q->queue_id = queue_id;
1857         q->tail_ptr = (volatile uint32_t *)
1858                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1859         if (handle_rxconf(q, conf))
1860                 return -EINVAL;
1861
1862         /* allocate memory for the software ring */
1863         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1864                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1865                         RTE_CACHE_LINE_SIZE, socket_id);
1866         if (q->sw_ring == NULL) {
1867                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1868                 rte_free(q);
1869                 return -ENOMEM;
1870         }
1871
1872         /*
1873          * allocate memory for the hardware descriptor ring. A memzone large
1874          * enough to hold the maximum ring size is requested to allow for
1875          * resizing in later calls to the queue setup function.
1876          */
1877         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1878                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1879                                       socket_id);
1880         if (mz == NULL) {
1881                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1882                 rte_free(q->sw_ring);
1883                 rte_free(q);
1884                 return -ENOMEM;
1885         }
1886         q->hw_ring = mz->addr;
1887         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1888
1889         /* Check if number of descs satisfied Vector requirement */
1890         if (!rte_is_power_of_2(nb_desc)) {
1891                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1892                                     "preconditions - canceling the feature for "
1893                                     "the whole port[%d]",
1894                              q->queue_id, q->port_id);
1895                 dev_info->rx_vec_allowed = false;
1896         } else
1897                 fm10k_rxq_vec_setup(q);
1898
1899         dev->data->rx_queues[queue_id] = q;
1900         return 0;
1901 }
1902
1903 static void
1904 fm10k_rx_queue_release(void *queue)
1905 {
1906         PMD_INIT_FUNC_TRACE();
1907
1908         rx_queue_free(queue);
1909 }
1910
1911 static inline int
1912 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1913 {
1914         uint16_t tx_free_thresh;
1915         uint16_t tx_rs_thresh;
1916
1917         /* constraint MACROs require that tx_free_thresh is configured
1918          * before tx_rs_thresh */
1919         if (conf->tx_free_thresh == 0)
1920                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1921         else
1922                 tx_free_thresh = conf->tx_free_thresh;
1923
1924         /* make sure the requested threshold satisfies the constraints */
1925         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1926                         FM10K_TX_FREE_THRESH_MAX(q),
1927                         FM10K_TX_FREE_THRESH_DIV(q),
1928                         tx_free_thresh)) {
1929                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1930                         "less than or equal to %u, "
1931                         "greater than or equal to %u, "
1932                         "and a divisor of %u",
1933                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1934                         FM10K_TX_FREE_THRESH_MIN(q),
1935                         FM10K_TX_FREE_THRESH_DIV(q));
1936                 return -EINVAL;
1937         }
1938
1939         q->free_thresh = tx_free_thresh;
1940
1941         if (conf->tx_rs_thresh == 0)
1942                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1943         else
1944                 tx_rs_thresh = conf->tx_rs_thresh;
1945
1946         q->tx_deferred_start = conf->tx_deferred_start;
1947
1948         /* make sure the requested threshold satisfies the constraints */
1949         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1950                         FM10K_TX_RS_THRESH_MAX(q),
1951                         FM10K_TX_RS_THRESH_DIV(q),
1952                         tx_rs_thresh)) {
1953                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1954                         "less than or equal to %u, "
1955                         "greater than or equal to %u, "
1956                         "and a divisor of %u",
1957                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1958                         FM10K_TX_RS_THRESH_MIN(q),
1959                         FM10K_TX_RS_THRESH_DIV(q));
1960                 return -EINVAL;
1961         }
1962
1963         q->rs_thresh = tx_rs_thresh;
1964
1965         return 0;
1966 }
1967
1968 static int
1969 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1970         uint16_t nb_desc, unsigned int socket_id,
1971         const struct rte_eth_txconf *conf)
1972 {
1973         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1974         struct fm10k_tx_queue *q;
1975         const struct rte_memzone *mz;
1976
1977         PMD_INIT_FUNC_TRACE();
1978
1979         /* make sure a valid number of descriptors have been requested */
1980         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1981                                 FM10K_MULT_TX_DESC, nb_desc)) {
1982                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1983                         "less than or equal to %"PRIu32", "
1984                         "greater than or equal to %u, "
1985                         "and a multiple of %u",
1986                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1987                         FM10K_MULT_TX_DESC);
1988                 return -EINVAL;
1989         }
1990
1991         /*
1992          * if this queue existed already, free the associated memory. The
1993          * queue cannot be reused in case we need to allocate memory on
1994          * different socket than was previously used.
1995          */
1996         if (dev->data->tx_queues[queue_id] != NULL) {
1997                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
1998
1999                 tx_queue_free(txq);
2000                 dev->data->tx_queues[queue_id] = NULL;
2001         }
2002
2003         /* allocate memory for the queue structure */
2004         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2005                                 socket_id);
2006         if (q == NULL) {
2007                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2008                 return -ENOMEM;
2009         }
2010
2011         /* setup queue */
2012         q->nb_desc = nb_desc;
2013         q->port_id = dev->data->port_id;
2014         q->queue_id = queue_id;
2015         q->txq_flags = conf->txq_flags;
2016         q->ops = &def_txq_ops;
2017         q->tail_ptr = (volatile uint32_t *)
2018                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2019         if (handle_txconf(q, conf))
2020                 return -EINVAL;
2021
2022         /* allocate memory for the software ring */
2023         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2024                                         nb_desc * sizeof(struct rte_mbuf *),
2025                                         RTE_CACHE_LINE_SIZE, socket_id);
2026         if (q->sw_ring == NULL) {
2027                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2028                 rte_free(q);
2029                 return -ENOMEM;
2030         }
2031
2032         /*
2033          * allocate memory for the hardware descriptor ring. A memzone large
2034          * enough to hold the maximum ring size is requested to allow for
2035          * resizing in later calls to the queue setup function.
2036          */
2037         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2038                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2039                                       socket_id);
2040         if (mz == NULL) {
2041                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2042                 rte_free(q->sw_ring);
2043                 rte_free(q);
2044                 return -ENOMEM;
2045         }
2046         q->hw_ring = mz->addr;
2047         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
2048
2049         /*
2050          * allocate memory for the RS bit tracker. Enough slots to hold the
2051          * descriptor index for each RS bit needing to be set are required.
2052          */
2053         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2054                                 ((nb_desc + 1) / q->rs_thresh) *
2055                                 sizeof(uint16_t),
2056                                 RTE_CACHE_LINE_SIZE, socket_id);
2057         if (q->rs_tracker.list == NULL) {
2058                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2059                 rte_free(q->sw_ring);
2060                 rte_free(q);
2061                 return -ENOMEM;
2062         }
2063
2064         dev->data->tx_queues[queue_id] = q;
2065         return 0;
2066 }
2067
2068 static void
2069 fm10k_tx_queue_release(void *queue)
2070 {
2071         struct fm10k_tx_queue *q = queue;
2072         PMD_INIT_FUNC_TRACE();
2073
2074         tx_queue_free(q);
2075 }
2076
2077 static int
2078 fm10k_reta_update(struct rte_eth_dev *dev,
2079                         struct rte_eth_rss_reta_entry64 *reta_conf,
2080                         uint16_t reta_size)
2081 {
2082         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2083         uint16_t i, j, idx, shift;
2084         uint8_t mask;
2085         uint32_t reta;
2086
2087         PMD_INIT_FUNC_TRACE();
2088
2089         if (reta_size > FM10K_MAX_RSS_INDICES) {
2090                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2091                         "(%d) doesn't match the number hardware can supported "
2092                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2093                 return -EINVAL;
2094         }
2095
2096         /*
2097          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2098          * 128-entries in 32 registers
2099          */
2100         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2101                 idx = i / RTE_RETA_GROUP_SIZE;
2102                 shift = i % RTE_RETA_GROUP_SIZE;
2103                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2104                                 BIT_MASK_PER_UINT32);
2105                 if (mask == 0)
2106                         continue;
2107
2108                 reta = 0;
2109                 if (mask != BIT_MASK_PER_UINT32)
2110                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2111
2112                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2113                         if (mask & (0x1 << j)) {
2114                                 if (mask != 0xF)
2115                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2116                                 reta |= reta_conf[idx].reta[shift + j] <<
2117                                                 (CHAR_BIT * j);
2118                         }
2119                 }
2120                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2121         }
2122
2123         return 0;
2124 }
2125
2126 static int
2127 fm10k_reta_query(struct rte_eth_dev *dev,
2128                         struct rte_eth_rss_reta_entry64 *reta_conf,
2129                         uint16_t reta_size)
2130 {
2131         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2132         uint16_t i, j, idx, shift;
2133         uint8_t mask;
2134         uint32_t reta;
2135
2136         PMD_INIT_FUNC_TRACE();
2137
2138         if (reta_size < FM10K_MAX_RSS_INDICES) {
2139                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2140                         "(%d) doesn't match the number hardware can supported "
2141                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2142                 return -EINVAL;
2143         }
2144
2145         /*
2146          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2147          * 128-entries in 32 registers
2148          */
2149         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2150                 idx = i / RTE_RETA_GROUP_SIZE;
2151                 shift = i % RTE_RETA_GROUP_SIZE;
2152                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2153                                 BIT_MASK_PER_UINT32);
2154                 if (mask == 0)
2155                         continue;
2156
2157                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2158                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2159                         if (mask & (0x1 << j))
2160                                 reta_conf[idx].reta[shift + j] = ((reta >>
2161                                         CHAR_BIT * j) & UINT8_MAX);
2162                 }
2163         }
2164
2165         return 0;
2166 }
2167
2168 static int
2169 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2170         struct rte_eth_rss_conf *rss_conf)
2171 {
2172         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2173         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2174         uint32_t mrqc;
2175         uint64_t hf = rss_conf->rss_hf;
2176         int i;
2177
2178         PMD_INIT_FUNC_TRACE();
2179
2180         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2181                                 FM10K_RSSRK_ENTRIES_PER_REG))
2182                 return -EINVAL;
2183
2184         if (hf == 0)
2185                 return -EINVAL;
2186
2187         mrqc = 0;
2188         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2189         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2190         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2191         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2192         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2193         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2194         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2195         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2196         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2197
2198         /* If the mapping doesn't fit any supported, return */
2199         if (mrqc == 0)
2200                 return -EINVAL;
2201
2202         if (key != NULL)
2203                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2204                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2205
2206         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2207
2208         return 0;
2209 }
2210
2211 static int
2212 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2213         struct rte_eth_rss_conf *rss_conf)
2214 {
2215         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2216         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2217         uint32_t mrqc;
2218         uint64_t hf;
2219         int i;
2220
2221         PMD_INIT_FUNC_TRACE();
2222
2223         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2224                                 FM10K_RSSRK_ENTRIES_PER_REG))
2225                 return -EINVAL;
2226
2227         if (key != NULL)
2228                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2229                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2230
2231         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2232         hf = 0;
2233         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2234         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2235         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2236         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2237         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2238         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2239         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2240         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2241         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2242
2243         rss_conf->rss_hf = hf;
2244
2245         return 0;
2246 }
2247
2248 static void
2249 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2250 {
2251         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2252         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2253
2254         /* Bind all local non-queue interrupt to vector 0 */
2255         int_map |= FM10K_MISC_VEC_ID;
2256
2257         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2258         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2259         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2260         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2261         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2262         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2263
2264         /* Enable misc causes */
2265         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2266                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2267                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2268                                 FM10K_EIMR_ENABLE(MAILBOX) |
2269                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2270                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2271                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2272                                 FM10K_EIMR_ENABLE(VFLR));
2273
2274         /* Enable ITR 0 */
2275         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2276                                         FM10K_ITR_MASK_CLEAR);
2277         FM10K_WRITE_FLUSH(hw);
2278 }
2279
2280 static void
2281 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2282 {
2283         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2284         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2285
2286         int_map |= FM10K_MISC_VEC_ID;
2287
2288         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2289         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2290         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2291         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2292         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2293         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2294
2295         /* Disable misc causes */
2296         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2297                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2298                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2299                                 FM10K_EIMR_DISABLE(MAILBOX) |
2300                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2301                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2302                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2303                                 FM10K_EIMR_DISABLE(VFLR));
2304
2305         /* Disable ITR 0 */
2306         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2307         FM10K_WRITE_FLUSH(hw);
2308 }
2309
2310 static void
2311 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2312 {
2313         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2314         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2315
2316         /* Bind all local non-queue interrupt to vector 0 */
2317         int_map |= FM10K_MISC_VEC_ID;
2318
2319         /* Only INT 0 available, other 15 are reserved. */
2320         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2321
2322         /* Enable ITR 0 */
2323         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2324                                         FM10K_ITR_MASK_CLEAR);
2325         FM10K_WRITE_FLUSH(hw);
2326 }
2327
2328 static void
2329 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2330 {
2331         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2332         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2333
2334         int_map |= FM10K_MISC_VEC_ID;
2335
2336         /* Only INT 0 available, other 15 are reserved. */
2337         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2338
2339         /* Disable ITR 0 */
2340         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2341         FM10K_WRITE_FLUSH(hw);
2342 }
2343
2344 static int
2345 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2346 {
2347         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2348         struct rte_pci_device *pdev = RTE_DEV_TO_PCI(dev->device);
2349
2350         /* Enable ITR */
2351         if (hw->mac.type == fm10k_mac_pf)
2352                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2353                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2354         else
2355                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2356                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2357         rte_intr_enable(&pdev->intr_handle);
2358         return 0;
2359 }
2360
2361 static int
2362 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2363 {
2364         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2365         struct rte_pci_device *pdev = RTE_DEV_TO_PCI(dev->device);
2366
2367         /* Disable ITR */
2368         if (hw->mac.type == fm10k_mac_pf)
2369                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2370                         FM10K_ITR_MASK_SET);
2371         else
2372                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2373                         FM10K_ITR_MASK_SET);
2374         return 0;
2375 }
2376
2377 static int
2378 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2379 {
2380         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2381         struct rte_pci_device *pdev = RTE_DEV_TO_PCI(dev->device);
2382         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2383         uint32_t intr_vector, vec;
2384         uint16_t queue_id;
2385         int result = 0;
2386
2387         /* fm10k needs one separate interrupt for mailbox,
2388          * so only drivers which support multiple interrupt vectors
2389          * e.g. vfio-pci can work for fm10k interrupt mode
2390          */
2391         if (!rte_intr_cap_multiple(intr_handle) ||
2392                         dev->data->dev_conf.intr_conf.rxq == 0)
2393                 return result;
2394
2395         intr_vector = dev->data->nb_rx_queues;
2396
2397         /* disable interrupt first */
2398         rte_intr_disable(intr_handle);
2399         if (hw->mac.type == fm10k_mac_pf)
2400                 fm10k_dev_disable_intr_pf(dev);
2401         else
2402                 fm10k_dev_disable_intr_vf(dev);
2403
2404         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2405                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2406                 result = -EIO;
2407         }
2408
2409         if (rte_intr_dp_is_en(intr_handle) && !result) {
2410                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2411                         dev->data->nb_rx_queues * sizeof(int), 0);
2412                 if (intr_handle->intr_vec) {
2413                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2414                                         queue_id < dev->data->nb_rx_queues;
2415                                         queue_id++) {
2416                                 intr_handle->intr_vec[queue_id] = vec;
2417                                 if (vec < intr_handle->nb_efd - 1
2418                                                 + FM10K_RX_VEC_START)
2419                                         vec++;
2420                         }
2421                 } else {
2422                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2423                                 " intr_vec", dev->data->nb_rx_queues);
2424                         rte_intr_efd_disable(intr_handle);
2425                         result = -ENOMEM;
2426                 }
2427         }
2428
2429         if (hw->mac.type == fm10k_mac_pf)
2430                 fm10k_dev_enable_intr_pf(dev);
2431         else
2432                 fm10k_dev_enable_intr_vf(dev);
2433         rte_intr_enable(intr_handle);
2434         hw->mac.ops.update_int_moderator(hw);
2435         return result;
2436 }
2437
2438 static int
2439 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2440 {
2441         struct fm10k_fault fault;
2442         int err;
2443         const char *estr = "Unknown error";
2444
2445         /* Process PCA fault */
2446         if (eicr & FM10K_EICR_PCA_FAULT) {
2447                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2448                 if (err)
2449                         goto error;
2450                 switch (fault.type) {
2451                 case PCA_NO_FAULT:
2452                         estr = "PCA_NO_FAULT"; break;
2453                 case PCA_UNMAPPED_ADDR:
2454                         estr = "PCA_UNMAPPED_ADDR"; break;
2455                 case PCA_BAD_QACCESS_PF:
2456                         estr = "PCA_BAD_QACCESS_PF"; break;
2457                 case PCA_BAD_QACCESS_VF:
2458                         estr = "PCA_BAD_QACCESS_VF"; break;
2459                 case PCA_MALICIOUS_REQ:
2460                         estr = "PCA_MALICIOUS_REQ"; break;
2461                 case PCA_POISONED_TLP:
2462                         estr = "PCA_POISONED_TLP"; break;
2463                 case PCA_TLP_ABORT:
2464                         estr = "PCA_TLP_ABORT"; break;
2465                 default:
2466                         goto error;
2467                 }
2468                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2469                         estr, fault.func ? "VF" : "PF", fault.func,
2470                         fault.address, fault.specinfo);
2471         }
2472
2473         /* Process THI fault */
2474         if (eicr & FM10K_EICR_THI_FAULT) {
2475                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2476                 if (err)
2477                         goto error;
2478                 switch (fault.type) {
2479                 case THI_NO_FAULT:
2480                         estr = "THI_NO_FAULT"; break;
2481                 case THI_MAL_DIS_Q_FAULT:
2482                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2483                 default:
2484                         goto error;
2485                 }
2486                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2487                         estr, fault.func ? "VF" : "PF", fault.func,
2488                         fault.address, fault.specinfo);
2489         }
2490
2491         /* Process FUM fault */
2492         if (eicr & FM10K_EICR_FUM_FAULT) {
2493                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2494                 if (err)
2495                         goto error;
2496                 switch (fault.type) {
2497                 case FUM_NO_FAULT:
2498                         estr = "FUM_NO_FAULT"; break;
2499                 case FUM_UNMAPPED_ADDR:
2500                         estr = "FUM_UNMAPPED_ADDR"; break;
2501                 case FUM_POISONED_TLP:
2502                         estr = "FUM_POISONED_TLP"; break;
2503                 case FUM_BAD_VF_QACCESS:
2504                         estr = "FUM_BAD_VF_QACCESS"; break;
2505                 case FUM_ADD_DECODE_ERR:
2506                         estr = "FUM_ADD_DECODE_ERR"; break;
2507                 case FUM_RO_ERROR:
2508                         estr = "FUM_RO_ERROR"; break;
2509                 case FUM_QPRC_CRC_ERROR:
2510                         estr = "FUM_QPRC_CRC_ERROR"; break;
2511                 case FUM_CSR_TIMEOUT:
2512                         estr = "FUM_CSR_TIMEOUT"; break;
2513                 case FUM_INVALID_TYPE:
2514                         estr = "FUM_INVALID_TYPE"; break;
2515                 case FUM_INVALID_LENGTH:
2516                         estr = "FUM_INVALID_LENGTH"; break;
2517                 case FUM_INVALID_BE:
2518                         estr = "FUM_INVALID_BE"; break;
2519                 case FUM_INVALID_ALIGN:
2520                         estr = "FUM_INVALID_ALIGN"; break;
2521                 default:
2522                         goto error;
2523                 }
2524                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2525                         estr, fault.func ? "VF" : "PF", fault.func,
2526                         fault.address, fault.specinfo);
2527         }
2528
2529         return 0;
2530 error:
2531         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2532         return err;
2533 }
2534
2535 /**
2536  * PF interrupt handler triggered by NIC for handling specific interrupt.
2537  *
2538  * @param handle
2539  *  Pointer to interrupt handle.
2540  * @param param
2541  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2542  *
2543  * @return
2544  *  void
2545  */
2546 static void
2547 fm10k_dev_interrupt_handler_pf(void *param)
2548 {
2549         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2550         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2551         uint32_t cause, status;
2552
2553         if (hw->mac.type != fm10k_mac_pf)
2554                 return;
2555
2556         cause = FM10K_READ_REG(hw, FM10K_EICR);
2557
2558         /* Handle PCI fault cases */
2559         if (cause & FM10K_EICR_FAULT_MASK) {
2560                 PMD_INIT_LOG(ERR, "INT: find fault!");
2561                 fm10k_dev_handle_fault(hw, cause);
2562         }
2563
2564         /* Handle switch up/down */
2565         if (cause & FM10K_EICR_SWITCHNOTREADY)
2566                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2567
2568         if (cause & FM10K_EICR_SWITCHREADY)
2569                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2570
2571         /* Handle mailbox message */
2572         fm10k_mbx_lock(hw);
2573         hw->mbx.ops.process(hw, &hw->mbx);
2574         fm10k_mbx_unlock(hw);
2575
2576         /* Handle SRAM error */
2577         if (cause & FM10K_EICR_SRAMERROR) {
2578                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2579
2580                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2581                 /* Write to clear pending bits */
2582                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2583
2584                 /* Todo: print out error message after shared code  updates */
2585         }
2586
2587         /* Clear these 3 events if having any */
2588         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2589                  FM10K_EICR_SWITCHREADY;
2590         if (cause)
2591                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2592
2593         /* Re-enable interrupt from device side */
2594         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2595                                         FM10K_ITR_MASK_CLEAR);
2596         /* Re-enable interrupt from host side */
2597         rte_intr_enable(dev->intr_handle);
2598 }
2599
2600 /**
2601  * VF interrupt handler triggered by NIC for handling specific interrupt.
2602  *
2603  * @param handle
2604  *  Pointer to interrupt handle.
2605  * @param param
2606  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2607  *
2608  * @return
2609  *  void
2610  */
2611 static void
2612 fm10k_dev_interrupt_handler_vf(void *param)
2613 {
2614         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2615         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2616
2617         if (hw->mac.type != fm10k_mac_vf)
2618                 return;
2619
2620         /* Handle mailbox message if lock is acquired */
2621         fm10k_mbx_lock(hw);
2622         hw->mbx.ops.process(hw, &hw->mbx);
2623         fm10k_mbx_unlock(hw);
2624
2625         /* Re-enable interrupt from device side */
2626         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2627                                         FM10K_ITR_MASK_CLEAR);
2628         /* Re-enable interrupt from host side */
2629         rte_intr_enable(dev->intr_handle);
2630 }
2631
2632 /* Mailbox message handler in VF */
2633 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2634         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2635         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2636         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2637         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2638 };
2639
2640 static int
2641 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2642 {
2643         int err = 0;
2644
2645         /* Initialize mailbox lock */
2646         fm10k_mbx_initlock(hw);
2647
2648         /* Replace default message handler with new ones */
2649         if (hw->mac.type == fm10k_mac_vf)
2650                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2651
2652         if (err) {
2653                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2654                                 err);
2655                 return err;
2656         }
2657         /* Connect to SM for PF device or PF for VF device */
2658         return hw->mbx.ops.connect(hw, &hw->mbx);
2659 }
2660
2661 static void
2662 fm10k_close_mbx_service(struct fm10k_hw *hw)
2663 {
2664         /* Disconnect from SM for PF device or PF for VF device */
2665         hw->mbx.ops.disconnect(hw, &hw->mbx);
2666 }
2667
2668 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2669         .dev_configure          = fm10k_dev_configure,
2670         .dev_start              = fm10k_dev_start,
2671         .dev_stop               = fm10k_dev_stop,
2672         .dev_close              = fm10k_dev_close,
2673         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2674         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2675         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2676         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2677         .stats_get              = fm10k_stats_get,
2678         .xstats_get             = fm10k_xstats_get,
2679         .xstats_get_names       = fm10k_xstats_get_names,
2680         .stats_reset            = fm10k_stats_reset,
2681         .xstats_reset           = fm10k_stats_reset,
2682         .link_update            = fm10k_link_update,
2683         .dev_infos_get          = fm10k_dev_infos_get,
2684         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2685         .vlan_filter_set        = fm10k_vlan_filter_set,
2686         .vlan_offload_set       = fm10k_vlan_offload_set,
2687         .mac_addr_add           = fm10k_macaddr_add,
2688         .mac_addr_remove        = fm10k_macaddr_remove,
2689         .rx_queue_start         = fm10k_dev_rx_queue_start,
2690         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2691         .tx_queue_start         = fm10k_dev_tx_queue_start,
2692         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2693         .rx_queue_setup         = fm10k_rx_queue_setup,
2694         .rx_queue_release       = fm10k_rx_queue_release,
2695         .tx_queue_setup         = fm10k_tx_queue_setup,
2696         .tx_queue_release       = fm10k_tx_queue_release,
2697         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2698         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2699         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2700         .reta_update            = fm10k_reta_update,
2701         .reta_query             = fm10k_reta_query,
2702         .rss_hash_update        = fm10k_rss_hash_update,
2703         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2704 };
2705
2706 static int ftag_check_handler(__rte_unused const char *key,
2707                 const char *value, __rte_unused void *opaque)
2708 {
2709         if (strcmp(value, "1"))
2710                 return -1;
2711
2712         return 0;
2713 }
2714
2715 static int
2716 fm10k_check_ftag(struct rte_devargs *devargs)
2717 {
2718         struct rte_kvargs *kvlist;
2719         const char *ftag_key = "enable_ftag";
2720
2721         if (devargs == NULL)
2722                 return 0;
2723
2724         kvlist = rte_kvargs_parse(devargs->args, NULL);
2725         if (kvlist == NULL)
2726                 return 0;
2727
2728         if (!rte_kvargs_count(kvlist, ftag_key)) {
2729                 rte_kvargs_free(kvlist);
2730                 return 0;
2731         }
2732         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2733         if (rte_kvargs_process(kvlist, ftag_key,
2734                                 ftag_check_handler, NULL) < 0) {
2735                 rte_kvargs_free(kvlist);
2736                 return 0;
2737         }
2738         rte_kvargs_free(kvlist);
2739
2740         return 1;
2741 }
2742
2743 static uint16_t
2744 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2745                     uint16_t nb_pkts)
2746 {
2747         uint16_t nb_tx = 0;
2748         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2749
2750         while (nb_pkts) {
2751                 uint16_t ret, num;
2752
2753                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2754                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2755                                                  num);
2756                 nb_tx += ret;
2757                 nb_pkts -= ret;
2758                 if (ret < num)
2759                         break;
2760         }
2761
2762         return nb_tx;
2763 }
2764
2765 static void __attribute__((cold))
2766 fm10k_set_tx_function(struct rte_eth_dev *dev)
2767 {
2768         struct fm10k_tx_queue *txq;
2769         int i;
2770         int use_sse = 1;
2771         uint16_t tx_ftag_en = 0;
2772
2773         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2774                 /* primary process has set the ftag flag and txq_flags */
2775                 txq = dev->data->tx_queues[0];
2776                 if (fm10k_tx_vec_condition_check(txq)) {
2777                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2778                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2779                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2780                 } else {
2781                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2782                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2783                         dev->tx_pkt_prepare = NULL;
2784                 }
2785                 return;
2786         }
2787
2788         if (fm10k_check_ftag(dev->device->devargs))
2789                 tx_ftag_en = 1;
2790
2791         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2792                 txq = dev->data->tx_queues[i];
2793                 txq->tx_ftag_en = tx_ftag_en;
2794                 /* Check if Vector Tx is satisfied */
2795                 if (fm10k_tx_vec_condition_check(txq))
2796                         use_sse = 0;
2797         }
2798
2799         if (use_sse) {
2800                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2801                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2802                         txq = dev->data->tx_queues[i];
2803                         fm10k_txq_vec_setup(txq);
2804                 }
2805                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2806                 dev->tx_pkt_prepare = NULL;
2807         } else {
2808                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2809                 dev->tx_pkt_prepare = fm10k_prep_pkts;
2810                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2811         }
2812 }
2813
2814 static void __attribute__((cold))
2815 fm10k_set_rx_function(struct rte_eth_dev *dev)
2816 {
2817         struct fm10k_dev_info *dev_info =
2818                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2819         uint16_t i, rx_using_sse;
2820         uint16_t rx_ftag_en = 0;
2821
2822         if (fm10k_check_ftag(dev->device->devargs))
2823                 rx_ftag_en = 1;
2824
2825         /* In order to allow Vector Rx there are a few configuration
2826          * conditions to be met.
2827          */
2828         if (!fm10k_rx_vec_condition_check(dev) &&
2829                         dev_info->rx_vec_allowed && !rx_ftag_en) {
2830                 if (dev->data->scattered_rx)
2831                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2832                 else
2833                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2834         } else if (dev->data->scattered_rx)
2835                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2836         else
2837                 dev->rx_pkt_burst = fm10k_recv_pkts;
2838
2839         rx_using_sse =
2840                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2841                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2842
2843         if (rx_using_sse)
2844                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
2845         else
2846                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
2847
2848         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2849                 return;
2850
2851         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2852                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2853
2854                 rxq->rx_using_sse = rx_using_sse;
2855                 rxq->rx_ftag_en = rx_ftag_en;
2856         }
2857 }
2858
2859 static void
2860 fm10k_params_init(struct rte_eth_dev *dev)
2861 {
2862         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2863         struct fm10k_dev_info *info =
2864                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2865
2866         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2867          * there is no way to get link status without reading BAR4.  Until this
2868          * works, assume we have maximum bandwidth.
2869          * @todo - fix bus info
2870          */
2871         hw->bus_caps.speed = fm10k_bus_speed_8000;
2872         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2873         hw->bus_caps.payload = fm10k_bus_payload_512;
2874         hw->bus.speed = fm10k_bus_speed_8000;
2875         hw->bus.width = fm10k_bus_width_pcie_x8;
2876         hw->bus.payload = fm10k_bus_payload_256;
2877
2878         info->rx_vec_allowed = true;
2879 }
2880
2881 static int
2882 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2883 {
2884         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2885         struct rte_pci_device *pdev = RTE_DEV_TO_PCI(dev->device);
2886         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2887         int diag, i;
2888         struct fm10k_macvlan_filter_info *macvlan;
2889
2890         PMD_INIT_FUNC_TRACE();
2891
2892         dev->dev_ops = &fm10k_eth_dev_ops;
2893         dev->rx_pkt_burst = &fm10k_recv_pkts;
2894         dev->tx_pkt_burst = &fm10k_xmit_pkts;
2895         dev->tx_pkt_prepare = &fm10k_prep_pkts;
2896
2897         /*
2898          * Primary process does the whole initialization, for secondary
2899          * processes, we just select the same Rx and Tx function as primary.
2900          */
2901         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2902                 fm10k_set_rx_function(dev);
2903                 fm10k_set_tx_function(dev);
2904                 return 0;
2905         }
2906
2907         rte_eth_copy_pci_info(dev, pdev);
2908         dev->data->dev_flags |= RTE_ETH_DEV_DETACHABLE;
2909
2910         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2911         memset(macvlan, 0, sizeof(*macvlan));
2912         /* Vendor and Device ID need to be set before init of shared code */
2913         memset(hw, 0, sizeof(*hw));
2914         hw->device_id = pdev->id.device_id;
2915         hw->vendor_id = pdev->id.vendor_id;
2916         hw->subsystem_device_id = pdev->id.subsystem_device_id;
2917         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
2918         hw->revision_id = 0;
2919         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
2920         if (hw->hw_addr == NULL) {
2921                 PMD_INIT_LOG(ERR, "Bad mem resource."
2922                         " Try to blacklist unused devices.");
2923                 return -EIO;
2924         }
2925
2926         /* Store fm10k_adapter pointer */
2927         hw->back = dev->data->dev_private;
2928
2929         /* Initialize the shared code */
2930         diag = fm10k_init_shared_code(hw);
2931         if (diag != FM10K_SUCCESS) {
2932                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2933                 return -EIO;
2934         }
2935
2936         /* Initialize parameters */
2937         fm10k_params_init(dev);
2938
2939         /* Initialize the hw */
2940         diag = fm10k_init_hw(hw);
2941         if (diag != FM10K_SUCCESS) {
2942                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2943                 return -EIO;
2944         }
2945
2946         /* Initialize MAC address(es) */
2947         dev->data->mac_addrs = rte_zmalloc("fm10k",
2948                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2949         if (dev->data->mac_addrs == NULL) {
2950                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2951                 return -ENOMEM;
2952         }
2953
2954         diag = fm10k_read_mac_addr(hw);
2955
2956         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2957                         &dev->data->mac_addrs[0]);
2958
2959         if (diag != FM10K_SUCCESS ||
2960                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2961
2962                 /* Generate a random addr */
2963                 eth_random_addr(hw->mac.addr);
2964                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2965                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2966                 &dev->data->mac_addrs[0]);
2967         }
2968
2969         /* Reset the hw statistics */
2970         fm10k_stats_reset(dev);
2971
2972         /* Reset the hw */
2973         diag = fm10k_reset_hw(hw);
2974         if (diag != FM10K_SUCCESS) {
2975                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2976                 return -EIO;
2977         }
2978
2979         /* Setup mailbox service */
2980         diag = fm10k_setup_mbx_service(hw);
2981         if (diag != FM10K_SUCCESS) {
2982                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2983                 return -EIO;
2984         }
2985
2986         /*PF/VF has different interrupt handling mechanism */
2987         if (hw->mac.type == fm10k_mac_pf) {
2988                 /* register callback func to eal lib */
2989                 rte_intr_callback_register(intr_handle,
2990                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2991
2992                 /* enable MISC interrupt */
2993                 fm10k_dev_enable_intr_pf(dev);
2994         } else { /* VF */
2995                 rte_intr_callback_register(intr_handle,
2996                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2997
2998                 fm10k_dev_enable_intr_vf(dev);
2999         }
3000
3001         /* Enable intr after callback registered */
3002         rte_intr_enable(intr_handle);
3003
3004         hw->mac.ops.update_int_moderator(hw);
3005
3006         /* Make sure Switch Manager is ready before going forward. */
3007         if (hw->mac.type == fm10k_mac_pf) {
3008                 int switch_ready = 0;
3009
3010                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3011                         fm10k_mbx_lock(hw);
3012                         hw->mac.ops.get_host_state(hw, &switch_ready);
3013                         fm10k_mbx_unlock(hw);
3014                         if (switch_ready)
3015                                 break;
3016                         /* Delay some time to acquire async LPORT_MAP info. */
3017                         rte_delay_us(WAIT_SWITCH_MSG_US);
3018                 }
3019
3020                 if (switch_ready == 0) {
3021                         PMD_INIT_LOG(ERR, "switch is not ready");
3022                         return -1;
3023                 }
3024         }
3025
3026         /*
3027          * Below function will trigger operations on mailbox, acquire lock to
3028          * avoid race condition from interrupt handler. Operations on mailbox
3029          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3030          * will handle and generate an interrupt to our side. Then,  FIFO in
3031          * mailbox will be touched.
3032          */
3033         fm10k_mbx_lock(hw);
3034         /* Enable port first */
3035         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3036                                         MAX_LPORT_NUM, 1);
3037
3038         /* Set unicast mode by default. App can change to other mode in other
3039          * API func.
3040          */
3041         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3042                                         FM10K_XCAST_MODE_NONE);
3043
3044         fm10k_mbx_unlock(hw);
3045
3046         /* Make sure default VID is ready before going forward. */
3047         if (hw->mac.type == fm10k_mac_pf) {
3048                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3049                         if (hw->mac.default_vid)
3050                                 break;
3051                         /* Delay some time to acquire async port VLAN info. */
3052                         rte_delay_us(WAIT_SWITCH_MSG_US);
3053                 }
3054
3055                 if (!hw->mac.default_vid) {
3056                         PMD_INIT_LOG(ERR, "default VID is not ready");
3057                         return -1;
3058                 }
3059         }
3060
3061         /* Add default mac address */
3062         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3063                 MAIN_VSI_POOL_NUMBER);
3064
3065         return 0;
3066 }
3067
3068 static int
3069 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3070 {
3071         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3072         struct rte_pci_device *pdev = RTE_DEV_TO_PCI(dev->device);
3073         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3074         PMD_INIT_FUNC_TRACE();
3075
3076         /* only uninitialize in the primary process */
3077         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3078                 return 0;
3079
3080         /* safe to close dev here */
3081         fm10k_dev_close(dev);
3082
3083         dev->dev_ops = NULL;
3084         dev->rx_pkt_burst = NULL;
3085         dev->tx_pkt_burst = NULL;
3086
3087         /* disable uio/vfio intr */
3088         rte_intr_disable(intr_handle);
3089
3090         /*PF/VF has different interrupt handling mechanism */
3091         if (hw->mac.type == fm10k_mac_pf) {
3092                 /* disable interrupt */
3093                 fm10k_dev_disable_intr_pf(dev);
3094
3095                 /* unregister callback func to eal lib */
3096                 rte_intr_callback_unregister(intr_handle,
3097                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3098         } else {
3099                 /* disable interrupt */
3100                 fm10k_dev_disable_intr_vf(dev);
3101
3102                 rte_intr_callback_unregister(intr_handle,
3103                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3104         }
3105
3106         /* free mac memory */
3107         if (dev->data->mac_addrs) {
3108                 rte_free(dev->data->mac_addrs);
3109                 dev->data->mac_addrs = NULL;
3110         }
3111
3112         memset(hw, 0, sizeof(*hw));
3113
3114         return 0;
3115 }
3116
3117 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3118         struct rte_pci_device *pci_dev)
3119 {
3120         return rte_eth_dev_pci_generic_probe(pci_dev,
3121                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3122 }
3123
3124 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3125 {
3126         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3127 }
3128
3129 /*
3130  * The set of PCI devices this driver supports. This driver will enable both PF
3131  * and SRIOV-VF devices.
3132  */
3133 static const struct rte_pci_id pci_id_fm10k_map[] = {
3134         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3135         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3136         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3137         { .vendor_id = 0, /* sentinel */ },
3138 };
3139
3140 static struct rte_pci_driver rte_pmd_fm10k = {
3141         .id_table = pci_id_fm10k_map,
3142         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
3143         .probe = eth_fm10k_pci_probe,
3144         .remove = eth_fm10k_pci_remove,
3145 };
3146
3147 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3148 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3149 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");