drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 /* Default delay to acquire mailbox lock */
  45 #define FM10K_MBXLOCK_DELAY_US 20
  46 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  47
  48 #define MAIN_VSI_POOL_NUMBER 0
  49
  50 /* Max try times to acquire switch status */
  51 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  52 /* Wait interval to get switch status */
  53 #define WAIT_SWITCH_MSG_US    100000
  54 /* Number of chars per uint32 type */
  55 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  56 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  57
  58 /* default 1:1 map from queue ID to interrupt vector ID */
  59 #define Q2V(dev, queue_id) (dev->pci_dev->intr_handle.intr_vec[queue_id])
  60
  61 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  62 #define MAX_LPORT_NUM    128
  63 #define GLORT_FD_Q_BASE  0x40
  64 #define GLORT_PF_MASK    0xFFC0
  65 #define GLORT_FD_MASK    GLORT_PF_MASK
  66 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  67
  68 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  69 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  70 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  71 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  72 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  73 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  74 static int
  75 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  76 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  77         const u8 *mac, bool add, uint32_t pool);
  78 static void fm10k_tx_queue_release(void *queue);
  79 static void fm10k_rx_queue_release(void *queue);
  80 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  81 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  82
  83 struct fm10k_xstats_name_off {
  84         char name[RTE_ETH_XSTATS_NAME_SIZE];
  85         unsigned offset;
  86 };
  87
  88 struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  89         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  90         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  91         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  92         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  93         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  94         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  95         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  96         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  97                 nodesc_drop)},
  98 };
  99
 100 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
 101                 sizeof(fm10k_hw_stats_strings[0]))
 102
 103 struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
 104         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
 105         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
 106         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
 107 };
 108
 109 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
 110                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
 111
 112 struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 113         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 114         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 115 };
 116
 117 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 118                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 119
 120 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 121                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 122 static int
 123 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 124
 125 static void
 126 fm10k_mbx_initlock(struct fm10k_hw *hw)
 127 {
 128         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 129 }
 130
 131 static void
 132 fm10k_mbx_lock(struct fm10k_hw *hw)
 133 {
 134         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 135                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 136 }
 137
 138 static void
 139 fm10k_mbx_unlock(struct fm10k_hw *hw)
 140 {
 141         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 142 }
 143
 144 /* Stubs needed for linkage when vPMD is disabled */
 145 int __attribute__((weak))
 146 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 147 {
 148         return -1;
 149 }
 150
 151 uint16_t __attribute__((weak))
 152 fm10k_recv_pkts_vec(
 153         __rte_unused void *rx_queue,
 154         __rte_unused struct rte_mbuf **rx_pkts,
 155         __rte_unused uint16_t nb_pkts)
 156 {
 157         return 0;
 158 }
 159
 160 uint16_t __attribute__((weak))
 161 fm10k_recv_scattered_pkts_vec(
 162                 __rte_unused void *rx_queue,
 163                 __rte_unused struct rte_mbuf **rx_pkts,
 164                 __rte_unused uint16_t nb_pkts)
 165 {
 166         return 0;
 167 }
 168
 169 int __attribute__((weak))
 170 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 171
 172 {
 173         return -1;
 174 }
 175
 176 void __attribute__((weak))
 177 fm10k_rx_queue_release_mbufs_vec(
 178                 __rte_unused struct fm10k_rx_queue *rxq)
 179 {
 180         return;
 181 }
 182
 183 void __attribute__((weak))
 184 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 185 {
 186         return;
 187 }
 188
 189 int __attribute__((weak))
 190 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 191 {
 192         return -1;
 193 }
 194
 195 uint16_t __attribute__((weak))
 196 fm10k_xmit_pkts_vec(__rte_unused void *tx_queue,
 197                 __rte_unused struct rte_mbuf **tx_pkts,
 198                 __rte_unused uint16_t nb_pkts)
 199 {
 200         return 0;
 201 }
 202
 203 /*
 204  * reset queue to initial state, allocate software buffers used when starting
 205  * device.
 206  * return 0 on success
 207  * return -ENOMEM if buffers cannot be allocated
 208  * return -EINVAL if buffers do not satisfy alignment condition
 209  */
 210 static inline int
 211 rx_queue_reset(struct fm10k_rx_queue *q)
 212 {
 213         static const union fm10k_rx_desc zero = {{0} };
 214         uint64_t dma_addr;
 215         int i, diag;
 216         PMD_INIT_FUNC_TRACE();
 217
 218         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 219         if (diag != 0)
 220                 return -ENOMEM;
 221
 222         for (i = 0; i < q->nb_desc; ++i) {
 223                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 224                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 225                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 226                                                 q->nb_desc);
 227                         return -EINVAL;
 228                 }
 229                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 230                 q->hw_ring[i].q.pkt_addr = dma_addr;
 231                 q->hw_ring[i].q.hdr_addr = dma_addr;
 232         }
 233
 234         /* initialize extra software ring entries. Space for these extra
 235          * entries is always allocated.
 236          */
 237         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 238         for (i = 0; i < q->nb_fake_desc; ++i) {
 239                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 240                 q->hw_ring[q->nb_desc + i] = zero;
 241         }
 242
 243         q->next_dd = 0;
 244         q->next_alloc = 0;
 245         q->next_trigger = q->alloc_thresh - 1;
 246         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 247         q->rxrearm_start = 0;
 248         q->rxrearm_nb = 0;
 249
 250         return 0;
 251 }
 252
 253 /*
 254  * clean queue, descriptor rings, free software buffers used when stopping
 255  * device.
 256  */
 257 static inline void
 258 rx_queue_clean(struct fm10k_rx_queue *q)
 259 {
 260         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 261         uint32_t i;
 262         PMD_INIT_FUNC_TRACE();
 263
 264         /* zero descriptor rings */
 265         for (i = 0; i < q->nb_desc; ++i)
 266                 q->hw_ring[i] = zero;
 267
 268         /* zero faked descriptors */
 269         for (i = 0; i < q->nb_fake_desc; ++i)
 270                 q->hw_ring[q->nb_desc + i] = zero;
 271
 272         /* vPMD driver has a different way of releasing mbufs. */
 273         if (q->rx_using_sse) {
 274                 fm10k_rx_queue_release_mbufs_vec(q);
 275                 return;
 276         }
 277
 278         /* free software buffers */
 279         for (i = 0; i < q->nb_desc; ++i) {
 280                 if (q->sw_ring[i]) {
 281                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 282                         q->sw_ring[i] = NULL;
 283                 }
 284         }
 285 }
 286
 287 /*
 288  * free all queue memory used when releasing the queue (i.e. configure)
 289  */
 290 static inline void
 291 rx_queue_free(struct fm10k_rx_queue *q)
 292 {
 293         PMD_INIT_FUNC_TRACE();
 294         if (q) {
 295                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 296                 rx_queue_clean(q);
 297                 if (q->sw_ring) {
 298                         rte_free(q->sw_ring);
 299                         q->sw_ring = NULL;
 300                 }
 301                 rte_free(q);
 302                 q = NULL;
 303         }
 304 }
 305
 306 /*
 307  * disable RX queue, wait unitl HW finished necessary flush operation
 308  */
 309 static inline int
 310 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 311 {
 312         uint32_t reg, i;
 313
 314         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 315         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 316                         reg & ~FM10K_RXQCTL_ENABLE);
 317
 318         /* Wait 100us at most */
 319         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 320                 rte_delay_us(1);
 321                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 322                 if (!(reg & FM10K_RXQCTL_ENABLE))
 323                         break;
 324         }
 325
 326         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 327                 return -1;
 328
 329         return 0;
 330 }
 331
 332 /*
 333  * reset queue to initial state, allocate software buffers used when starting
 334  * device
 335  */
 336 static inline void
 337 tx_queue_reset(struct fm10k_tx_queue *q)
 338 {
 339         PMD_INIT_FUNC_TRACE();
 340         q->last_free = 0;
 341         q->next_free = 0;
 342         q->nb_used = 0;
 343         q->nb_free = q->nb_desc - 1;
 344         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 345         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 346 }
 347
 348 /*
 349  * clean queue, descriptor rings, free software buffers used when stopping
 350  * device
 351  */
 352 static inline void
 353 tx_queue_clean(struct fm10k_tx_queue *q)
 354 {
 355         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 356         uint32_t i;
 357         PMD_INIT_FUNC_TRACE();
 358
 359         /* zero descriptor rings */
 360         for (i = 0; i < q->nb_desc; ++i)
 361                 q->hw_ring[i] = zero;
 362
 363         /* free software buffers */
 364         for (i = 0; i < q->nb_desc; ++i) {
 365                 if (q->sw_ring[i]) {
 366                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 367                         q->sw_ring[i] = NULL;
 368                 }
 369         }
 370 }
 371
 372 /*
 373  * free all queue memory used when releasing the queue (i.e. configure)
 374  */
 375 static inline void
 376 tx_queue_free(struct fm10k_tx_queue *q)
 377 {
 378         PMD_INIT_FUNC_TRACE();
 379         if (q) {
 380                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 381                 tx_queue_clean(q);
 382                 if (q->rs_tracker.list) {
 383                         rte_free(q->rs_tracker.list);
 384                         q->rs_tracker.list = NULL;
 385                 }
 386                 if (q->sw_ring) {
 387                         rte_free(q->sw_ring);
 388                         q->sw_ring = NULL;
 389                 }
 390                 rte_free(q);
 391                 q = NULL;
 392         }
 393 }
 394
 395 /*
 396  * disable TX queue, wait unitl HW finished necessary flush operation
 397  */
 398 static inline int
 399 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 400 {
 401         uint32_t reg, i;
 402
 403         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 404         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 405                         reg & ~FM10K_TXDCTL_ENABLE);
 406
 407         /* Wait 100us at most */
 408         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 409                 rte_delay_us(1);
 410                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 411                 if (!(reg & FM10K_TXDCTL_ENABLE))
 412                         break;
 413         }
 414
 415         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 416                 return -1;
 417
 418         return 0;
 419 }
 420
 421 static int
 422 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 423 {
 424         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 425         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 426         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 427         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 428
 429         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 430
 431         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 432                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 433                 return -EINVAL;
 434         }
 435
 436         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 437                 return 0;
 438
 439         if (hw->mac.type == fm10k_mac_vf) {
 440                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 441                 return -EINVAL;
 442         }
 443
 444         /* Check VMDQ queue pool number */
 445         if (vmdq_conf->nb_queue_pools >
 446                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 447                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 448                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 449                         vmdq_conf->nb_queue_pools);
 450                 return -EINVAL;
 451         }
 452
 453         return 0;
 454 }
 455
 456 static const struct fm10k_txq_ops def_txq_ops = {
 457         .reset = tx_queue_reset,
 458 };
 459
 460 static int
 461 fm10k_dev_configure(struct rte_eth_dev *dev)
 462 {
 463         int ret;
 464
 465         PMD_INIT_FUNC_TRACE();
 466
 467         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 468                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 469         /* multipe queue mode checking */
 470         ret  = fm10k_check_mq_mode(dev);
 471         if (ret != 0) {
 472                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 473                             ret);
 474                 return ret;
 475         }
 476
 477         return 0;
 478 }
 479
 480 /* fls = find last set bit = 32 minus the number of leading zeros */
 481 #ifndef fls
 482 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 483 #endif
 484
 485 static void
 486 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 487 {
 488         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 489         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 490         uint32_t i;
 491
 492         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 493
 494         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 495                 if (!vmdq_conf->pool_map[i].pools)
 496                         continue;
 497                 fm10k_mbx_lock(hw);
 498                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 499                 fm10k_mbx_unlock(hw);
 500         }
 501 }
 502
 503 static void
 504 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 505 {
 506         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 507
 508         /* Add default mac address */
 509         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 510                 MAIN_VSI_POOL_NUMBER);
 511 }
 512
 513 static void
 514 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 515 {
 516         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 517         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 518         uint32_t mrqc, *key, i, reta, j;
 519         uint64_t hf;
 520
 521 #define RSS_KEY_SIZE 40
 522         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 523                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 524                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 525                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 526                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 527                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 528         };
 529
 530         if (dev->data->nb_rx_queues == 1 ||
 531             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 532             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 533                 return;
 534
 535         /* random key is rss_intel_key (default) or user provided (rss_key) */
 536         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 537                 key = (uint32_t *)rss_intel_key;
 538         else
 539                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 540
 541         /* Now fill our hash function seeds, 4 bytes at a time */
 542         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 543                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 544
 545         /*
 546          * Fill in redirection table
 547          * The byte-swap is needed because NIC registers are in
 548          * little-endian order.
 549          */
 550         reta = 0;
 551         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 552                 if (j == dev->data->nb_rx_queues)
 553                         j = 0;
 554                 reta = (reta << CHAR_BIT) | j;
 555                 if ((i & 3) == 3)
 556                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 557                                         rte_bswap32(reta));
 558         }
 559
 560         /*
 561          * Generate RSS hash based on packet types, TCP/UDP
 562          * port numbers and/or IPv4/v6 src and dst addresses
 563          */
 564         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 565         mrqc = 0;
 566         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 567         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 568         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 569         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 570         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 571         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 572         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 573         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 574         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 575
 576         if (mrqc == 0) {
 577                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 578                         "supported", hf);
 579                 return;
 580         }
 581
 582         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 583 }
 584
 585 static void
 586 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 587 {
 588         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 589         uint32_t i;
 590
 591         for (i = 0; i < nb_lport_new; i++) {
 592                 /* Set unicast mode by default. App can change
 593                  * to other mode in other API func.
 594                  */
 595                 fm10k_mbx_lock(hw);
 596                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 597                         FM10K_XCAST_MODE_NONE);
 598                 fm10k_mbx_unlock(hw);
 599         }
 600 }
 601
 602 static void
 603 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 604 {
 605         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 606         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 607         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 608         struct fm10k_macvlan_filter_info *macvlan;
 609         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 610         uint16_t nb_lport_new;
 611
 612         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 613         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 614
 615         fm10k_dev_rss_configure(dev);
 616
 617         /* only PF supports VMDQ */
 618         if (hw->mac.type != fm10k_mac_pf)
 619                 return;
 620
 621         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 622                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 623
 624         /* no pool number change, no need to update logic port and VLAN/MAC */
 625         if (macvlan->nb_queue_pools == nb_queue_pools)
 626                 return;
 627
 628         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 629         fm10k_dev_logic_port_update(dev, nb_lport_new);
 630
 631         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 632         memset(dev->data->mac_addrs, 0,
 633                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 634         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 635                 &dev->data->mac_addrs[0]);
 636         memset(macvlan, 0, sizeof(*macvlan));
 637         macvlan->nb_queue_pools = nb_queue_pools;
 638
 639         if (nb_queue_pools)
 640                 fm10k_dev_vmdq_rx_configure(dev);
 641         else
 642                 fm10k_dev_pf_main_vsi_reset(dev);
 643 }
 644
 645 static int
 646 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 647 {
 648         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 649         int i, ret;
 650         struct fm10k_tx_queue *txq;
 651         uint64_t base_addr;
 652         uint32_t size;
 653
 654         /* Disable TXINT to avoid possible interrupt */
 655         for (i = 0; i < hw->mac.max_queues; i++)
 656                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 657                                 3 << FM10K_TXINT_TIMER_SHIFT);
 658
 659         /* Setup TX queue */
 660         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 661                 txq = dev->data->tx_queues[i];
 662                 base_addr = txq->hw_ring_phys_addr;
 663                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 664
 665                 /* disable queue to avoid issues while updating state */
 666                 ret = tx_queue_disable(hw, i);
 667                 if (ret) {
 668                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 669                         return -1;
 670                 }
 671
 672                 /* set location and size for descriptor ring */
 673                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 674                                 base_addr & UINT64_LOWER_32BITS_MASK);
 675                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 676                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 677                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 678         }
 679
 680         /* set up vector or scalar TX function as appropriate */
 681         fm10k_set_tx_function(dev);
 682
 683         return 0;
 684 }
 685
 686 static int
 687 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 688 {
 689         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 690         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
 691         int i, ret;
 692         struct fm10k_rx_queue *rxq;
 693         uint64_t base_addr;
 694         uint32_t size;
 695         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 696         uint16_t buf_size;
 697
 698         /* enable RXINT for interrupt mode */
 699         i = 0;
 700         if (rte_intr_dp_is_en(intr_handle)) {
 701                 for (; i < dev->data->nb_rx_queues; i++) {
 702                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(dev, i));
 703                         if (hw->mac.type == fm10k_mac_pf)
 704                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
 705                                         FM10K_ITR_AUTOMASK |
 706                                         FM10K_ITR_MASK_CLEAR);
 707                         else
 708                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
 709                                         FM10K_ITR_AUTOMASK |
 710                                         FM10K_ITR_MASK_CLEAR);
 711                 }
 712         }
 713         /* Disable other RXINT to avoid possible interrupt */
 714         for (; i < hw->mac.max_queues; i++)
 715                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 716                         3 << FM10K_RXINT_TIMER_SHIFT);
 717
 718         /* Setup RX queues */
 719         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 720                 rxq = dev->data->rx_queues[i];
 721                 base_addr = rxq->hw_ring_phys_addr;
 722                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 723
 724                 /* disable queue to avoid issues while updating state */
 725                 ret = rx_queue_disable(hw, i);
 726                 if (ret) {
 727                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 728                         return -1;
 729                 }
 730
 731                 /* Setup the Base and Length of the Rx Descriptor Ring */
 732                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 733                                 base_addr & UINT64_LOWER_32BITS_MASK);
 734                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 735                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 736                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 737
 738                 /* Configure the Rx buffer size for one buff without split */
 739                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 740                         RTE_PKTMBUF_HEADROOM);
 741                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 742                  * reserved for this purpose, and the worst case could be 511B.
 743                  * But SRR reg assumes all buffers have the same size. In order
 744                  * to fill the gap, we'll have to consider the worst case and
 745                  * assume 512B is reserved. If we don't do so, it's possible
 746                  * for HW to overwrite data to next mbuf.
 747                  */
 748                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 749
 750                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 751                                 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
 752
 753                 /* It adds dual VLAN length for supporting dual VLAN */
 754                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 755                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 756                         dev->data->dev_conf.rxmode.enable_scatter) {
 757                         uint32_t reg;
 758                         dev->data->scattered_rx = 1;
 759                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 760                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 761                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 762                 }
 763
 764                 /* Enable drop on empty, it's RO for VF */
 765                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 766                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 767
 768                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 769                 FM10K_WRITE_FLUSH(hw);
 770         }
 771
 772         /* Configure VMDQ/RSS if applicable */
 773         fm10k_dev_mq_rx_configure(dev);
 774
 775         /* Decide the best RX function */
 776         fm10k_set_rx_function(dev);
 777
 778         return 0;
 779 }
 780
 781 static int
 782 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 783 {
 784         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 785         int err = -1;
 786         uint32_t reg;
 787         struct fm10k_rx_queue *rxq;
 788
 789         PMD_INIT_FUNC_TRACE();
 790
 791         if (rx_queue_id < dev->data->nb_rx_queues) {
 792                 rxq = dev->data->rx_queues[rx_queue_id];
 793                 err = rx_queue_reset(rxq);
 794                 if (err == -ENOMEM) {
 795                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 796                         return err;
 797                 } else if (err == -EINVAL) {
 798                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 799                                 " %d", err);
 800                         return err;
 801                 }
 802
 803                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 804                  * Note: this must be done AFTER the queue is enabled on real
 805                  * hardware, but BEFORE the queue is enabled when using the
 806                  * emulation platform. Do it in both places for now and remove
 807                  * this comment and the following two register writes when the
 808                  * emulation platform is no longer being used.
 809                  */
 810                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 811                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 812
 813                 /* Set PF ownership flag for PF devices */
 814                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 815                 if (hw->mac.type == fm10k_mac_pf)
 816                         reg |= FM10K_RXQCTL_PF;
 817                 reg |= FM10K_RXQCTL_ENABLE;
 818                 /* enable RX queue */
 819                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 820                 FM10K_WRITE_FLUSH(hw);
 821
 822                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 823                  * Note: this must be done AFTER the queue is enabled
 824                  */
 825                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 826                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 827                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 828         }
 829
 830         return err;
 831 }
 832
 833 static int
 834 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 835 {
 836         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 837
 838         PMD_INIT_FUNC_TRACE();
 839
 840         if (rx_queue_id < dev->data->nb_rx_queues) {
 841                 /* Disable RX queue */
 842                 rx_queue_disable(hw, rx_queue_id);
 843
 844                 /* Free mbuf and clean HW ring */
 845                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 846                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 847         }
 848
 849         return 0;
 850 }
 851
 852 static int
 853 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 854 {
 855         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 856         /** @todo - this should be defined in the shared code */
 857 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 858         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 859         int err = 0;
 860
 861         PMD_INIT_FUNC_TRACE();
 862
 863         if (tx_queue_id < dev->data->nb_tx_queues) {
 864                 struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 865
 866                 q->ops->reset(q);
 867
 868                 /* reset head and tail pointers */
 869                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 870                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 871
 872                 /* enable TX queue */
 873                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 874                                         FM10K_TXDCTL_ENABLE | txdctl);
 875                 FM10K_WRITE_FLUSH(hw);
 876                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 877         } else
 878                 err = -1;
 879
 880         return err;
 881 }
 882
 883 static int
 884 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 885 {
 886         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 887
 888         PMD_INIT_FUNC_TRACE();
 889
 890         if (tx_queue_id < dev->data->nb_tx_queues) {
 891                 tx_queue_disable(hw, tx_queue_id);
 892                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 893                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 894         }
 895
 896         return 0;
 897 }
 898
 899 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 900 {
 901         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 902                 != FM10K_DGLORTMAP_NONE);
 903 }
 904
 905 static void
 906 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 907 {
 908         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 909         int status;
 910
 911         PMD_INIT_FUNC_TRACE();
 912
 913         /* Return if it didn't acquire valid glort range */
 914         if (!fm10k_glort_valid(hw))
 915                 return;
 916
 917         fm10k_mbx_lock(hw);
 918         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 919                                 FM10K_XCAST_MODE_PROMISC);
 920         fm10k_mbx_unlock(hw);
 921
 922         if (status != FM10K_SUCCESS)
 923                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 924 }
 925
 926 static void
 927 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 928 {
 929         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 930         uint8_t mode;
 931         int status;
 932
 933         PMD_INIT_FUNC_TRACE();
 934
 935         /* Return if it didn't acquire valid glort range */
 936         if (!fm10k_glort_valid(hw))
 937                 return;
 938
 939         if (dev->data->all_multicast == 1)
 940                 mode = FM10K_XCAST_MODE_ALLMULTI;
 941         else
 942                 mode = FM10K_XCAST_MODE_NONE;
 943
 944         fm10k_mbx_lock(hw);
 945         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 946                                 mode);
 947         fm10k_mbx_unlock(hw);
 948
 949         if (status != FM10K_SUCCESS)
 950                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 951 }
 952
 953 static void
 954 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 955 {
 956         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 957         int status;
 958
 959         PMD_INIT_FUNC_TRACE();
 960
 961         /* Return if it didn't acquire valid glort range */
 962         if (!fm10k_glort_valid(hw))
 963                 return;
 964
 965         /* If promiscuous mode is enabled, it doesn't make sense to enable
 966          * allmulticast and disable promiscuous since fm10k only can select
 967          * one of the modes.
 968          */
 969         if (dev->data->promiscuous) {
 970                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 971                         "needn't enable allmulticast");
 972                 return;
 973         }
 974
 975         fm10k_mbx_lock(hw);
 976         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 977                                 FM10K_XCAST_MODE_ALLMULTI);
 978         fm10k_mbx_unlock(hw);
 979
 980         if (status != FM10K_SUCCESS)
 981                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 982 }
 983
 984 static void
 985 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 986 {
 987         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 988         int status;
 989
 990         PMD_INIT_FUNC_TRACE();
 991
 992         /* Return if it didn't acquire valid glort range */
 993         if (!fm10k_glort_valid(hw))
 994                 return;
 995
 996         if (dev->data->promiscuous) {
 997                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
 998                         "since promisc mode is enabled");
 999                 return;
1000         }
1001
1002         fm10k_mbx_lock(hw);
1003         /* Change mode to unicast mode */
1004         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1005                                 FM10K_XCAST_MODE_NONE);
1006         fm10k_mbx_unlock(hw);
1007
1008         if (status != FM10K_SUCCESS)
1009                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1010 }
1011
1012 static void
1013 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1014 {
1015         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1016         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1017         uint16_t nb_queue_pools;
1018         struct fm10k_macvlan_filter_info *macvlan;
1019
1020         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1021         nb_queue_pools = macvlan->nb_queue_pools;
1022         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
1023         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
1024
1025         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1026         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1027         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1028                         hw->mac.dglort_map;
1029         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1030         /* Configure VMDQ/RSS DGlort Decoder */
1031         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1032
1033         /* Flow Director configurations, only queue number is valid. */
1034         dglortdec = fls(dev->data->nb_rx_queues - 1);
1035         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1036                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1037         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1038         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1039
1040         /* Invalidate all other GLORT entries */
1041         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1042                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1043                                 FM10K_DGLORTMAP_NONE);
1044 }
1045
1046 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1047 static int
1048 fm10k_dev_start(struct rte_eth_dev *dev)
1049 {
1050         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1051         int i, diag;
1052
1053         PMD_INIT_FUNC_TRACE();
1054
1055         /* stop, init, then start the hw */
1056         diag = fm10k_stop_hw(hw);
1057         if (diag != FM10K_SUCCESS) {
1058                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1059                 return -EIO;
1060         }
1061
1062         diag = fm10k_init_hw(hw);
1063         if (diag != FM10K_SUCCESS) {
1064                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1065                 return -EIO;
1066         }
1067
1068         diag = fm10k_start_hw(hw);
1069         if (diag != FM10K_SUCCESS) {
1070                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1071                 return -EIO;
1072         }
1073
1074         diag = fm10k_dev_tx_init(dev);
1075         if (diag) {
1076                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1077                 return diag;
1078         }
1079
1080         if (fm10k_dev_rxq_interrupt_setup(dev))
1081                 return -EIO;
1082
1083         diag = fm10k_dev_rx_init(dev);
1084         if (diag) {
1085                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1086                 return diag;
1087         }
1088
1089         if (hw->mac.type == fm10k_mac_pf)
1090                 fm10k_dev_dglort_map_configure(dev);
1091
1092         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1093                 struct fm10k_rx_queue *rxq;
1094                 rxq = dev->data->rx_queues[i];
1095
1096                 if (rxq->rx_deferred_start)
1097                         continue;
1098                 diag = fm10k_dev_rx_queue_start(dev, i);
1099                 if (diag != 0) {
1100                         int j;
1101                         for (j = 0; j < i; ++j)
1102                                 rx_queue_clean(dev->data->rx_queues[j]);
1103                         return diag;
1104                 }
1105         }
1106
1107         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1108                 struct fm10k_tx_queue *txq;
1109                 txq = dev->data->tx_queues[i];
1110
1111                 if (txq->tx_deferred_start)
1112                         continue;
1113                 diag = fm10k_dev_tx_queue_start(dev, i);
1114                 if (diag != 0) {
1115                         int j;
1116                         for (j = 0; j < i; ++j)
1117                                 tx_queue_clean(dev->data->tx_queues[j]);
1118                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1119                                 rx_queue_clean(dev->data->rx_queues[j]);
1120                         return diag;
1121                 }
1122         }
1123
1124         /* Update default vlan when not in VMDQ mode */
1125         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1126                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1127
1128         return 0;
1129 }
1130
1131 static void
1132 fm10k_dev_stop(struct rte_eth_dev *dev)
1133 {
1134         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1135         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1136         int i;
1137
1138         PMD_INIT_FUNC_TRACE();
1139
1140         if (dev->data->tx_queues)
1141                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1142                         fm10k_dev_tx_queue_stop(dev, i);
1143
1144         if (dev->data->rx_queues)
1145                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1146                         fm10k_dev_rx_queue_stop(dev, i);
1147
1148         /* Disable datapath event */
1149         if (rte_intr_dp_is_en(intr_handle)) {
1150                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1151                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1152                                 3 << FM10K_RXINT_TIMER_SHIFT);
1153                         if (hw->mac.type == fm10k_mac_pf)
1154                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
1155                                         FM10K_ITR_MASK_SET);
1156                         else
1157                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
1158                                         FM10K_ITR_MASK_SET);
1159                 }
1160         }
1161         /* Clean datapath event and queue/vec mapping */
1162         rte_intr_efd_disable(intr_handle);
1163         rte_free(intr_handle->intr_vec);
1164         intr_handle->intr_vec = NULL;
1165 }
1166
1167 static void
1168 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1169 {
1170         int i;
1171
1172         PMD_INIT_FUNC_TRACE();
1173
1174         if (dev->data->tx_queues) {
1175                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1176                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1177
1178                         tx_queue_free(txq);
1179                 }
1180         }
1181
1182         if (dev->data->rx_queues) {
1183                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1184                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1185         }
1186 }
1187
1188 static void
1189 fm10k_dev_close(struct rte_eth_dev *dev)
1190 {
1191         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1192
1193         PMD_INIT_FUNC_TRACE();
1194
1195         fm10k_mbx_lock(hw);
1196         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1197                 MAX_LPORT_NUM, false);
1198         fm10k_mbx_unlock(hw);
1199
1200         /* Stop mailbox service first */
1201         fm10k_close_mbx_service(hw);
1202         fm10k_dev_stop(dev);
1203         fm10k_dev_queue_release(dev);
1204         fm10k_stop_hw(hw);
1205 }
1206
1207 static int
1208 fm10k_link_update(struct rte_eth_dev *dev,
1209         __rte_unused int wait_to_complete)
1210 {
1211         PMD_INIT_FUNC_TRACE();
1212
1213         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
1214          * x8 PCIe interface. For now, we leave the speed undefined since there
1215          * is no 50Gbps Ethernet. */
1216         dev->data->dev_link.link_speed  = 0;
1217         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1218         dev->data->dev_link.link_status = 1;
1219
1220         return 0;
1221 }
1222
1223 static int
1224 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstats *xstats,
1225                  unsigned n)
1226 {
1227         struct fm10k_hw_stats *hw_stats =
1228                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1229         unsigned i, q, count = 0;
1230
1231         if (n < FM10K_NB_XSTATS)
1232                 return FM10K_NB_XSTATS;
1233
1234         /* Global stats */
1235         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1236                 snprintf(xstats[count].name, sizeof(xstats[count].name),
1237                          "%s", fm10k_hw_stats_strings[count].name);
1238                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1239                         fm10k_hw_stats_strings[count].offset);
1240                 count++;
1241         }
1242
1243         /* PF queue stats */
1244         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1245                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1246                         snprintf(xstats[count].name, sizeof(xstats[count].name),
1247                                  "rx_q%u_%s", q,
1248                                  fm10k_hw_stats_rx_q_strings[i].name);
1249                         xstats[count].value =
1250                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1251                                 fm10k_hw_stats_rx_q_strings[i].offset);
1252                         count++;
1253                 }
1254                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1255                         snprintf(xstats[count].name, sizeof(xstats[count].name),
1256                                  "tx_q%u_%s", q,
1257                                  fm10k_hw_stats_tx_q_strings[i].name);
1258                         xstats[count].value =
1259                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1260                                 fm10k_hw_stats_tx_q_strings[i].offset);
1261                         count++;
1262                 }
1263         }
1264
1265         return FM10K_NB_XSTATS;
1266 }
1267
1268 static void
1269 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1270 {
1271         uint64_t ipackets, opackets, ibytes, obytes;
1272         struct fm10k_hw *hw =
1273                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1274         struct fm10k_hw_stats *hw_stats =
1275                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1276         int i;
1277
1278         PMD_INIT_FUNC_TRACE();
1279
1280         fm10k_update_hw_stats(hw, hw_stats);
1281
1282         ipackets = opackets = ibytes = obytes = 0;
1283         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1284                 (i < hw->mac.max_queues); ++i) {
1285                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1286                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1287                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1288                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1289                 ipackets += stats->q_ipackets[i];
1290                 opackets += stats->q_opackets[i];
1291                 ibytes   += stats->q_ibytes[i];
1292                 obytes   += stats->q_obytes[i];
1293         }
1294         stats->ipackets = ipackets;
1295         stats->opackets = opackets;
1296         stats->ibytes = ibytes;
1297         stats->obytes = obytes;
1298 }
1299
1300 static void
1301 fm10k_stats_reset(struct rte_eth_dev *dev)
1302 {
1303         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1304         struct fm10k_hw_stats *hw_stats =
1305                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1306
1307         PMD_INIT_FUNC_TRACE();
1308
1309         memset(hw_stats, 0, sizeof(*hw_stats));
1310         fm10k_rebind_hw_stats(hw, hw_stats);
1311 }
1312
1313 static void
1314 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1315         struct rte_eth_dev_info *dev_info)
1316 {
1317         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1318
1319         PMD_INIT_FUNC_TRACE();
1320
1321         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1322         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1323         dev_info->max_rx_queues      = hw->mac.max_queues;
1324         dev_info->max_tx_queues      = hw->mac.max_queues;
1325         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1326         dev_info->max_hash_mac_addrs = 0;
1327         dev_info->max_vfs            = dev->pci_dev->max_vfs;
1328         dev_info->vmdq_pool_base     = 0;
1329         dev_info->vmdq_queue_base    = 0;
1330         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1331         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1332         dev_info->rx_offload_capa =
1333                 DEV_RX_OFFLOAD_VLAN_STRIP |
1334                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1335                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1336                 DEV_RX_OFFLOAD_TCP_CKSUM;
1337         dev_info->tx_offload_capa =
1338                 DEV_TX_OFFLOAD_VLAN_INSERT |
1339                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1340                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1341                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1342                 DEV_TX_OFFLOAD_TCP_TSO;
1343
1344         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1345         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1346
1347         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1348                 .rx_thresh = {
1349                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1350                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1351                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1352                 },
1353                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1354                 .rx_drop_en = 0,
1355         };
1356
1357         dev_info->default_txconf = (struct rte_eth_txconf) {
1358                 .tx_thresh = {
1359                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1360                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1361                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1362                 },
1363                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1364                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1365                 .txq_flags = FM10K_SIMPLE_TX_FLAG,
1366         };
1367
1368         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1369                 .nb_max = FM10K_MAX_RX_DESC,
1370                 .nb_min = FM10K_MIN_RX_DESC,
1371                 .nb_align = FM10K_MULT_RX_DESC,
1372         };
1373
1374         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1375                 .nb_max = FM10K_MAX_TX_DESC,
1376                 .nb_min = FM10K_MIN_TX_DESC,
1377                 .nb_align = FM10K_MULT_TX_DESC,
1378         };
1379 }
1380
1381 static int
1382 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1383 {
1384         s32 result;
1385         uint16_t mac_num = 0;
1386         uint32_t vid_idx, vid_bit, mac_index;
1387         struct fm10k_hw *hw;
1388         struct fm10k_macvlan_filter_info *macvlan;
1389         struct rte_eth_dev_data *data = dev->data;
1390
1391         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1392         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1393
1394         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1395                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1396                 return -EINVAL;
1397         }
1398
1399         if (vlan_id > ETH_VLAN_ID_MAX) {
1400                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1401                 return -EINVAL;
1402         }
1403
1404         vid_idx = FM10K_VFTA_IDX(vlan_id);
1405         vid_bit = FM10K_VFTA_BIT(vlan_id);
1406         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1407         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1408                 return 0;
1409         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1410         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1411                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1412                         "in the VLAN filter table");
1413                 return -EINVAL;
1414         }
1415
1416         fm10k_mbx_lock(hw);
1417         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1418         fm10k_mbx_unlock(hw);
1419         if (result != FM10K_SUCCESS) {
1420                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1421                 return -EIO;
1422         }
1423
1424         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1425                         (result == FM10K_SUCCESS); mac_index++) {
1426                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1427                         continue;
1428                 if (mac_num > macvlan->mac_num - 1) {
1429                         PMD_INIT_LOG(ERR, "MAC address number "
1430                                         "not match");
1431                         break;
1432                 }
1433                 fm10k_mbx_lock(hw);
1434                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1435                         data->mac_addrs[mac_index].addr_bytes,
1436                         vlan_id, on, 0);
1437                 fm10k_mbx_unlock(hw);
1438                 mac_num++;
1439         }
1440         if (result != FM10K_SUCCESS) {
1441                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1442                 return -EIO;
1443         }
1444
1445         if (on) {
1446                 macvlan->vlan_num++;
1447                 macvlan->vfta[vid_idx] |= vid_bit;
1448         } else {
1449                 macvlan->vlan_num--;
1450                 macvlan->vfta[vid_idx] &= ~vid_bit;
1451         }
1452         return 0;
1453 }
1454
1455 static void
1456 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1457 {
1458         if (mask & ETH_VLAN_STRIP_MASK) {
1459                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1460                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1461                                         "always on in fm10k");
1462         }
1463
1464         if (mask & ETH_VLAN_EXTEND_MASK) {
1465                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1466                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1467                                         "supported in fm10k");
1468         }
1469
1470         if (mask & ETH_VLAN_FILTER_MASK) {
1471                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1472                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1473         }
1474 }
1475
1476 /* Add/Remove a MAC address, and update filters to main VSI */
1477 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1478                 const u8 *mac, bool add, uint32_t pool)
1479 {
1480         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1481         struct fm10k_macvlan_filter_info *macvlan;
1482         uint32_t i, j, k;
1483
1484         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1485
1486         if (pool != MAIN_VSI_POOL_NUMBER) {
1487                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1488                         "mac to pool %u", pool);
1489                 return;
1490         }
1491         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1492                 if (!macvlan->vfta[j])
1493                         continue;
1494                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1495                         if (!(macvlan->vfta[j] & (1 << k)))
1496                                 continue;
1497                         if (i + 1 > macvlan->vlan_num) {
1498                                 PMD_INIT_LOG(ERR, "vlan number not match");
1499                                 return;
1500                         }
1501                         fm10k_mbx_lock(hw);
1502                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1503                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1504                         fm10k_mbx_unlock(hw);
1505                         i++;
1506                 }
1507         }
1508 }
1509
1510 /* Add/Remove a MAC address, and update filters to VMDQ */
1511 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1512                 const u8 *mac, bool add, uint32_t pool)
1513 {
1514         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1515         struct fm10k_macvlan_filter_info *macvlan;
1516         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1517         uint32_t i;
1518
1519         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1520         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1521
1522         if (pool > macvlan->nb_queue_pools) {
1523                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1524                         " Max pool is %u",
1525                         pool, macvlan->nb_queue_pools);
1526                 return;
1527         }
1528         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1529                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1530                         continue;
1531                 fm10k_mbx_lock(hw);
1532                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1533                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1534                 fm10k_mbx_unlock(hw);
1535         }
1536 }
1537
1538 /* Add/Remove a MAC address, and update filters */
1539 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1540                 const u8 *mac, bool add, uint32_t pool)
1541 {
1542         struct fm10k_macvlan_filter_info *macvlan;
1543
1544         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1545
1546         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1547                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1548         else
1549                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1550
1551         if (add)
1552                 macvlan->mac_num++;
1553         else
1554                 macvlan->mac_num--;
1555 }
1556
1557 /* Add a MAC address, and update filters */
1558 static void
1559 fm10k_macaddr_add(struct rte_eth_dev *dev,
1560                 struct ether_addr *mac_addr,
1561                 uint32_t index,
1562                 uint32_t pool)
1563 {
1564         struct fm10k_macvlan_filter_info *macvlan;
1565
1566         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1567         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1568         macvlan->mac_vmdq_id[index] = pool;
1569 }
1570
1571 /* Remove a MAC address, and update filters */
1572 static void
1573 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1574 {
1575         struct rte_eth_dev_data *data = dev->data;
1576         struct fm10k_macvlan_filter_info *macvlan;
1577
1578         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1579         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1580                         FALSE, macvlan->mac_vmdq_id[index]);
1581         macvlan->mac_vmdq_id[index] = 0;
1582 }
1583
1584 static inline int
1585 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1586 {
1587         if ((request < min) || (request > max) || ((request % mult) != 0))
1588                 return -1;
1589         else
1590                 return 0;
1591 }
1592
1593
1594 static inline int
1595 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1596 {
1597         if ((request < min) || (request > max) || ((div % request) != 0))
1598                 return -1;
1599         else
1600                 return 0;
1601 }
1602
1603 static inline int
1604 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1605 {
1606         uint16_t rx_free_thresh;
1607
1608         if (conf->rx_free_thresh == 0)
1609                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1610         else
1611                 rx_free_thresh = conf->rx_free_thresh;
1612
1613         /* make sure the requested threshold satisfies the constraints */
1614         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1615                         FM10K_RX_FREE_THRESH_MAX(q),
1616                         FM10K_RX_FREE_THRESH_DIV(q),
1617                         rx_free_thresh)) {
1618                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1619                         "less than or equal to %u, "
1620                         "greater than or equal to %u, "
1621                         "and a divisor of %u",
1622                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1623                         FM10K_RX_FREE_THRESH_MIN(q),
1624                         FM10K_RX_FREE_THRESH_DIV(q));
1625                 return -EINVAL;
1626         }
1627
1628         q->alloc_thresh = rx_free_thresh;
1629         q->drop_en = conf->rx_drop_en;
1630         q->rx_deferred_start = conf->rx_deferred_start;
1631
1632         return 0;
1633 }
1634
1635 /*
1636  * Hardware requires specific alignment for Rx packet buffers. At
1637  * least one of the following two conditions must be satisfied.
1638  *  1. Address is 512B aligned
1639  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1640  *
1641  * As such, the driver may need to adjust the DMA address within the
1642  * buffer by up to 512B.
1643  *
1644  * return 1 if the element size is valid, otherwise return 0.
1645  */
1646 static int
1647 mempool_element_size_valid(struct rte_mempool *mp)
1648 {
1649         uint32_t min_size;
1650
1651         /* elt_size includes mbuf header and headroom */
1652         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1653                         RTE_PKTMBUF_HEADROOM;
1654
1655         /* account for up to 512B of alignment */
1656         min_size -= FM10K_RX_DATABUF_ALIGN;
1657
1658         /* sanity check for overflow */
1659         if (min_size > mp->elt_size)
1660                 return 0;
1661
1662         /* size is valid */
1663         return 1;
1664 }
1665
1666 static int
1667 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1668         uint16_t nb_desc, unsigned int socket_id,
1669         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1670 {
1671         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1672         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
1673         struct fm10k_rx_queue *q;
1674         const struct rte_memzone *mz;
1675
1676         PMD_INIT_FUNC_TRACE();
1677
1678         /* make sure the mempool element size can account for alignment. */
1679         if (!mempool_element_size_valid(mp)) {
1680                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1681                 return -EINVAL;
1682         }
1683
1684         /* make sure a valid number of descriptors have been requested */
1685         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1686                                 FM10K_MULT_RX_DESC, nb_desc)) {
1687                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1688                         "less than or equal to %"PRIu32", "
1689                         "greater than or equal to %u, "
1690                         "and a multiple of %u",
1691                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1692                         FM10K_MULT_RX_DESC);
1693                 return -EINVAL;
1694         }
1695
1696         /*
1697          * if this queue existed already, free the associated memory. The
1698          * queue cannot be reused in case we need to allocate memory on
1699          * different socket than was previously used.
1700          */
1701         if (dev->data->rx_queues[queue_id] != NULL) {
1702                 rx_queue_free(dev->data->rx_queues[queue_id]);
1703                 dev->data->rx_queues[queue_id] = NULL;
1704         }
1705
1706         /* allocate memory for the queue structure */
1707         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1708                                 socket_id);
1709         if (q == NULL) {
1710                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1711                 return -ENOMEM;
1712         }
1713
1714         /* setup queue */
1715         q->mp = mp;
1716         q->nb_desc = nb_desc;
1717         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1718         q->port_id = dev->data->port_id;
1719         q->queue_id = queue_id;
1720         q->tail_ptr = (volatile uint32_t *)
1721                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1722         if (handle_rxconf(q, conf))
1723                 return -EINVAL;
1724
1725         /* allocate memory for the software ring */
1726         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1727                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1728                         RTE_CACHE_LINE_SIZE, socket_id);
1729         if (q->sw_ring == NULL) {
1730                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1731                 rte_free(q);
1732                 return -ENOMEM;
1733         }
1734
1735         /*
1736          * allocate memory for the hardware descriptor ring. A memzone large
1737          * enough to hold the maximum ring size is requested to allow for
1738          * resizing in later calls to the queue setup function.
1739          */
1740         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1741                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1742                                       socket_id);
1743         if (mz == NULL) {
1744                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1745                 rte_free(q->sw_ring);
1746                 rte_free(q);
1747                 return -ENOMEM;
1748         }
1749         q->hw_ring = mz->addr;
1750         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1751
1752         /* Check if number of descs satisfied Vector requirement */
1753         if (!rte_is_power_of_2(nb_desc)) {
1754                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1755                                     "preconditions - canceling the feature for "
1756                                     "the whole port[%d]",
1757                              q->queue_id, q->port_id);
1758                 dev_info->rx_vec_allowed = false;
1759         } else
1760                 fm10k_rxq_vec_setup(q);
1761
1762         dev->data->rx_queues[queue_id] = q;
1763         return 0;
1764 }
1765
1766 static void
1767 fm10k_rx_queue_release(void *queue)
1768 {
1769         PMD_INIT_FUNC_TRACE();
1770
1771         rx_queue_free(queue);
1772 }
1773
1774 static inline int
1775 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1776 {
1777         uint16_t tx_free_thresh;
1778         uint16_t tx_rs_thresh;
1779
1780         /* constraint MACROs require that tx_free_thresh is configured
1781          * before tx_rs_thresh */
1782         if (conf->tx_free_thresh == 0)
1783                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1784         else
1785                 tx_free_thresh = conf->tx_free_thresh;
1786
1787         /* make sure the requested threshold satisfies the constraints */
1788         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1789                         FM10K_TX_FREE_THRESH_MAX(q),
1790                         FM10K_TX_FREE_THRESH_DIV(q),
1791                         tx_free_thresh)) {
1792                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1793                         "less than or equal to %u, "
1794                         "greater than or equal to %u, "
1795                         "and a divisor of %u",
1796                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1797                         FM10K_TX_FREE_THRESH_MIN(q),
1798                         FM10K_TX_FREE_THRESH_DIV(q));
1799                 return -EINVAL;
1800         }
1801
1802         q->free_thresh = tx_free_thresh;
1803
1804         if (conf->tx_rs_thresh == 0)
1805                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1806         else
1807                 tx_rs_thresh = conf->tx_rs_thresh;
1808
1809         q->tx_deferred_start = conf->tx_deferred_start;
1810
1811         /* make sure the requested threshold satisfies the constraints */
1812         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1813                         FM10K_TX_RS_THRESH_MAX(q),
1814                         FM10K_TX_RS_THRESH_DIV(q),
1815                         tx_rs_thresh)) {
1816                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1817                         "less than or equal to %u, "
1818                         "greater than or equal to %u, "
1819                         "and a divisor of %u",
1820                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1821                         FM10K_TX_RS_THRESH_MIN(q),
1822                         FM10K_TX_RS_THRESH_DIV(q));
1823                 return -EINVAL;
1824         }
1825
1826         q->rs_thresh = tx_rs_thresh;
1827
1828         return 0;
1829 }
1830
1831 static int
1832 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1833         uint16_t nb_desc, unsigned int socket_id,
1834         const struct rte_eth_txconf *conf)
1835 {
1836         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1837         struct fm10k_tx_queue *q;
1838         const struct rte_memzone *mz;
1839
1840         PMD_INIT_FUNC_TRACE();
1841
1842         /* make sure a valid number of descriptors have been requested */
1843         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1844                                 FM10K_MULT_TX_DESC, nb_desc)) {
1845                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1846                         "less than or equal to %"PRIu32", "
1847                         "greater than or equal to %u, "
1848                         "and a multiple of %u",
1849                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1850                         FM10K_MULT_TX_DESC);
1851                 return -EINVAL;
1852         }
1853
1854         /*
1855          * if this queue existed already, free the associated memory. The
1856          * queue cannot be reused in case we need to allocate memory on
1857          * different socket than was previously used.
1858          */
1859         if (dev->data->tx_queues[queue_id] != NULL) {
1860                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
1861
1862                 tx_queue_free(txq);
1863                 dev->data->tx_queues[queue_id] = NULL;
1864         }
1865
1866         /* allocate memory for the queue structure */
1867         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1868                                 socket_id);
1869         if (q == NULL) {
1870                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1871                 return -ENOMEM;
1872         }
1873
1874         /* setup queue */
1875         q->nb_desc = nb_desc;
1876         q->port_id = dev->data->port_id;
1877         q->queue_id = queue_id;
1878         q->txq_flags = conf->txq_flags;
1879         q->ops = &def_txq_ops;
1880         q->tail_ptr = (volatile uint32_t *)
1881                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1882         if (handle_txconf(q, conf))
1883                 return -EINVAL;
1884
1885         /* allocate memory for the software ring */
1886         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1887                                         nb_desc * sizeof(struct rte_mbuf *),
1888                                         RTE_CACHE_LINE_SIZE, socket_id);
1889         if (q->sw_ring == NULL) {
1890                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1891                 rte_free(q);
1892                 return -ENOMEM;
1893         }
1894
1895         /*
1896          * allocate memory for the hardware descriptor ring. A memzone large
1897          * enough to hold the maximum ring size is requested to allow for
1898          * resizing in later calls to the queue setup function.
1899          */
1900         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
1901                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
1902                                       socket_id);
1903         if (mz == NULL) {
1904                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1905                 rte_free(q->sw_ring);
1906                 rte_free(q);
1907                 return -ENOMEM;
1908         }
1909         q->hw_ring = mz->addr;
1910         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1911
1912         /*
1913          * allocate memory for the RS bit tracker. Enough slots to hold the
1914          * descriptor index for each RS bit needing to be set are required.
1915          */
1916         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1917                                 ((nb_desc + 1) / q->rs_thresh) *
1918                                 sizeof(uint16_t),
1919                                 RTE_CACHE_LINE_SIZE, socket_id);
1920         if (q->rs_tracker.list == NULL) {
1921                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1922                 rte_free(q->sw_ring);
1923                 rte_free(q);
1924                 return -ENOMEM;
1925         }
1926
1927         dev->data->tx_queues[queue_id] = q;
1928         return 0;
1929 }
1930
1931 static void
1932 fm10k_tx_queue_release(void *queue)
1933 {
1934         struct fm10k_tx_queue *q = queue;
1935         PMD_INIT_FUNC_TRACE();
1936
1937         tx_queue_free(q);
1938 }
1939
1940 static int
1941 fm10k_reta_update(struct rte_eth_dev *dev,
1942                         struct rte_eth_rss_reta_entry64 *reta_conf,
1943                         uint16_t reta_size)
1944 {
1945         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1946         uint16_t i, j, idx, shift;
1947         uint8_t mask;
1948         uint32_t reta;
1949
1950         PMD_INIT_FUNC_TRACE();
1951
1952         if (reta_size > FM10K_MAX_RSS_INDICES) {
1953                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1954                         "(%d) doesn't match the number hardware can supported "
1955                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1956                 return -EINVAL;
1957         }
1958
1959         /*
1960          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1961          * 128-entries in 32 registers
1962          */
1963         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1964                 idx = i / RTE_RETA_GROUP_SIZE;
1965                 shift = i % RTE_RETA_GROUP_SIZE;
1966                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1967                                 BIT_MASK_PER_UINT32);
1968                 if (mask == 0)
1969                         continue;
1970
1971                 reta = 0;
1972                 if (mask != BIT_MASK_PER_UINT32)
1973                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1974
1975                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1976                         if (mask & (0x1 << j)) {
1977                                 if (mask != 0xF)
1978                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1979                                 reta |= reta_conf[idx].reta[shift + j] <<
1980                                                 (CHAR_BIT * j);
1981                         }
1982                 }
1983                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1984         }
1985
1986         return 0;
1987 }
1988
1989 static int
1990 fm10k_reta_query(struct rte_eth_dev *dev,
1991                         struct rte_eth_rss_reta_entry64 *reta_conf,
1992                         uint16_t reta_size)
1993 {
1994         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1995         uint16_t i, j, idx, shift;
1996         uint8_t mask;
1997         uint32_t reta;
1998
1999         PMD_INIT_FUNC_TRACE();
2000
2001         if (reta_size < FM10K_MAX_RSS_INDICES) {
2002                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2003                         "(%d) doesn't match the number hardware can supported "
2004                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2005                 return -EINVAL;
2006         }
2007
2008         /*
2009          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2010          * 128-entries in 32 registers
2011          */
2012         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2013                 idx = i / RTE_RETA_GROUP_SIZE;
2014                 shift = i % RTE_RETA_GROUP_SIZE;
2015                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2016                                 BIT_MASK_PER_UINT32);
2017                 if (mask == 0)
2018                         continue;
2019
2020                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2021                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2022                         if (mask & (0x1 << j))
2023                                 reta_conf[idx].reta[shift + j] = ((reta >>
2024                                         CHAR_BIT * j) & UINT8_MAX);
2025                 }
2026         }
2027
2028         return 0;
2029 }
2030
2031 static int
2032 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2033         struct rte_eth_rss_conf *rss_conf)
2034 {
2035         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2036         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2037         uint32_t mrqc;
2038         uint64_t hf = rss_conf->rss_hf;
2039         int i;
2040
2041         PMD_INIT_FUNC_TRACE();
2042
2043         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2044                 FM10K_RSSRK_ENTRIES_PER_REG)
2045                 return -EINVAL;
2046
2047         if (hf == 0)
2048                 return -EINVAL;
2049
2050         mrqc = 0;
2051         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2052         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2053         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2054         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2055         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2056         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2057         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2058         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2059         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2060
2061         /* If the mapping doesn't fit any supported, return */
2062         if (mrqc == 0)
2063                 return -EINVAL;
2064
2065         if (key != NULL)
2066                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2067                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2068
2069         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2070
2071         return 0;
2072 }
2073
2074 static int
2075 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2076         struct rte_eth_rss_conf *rss_conf)
2077 {
2078         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2079         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2080         uint32_t mrqc;
2081         uint64_t hf;
2082         int i;
2083
2084         PMD_INIT_FUNC_TRACE();
2085
2086         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2087                                 FM10K_RSSRK_ENTRIES_PER_REG)
2088                 return -EINVAL;
2089
2090         if (key != NULL)
2091                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2092                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2093
2094         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2095         hf = 0;
2096         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2097         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2098         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2099         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2100         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2101         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2102         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2103         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2104         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2105
2106         rss_conf->rss_hf = hf;
2107
2108         return 0;
2109 }
2110
2111 static void
2112 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2113 {
2114         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2115         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2116
2117         /* Bind all local non-queue interrupt to vector 0 */
2118         int_map |= FM10K_MISC_VEC_ID;
2119
2120         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
2121         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
2122         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
2123         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
2124         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
2125         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
2126
2127         /* Enable misc causes */
2128         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2129                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2130                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2131                                 FM10K_EIMR_ENABLE(MAILBOX) |
2132                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2133                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2134                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2135                                 FM10K_EIMR_ENABLE(VFLR));
2136
2137         /* Enable ITR 0 */
2138         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2139                                         FM10K_ITR_MASK_CLEAR);
2140         FM10K_WRITE_FLUSH(hw);
2141 }
2142
2143 static void
2144 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2145 {
2146         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2147         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2148
2149         int_map |= FM10K_MISC_VEC_ID;
2150
2151         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
2152         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
2153         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
2154         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
2155         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
2156         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
2157
2158         /* Disable misc causes */
2159         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2160                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2161                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2162                                 FM10K_EIMR_DISABLE(MAILBOX) |
2163                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2164                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2165                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2166                                 FM10K_EIMR_DISABLE(VFLR));
2167
2168         /* Disable ITR 0 */
2169         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2170         FM10K_WRITE_FLUSH(hw);
2171 }
2172
2173 static void
2174 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2175 {
2176         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2177         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2178
2179         /* Bind all local non-queue interrupt to vector 0 */
2180         int_map |= FM10K_MISC_VEC_ID;
2181
2182         /* Only INT 0 available, other 15 are reserved. */
2183         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2184
2185         /* Enable ITR 0 */
2186         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2187                                         FM10K_ITR_MASK_CLEAR);
2188         FM10K_WRITE_FLUSH(hw);
2189 }
2190
2191 static void
2192 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2193 {
2194         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2195         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2196
2197         int_map |= FM10K_MISC_VEC_ID;
2198
2199         /* Only INT 0 available, other 15 are reserved. */
2200         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2201
2202         /* Disable ITR 0 */
2203         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2204         FM10K_WRITE_FLUSH(hw);
2205 }
2206
2207 static int
2208 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2209 {
2210         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2211
2212         /* Enable ITR */
2213         if (hw->mac.type == fm10k_mac_pf)
2214                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
2215                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2216         else
2217                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
2218                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2219         rte_intr_enable(&dev->pci_dev->intr_handle);
2220         return 0;
2221 }
2222
2223 static int
2224 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2225 {
2226         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2227
2228         /* Disable ITR */
2229         if (hw->mac.type == fm10k_mac_pf)
2230                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
2231                         FM10K_ITR_MASK_SET);
2232         else
2233                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
2234                         FM10K_ITR_MASK_SET);
2235         return 0;
2236 }
2237
2238 static int
2239 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2240 {
2241         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2242         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
2243         uint32_t intr_vector, vec;
2244         uint16_t queue_id;
2245         int result = 0;
2246
2247         /* fm10k needs one separate interrupt for mailbox,
2248          * so only drivers which support multiple interrupt vectors
2249          * e.g. vfio-pci can work for fm10k interrupt mode
2250          */
2251         if (!rte_intr_cap_multiple(intr_handle) ||
2252                         dev->data->dev_conf.intr_conf.rxq == 0)
2253                 return result;
2254
2255         intr_vector = dev->data->nb_rx_queues;
2256
2257         /* disable interrupt first */
2258         rte_intr_disable(&dev->pci_dev->intr_handle);
2259         if (hw->mac.type == fm10k_mac_pf)
2260                 fm10k_dev_disable_intr_pf(dev);
2261         else
2262                 fm10k_dev_disable_intr_vf(dev);
2263
2264         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2265                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2266                 result = -EIO;
2267         }
2268
2269         if (rte_intr_dp_is_en(intr_handle) && !result) {
2270                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2271                         dev->data->nb_rx_queues * sizeof(int), 0);
2272                 if (intr_handle->intr_vec) {
2273                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2274                                         queue_id < dev->data->nb_rx_queues;
2275                                         queue_id++) {
2276                                 intr_handle->intr_vec[queue_id] = vec;
2277                                 if (vec < intr_handle->nb_efd - 1
2278                                                 + FM10K_RX_VEC_START)
2279                                         vec++;
2280                         }
2281                 } else {
2282                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2283                                 " intr_vec", dev->data->nb_rx_queues);
2284                         rte_intr_efd_disable(intr_handle);
2285                         result = -ENOMEM;
2286                 }
2287         }
2288
2289         if (hw->mac.type == fm10k_mac_pf)
2290                 fm10k_dev_enable_intr_pf(dev);
2291         else
2292                 fm10k_dev_enable_intr_vf(dev);
2293         rte_intr_enable(&dev->pci_dev->intr_handle);
2294         hw->mac.ops.update_int_moderator(hw);
2295         return result;
2296 }
2297
2298 static int
2299 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2300 {
2301         struct fm10k_fault fault;
2302         int err;
2303         const char *estr = "Unknown error";
2304
2305         /* Process PCA fault */
2306         if (eicr & FM10K_EICR_PCA_FAULT) {
2307                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2308                 if (err)
2309                         goto error;
2310                 switch (fault.type) {
2311                 case PCA_NO_FAULT:
2312                         estr = "PCA_NO_FAULT"; break;
2313                 case PCA_UNMAPPED_ADDR:
2314                         estr = "PCA_UNMAPPED_ADDR"; break;
2315                 case PCA_BAD_QACCESS_PF:
2316                         estr = "PCA_BAD_QACCESS_PF"; break;
2317                 case PCA_BAD_QACCESS_VF:
2318                         estr = "PCA_BAD_QACCESS_VF"; break;
2319                 case PCA_MALICIOUS_REQ:
2320                         estr = "PCA_MALICIOUS_REQ"; break;
2321                 case PCA_POISONED_TLP:
2322                         estr = "PCA_POISONED_TLP"; break;
2323                 case PCA_TLP_ABORT:
2324                         estr = "PCA_TLP_ABORT"; break;
2325                 default:
2326                         goto error;
2327                 }
2328                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2329                         estr, fault.func ? "VF" : "PF", fault.func,
2330                         fault.address, fault.specinfo);
2331         }
2332
2333         /* Process THI fault */
2334         if (eicr & FM10K_EICR_THI_FAULT) {
2335                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2336                 if (err)
2337                         goto error;
2338                 switch (fault.type) {
2339                 case THI_NO_FAULT:
2340                         estr = "THI_NO_FAULT"; break;
2341                 case THI_MAL_DIS_Q_FAULT:
2342                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2343                 default:
2344                         goto error;
2345                 }
2346                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2347                         estr, fault.func ? "VF" : "PF", fault.func,
2348                         fault.address, fault.specinfo);
2349         }
2350
2351         /* Process FUM fault */
2352         if (eicr & FM10K_EICR_FUM_FAULT) {
2353                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2354                 if (err)
2355                         goto error;
2356                 switch (fault.type) {
2357                 case FUM_NO_FAULT:
2358                         estr = "FUM_NO_FAULT"; break;
2359                 case FUM_UNMAPPED_ADDR:
2360                         estr = "FUM_UNMAPPED_ADDR"; break;
2361                 case FUM_POISONED_TLP:
2362                         estr = "FUM_POISONED_TLP"; break;
2363                 case FUM_BAD_VF_QACCESS:
2364                         estr = "FUM_BAD_VF_QACCESS"; break;
2365                 case FUM_ADD_DECODE_ERR:
2366                         estr = "FUM_ADD_DECODE_ERR"; break;
2367                 case FUM_RO_ERROR:
2368                         estr = "FUM_RO_ERROR"; break;
2369                 case FUM_QPRC_CRC_ERROR:
2370                         estr = "FUM_QPRC_CRC_ERROR"; break;
2371                 case FUM_CSR_TIMEOUT:
2372                         estr = "FUM_CSR_TIMEOUT"; break;
2373                 case FUM_INVALID_TYPE:
2374                         estr = "FUM_INVALID_TYPE"; break;
2375                 case FUM_INVALID_LENGTH:
2376                         estr = "FUM_INVALID_LENGTH"; break;
2377                 case FUM_INVALID_BE:
2378                         estr = "FUM_INVALID_BE"; break;
2379                 case FUM_INVALID_ALIGN:
2380                         estr = "FUM_INVALID_ALIGN"; break;
2381                 default:
2382                         goto error;
2383                 }
2384                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2385                         estr, fault.func ? "VF" : "PF", fault.func,
2386                         fault.address, fault.specinfo);
2387         }
2388
2389         return 0;
2390 error:
2391         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2392         return err;
2393 }
2394
2395 /**
2396  * PF interrupt handler triggered by NIC for handling specific interrupt.
2397  *
2398  * @param handle
2399  *  Pointer to interrupt handle.
2400  * @param param
2401  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2402  *
2403  * @return
2404  *  void
2405  */
2406 static void
2407 fm10k_dev_interrupt_handler_pf(
2408                         __rte_unused struct rte_intr_handle *handle,
2409                         void *param)
2410 {
2411         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2412         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2413         uint32_t cause, status;
2414
2415         if (hw->mac.type != fm10k_mac_pf)
2416                 return;
2417
2418         cause = FM10K_READ_REG(hw, FM10K_EICR);
2419
2420         /* Handle PCI fault cases */
2421         if (cause & FM10K_EICR_FAULT_MASK) {
2422                 PMD_INIT_LOG(ERR, "INT: find fault!");
2423                 fm10k_dev_handle_fault(hw, cause);
2424         }
2425
2426         /* Handle switch up/down */
2427         if (cause & FM10K_EICR_SWITCHNOTREADY)
2428                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2429
2430         if (cause & FM10K_EICR_SWITCHREADY)
2431                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2432
2433         /* Handle mailbox message */
2434         fm10k_mbx_lock(hw);
2435         hw->mbx.ops.process(hw, &hw->mbx);
2436         fm10k_mbx_unlock(hw);
2437
2438         /* Handle SRAM error */
2439         if (cause & FM10K_EICR_SRAMERROR) {
2440                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2441
2442                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2443                 /* Write to clear pending bits */
2444                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2445
2446                 /* Todo: print out error message after shared code  updates */
2447         }
2448
2449         /* Clear these 3 events if having any */
2450         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2451                  FM10K_EICR_SWITCHREADY;
2452         if (cause)
2453                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2454
2455         /* Re-enable interrupt from device side */
2456         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2457                                         FM10K_ITR_MASK_CLEAR);
2458         /* Re-enable interrupt from host side */
2459         rte_intr_enable(&(dev->pci_dev->intr_handle));
2460 }
2461
2462 /**
2463  * VF interrupt handler triggered by NIC for handling specific interrupt.
2464  *
2465  * @param handle
2466  *  Pointer to interrupt handle.
2467  * @param param
2468  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2469  *
2470  * @return
2471  *  void
2472  */
2473 static void
2474 fm10k_dev_interrupt_handler_vf(
2475                         __rte_unused struct rte_intr_handle *handle,
2476                         void *param)
2477 {
2478         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2479         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2480
2481         if (hw->mac.type != fm10k_mac_vf)
2482                 return;
2483
2484         /* Handle mailbox message if lock is acquired */
2485         fm10k_mbx_lock(hw);
2486         hw->mbx.ops.process(hw, &hw->mbx);
2487         fm10k_mbx_unlock(hw);
2488
2489         /* Re-enable interrupt from device side */
2490         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2491                                         FM10K_ITR_MASK_CLEAR);
2492         /* Re-enable interrupt from host side */
2493         rte_intr_enable(&(dev->pci_dev->intr_handle));
2494 }
2495
2496 /* Mailbox message handler in VF */
2497 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2498         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2499         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2500         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2501         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2502 };
2503
2504 /* Mailbox message handler in PF */
2505 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
2506         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
2507         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
2508         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
2509         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
2510         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
2511         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
2512         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2513 };
2514
2515 static int
2516 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2517 {
2518         int err;
2519
2520         /* Initialize mailbox lock */
2521         fm10k_mbx_initlock(hw);
2522
2523         /* Replace default message handler with new ones */
2524         if (hw->mac.type == fm10k_mac_pf)
2525                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
2526         else
2527                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2528
2529         if (err) {
2530                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2531                                 err);
2532                 return err;
2533         }
2534         /* Connect to SM for PF device or PF for VF device */
2535         return hw->mbx.ops.connect(hw, &hw->mbx);
2536 }
2537
2538 static void
2539 fm10k_close_mbx_service(struct fm10k_hw *hw)
2540 {
2541         /* Disconnect from SM for PF device or PF for VF device */
2542         hw->mbx.ops.disconnect(hw, &hw->mbx);
2543 }
2544
2545 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2546         .dev_configure          = fm10k_dev_configure,
2547         .dev_start              = fm10k_dev_start,
2548         .dev_stop               = fm10k_dev_stop,
2549         .dev_close              = fm10k_dev_close,
2550         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2551         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2552         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2553         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2554         .stats_get              = fm10k_stats_get,
2555         .xstats_get             = fm10k_xstats_get,
2556         .stats_reset            = fm10k_stats_reset,
2557         .xstats_reset           = fm10k_stats_reset,
2558         .link_update            = fm10k_link_update,
2559         .dev_infos_get          = fm10k_dev_infos_get,
2560         .vlan_filter_set        = fm10k_vlan_filter_set,
2561         .vlan_offload_set       = fm10k_vlan_offload_set,
2562         .mac_addr_add           = fm10k_macaddr_add,
2563         .mac_addr_remove        = fm10k_macaddr_remove,
2564         .rx_queue_start         = fm10k_dev_rx_queue_start,
2565         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2566         .tx_queue_start         = fm10k_dev_tx_queue_start,
2567         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2568         .rx_queue_setup         = fm10k_rx_queue_setup,
2569         .rx_queue_release       = fm10k_rx_queue_release,
2570         .tx_queue_setup         = fm10k_tx_queue_setup,
2571         .tx_queue_release       = fm10k_tx_queue_release,
2572         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2573         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2574         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2575         .reta_update            = fm10k_reta_update,
2576         .reta_query             = fm10k_reta_query,
2577         .rss_hash_update        = fm10k_rss_hash_update,
2578         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2579 };
2580
2581 static void __attribute__((cold))
2582 fm10k_set_tx_function(struct rte_eth_dev *dev)
2583 {
2584         struct fm10k_tx_queue *txq;
2585         int i;
2586         int use_sse = 1;
2587
2588         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2589                 txq = dev->data->tx_queues[i];
2590                 /* Check if Vector Tx is satisfied */
2591                 if (fm10k_tx_vec_condition_check(txq)) {
2592                         use_sse = 0;
2593                         break;
2594                 }
2595         }
2596
2597         if (use_sse) {
2598                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2599                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2600                         txq = dev->data->tx_queues[i];
2601                         fm10k_txq_vec_setup(txq);
2602                 }
2603                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2604         } else {
2605                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2606                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2607         }
2608 }
2609
2610 static void __attribute__((cold))
2611 fm10k_set_rx_function(struct rte_eth_dev *dev)
2612 {
2613         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2614         uint16_t i, rx_using_sse;
2615
2616         /* In order to allow Vector Rx there are a few configuration
2617          * conditions to be met.
2618          */
2619         if (!fm10k_rx_vec_condition_check(dev) && dev_info->rx_vec_allowed) {
2620                 if (dev->data->scattered_rx)
2621                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2622                 else
2623                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2624         } else if (dev->data->scattered_rx)
2625                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2626         else
2627                 dev->rx_pkt_burst = fm10k_recv_pkts;
2628
2629         rx_using_sse =
2630                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2631                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2632
2633         if (rx_using_sse)
2634                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
2635         else
2636                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
2637
2638         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2639                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2640
2641                 rxq->rx_using_sse = rx_using_sse;
2642         }
2643 }
2644
2645 static void
2646 fm10k_params_init(struct rte_eth_dev *dev)
2647 {
2648         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2649         struct fm10k_dev_info *info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2650
2651         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2652          * there is no way to get link status without reading BAR4.  Until this
2653          * works, assume we have maximum bandwidth.
2654          * @todo - fix bus info
2655          */
2656         hw->bus_caps.speed = fm10k_bus_speed_8000;
2657         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2658         hw->bus_caps.payload = fm10k_bus_payload_512;
2659         hw->bus.speed = fm10k_bus_speed_8000;
2660         hw->bus.width = fm10k_bus_width_pcie_x8;
2661         hw->bus.payload = fm10k_bus_payload_256;
2662
2663         info->rx_vec_allowed = true;
2664 }
2665
2666 static int
2667 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2668 {
2669         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2670         int diag, i;
2671         struct fm10k_macvlan_filter_info *macvlan;
2672
2673         PMD_INIT_FUNC_TRACE();
2674
2675         dev->dev_ops = &fm10k_eth_dev_ops;
2676         dev->rx_pkt_burst = &fm10k_recv_pkts;
2677         dev->tx_pkt_burst = &fm10k_xmit_pkts;
2678
2679         /* only initialize in the primary process */
2680         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2681                 return 0;
2682
2683         rte_eth_copy_pci_info(dev, dev->pci_dev);
2684
2685         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2686         memset(macvlan, 0, sizeof(*macvlan));
2687         /* Vendor and Device ID need to be set before init of shared code */
2688         memset(hw, 0, sizeof(*hw));
2689         hw->device_id = dev->pci_dev->id.device_id;
2690         hw->vendor_id = dev->pci_dev->id.vendor_id;
2691         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
2692         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
2693         hw->revision_id = 0;
2694         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2695         if (hw->hw_addr == NULL) {
2696                 PMD_INIT_LOG(ERR, "Bad mem resource."
2697                         " Try to blacklist unused devices.");
2698                 return -EIO;
2699         }
2700
2701         /* Store fm10k_adapter pointer */
2702         hw->back = dev->data->dev_private;
2703
2704         /* Initialize the shared code */
2705         diag = fm10k_init_shared_code(hw);
2706         if (diag != FM10K_SUCCESS) {
2707                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2708                 return -EIO;
2709         }
2710
2711         /* Initialize parameters */
2712         fm10k_params_init(dev);
2713
2714         /* Initialize the hw */
2715         diag = fm10k_init_hw(hw);
2716         if (diag != FM10K_SUCCESS) {
2717                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2718                 return -EIO;
2719         }
2720
2721         /* Initialize MAC address(es) */
2722         dev->data->mac_addrs = rte_zmalloc("fm10k",
2723                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2724         if (dev->data->mac_addrs == NULL) {
2725                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2726                 return -ENOMEM;
2727         }
2728
2729         diag = fm10k_read_mac_addr(hw);
2730
2731         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2732                         &dev->data->mac_addrs[0]);
2733
2734         if (diag != FM10K_SUCCESS ||
2735                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2736
2737                 /* Generate a random addr */
2738                 eth_random_addr(hw->mac.addr);
2739                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2740                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2741                 &dev->data->mac_addrs[0]);
2742         }
2743
2744         /* Reset the hw statistics */
2745         fm10k_stats_reset(dev);
2746
2747         /* Reset the hw */
2748         diag = fm10k_reset_hw(hw);
2749         if (diag != FM10K_SUCCESS) {
2750                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2751                 return -EIO;
2752         }
2753
2754         /* Setup mailbox service */
2755         diag = fm10k_setup_mbx_service(hw);
2756         if (diag != FM10K_SUCCESS) {
2757                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2758                 return -EIO;
2759         }
2760
2761         /*PF/VF has different interrupt handling mechanism */
2762         if (hw->mac.type == fm10k_mac_pf) {
2763                 /* register callback func to eal lib */
2764                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2765                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2766
2767                 /* enable MISC interrupt */
2768                 fm10k_dev_enable_intr_pf(dev);
2769         } else { /* VF */
2770                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2771                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2772
2773                 fm10k_dev_enable_intr_vf(dev);
2774         }
2775
2776         /* Enable intr after callback registered */
2777         rte_intr_enable(&(dev->pci_dev->intr_handle));
2778
2779         hw->mac.ops.update_int_moderator(hw);
2780
2781         /* Make sure Switch Manager is ready before going forward. */
2782         if (hw->mac.type == fm10k_mac_pf) {
2783                 int switch_ready = 0;
2784
2785                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2786                         fm10k_mbx_lock(hw);
2787                         hw->mac.ops.get_host_state(hw, &switch_ready);
2788                         fm10k_mbx_unlock(hw);
2789                         if (switch_ready)
2790                                 break;
2791                         /* Delay some time to acquire async LPORT_MAP info. */
2792                         rte_delay_us(WAIT_SWITCH_MSG_US);
2793                 }
2794
2795                 if (switch_ready == 0) {
2796                         PMD_INIT_LOG(ERR, "switch is not ready");
2797                         return -1;
2798                 }
2799         }
2800
2801         /*
2802          * Below function will trigger operations on mailbox, acquire lock to
2803          * avoid race condition from interrupt handler. Operations on mailbox
2804          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2805          * will handle and generate an interrupt to our side. Then,  FIFO in
2806          * mailbox will be touched.
2807          */
2808         fm10k_mbx_lock(hw);
2809         /* Enable port first */
2810         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2811                                         MAX_LPORT_NUM, 1);
2812
2813         /* Set unicast mode by default. App can change to other mode in other
2814          * API func.
2815          */
2816         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2817                                         FM10K_XCAST_MODE_NONE);
2818
2819         fm10k_mbx_unlock(hw);
2820
2821         /* Add default mac address */
2822         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2823                 MAIN_VSI_POOL_NUMBER);
2824
2825         return 0;
2826 }
2827
2828 static int
2829 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
2830 {
2831         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2832
2833         PMD_INIT_FUNC_TRACE();
2834
2835         /* only uninitialize in the primary process */
2836         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2837                 return 0;
2838
2839         /* safe to close dev here */
2840         fm10k_dev_close(dev);
2841
2842         dev->dev_ops = NULL;
2843         dev->rx_pkt_burst = NULL;
2844         dev->tx_pkt_burst = NULL;
2845
2846         /* disable uio/vfio intr */
2847         rte_intr_disable(&(dev->pci_dev->intr_handle));
2848
2849         /*PF/VF has different interrupt handling mechanism */
2850         if (hw->mac.type == fm10k_mac_pf) {
2851                 /* disable interrupt */
2852                 fm10k_dev_disable_intr_pf(dev);
2853
2854                 /* unregister callback func to eal lib */
2855                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2856                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2857         } else {
2858                 /* disable interrupt */
2859                 fm10k_dev_disable_intr_vf(dev);
2860
2861                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2862                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2863         }
2864
2865         /* free mac memory */
2866         if (dev->data->mac_addrs) {
2867                 rte_free(dev->data->mac_addrs);
2868                 dev->data->mac_addrs = NULL;
2869         }
2870
2871         memset(hw, 0, sizeof(*hw));
2872
2873         return 0;
2874 }
2875
2876 /*
2877  * The set of PCI devices this driver supports. This driver will enable both PF
2878  * and SRIOV-VF devices.
2879  */
2880 static const struct rte_pci_id pci_id_fm10k_map[] = {
2881 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2882 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2883 #include "rte_pci_dev_ids.h"
2884         { .vendor_id = 0, /* sentinel */ },
2885 };
2886
2887 static struct eth_driver rte_pmd_fm10k = {
2888         .pci_drv = {
2889                 .name = "rte_pmd_fm10k",
2890                 .id_table = pci_id_fm10k_map,
2891                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
2892                         RTE_PCI_DRV_DETACHABLE,
2893         },
2894         .eth_dev_init = eth_fm10k_dev_init,
2895         .eth_dev_uninit = eth_fm10k_dev_uninit,
2896         .dev_private_size = sizeof(struct fm10k_adapter),
2897 };
2898
2899 /*
2900  * Driver initialization routine.
2901  * Invoked once at EAL init time.
2902  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2903  */
2904 static int
2905 rte_pmd_fm10k_init(__rte_unused const char *name,
2906         __rte_unused const char *params)
2907 {
2908         PMD_INIT_FUNC_TRACE();
2909         rte_eth_driver_register(&rte_pmd_fm10k);
2910         return 0;
2911 }
2912
2913 static struct rte_driver rte_fm10k_driver = {
2914         .type = PMD_PDEV,
2915         .init = rte_pmd_fm10k_init,
2916 };
2917
2918 PMD_REGISTER_DRIVER(rte_fm10k_driver);