drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 /* Default delay to acquire mailbox lock */
  45 #define FM10K_MBXLOCK_DELAY_US 20
  46 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  47
  48 #define MAIN_VSI_POOL_NUMBER 0
  49
  50 /* Max try times to acquire switch status */
  51 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  52 /* Wait interval to get switch status */
  53 #define WAIT_SWITCH_MSG_US    100000
  54 /* Number of chars per uint32 type */
  55 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  56 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  57
  58 /* default 1:1 map from queue ID to interrupt vector ID */
  59 #define Q2V(dev, queue_id) (dev->pci_dev->intr_handle.intr_vec[queue_id])
  60
  61 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  62 #define MAX_LPORT_NUM    128
  63 #define GLORT_FD_Q_BASE  0x40
  64 #define GLORT_PF_MASK    0xFFC0
  65 #define GLORT_FD_MASK    GLORT_PF_MASK
  66 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  67
  68 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  69 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  70 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  71 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  72 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  73 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  74 static int
  75 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  76 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  77         const u8 *mac, bool add, uint32_t pool);
  78 static void fm10k_tx_queue_release(void *queue);
  79 static void fm10k_rx_queue_release(void *queue);
  80 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  81 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  82
  83 struct fm10k_xstats_name_off {
  84         char name[RTE_ETH_XSTATS_NAME_SIZE];
  85         unsigned offset;
  86 };
  87
  88 struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  89         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  90         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  91         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  92         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  93         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  94         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  95         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  96         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  97                 nodesc_drop)},
  98 };
  99
 100 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
 101                 sizeof(fm10k_hw_stats_strings[0]))
 102
 103 struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
 104         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
 105         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
 106         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
 107 };
 108
 109 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
 110                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
 111
 112 struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 113         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 114         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 115 };
 116
 117 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 118                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 119
 120 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 121                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 122 static int
 123 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 124
 125 static void
 126 fm10k_mbx_initlock(struct fm10k_hw *hw)
 127 {
 128         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 129 }
 130
 131 static void
 132 fm10k_mbx_lock(struct fm10k_hw *hw)
 133 {
 134         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 135                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 136 }
 137
 138 static void
 139 fm10k_mbx_unlock(struct fm10k_hw *hw)
 140 {
 141         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 142 }
 143
 144 /* Stubs needed for linkage when vPMD is disabled */
 145 int __attribute__((weak))
 146 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 147 {
 148         return -1;
 149 }
 150
 151 uint16_t __attribute__((weak))
 152 fm10k_recv_pkts_vec(
 153         __rte_unused void *rx_queue,
 154         __rte_unused struct rte_mbuf **rx_pkts,
 155         __rte_unused uint16_t nb_pkts)
 156 {
 157         return 0;
 158 }
 159
 160 uint16_t __attribute__((weak))
 161 fm10k_recv_scattered_pkts_vec(
 162                 __rte_unused void *rx_queue,
 163                 __rte_unused struct rte_mbuf **rx_pkts,
 164                 __rte_unused uint16_t nb_pkts)
 165 {
 166         return 0;
 167 }
 168
 169 int __attribute__((weak))
 170 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 171
 172 {
 173         return -1;
 174 }
 175
 176 void __attribute__((weak))
 177 fm10k_rx_queue_release_mbufs_vec(
 178                 __rte_unused struct fm10k_rx_queue *rxq)
 179 {
 180         return;
 181 }
 182
 183 void __attribute__((weak))
 184 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 185 {
 186         return;
 187 }
 188
 189 int __attribute__((weak))
 190 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 191 {
 192         return -1;
 193 }
 194
 195 uint16_t __attribute__((weak))
 196 fm10k_xmit_pkts_vec(__rte_unused void *tx_queue,
 197                 __rte_unused struct rte_mbuf **tx_pkts,
 198                 __rte_unused uint16_t nb_pkts)
 199 {
 200         return 0;
 201 }
 202
 203 /*
 204  * reset queue to initial state, allocate software buffers used when starting
 205  * device.
 206  * return 0 on success
 207  * return -ENOMEM if buffers cannot be allocated
 208  * return -EINVAL if buffers do not satisfy alignment condition
 209  */
 210 static inline int
 211 rx_queue_reset(struct fm10k_rx_queue *q)
 212 {
 213         static const union fm10k_rx_desc zero = {{0} };
 214         uint64_t dma_addr;
 215         int i, diag;
 216         PMD_INIT_FUNC_TRACE();
 217
 218         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 219         if (diag != 0)
 220                 return -ENOMEM;
 221
 222         for (i = 0; i < q->nb_desc; ++i) {
 223                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 224                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 225                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 226                                                 q->nb_desc);
 227                         return -EINVAL;
 228                 }
 229                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 230                 q->hw_ring[i].q.pkt_addr = dma_addr;
 231                 q->hw_ring[i].q.hdr_addr = dma_addr;
 232         }
 233
 234         /* initialize extra software ring entries. Space for these extra
 235          * entries is always allocated.
 236          */
 237         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 238         for (i = 0; i < q->nb_fake_desc; ++i) {
 239                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 240                 q->hw_ring[q->nb_desc + i] = zero;
 241         }
 242
 243         q->next_dd = 0;
 244         q->next_alloc = 0;
 245         q->next_trigger = q->alloc_thresh - 1;
 246         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 247         q->rxrearm_start = 0;
 248         q->rxrearm_nb = 0;
 249
 250         return 0;
 251 }
 252
 253 /*
 254  * clean queue, descriptor rings, free software buffers used when stopping
 255  * device.
 256  */
 257 static inline void
 258 rx_queue_clean(struct fm10k_rx_queue *q)
 259 {
 260         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 261         uint32_t i;
 262         PMD_INIT_FUNC_TRACE();
 263
 264         /* zero descriptor rings */
 265         for (i = 0; i < q->nb_desc; ++i)
 266                 q->hw_ring[i] = zero;
 267
 268         /* zero faked descriptors */
 269         for (i = 0; i < q->nb_fake_desc; ++i)
 270                 q->hw_ring[q->nb_desc + i] = zero;
 271
 272         /* vPMD driver has a different way of releasing mbufs. */
 273         if (q->rx_using_sse) {
 274                 fm10k_rx_queue_release_mbufs_vec(q);
 275                 return;
 276         }
 277
 278         /* free software buffers */
 279         for (i = 0; i < q->nb_desc; ++i) {
 280                 if (q->sw_ring[i]) {
 281                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 282                         q->sw_ring[i] = NULL;
 283                 }
 284         }
 285 }
 286
 287 /*
 288  * free all queue memory used when releasing the queue (i.e. configure)
 289  */
 290 static inline void
 291 rx_queue_free(struct fm10k_rx_queue *q)
 292 {
 293         PMD_INIT_FUNC_TRACE();
 294         if (q) {
 295                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 296                 rx_queue_clean(q);
 297                 if (q->sw_ring) {
 298                         rte_free(q->sw_ring);
 299                         q->sw_ring = NULL;
 300                 }
 301                 rte_free(q);
 302                 q = NULL;
 303         }
 304 }
 305
 306 /*
 307  * disable RX queue, wait unitl HW finished necessary flush operation
 308  */
 309 static inline int
 310 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 311 {
 312         uint32_t reg, i;
 313
 314         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 315         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 316                         reg & ~FM10K_RXQCTL_ENABLE);
 317
 318         /* Wait 100us at most */
 319         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 320                 rte_delay_us(1);
 321                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 322                 if (!(reg & FM10K_RXQCTL_ENABLE))
 323                         break;
 324         }
 325
 326         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 327                 return -1;
 328
 329         return 0;
 330 }
 331
 332 /*
 333  * reset queue to initial state, allocate software buffers used when starting
 334  * device
 335  */
 336 static inline void
 337 tx_queue_reset(struct fm10k_tx_queue *q)
 338 {
 339         PMD_INIT_FUNC_TRACE();
 340         q->last_free = 0;
 341         q->next_free = 0;
 342         q->nb_used = 0;
 343         q->nb_free = q->nb_desc - 1;
 344         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 345         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 346 }
 347
 348 /*
 349  * clean queue, descriptor rings, free software buffers used when stopping
 350  * device
 351  */
 352 static inline void
 353 tx_queue_clean(struct fm10k_tx_queue *q)
 354 {
 355         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 356         uint32_t i;
 357         PMD_INIT_FUNC_TRACE();
 358
 359         /* zero descriptor rings */
 360         for (i = 0; i < q->nb_desc; ++i)
 361                 q->hw_ring[i] = zero;
 362
 363         /* free software buffers */
 364         for (i = 0; i < q->nb_desc; ++i) {
 365                 if (q->sw_ring[i]) {
 366                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 367                         q->sw_ring[i] = NULL;
 368                 }
 369         }
 370 }
 371
 372 /*
 373  * free all queue memory used when releasing the queue (i.e. configure)
 374  */
 375 static inline void
 376 tx_queue_free(struct fm10k_tx_queue *q)
 377 {
 378         PMD_INIT_FUNC_TRACE();
 379         if (q) {
 380                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 381                 tx_queue_clean(q);
 382                 if (q->rs_tracker.list) {
 383                         rte_free(q->rs_tracker.list);
 384                         q->rs_tracker.list = NULL;
 385                 }
 386                 if (q->sw_ring) {
 387                         rte_free(q->sw_ring);
 388                         q->sw_ring = NULL;
 389                 }
 390                 rte_free(q);
 391                 q = NULL;
 392         }
 393 }
 394
 395 /*
 396  * disable TX queue, wait unitl HW finished necessary flush operation
 397  */
 398 static inline int
 399 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 400 {
 401         uint32_t reg, i;
 402
 403         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 404         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 405                         reg & ~FM10K_TXDCTL_ENABLE);
 406
 407         /* Wait 100us at most */
 408         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 409                 rte_delay_us(1);
 410                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 411                 if (!(reg & FM10K_TXDCTL_ENABLE))
 412                         break;
 413         }
 414
 415         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 416                 return -1;
 417
 418         return 0;
 419 }
 420
 421 static int
 422 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 423 {
 424         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 425         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 426         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 427         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 428
 429         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 430
 431         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 432                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 433                 return -EINVAL;
 434         }
 435
 436         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 437                 return 0;
 438
 439         if (hw->mac.type == fm10k_mac_vf) {
 440                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 441                 return -EINVAL;
 442         }
 443
 444         /* Check VMDQ queue pool number */
 445         if (vmdq_conf->nb_queue_pools >
 446                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 447                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 448                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 449                         vmdq_conf->nb_queue_pools);
 450                 return -EINVAL;
 451         }
 452
 453         return 0;
 454 }
 455
 456 static const struct fm10k_txq_ops def_txq_ops = {
 457         .reset = tx_queue_reset,
 458 };
 459
 460 static int
 461 fm10k_dev_configure(struct rte_eth_dev *dev)
 462 {
 463         int ret;
 464
 465         PMD_INIT_FUNC_TRACE();
 466
 467         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 468                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 469         /* multipe queue mode checking */
 470         ret  = fm10k_check_mq_mode(dev);
 471         if (ret != 0) {
 472                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 473                             ret);
 474                 return ret;
 475         }
 476
 477         return 0;
 478 }
 479
 480 /* fls = find last set bit = 32 minus the number of leading zeros */
 481 #ifndef fls
 482 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 483 #endif
 484
 485 static void
 486 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 487 {
 488         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 489         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 490         uint32_t i;
 491
 492         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 493
 494         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 495                 if (!vmdq_conf->pool_map[i].pools)
 496                         continue;
 497                 fm10k_mbx_lock(hw);
 498                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 499                 fm10k_mbx_unlock(hw);
 500         }
 501 }
 502
 503 static void
 504 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 505 {
 506         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 507
 508         /* Add default mac address */
 509         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 510                 MAIN_VSI_POOL_NUMBER);
 511 }
 512
 513 static void
 514 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 515 {
 516         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 517         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 518         uint32_t mrqc, *key, i, reta, j;
 519         uint64_t hf;
 520
 521 #define RSS_KEY_SIZE 40
 522         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 523                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 524                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 525                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 526                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 527                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 528         };
 529
 530         if (dev->data->nb_rx_queues == 1 ||
 531             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 532             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 533                 return;
 534
 535         /* random key is rss_intel_key (default) or user provided (rss_key) */
 536         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 537                 key = (uint32_t *)rss_intel_key;
 538         else
 539                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 540
 541         /* Now fill our hash function seeds, 4 bytes at a time */
 542         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 543                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 544
 545         /*
 546          * Fill in redirection table
 547          * The byte-swap is needed because NIC registers are in
 548          * little-endian order.
 549          */
 550         reta = 0;
 551         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 552                 if (j == dev->data->nb_rx_queues)
 553                         j = 0;
 554                 reta = (reta << CHAR_BIT) | j;
 555                 if ((i & 3) == 3)
 556                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 557                                         rte_bswap32(reta));
 558         }
 559
 560         /*
 561          * Generate RSS hash based on packet types, TCP/UDP
 562          * port numbers and/or IPv4/v6 src and dst addresses
 563          */
 564         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 565         mrqc = 0;
 566         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 567         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 568         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 569         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 570         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 571         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 572         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 573         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 574         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 575
 576         if (mrqc == 0) {
 577                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 578                         "supported", hf);
 579                 return;
 580         }
 581
 582         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 583 }
 584
 585 static void
 586 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 587 {
 588         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 589         uint32_t i;
 590
 591         for (i = 0; i < nb_lport_new; i++) {
 592                 /* Set unicast mode by default. App can change
 593                  * to other mode in other API func.
 594                  */
 595                 fm10k_mbx_lock(hw);
 596                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 597                         FM10K_XCAST_MODE_NONE);
 598                 fm10k_mbx_unlock(hw);
 599         }
 600 }
 601
 602 static void
 603 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 604 {
 605         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 606         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 607         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 608         struct fm10k_macvlan_filter_info *macvlan;
 609         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 610         uint16_t nb_lport_new;
 611
 612         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 613         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 614
 615         fm10k_dev_rss_configure(dev);
 616
 617         /* only PF supports VMDQ */
 618         if (hw->mac.type != fm10k_mac_pf)
 619                 return;
 620
 621         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 622                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 623
 624         /* no pool number change, no need to update logic port and VLAN/MAC */
 625         if (macvlan->nb_queue_pools == nb_queue_pools)
 626                 return;
 627
 628         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 629         fm10k_dev_logic_port_update(dev, nb_lport_new);
 630
 631         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 632         memset(dev->data->mac_addrs, 0,
 633                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 634         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 635                 &dev->data->mac_addrs[0]);
 636         memset(macvlan, 0, sizeof(*macvlan));
 637         macvlan->nb_queue_pools = nb_queue_pools;
 638
 639         if (nb_queue_pools)
 640                 fm10k_dev_vmdq_rx_configure(dev);
 641         else
 642                 fm10k_dev_pf_main_vsi_reset(dev);
 643 }
 644
 645 static int
 646 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 647 {
 648         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 649         int i, ret;
 650         struct fm10k_tx_queue *txq;
 651         uint64_t base_addr;
 652         uint32_t size;
 653
 654         /* Disable TXINT to avoid possible interrupt */
 655         for (i = 0; i < hw->mac.max_queues; i++)
 656                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 657                                 3 << FM10K_TXINT_TIMER_SHIFT);
 658
 659         /* Setup TX queue */
 660         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 661                 txq = dev->data->tx_queues[i];
 662                 base_addr = txq->hw_ring_phys_addr;
 663                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 664
 665                 /* disable queue to avoid issues while updating state */
 666                 ret = tx_queue_disable(hw, i);
 667                 if (ret) {
 668                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 669                         return -1;
 670                 }
 671
 672                 /* set location and size for descriptor ring */
 673                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 674                                 base_addr & UINT64_LOWER_32BITS_MASK);
 675                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 676                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 677                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 678         }
 679
 680         /* set up vector or scalar TX function as appropriate */
 681         fm10k_set_tx_function(dev);
 682
 683         return 0;
 684 }
 685
 686 static int
 687 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 688 {
 689         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 690         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
 691         int i, ret;
 692         struct fm10k_rx_queue *rxq;
 693         uint64_t base_addr;
 694         uint32_t size;
 695         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 696         uint16_t buf_size;
 697
 698         /* enable RXINT for interrupt mode */
 699         i = 0;
 700         if (rte_intr_dp_is_en(intr_handle)) {
 701                 for (; i < dev->data->nb_rx_queues; i++) {
 702                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(dev, i));
 703                         if (hw->mac.type == fm10k_mac_pf)
 704                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
 705                                         FM10K_ITR_AUTOMASK |
 706                                         FM10K_ITR_MASK_CLEAR);
 707                         else
 708                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
 709                                         FM10K_ITR_AUTOMASK |
 710                                         FM10K_ITR_MASK_CLEAR);
 711                 }
 712         }
 713         /* Disable other RXINT to avoid possible interrupt */
 714         for (; i < hw->mac.max_queues; i++)
 715                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 716                         3 << FM10K_RXINT_TIMER_SHIFT);
 717
 718         /* Setup RX queues */
 719         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 720                 rxq = dev->data->rx_queues[i];
 721                 base_addr = rxq->hw_ring_phys_addr;
 722                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 723
 724                 /* disable queue to avoid issues while updating state */
 725                 ret = rx_queue_disable(hw, i);
 726                 if (ret) {
 727                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 728                         return -1;
 729                 }
 730
 731                 /* Setup the Base and Length of the Rx Descriptor Ring */
 732                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 733                                 base_addr & UINT64_LOWER_32BITS_MASK);
 734                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 735                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 736                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 737
 738                 /* Configure the Rx buffer size for one buff without split */
 739                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 740                         RTE_PKTMBUF_HEADROOM);
 741                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 742                  * reserved for this purpose, and the worst case could be 511B.
 743                  * But SRR reg assumes all buffers have the same size. In order
 744                  * to fill the gap, we'll have to consider the worst case and
 745                  * assume 512B is reserved. If we don't do so, it's possible
 746                  * for HW to overwrite data to next mbuf.
 747                  */
 748                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 749
 750                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 751                                 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
 752
 753                 /* It adds dual VLAN length for supporting dual VLAN */
 754                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 755                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 756                         dev->data->dev_conf.rxmode.enable_scatter) {
 757                         uint32_t reg;
 758                         dev->data->scattered_rx = 1;
 759                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 760                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 761                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 762                 }
 763
 764                 /* Enable drop on empty, it's RO for VF */
 765                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 766                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 767
 768                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 769                 FM10K_WRITE_FLUSH(hw);
 770         }
 771
 772         /* Configure VMDQ/RSS if applicable */
 773         fm10k_dev_mq_rx_configure(dev);
 774
 775         /* Decide the best RX function */
 776         fm10k_set_rx_function(dev);
 777
 778         return 0;
 779 }
 780
 781 static int
 782 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 783 {
 784         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 785         int err = -1;
 786         uint32_t reg;
 787         struct fm10k_rx_queue *rxq;
 788
 789         PMD_INIT_FUNC_TRACE();
 790
 791         if (rx_queue_id < dev->data->nb_rx_queues) {
 792                 rxq = dev->data->rx_queues[rx_queue_id];
 793                 err = rx_queue_reset(rxq);
 794                 if (err == -ENOMEM) {
 795                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 796                         return err;
 797                 } else if (err == -EINVAL) {
 798                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 799                                 " %d", err);
 800                         return err;
 801                 }
 802
 803                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 804                  * Note: this must be done AFTER the queue is enabled on real
 805                  * hardware, but BEFORE the queue is enabled when using the
 806                  * emulation platform. Do it in both places for now and remove
 807                  * this comment and the following two register writes when the
 808                  * emulation platform is no longer being used.
 809                  */
 810                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 811                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 812
 813                 /* Set PF ownership flag for PF devices */
 814                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 815                 if (hw->mac.type == fm10k_mac_pf)
 816                         reg |= FM10K_RXQCTL_PF;
 817                 reg |= FM10K_RXQCTL_ENABLE;
 818                 /* enable RX queue */
 819                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 820                 FM10K_WRITE_FLUSH(hw);
 821
 822                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 823                  * Note: this must be done AFTER the queue is enabled
 824                  */
 825                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 826                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 827                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 828         }
 829
 830         return err;
 831 }
 832
 833 static int
 834 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 835 {
 836         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 837
 838         PMD_INIT_FUNC_TRACE();
 839
 840         if (rx_queue_id < dev->data->nb_rx_queues) {
 841                 /* Disable RX queue */
 842                 rx_queue_disable(hw, rx_queue_id);
 843
 844                 /* Free mbuf and clean HW ring */
 845                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 846                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 847         }
 848
 849         return 0;
 850 }
 851
 852 static int
 853 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 854 {
 855         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 856         /** @todo - this should be defined in the shared code */
 857 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 858         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 859         int err = 0;
 860
 861         PMD_INIT_FUNC_TRACE();
 862
 863         if (tx_queue_id < dev->data->nb_tx_queues) {
 864                 struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 865
 866                 q->ops->reset(q);
 867
 868                 /* reset head and tail pointers */
 869                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 870                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 871
 872                 /* enable TX queue */
 873                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 874                                         FM10K_TXDCTL_ENABLE | txdctl);
 875                 FM10K_WRITE_FLUSH(hw);
 876                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 877         } else
 878                 err = -1;
 879
 880         return err;
 881 }
 882
 883 static int
 884 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 885 {
 886         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 887
 888         PMD_INIT_FUNC_TRACE();
 889
 890         if (tx_queue_id < dev->data->nb_tx_queues) {
 891                 tx_queue_disable(hw, tx_queue_id);
 892                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 893                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 894         }
 895
 896         return 0;
 897 }
 898
 899 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 900 {
 901         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 902                 != FM10K_DGLORTMAP_NONE);
 903 }
 904
 905 static void
 906 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 907 {
 908         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 909         int status;
 910
 911         PMD_INIT_FUNC_TRACE();
 912
 913         /* Return if it didn't acquire valid glort range */
 914         if (!fm10k_glort_valid(hw))
 915                 return;
 916
 917         fm10k_mbx_lock(hw);
 918         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 919                                 FM10K_XCAST_MODE_PROMISC);
 920         fm10k_mbx_unlock(hw);
 921
 922         if (status != FM10K_SUCCESS)
 923                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 924 }
 925
 926 static void
 927 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 928 {
 929         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 930         uint8_t mode;
 931         int status;
 932
 933         PMD_INIT_FUNC_TRACE();
 934
 935         /* Return if it didn't acquire valid glort range */
 936         if (!fm10k_glort_valid(hw))
 937                 return;
 938
 939         if (dev->data->all_multicast == 1)
 940                 mode = FM10K_XCAST_MODE_ALLMULTI;
 941         else
 942                 mode = FM10K_XCAST_MODE_NONE;
 943
 944         fm10k_mbx_lock(hw);
 945         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 946                                 mode);
 947         fm10k_mbx_unlock(hw);
 948
 949         if (status != FM10K_SUCCESS)
 950                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 951 }
 952
 953 static void
 954 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 955 {
 956         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 957         int status;
 958
 959         PMD_INIT_FUNC_TRACE();
 960
 961         /* Return if it didn't acquire valid glort range */
 962         if (!fm10k_glort_valid(hw))
 963                 return;
 964
 965         /* If promiscuous mode is enabled, it doesn't make sense to enable
 966          * allmulticast and disable promiscuous since fm10k only can select
 967          * one of the modes.
 968          */
 969         if (dev->data->promiscuous) {
 970                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 971                         "needn't enable allmulticast");
 972                 return;
 973         }
 974
 975         fm10k_mbx_lock(hw);
 976         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 977                                 FM10K_XCAST_MODE_ALLMULTI);
 978         fm10k_mbx_unlock(hw);
 979
 980         if (status != FM10K_SUCCESS)
 981                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 982 }
 983
 984 static void
 985 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 986 {
 987         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 988         int status;
 989
 990         PMD_INIT_FUNC_TRACE();
 991
 992         /* Return if it didn't acquire valid glort range */
 993         if (!fm10k_glort_valid(hw))
 994                 return;
 995
 996         if (dev->data->promiscuous) {
 997                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
 998                         "since promisc mode is enabled");
 999                 return;
1000         }
1001
1002         fm10k_mbx_lock(hw);
1003         /* Change mode to unicast mode */
1004         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1005                                 FM10K_XCAST_MODE_NONE);
1006         fm10k_mbx_unlock(hw);
1007
1008         if (status != FM10K_SUCCESS)
1009                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1010 }
1011
1012 static void
1013 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1014 {
1015         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1016         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1017         uint16_t nb_queue_pools;
1018         struct fm10k_macvlan_filter_info *macvlan;
1019
1020         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1021         nb_queue_pools = macvlan->nb_queue_pools;
1022         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
1023         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
1024
1025         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1026         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1027         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1028                         hw->mac.dglort_map;
1029         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1030         /* Configure VMDQ/RSS DGlort Decoder */
1031         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1032
1033         /* Flow Director configurations, only queue number is valid. */
1034         dglortdec = fls(dev->data->nb_rx_queues - 1);
1035         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1036                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1037         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1038         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1039
1040         /* Invalidate all other GLORT entries */
1041         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1042                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1043                                 FM10K_DGLORTMAP_NONE);
1044 }
1045
1046 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1047 static int
1048 fm10k_dev_start(struct rte_eth_dev *dev)
1049 {
1050         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1051         int i, diag;
1052
1053         PMD_INIT_FUNC_TRACE();
1054
1055         /* stop, init, then start the hw */
1056         diag = fm10k_stop_hw(hw);
1057         if (diag != FM10K_SUCCESS) {
1058                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1059                 return -EIO;
1060         }
1061
1062         diag = fm10k_init_hw(hw);
1063         if (diag != FM10K_SUCCESS) {
1064                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1065                 return -EIO;
1066         }
1067
1068         diag = fm10k_start_hw(hw);
1069         if (diag != FM10K_SUCCESS) {
1070                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1071                 return -EIO;
1072         }
1073
1074         diag = fm10k_dev_tx_init(dev);
1075         if (diag) {
1076                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1077                 return diag;
1078         }
1079
1080         if (fm10k_dev_rxq_interrupt_setup(dev))
1081                 return -EIO;
1082
1083         diag = fm10k_dev_rx_init(dev);
1084         if (diag) {
1085                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1086                 return diag;
1087         }
1088
1089         if (hw->mac.type == fm10k_mac_pf)
1090                 fm10k_dev_dglort_map_configure(dev);
1091
1092         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1093                 struct fm10k_rx_queue *rxq;
1094                 rxq = dev->data->rx_queues[i];
1095
1096                 if (rxq->rx_deferred_start)
1097                         continue;
1098                 diag = fm10k_dev_rx_queue_start(dev, i);
1099                 if (diag != 0) {
1100                         int j;
1101                         for (j = 0; j < i; ++j)
1102                                 rx_queue_clean(dev->data->rx_queues[j]);
1103                         return diag;
1104                 }
1105         }
1106
1107         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1108                 struct fm10k_tx_queue *txq;
1109                 txq = dev->data->tx_queues[i];
1110
1111                 if (txq->tx_deferred_start)
1112                         continue;
1113                 diag = fm10k_dev_tx_queue_start(dev, i);
1114                 if (diag != 0) {
1115                         int j;
1116                         for (j = 0; j < i; ++j)
1117                                 tx_queue_clean(dev->data->tx_queues[j]);
1118                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1119                                 rx_queue_clean(dev->data->rx_queues[j]);
1120                         return diag;
1121                 }
1122         }
1123
1124         /* Update default vlan when not in VMDQ mode */
1125         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1126                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1127
1128         return 0;
1129 }
1130
1131 static void
1132 fm10k_dev_stop(struct rte_eth_dev *dev)
1133 {
1134         int i;
1135
1136         PMD_INIT_FUNC_TRACE();
1137
1138         if (dev->data->tx_queues)
1139                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1140                         fm10k_dev_tx_queue_stop(dev, i);
1141
1142         if (dev->data->rx_queues)
1143                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1144                         fm10k_dev_rx_queue_stop(dev, i);
1145 }
1146
1147 static void
1148 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1149 {
1150         int i;
1151
1152         PMD_INIT_FUNC_TRACE();
1153
1154         if (dev->data->tx_queues) {
1155                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1156                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1157
1158                         tx_queue_free(txq);
1159                 }
1160         }
1161
1162         if (dev->data->rx_queues) {
1163                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1164                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1165         }
1166 }
1167
1168 static void
1169 fm10k_dev_close(struct rte_eth_dev *dev)
1170 {
1171         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1172
1173         PMD_INIT_FUNC_TRACE();
1174
1175         fm10k_mbx_lock(hw);
1176         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1177                 MAX_LPORT_NUM, false);
1178         fm10k_mbx_unlock(hw);
1179
1180         /* Stop mailbox service first */
1181         fm10k_close_mbx_service(hw);
1182         fm10k_dev_stop(dev);
1183         fm10k_dev_queue_release(dev);
1184         fm10k_stop_hw(hw);
1185 }
1186
1187 static int
1188 fm10k_link_update(struct rte_eth_dev *dev,
1189         __rte_unused int wait_to_complete)
1190 {
1191         PMD_INIT_FUNC_TRACE();
1192
1193         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
1194          * x8 PCIe interface. For now, we leave the speed undefined since there
1195          * is no 50Gbps Ethernet. */
1196         dev->data->dev_link.link_speed  = 0;
1197         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1198         dev->data->dev_link.link_status = 1;
1199
1200         return 0;
1201 }
1202
1203 static int
1204 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstats *xstats,
1205                  unsigned n)
1206 {
1207         struct fm10k_hw_stats *hw_stats =
1208                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1209         unsigned i, q, count = 0;
1210
1211         if (n < FM10K_NB_XSTATS)
1212                 return FM10K_NB_XSTATS;
1213
1214         /* Global stats */
1215         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1216                 snprintf(xstats[count].name, sizeof(xstats[count].name),
1217                          "%s", fm10k_hw_stats_strings[count].name);
1218                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1219                         fm10k_hw_stats_strings[count].offset);
1220                 count++;
1221         }
1222
1223         /* PF queue stats */
1224         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1225                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1226                         snprintf(xstats[count].name, sizeof(xstats[count].name),
1227                                  "rx_q%u_%s", q,
1228                                  fm10k_hw_stats_rx_q_strings[i].name);
1229                         xstats[count].value =
1230                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1231                                 fm10k_hw_stats_rx_q_strings[i].offset);
1232                         count++;
1233                 }
1234                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1235                         snprintf(xstats[count].name, sizeof(xstats[count].name),
1236                                  "tx_q%u_%s", q,
1237                                  fm10k_hw_stats_tx_q_strings[i].name);
1238                         xstats[count].value =
1239                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1240                                 fm10k_hw_stats_tx_q_strings[i].offset);
1241                         count++;
1242                 }
1243         }
1244
1245         return FM10K_NB_XSTATS;
1246 }
1247
1248 static void
1249 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1250 {
1251         uint64_t ipackets, opackets, ibytes, obytes;
1252         struct fm10k_hw *hw =
1253                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1254         struct fm10k_hw_stats *hw_stats =
1255                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1256         int i;
1257
1258         PMD_INIT_FUNC_TRACE();
1259
1260         fm10k_update_hw_stats(hw, hw_stats);
1261
1262         ipackets = opackets = ibytes = obytes = 0;
1263         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1264                 (i < hw->mac.max_queues); ++i) {
1265                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1266                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1267                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1268                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1269                 ipackets += stats->q_ipackets[i];
1270                 opackets += stats->q_opackets[i];
1271                 ibytes   += stats->q_ibytes[i];
1272                 obytes   += stats->q_obytes[i];
1273         }
1274         stats->ipackets = ipackets;
1275         stats->opackets = opackets;
1276         stats->ibytes = ibytes;
1277         stats->obytes = obytes;
1278 }
1279
1280 static void
1281 fm10k_stats_reset(struct rte_eth_dev *dev)
1282 {
1283         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1284         struct fm10k_hw_stats *hw_stats =
1285                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1286
1287         PMD_INIT_FUNC_TRACE();
1288
1289         memset(hw_stats, 0, sizeof(*hw_stats));
1290         fm10k_rebind_hw_stats(hw, hw_stats);
1291 }
1292
1293 static void
1294 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1295         struct rte_eth_dev_info *dev_info)
1296 {
1297         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1298
1299         PMD_INIT_FUNC_TRACE();
1300
1301         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1302         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1303         dev_info->max_rx_queues      = hw->mac.max_queues;
1304         dev_info->max_tx_queues      = hw->mac.max_queues;
1305         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1306         dev_info->max_hash_mac_addrs = 0;
1307         dev_info->max_vfs            = dev->pci_dev->max_vfs;
1308         dev_info->vmdq_pool_base     = 0;
1309         dev_info->vmdq_queue_base    = 0;
1310         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1311         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1312         dev_info->rx_offload_capa =
1313                 DEV_RX_OFFLOAD_VLAN_STRIP |
1314                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1315                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1316                 DEV_RX_OFFLOAD_TCP_CKSUM;
1317         dev_info->tx_offload_capa =
1318                 DEV_TX_OFFLOAD_VLAN_INSERT |
1319                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1320                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1321                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1322                 DEV_TX_OFFLOAD_TCP_TSO;
1323
1324         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1325         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1326
1327         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1328                 .rx_thresh = {
1329                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1330                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1331                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1332                 },
1333                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1334                 .rx_drop_en = 0,
1335         };
1336
1337         dev_info->default_txconf = (struct rte_eth_txconf) {
1338                 .tx_thresh = {
1339                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1340                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1341                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1342                 },
1343                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1344                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1345                 .txq_flags = FM10K_SIMPLE_TX_FLAG,
1346         };
1347
1348         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1349                 .nb_max = FM10K_MAX_RX_DESC,
1350                 .nb_min = FM10K_MIN_RX_DESC,
1351                 .nb_align = FM10K_MULT_RX_DESC,
1352         };
1353
1354         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1355                 .nb_max = FM10K_MAX_TX_DESC,
1356                 .nb_min = FM10K_MIN_TX_DESC,
1357                 .nb_align = FM10K_MULT_TX_DESC,
1358         };
1359 }
1360
1361 static int
1362 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1363 {
1364         s32 result;
1365         uint16_t mac_num = 0;
1366         uint32_t vid_idx, vid_bit, mac_index;
1367         struct fm10k_hw *hw;
1368         struct fm10k_macvlan_filter_info *macvlan;
1369         struct rte_eth_dev_data *data = dev->data;
1370
1371         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1372         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1373
1374         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1375                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1376                 return -EINVAL;
1377         }
1378
1379         if (vlan_id > ETH_VLAN_ID_MAX) {
1380                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1381                 return -EINVAL;
1382         }
1383
1384         vid_idx = FM10K_VFTA_IDX(vlan_id);
1385         vid_bit = FM10K_VFTA_BIT(vlan_id);
1386         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1387         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1388                 return 0;
1389         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1390         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1391                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1392                         "in the VLAN filter table");
1393                 return -EINVAL;
1394         }
1395
1396         fm10k_mbx_lock(hw);
1397         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1398         fm10k_mbx_unlock(hw);
1399         if (result != FM10K_SUCCESS) {
1400                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1401                 return -EIO;
1402         }
1403
1404         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1405                         (result == FM10K_SUCCESS); mac_index++) {
1406                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1407                         continue;
1408                 if (mac_num > macvlan->mac_num - 1) {
1409                         PMD_INIT_LOG(ERR, "MAC address number "
1410                                         "not match");
1411                         break;
1412                 }
1413                 fm10k_mbx_lock(hw);
1414                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1415                         data->mac_addrs[mac_index].addr_bytes,
1416                         vlan_id, on, 0);
1417                 fm10k_mbx_unlock(hw);
1418                 mac_num++;
1419         }
1420         if (result != FM10K_SUCCESS) {
1421                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1422                 return -EIO;
1423         }
1424
1425         if (on) {
1426                 macvlan->vlan_num++;
1427                 macvlan->vfta[vid_idx] |= vid_bit;
1428         } else {
1429                 macvlan->vlan_num--;
1430                 macvlan->vfta[vid_idx] &= ~vid_bit;
1431         }
1432         return 0;
1433 }
1434
1435 static void
1436 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1437 {
1438         if (mask & ETH_VLAN_STRIP_MASK) {
1439                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1440                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1441                                         "always on in fm10k");
1442         }
1443
1444         if (mask & ETH_VLAN_EXTEND_MASK) {
1445                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1446                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1447                                         "supported in fm10k");
1448         }
1449
1450         if (mask & ETH_VLAN_FILTER_MASK) {
1451                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1452                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1453         }
1454 }
1455
1456 /* Add/Remove a MAC address, and update filters to main VSI */
1457 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1458                 const u8 *mac, bool add, uint32_t pool)
1459 {
1460         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1461         struct fm10k_macvlan_filter_info *macvlan;
1462         uint32_t i, j, k;
1463
1464         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1465
1466         if (pool != MAIN_VSI_POOL_NUMBER) {
1467                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1468                         "mac to pool %u", pool);
1469                 return;
1470         }
1471         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1472                 if (!macvlan->vfta[j])
1473                         continue;
1474                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1475                         if (!(macvlan->vfta[j] & (1 << k)))
1476                                 continue;
1477                         if (i + 1 > macvlan->vlan_num) {
1478                                 PMD_INIT_LOG(ERR, "vlan number not match");
1479                                 return;
1480                         }
1481                         fm10k_mbx_lock(hw);
1482                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1483                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1484                         fm10k_mbx_unlock(hw);
1485                         i++;
1486                 }
1487         }
1488 }
1489
1490 /* Add/Remove a MAC address, and update filters to VMDQ */
1491 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1492                 const u8 *mac, bool add, uint32_t pool)
1493 {
1494         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1495         struct fm10k_macvlan_filter_info *macvlan;
1496         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1497         uint32_t i;
1498
1499         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1500         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1501
1502         if (pool > macvlan->nb_queue_pools) {
1503                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1504                         " Max pool is %u",
1505                         pool, macvlan->nb_queue_pools);
1506                 return;
1507         }
1508         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1509                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1510                         continue;
1511                 fm10k_mbx_lock(hw);
1512                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1513                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1514                 fm10k_mbx_unlock(hw);
1515         }
1516 }
1517
1518 /* Add/Remove a MAC address, and update filters */
1519 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1520                 const u8 *mac, bool add, uint32_t pool)
1521 {
1522         struct fm10k_macvlan_filter_info *macvlan;
1523
1524         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1525
1526         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1527                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1528         else
1529                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1530
1531         if (add)
1532                 macvlan->mac_num++;
1533         else
1534                 macvlan->mac_num--;
1535 }
1536
1537 /* Add a MAC address, and update filters */
1538 static void
1539 fm10k_macaddr_add(struct rte_eth_dev *dev,
1540                 struct ether_addr *mac_addr,
1541                 uint32_t index,
1542                 uint32_t pool)
1543 {
1544         struct fm10k_macvlan_filter_info *macvlan;
1545
1546         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1547         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1548         macvlan->mac_vmdq_id[index] = pool;
1549 }
1550
1551 /* Remove a MAC address, and update filters */
1552 static void
1553 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1554 {
1555         struct rte_eth_dev_data *data = dev->data;
1556         struct fm10k_macvlan_filter_info *macvlan;
1557
1558         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1559         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1560                         FALSE, macvlan->mac_vmdq_id[index]);
1561         macvlan->mac_vmdq_id[index] = 0;
1562 }
1563
1564 static inline int
1565 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1566 {
1567         if ((request < min) || (request > max) || ((request % mult) != 0))
1568                 return -1;
1569         else
1570                 return 0;
1571 }
1572
1573
1574 static inline int
1575 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1576 {
1577         if ((request < min) || (request > max) || ((div % request) != 0))
1578                 return -1;
1579         else
1580                 return 0;
1581 }
1582
1583 static inline int
1584 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1585 {
1586         uint16_t rx_free_thresh;
1587
1588         if (conf->rx_free_thresh == 0)
1589                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1590         else
1591                 rx_free_thresh = conf->rx_free_thresh;
1592
1593         /* make sure the requested threshold satisfies the constraints */
1594         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1595                         FM10K_RX_FREE_THRESH_MAX(q),
1596                         FM10K_RX_FREE_THRESH_DIV(q),
1597                         rx_free_thresh)) {
1598                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1599                         "less than or equal to %u, "
1600                         "greater than or equal to %u, "
1601                         "and a divisor of %u",
1602                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1603                         FM10K_RX_FREE_THRESH_MIN(q),
1604                         FM10K_RX_FREE_THRESH_DIV(q));
1605                 return -EINVAL;
1606         }
1607
1608         q->alloc_thresh = rx_free_thresh;
1609         q->drop_en = conf->rx_drop_en;
1610         q->rx_deferred_start = conf->rx_deferred_start;
1611
1612         return 0;
1613 }
1614
1615 /*
1616  * Hardware requires specific alignment for Rx packet buffers. At
1617  * least one of the following two conditions must be satisfied.
1618  *  1. Address is 512B aligned
1619  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1620  *
1621  * As such, the driver may need to adjust the DMA address within the
1622  * buffer by up to 512B.
1623  *
1624  * return 1 if the element size is valid, otherwise return 0.
1625  */
1626 static int
1627 mempool_element_size_valid(struct rte_mempool *mp)
1628 {
1629         uint32_t min_size;
1630
1631         /* elt_size includes mbuf header and headroom */
1632         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1633                         RTE_PKTMBUF_HEADROOM;
1634
1635         /* account for up to 512B of alignment */
1636         min_size -= FM10K_RX_DATABUF_ALIGN;
1637
1638         /* sanity check for overflow */
1639         if (min_size > mp->elt_size)
1640                 return 0;
1641
1642         /* size is valid */
1643         return 1;
1644 }
1645
1646 static int
1647 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1648         uint16_t nb_desc, unsigned int socket_id,
1649         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1650 {
1651         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1652         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
1653         struct fm10k_rx_queue *q;
1654         const struct rte_memzone *mz;
1655
1656         PMD_INIT_FUNC_TRACE();
1657
1658         /* make sure the mempool element size can account for alignment. */
1659         if (!mempool_element_size_valid(mp)) {
1660                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1661                 return -EINVAL;
1662         }
1663
1664         /* make sure a valid number of descriptors have been requested */
1665         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1666                                 FM10K_MULT_RX_DESC, nb_desc)) {
1667                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1668                         "less than or equal to %"PRIu32", "
1669                         "greater than or equal to %u, "
1670                         "and a multiple of %u",
1671                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1672                         FM10K_MULT_RX_DESC);
1673                 return -EINVAL;
1674         }
1675
1676         /*
1677          * if this queue existed already, free the associated memory. The
1678          * queue cannot be reused in case we need to allocate memory on
1679          * different socket than was previously used.
1680          */
1681         if (dev->data->rx_queues[queue_id] != NULL) {
1682                 rx_queue_free(dev->data->rx_queues[queue_id]);
1683                 dev->data->rx_queues[queue_id] = NULL;
1684         }
1685
1686         /* allocate memory for the queue structure */
1687         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1688                                 socket_id);
1689         if (q == NULL) {
1690                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1691                 return -ENOMEM;
1692         }
1693
1694         /* setup queue */
1695         q->mp = mp;
1696         q->nb_desc = nb_desc;
1697         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1698         q->port_id = dev->data->port_id;
1699         q->queue_id = queue_id;
1700         q->tail_ptr = (volatile uint32_t *)
1701                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1702         if (handle_rxconf(q, conf))
1703                 return -EINVAL;
1704
1705         /* allocate memory for the software ring */
1706         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1707                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1708                         RTE_CACHE_LINE_SIZE, socket_id);
1709         if (q->sw_ring == NULL) {
1710                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1711                 rte_free(q);
1712                 return -ENOMEM;
1713         }
1714
1715         /*
1716          * allocate memory for the hardware descriptor ring. A memzone large
1717          * enough to hold the maximum ring size is requested to allow for
1718          * resizing in later calls to the queue setup function.
1719          */
1720         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1721                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1722                                       socket_id);
1723         if (mz == NULL) {
1724                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1725                 rte_free(q->sw_ring);
1726                 rte_free(q);
1727                 return -ENOMEM;
1728         }
1729         q->hw_ring = mz->addr;
1730         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1731
1732         /* Check if number of descs satisfied Vector requirement */
1733         if (!rte_is_power_of_2(nb_desc)) {
1734                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1735                                     "preconditions - canceling the feature for "
1736                                     "the whole port[%d]",
1737                              q->queue_id, q->port_id);
1738                 dev_info->rx_vec_allowed = false;
1739         } else
1740                 fm10k_rxq_vec_setup(q);
1741
1742         dev->data->rx_queues[queue_id] = q;
1743         return 0;
1744 }
1745
1746 static void
1747 fm10k_rx_queue_release(void *queue)
1748 {
1749         PMD_INIT_FUNC_TRACE();
1750
1751         rx_queue_free(queue);
1752 }
1753
1754 static inline int
1755 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1756 {
1757         uint16_t tx_free_thresh;
1758         uint16_t tx_rs_thresh;
1759
1760         /* constraint MACROs require that tx_free_thresh is configured
1761          * before tx_rs_thresh */
1762         if (conf->tx_free_thresh == 0)
1763                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1764         else
1765                 tx_free_thresh = conf->tx_free_thresh;
1766
1767         /* make sure the requested threshold satisfies the constraints */
1768         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1769                         FM10K_TX_FREE_THRESH_MAX(q),
1770                         FM10K_TX_FREE_THRESH_DIV(q),
1771                         tx_free_thresh)) {
1772                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1773                         "less than or equal to %u, "
1774                         "greater than or equal to %u, "
1775                         "and a divisor of %u",
1776                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1777                         FM10K_TX_FREE_THRESH_MIN(q),
1778                         FM10K_TX_FREE_THRESH_DIV(q));
1779                 return -EINVAL;
1780         }
1781
1782         q->free_thresh = tx_free_thresh;
1783
1784         if (conf->tx_rs_thresh == 0)
1785                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1786         else
1787                 tx_rs_thresh = conf->tx_rs_thresh;
1788
1789         q->tx_deferred_start = conf->tx_deferred_start;
1790
1791         /* make sure the requested threshold satisfies the constraints */
1792         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1793                         FM10K_TX_RS_THRESH_MAX(q),
1794                         FM10K_TX_RS_THRESH_DIV(q),
1795                         tx_rs_thresh)) {
1796                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1797                         "less than or equal to %u, "
1798                         "greater than or equal to %u, "
1799                         "and a divisor of %u",
1800                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1801                         FM10K_TX_RS_THRESH_MIN(q),
1802                         FM10K_TX_RS_THRESH_DIV(q));
1803                 return -EINVAL;
1804         }
1805
1806         q->rs_thresh = tx_rs_thresh;
1807
1808         return 0;
1809 }
1810
1811 static int
1812 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1813         uint16_t nb_desc, unsigned int socket_id,
1814         const struct rte_eth_txconf *conf)
1815 {
1816         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1817         struct fm10k_tx_queue *q;
1818         const struct rte_memzone *mz;
1819
1820         PMD_INIT_FUNC_TRACE();
1821
1822         /* make sure a valid number of descriptors have been requested */
1823         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1824                                 FM10K_MULT_TX_DESC, nb_desc)) {
1825                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1826                         "less than or equal to %"PRIu32", "
1827                         "greater than or equal to %u, "
1828                         "and a multiple of %u",
1829                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1830                         FM10K_MULT_TX_DESC);
1831                 return -EINVAL;
1832         }
1833
1834         /*
1835          * if this queue existed already, free the associated memory. The
1836          * queue cannot be reused in case we need to allocate memory on
1837          * different socket than was previously used.
1838          */
1839         if (dev->data->tx_queues[queue_id] != NULL) {
1840                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
1841
1842                 tx_queue_free(txq);
1843                 dev->data->tx_queues[queue_id] = NULL;
1844         }
1845
1846         /* allocate memory for the queue structure */
1847         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1848                                 socket_id);
1849         if (q == NULL) {
1850                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1851                 return -ENOMEM;
1852         }
1853
1854         /* setup queue */
1855         q->nb_desc = nb_desc;
1856         q->port_id = dev->data->port_id;
1857         q->queue_id = queue_id;
1858         q->txq_flags = conf->txq_flags;
1859         q->ops = &def_txq_ops;
1860         q->tail_ptr = (volatile uint32_t *)
1861                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1862         if (handle_txconf(q, conf))
1863                 return -EINVAL;
1864
1865         /* allocate memory for the software ring */
1866         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1867                                         nb_desc * sizeof(struct rte_mbuf *),
1868                                         RTE_CACHE_LINE_SIZE, socket_id);
1869         if (q->sw_ring == NULL) {
1870                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1871                 rte_free(q);
1872                 return -ENOMEM;
1873         }
1874
1875         /*
1876          * allocate memory for the hardware descriptor ring. A memzone large
1877          * enough to hold the maximum ring size is requested to allow for
1878          * resizing in later calls to the queue setup function.
1879          */
1880         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
1881                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
1882                                       socket_id);
1883         if (mz == NULL) {
1884                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1885                 rte_free(q->sw_ring);
1886                 rte_free(q);
1887                 return -ENOMEM;
1888         }
1889         q->hw_ring = mz->addr;
1890         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1891
1892         /*
1893          * allocate memory for the RS bit tracker. Enough slots to hold the
1894          * descriptor index for each RS bit needing to be set are required.
1895          */
1896         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1897                                 ((nb_desc + 1) / q->rs_thresh) *
1898                                 sizeof(uint16_t),
1899                                 RTE_CACHE_LINE_SIZE, socket_id);
1900         if (q->rs_tracker.list == NULL) {
1901                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1902                 rte_free(q->sw_ring);
1903                 rte_free(q);
1904                 return -ENOMEM;
1905         }
1906
1907         dev->data->tx_queues[queue_id] = q;
1908         return 0;
1909 }
1910
1911 static void
1912 fm10k_tx_queue_release(void *queue)
1913 {
1914         struct fm10k_tx_queue *q = queue;
1915         PMD_INIT_FUNC_TRACE();
1916
1917         tx_queue_free(q);
1918 }
1919
1920 static int
1921 fm10k_reta_update(struct rte_eth_dev *dev,
1922                         struct rte_eth_rss_reta_entry64 *reta_conf,
1923                         uint16_t reta_size)
1924 {
1925         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1926         uint16_t i, j, idx, shift;
1927         uint8_t mask;
1928         uint32_t reta;
1929
1930         PMD_INIT_FUNC_TRACE();
1931
1932         if (reta_size > FM10K_MAX_RSS_INDICES) {
1933                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1934                         "(%d) doesn't match the number hardware can supported "
1935                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1936                 return -EINVAL;
1937         }
1938
1939         /*
1940          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1941          * 128-entries in 32 registers
1942          */
1943         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1944                 idx = i / RTE_RETA_GROUP_SIZE;
1945                 shift = i % RTE_RETA_GROUP_SIZE;
1946                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1947                                 BIT_MASK_PER_UINT32);
1948                 if (mask == 0)
1949                         continue;
1950
1951                 reta = 0;
1952                 if (mask != BIT_MASK_PER_UINT32)
1953                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1954
1955                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1956                         if (mask & (0x1 << j)) {
1957                                 if (mask != 0xF)
1958                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1959                                 reta |= reta_conf[idx].reta[shift + j] <<
1960                                                 (CHAR_BIT * j);
1961                         }
1962                 }
1963                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1964         }
1965
1966         return 0;
1967 }
1968
1969 static int
1970 fm10k_reta_query(struct rte_eth_dev *dev,
1971                         struct rte_eth_rss_reta_entry64 *reta_conf,
1972                         uint16_t reta_size)
1973 {
1974         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1975         uint16_t i, j, idx, shift;
1976         uint8_t mask;
1977         uint32_t reta;
1978
1979         PMD_INIT_FUNC_TRACE();
1980
1981         if (reta_size < FM10K_MAX_RSS_INDICES) {
1982                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1983                         "(%d) doesn't match the number hardware can supported "
1984                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1985                 return -EINVAL;
1986         }
1987
1988         /*
1989          * Read Redirection Table RETA[n], n=0..31. The redirection table has
1990          * 128-entries in 32 registers
1991          */
1992         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1993                 idx = i / RTE_RETA_GROUP_SIZE;
1994                 shift = i % RTE_RETA_GROUP_SIZE;
1995                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1996                                 BIT_MASK_PER_UINT32);
1997                 if (mask == 0)
1998                         continue;
1999
2000                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2001                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2002                         if (mask & (0x1 << j))
2003                                 reta_conf[idx].reta[shift + j] = ((reta >>
2004                                         CHAR_BIT * j) & UINT8_MAX);
2005                 }
2006         }
2007
2008         return 0;
2009 }
2010
2011 static int
2012 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2013         struct rte_eth_rss_conf *rss_conf)
2014 {
2015         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2016         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2017         uint32_t mrqc;
2018         uint64_t hf = rss_conf->rss_hf;
2019         int i;
2020
2021         PMD_INIT_FUNC_TRACE();
2022
2023         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2024                 FM10K_RSSRK_ENTRIES_PER_REG)
2025                 return -EINVAL;
2026
2027         if (hf == 0)
2028                 return -EINVAL;
2029
2030         mrqc = 0;
2031         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2032         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2033         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2034         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2035         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2036         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2037         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2038         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2039         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2040
2041         /* If the mapping doesn't fit any supported, return */
2042         if (mrqc == 0)
2043                 return -EINVAL;
2044
2045         if (key != NULL)
2046                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2047                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2048
2049         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2050
2051         return 0;
2052 }
2053
2054 static int
2055 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2056         struct rte_eth_rss_conf *rss_conf)
2057 {
2058         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2059         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2060         uint32_t mrqc;
2061         uint64_t hf;
2062         int i;
2063
2064         PMD_INIT_FUNC_TRACE();
2065
2066         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2067                                 FM10K_RSSRK_ENTRIES_PER_REG)
2068                 return -EINVAL;
2069
2070         if (key != NULL)
2071                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2072                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2073
2074         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2075         hf = 0;
2076         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2077         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2078         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2079         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2080         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2081         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2082         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2083         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2084         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2085
2086         rss_conf->rss_hf = hf;
2087
2088         return 0;
2089 }
2090
2091 static void
2092 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2093 {
2094         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2095         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2096
2097         /* Bind all local non-queue interrupt to vector 0 */
2098         int_map |= FM10K_MISC_VEC_ID;
2099
2100         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
2101         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
2102         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
2103         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
2104         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
2105         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
2106
2107         /* Enable misc causes */
2108         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2109                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2110                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2111                                 FM10K_EIMR_ENABLE(MAILBOX) |
2112                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2113                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2114                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2115                                 FM10K_EIMR_ENABLE(VFLR));
2116
2117         /* Enable ITR 0 */
2118         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2119                                         FM10K_ITR_MASK_CLEAR);
2120         FM10K_WRITE_FLUSH(hw);
2121 }
2122
2123 static void
2124 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2125 {
2126         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2127         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2128
2129         int_map |= FM10K_MISC_VEC_ID;
2130
2131         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
2132         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
2133         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
2134         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
2135         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
2136         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
2137
2138         /* Disable misc causes */
2139         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2140                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2141                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2142                                 FM10K_EIMR_DISABLE(MAILBOX) |
2143                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2144                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2145                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2146                                 FM10K_EIMR_DISABLE(VFLR));
2147
2148         /* Disable ITR 0 */
2149         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2150         FM10K_WRITE_FLUSH(hw);
2151 }
2152
2153 static void
2154 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2155 {
2156         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2157         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2158
2159         /* Bind all local non-queue interrupt to vector 0 */
2160         int_map |= FM10K_MISC_VEC_ID;
2161
2162         /* Only INT 0 available, other 15 are reserved. */
2163         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2164
2165         /* Enable ITR 0 */
2166         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2167                                         FM10K_ITR_MASK_CLEAR);
2168         FM10K_WRITE_FLUSH(hw);
2169 }
2170
2171 static void
2172 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2173 {
2174         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2175         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2176
2177         int_map |= FM10K_MISC_VEC_ID;
2178
2179         /* Only INT 0 available, other 15 are reserved. */
2180         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2181
2182         /* Disable ITR 0 */
2183         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2184         FM10K_WRITE_FLUSH(hw);
2185 }
2186
2187 static int
2188 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2189 {
2190         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2191         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
2192         uint32_t intr_vector, vec;
2193         uint16_t queue_id;
2194         int result = 0;
2195
2196         /* fm10k needs one separate interrupt for mailbox,
2197          * so only drivers which support multiple interrupt vectors
2198          * e.g. vfio-pci can work for fm10k interrupt mode
2199          */
2200         if (!rte_intr_cap_multiple(intr_handle) ||
2201                         dev->data->dev_conf.intr_conf.rxq == 0)
2202                 return result;
2203
2204         intr_vector = dev->data->nb_rx_queues;
2205
2206         /* disable interrupt first */
2207         rte_intr_disable(&dev->pci_dev->intr_handle);
2208         if (hw->mac.type == fm10k_mac_pf)
2209                 fm10k_dev_disable_intr_pf(dev);
2210         else
2211                 fm10k_dev_disable_intr_vf(dev);
2212
2213         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2214                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2215                 result = -EIO;
2216         }
2217
2218         if (rte_intr_dp_is_en(intr_handle) && !result) {
2219                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2220                         dev->data->nb_rx_queues * sizeof(int), 0);
2221                 if (intr_handle->intr_vec) {
2222                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2223                                         queue_id < dev->data->nb_rx_queues;
2224                                         queue_id++) {
2225                                 intr_handle->intr_vec[queue_id] = vec;
2226                                 if (vec < intr_handle->nb_efd - 1
2227                                                 + FM10K_RX_VEC_START)
2228                                         vec++;
2229                         }
2230                 } else {
2231                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2232                                 " intr_vec", dev->data->nb_rx_queues);
2233                         rte_intr_efd_disable(intr_handle);
2234                         result = -ENOMEM;
2235                 }
2236         }
2237
2238         if (hw->mac.type == fm10k_mac_pf)
2239                 fm10k_dev_enable_intr_pf(dev);
2240         else
2241                 fm10k_dev_enable_intr_vf(dev);
2242         rte_intr_enable(&dev->pci_dev->intr_handle);
2243         hw->mac.ops.update_int_moderator(hw);
2244         return result;
2245 }
2246
2247 static int
2248 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2249 {
2250         struct fm10k_fault fault;
2251         int err;
2252         const char *estr = "Unknown error";
2253
2254         /* Process PCA fault */
2255         if (eicr & FM10K_EICR_PCA_FAULT) {
2256                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2257                 if (err)
2258                         goto error;
2259                 switch (fault.type) {
2260                 case PCA_NO_FAULT:
2261                         estr = "PCA_NO_FAULT"; break;
2262                 case PCA_UNMAPPED_ADDR:
2263                         estr = "PCA_UNMAPPED_ADDR"; break;
2264                 case PCA_BAD_QACCESS_PF:
2265                         estr = "PCA_BAD_QACCESS_PF"; break;
2266                 case PCA_BAD_QACCESS_VF:
2267                         estr = "PCA_BAD_QACCESS_VF"; break;
2268                 case PCA_MALICIOUS_REQ:
2269                         estr = "PCA_MALICIOUS_REQ"; break;
2270                 case PCA_POISONED_TLP:
2271                         estr = "PCA_POISONED_TLP"; break;
2272                 case PCA_TLP_ABORT:
2273                         estr = "PCA_TLP_ABORT"; break;
2274                 default:
2275                         goto error;
2276                 }
2277                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2278                         estr, fault.func ? "VF" : "PF", fault.func,
2279                         fault.address, fault.specinfo);
2280         }
2281
2282         /* Process THI fault */
2283         if (eicr & FM10K_EICR_THI_FAULT) {
2284                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2285                 if (err)
2286                         goto error;
2287                 switch (fault.type) {
2288                 case THI_NO_FAULT:
2289                         estr = "THI_NO_FAULT"; break;
2290                 case THI_MAL_DIS_Q_FAULT:
2291                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2292                 default:
2293                         goto error;
2294                 }
2295                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2296                         estr, fault.func ? "VF" : "PF", fault.func,
2297                         fault.address, fault.specinfo);
2298         }
2299
2300         /* Process FUM fault */
2301         if (eicr & FM10K_EICR_FUM_FAULT) {
2302                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2303                 if (err)
2304                         goto error;
2305                 switch (fault.type) {
2306                 case FUM_NO_FAULT:
2307                         estr = "FUM_NO_FAULT"; break;
2308                 case FUM_UNMAPPED_ADDR:
2309                         estr = "FUM_UNMAPPED_ADDR"; break;
2310                 case FUM_POISONED_TLP:
2311                         estr = "FUM_POISONED_TLP"; break;
2312                 case FUM_BAD_VF_QACCESS:
2313                         estr = "FUM_BAD_VF_QACCESS"; break;
2314                 case FUM_ADD_DECODE_ERR:
2315                         estr = "FUM_ADD_DECODE_ERR"; break;
2316                 case FUM_RO_ERROR:
2317                         estr = "FUM_RO_ERROR"; break;
2318                 case FUM_QPRC_CRC_ERROR:
2319                         estr = "FUM_QPRC_CRC_ERROR"; break;
2320                 case FUM_CSR_TIMEOUT:
2321                         estr = "FUM_CSR_TIMEOUT"; break;
2322                 case FUM_INVALID_TYPE:
2323                         estr = "FUM_INVALID_TYPE"; break;
2324                 case FUM_INVALID_LENGTH:
2325                         estr = "FUM_INVALID_LENGTH"; break;
2326                 case FUM_INVALID_BE:
2327                         estr = "FUM_INVALID_BE"; break;
2328                 case FUM_INVALID_ALIGN:
2329                         estr = "FUM_INVALID_ALIGN"; break;
2330                 default:
2331                         goto error;
2332                 }
2333                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2334                         estr, fault.func ? "VF" : "PF", fault.func,
2335                         fault.address, fault.specinfo);
2336         }
2337
2338         return 0;
2339 error:
2340         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2341         return err;
2342 }
2343
2344 /**
2345  * PF interrupt handler triggered by NIC for handling specific interrupt.
2346  *
2347  * @param handle
2348  *  Pointer to interrupt handle.
2349  * @param param
2350  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2351  *
2352  * @return
2353  *  void
2354  */
2355 static void
2356 fm10k_dev_interrupt_handler_pf(
2357                         __rte_unused struct rte_intr_handle *handle,
2358                         void *param)
2359 {
2360         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2361         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2362         uint32_t cause, status;
2363
2364         if (hw->mac.type != fm10k_mac_pf)
2365                 return;
2366
2367         cause = FM10K_READ_REG(hw, FM10K_EICR);
2368
2369         /* Handle PCI fault cases */
2370         if (cause & FM10K_EICR_FAULT_MASK) {
2371                 PMD_INIT_LOG(ERR, "INT: find fault!");
2372                 fm10k_dev_handle_fault(hw, cause);
2373         }
2374
2375         /* Handle switch up/down */
2376         if (cause & FM10K_EICR_SWITCHNOTREADY)
2377                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2378
2379         if (cause & FM10K_EICR_SWITCHREADY)
2380                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2381
2382         /* Handle mailbox message */
2383         fm10k_mbx_lock(hw);
2384         hw->mbx.ops.process(hw, &hw->mbx);
2385         fm10k_mbx_unlock(hw);
2386
2387         /* Handle SRAM error */
2388         if (cause & FM10K_EICR_SRAMERROR) {
2389                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2390
2391                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2392                 /* Write to clear pending bits */
2393                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2394
2395                 /* Todo: print out error message after shared code  updates */
2396         }
2397
2398         /* Clear these 3 events if having any */
2399         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2400                  FM10K_EICR_SWITCHREADY;
2401         if (cause)
2402                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2403
2404         /* Re-enable interrupt from device side */
2405         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2406                                         FM10K_ITR_MASK_CLEAR);
2407         /* Re-enable interrupt from host side */
2408         rte_intr_enable(&(dev->pci_dev->intr_handle));
2409 }
2410
2411 /**
2412  * VF interrupt handler triggered by NIC for handling specific interrupt.
2413  *
2414  * @param handle
2415  *  Pointer to interrupt handle.
2416  * @param param
2417  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2418  *
2419  * @return
2420  *  void
2421  */
2422 static void
2423 fm10k_dev_interrupt_handler_vf(
2424                         __rte_unused struct rte_intr_handle *handle,
2425                         void *param)
2426 {
2427         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2428         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2429
2430         if (hw->mac.type != fm10k_mac_vf)
2431                 return;
2432
2433         /* Handle mailbox message if lock is acquired */
2434         fm10k_mbx_lock(hw);
2435         hw->mbx.ops.process(hw, &hw->mbx);
2436         fm10k_mbx_unlock(hw);
2437
2438         /* Re-enable interrupt from device side */
2439         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2440                                         FM10K_ITR_MASK_CLEAR);
2441         /* Re-enable interrupt from host side */
2442         rte_intr_enable(&(dev->pci_dev->intr_handle));
2443 }
2444
2445 /* Mailbox message handler in VF */
2446 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2447         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2448         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2449         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2450         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2451 };
2452
2453 /* Mailbox message handler in PF */
2454 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
2455         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
2456         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
2457         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
2458         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
2459         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
2460         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
2461         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2462 };
2463
2464 static int
2465 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2466 {
2467         int err;
2468
2469         /* Initialize mailbox lock */
2470         fm10k_mbx_initlock(hw);
2471
2472         /* Replace default message handler with new ones */
2473         if (hw->mac.type == fm10k_mac_pf)
2474                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
2475         else
2476                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2477
2478         if (err) {
2479                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2480                                 err);
2481                 return err;
2482         }
2483         /* Connect to SM for PF device or PF for VF device */
2484         return hw->mbx.ops.connect(hw, &hw->mbx);
2485 }
2486
2487 static void
2488 fm10k_close_mbx_service(struct fm10k_hw *hw)
2489 {
2490         /* Disconnect from SM for PF device or PF for VF device */
2491         hw->mbx.ops.disconnect(hw, &hw->mbx);
2492 }
2493
2494 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2495         .dev_configure          = fm10k_dev_configure,
2496         .dev_start              = fm10k_dev_start,
2497         .dev_stop               = fm10k_dev_stop,
2498         .dev_close              = fm10k_dev_close,
2499         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2500         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2501         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2502         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2503         .stats_get              = fm10k_stats_get,
2504         .xstats_get             = fm10k_xstats_get,
2505         .stats_reset            = fm10k_stats_reset,
2506         .xstats_reset           = fm10k_stats_reset,
2507         .link_update            = fm10k_link_update,
2508         .dev_infos_get          = fm10k_dev_infos_get,
2509         .vlan_filter_set        = fm10k_vlan_filter_set,
2510         .vlan_offload_set       = fm10k_vlan_offload_set,
2511         .mac_addr_add           = fm10k_macaddr_add,
2512         .mac_addr_remove        = fm10k_macaddr_remove,
2513         .rx_queue_start         = fm10k_dev_rx_queue_start,
2514         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2515         .tx_queue_start         = fm10k_dev_tx_queue_start,
2516         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2517         .rx_queue_setup         = fm10k_rx_queue_setup,
2518         .rx_queue_release       = fm10k_rx_queue_release,
2519         .tx_queue_setup         = fm10k_tx_queue_setup,
2520         .tx_queue_release       = fm10k_tx_queue_release,
2521         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2522         .reta_update            = fm10k_reta_update,
2523         .reta_query             = fm10k_reta_query,
2524         .rss_hash_update        = fm10k_rss_hash_update,
2525         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2526 };
2527
2528 static void __attribute__((cold))
2529 fm10k_set_tx_function(struct rte_eth_dev *dev)
2530 {
2531         struct fm10k_tx_queue *txq;
2532         int i;
2533         int use_sse = 1;
2534
2535         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2536                 txq = dev->data->tx_queues[i];
2537                 /* Check if Vector Tx is satisfied */
2538                 if (fm10k_tx_vec_condition_check(txq)) {
2539                         use_sse = 0;
2540                         break;
2541                 }
2542         }
2543
2544         if (use_sse) {
2545                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2546                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2547                         txq = dev->data->tx_queues[i];
2548                         fm10k_txq_vec_setup(txq);
2549                 }
2550                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2551         } else {
2552                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2553                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2554         }
2555 }
2556
2557 static void __attribute__((cold))
2558 fm10k_set_rx_function(struct rte_eth_dev *dev)
2559 {
2560         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2561         uint16_t i, rx_using_sse;
2562
2563         /* In order to allow Vector Rx there are a few configuration
2564          * conditions to be met.
2565          */
2566         if (!fm10k_rx_vec_condition_check(dev) && dev_info->rx_vec_allowed) {
2567                 if (dev->data->scattered_rx)
2568                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2569                 else
2570                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2571         } else if (dev->data->scattered_rx)
2572                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2573         else
2574                 dev->rx_pkt_burst = fm10k_recv_pkts;
2575
2576         rx_using_sse =
2577                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2578                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2579
2580         if (rx_using_sse)
2581                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
2582         else
2583                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
2584
2585         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2586                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2587
2588                 rxq->rx_using_sse = rx_using_sse;
2589         }
2590 }
2591
2592 static void
2593 fm10k_params_init(struct rte_eth_dev *dev)
2594 {
2595         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2596         struct fm10k_dev_info *info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2597
2598         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2599          * there is no way to get link status without reading BAR4.  Until this
2600          * works, assume we have maximum bandwidth.
2601          * @todo - fix bus info
2602          */
2603         hw->bus_caps.speed = fm10k_bus_speed_8000;
2604         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2605         hw->bus_caps.payload = fm10k_bus_payload_512;
2606         hw->bus.speed = fm10k_bus_speed_8000;
2607         hw->bus.width = fm10k_bus_width_pcie_x8;
2608         hw->bus.payload = fm10k_bus_payload_256;
2609
2610         info->rx_vec_allowed = true;
2611 }
2612
2613 static int
2614 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2615 {
2616         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2617         int diag, i;
2618         struct fm10k_macvlan_filter_info *macvlan;
2619
2620         PMD_INIT_FUNC_TRACE();
2621
2622         dev->dev_ops = &fm10k_eth_dev_ops;
2623         dev->rx_pkt_burst = &fm10k_recv_pkts;
2624         dev->tx_pkt_burst = &fm10k_xmit_pkts;
2625
2626         /* only initialize in the primary process */
2627         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2628                 return 0;
2629
2630         rte_eth_copy_pci_info(dev, dev->pci_dev);
2631
2632         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2633         memset(macvlan, 0, sizeof(*macvlan));
2634         /* Vendor and Device ID need to be set before init of shared code */
2635         memset(hw, 0, sizeof(*hw));
2636         hw->device_id = dev->pci_dev->id.device_id;
2637         hw->vendor_id = dev->pci_dev->id.vendor_id;
2638         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
2639         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
2640         hw->revision_id = 0;
2641         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2642         if (hw->hw_addr == NULL) {
2643                 PMD_INIT_LOG(ERR, "Bad mem resource."
2644                         " Try to blacklist unused devices.");
2645                 return -EIO;
2646         }
2647
2648         /* Store fm10k_adapter pointer */
2649         hw->back = dev->data->dev_private;
2650
2651         /* Initialize the shared code */
2652         diag = fm10k_init_shared_code(hw);
2653         if (diag != FM10K_SUCCESS) {
2654                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2655                 return -EIO;
2656         }
2657
2658         /* Initialize parameters */
2659         fm10k_params_init(dev);
2660
2661         /* Initialize the hw */
2662         diag = fm10k_init_hw(hw);
2663         if (diag != FM10K_SUCCESS) {
2664                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2665                 return -EIO;
2666         }
2667
2668         /* Initialize MAC address(es) */
2669         dev->data->mac_addrs = rte_zmalloc("fm10k",
2670                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2671         if (dev->data->mac_addrs == NULL) {
2672                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2673                 return -ENOMEM;
2674         }
2675
2676         diag = fm10k_read_mac_addr(hw);
2677
2678         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2679                         &dev->data->mac_addrs[0]);
2680
2681         if (diag != FM10K_SUCCESS ||
2682                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2683
2684                 /* Generate a random addr */
2685                 eth_random_addr(hw->mac.addr);
2686                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2687                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2688                 &dev->data->mac_addrs[0]);
2689         }
2690
2691         /* Reset the hw statistics */
2692         fm10k_stats_reset(dev);
2693
2694         /* Reset the hw */
2695         diag = fm10k_reset_hw(hw);
2696         if (diag != FM10K_SUCCESS) {
2697                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2698                 return -EIO;
2699         }
2700
2701         /* Setup mailbox service */
2702         diag = fm10k_setup_mbx_service(hw);
2703         if (diag != FM10K_SUCCESS) {
2704                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2705                 return -EIO;
2706         }
2707
2708         /*PF/VF has different interrupt handling mechanism */
2709         if (hw->mac.type == fm10k_mac_pf) {
2710                 /* register callback func to eal lib */
2711                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2712                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2713
2714                 /* enable MISC interrupt */
2715                 fm10k_dev_enable_intr_pf(dev);
2716         } else { /* VF */
2717                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2718                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2719
2720                 fm10k_dev_enable_intr_vf(dev);
2721         }
2722
2723         /* Enable intr after callback registered */
2724         rte_intr_enable(&(dev->pci_dev->intr_handle));
2725
2726         hw->mac.ops.update_int_moderator(hw);
2727
2728         /* Make sure Switch Manager is ready before going forward. */
2729         if (hw->mac.type == fm10k_mac_pf) {
2730                 int switch_ready = 0;
2731
2732                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2733                         fm10k_mbx_lock(hw);
2734                         hw->mac.ops.get_host_state(hw, &switch_ready);
2735                         fm10k_mbx_unlock(hw);
2736                         if (switch_ready)
2737                                 break;
2738                         /* Delay some time to acquire async LPORT_MAP info. */
2739                         rte_delay_us(WAIT_SWITCH_MSG_US);
2740                 }
2741
2742                 if (switch_ready == 0) {
2743                         PMD_INIT_LOG(ERR, "switch is not ready");
2744                         return -1;
2745                 }
2746         }
2747
2748         /*
2749          * Below function will trigger operations on mailbox, acquire lock to
2750          * avoid race condition from interrupt handler. Operations on mailbox
2751          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2752          * will handle and generate an interrupt to our side. Then,  FIFO in
2753          * mailbox will be touched.
2754          */
2755         fm10k_mbx_lock(hw);
2756         /* Enable port first */
2757         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2758                                         MAX_LPORT_NUM, 1);
2759
2760         /* Set unicast mode by default. App can change to other mode in other
2761          * API func.
2762          */
2763         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2764                                         FM10K_XCAST_MODE_NONE);
2765
2766         fm10k_mbx_unlock(hw);
2767
2768         /* Add default mac address */
2769         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2770                 MAIN_VSI_POOL_NUMBER);
2771
2772         return 0;
2773 }
2774
2775 static int
2776 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
2777 {
2778         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2779
2780         PMD_INIT_FUNC_TRACE();
2781
2782         /* only uninitialize in the primary process */
2783         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2784                 return 0;
2785
2786         /* safe to close dev here */
2787         fm10k_dev_close(dev);
2788
2789         dev->dev_ops = NULL;
2790         dev->rx_pkt_burst = NULL;
2791         dev->tx_pkt_burst = NULL;
2792
2793         /* disable uio/vfio intr */
2794         rte_intr_disable(&(dev->pci_dev->intr_handle));
2795
2796         /*PF/VF has different interrupt handling mechanism */
2797         if (hw->mac.type == fm10k_mac_pf) {
2798                 /* disable interrupt */
2799                 fm10k_dev_disable_intr_pf(dev);
2800
2801                 /* unregister callback func to eal lib */
2802                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2803                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2804         } else {
2805                 /* disable interrupt */
2806                 fm10k_dev_disable_intr_vf(dev);
2807
2808                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2809                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2810         }
2811
2812         /* free mac memory */
2813         if (dev->data->mac_addrs) {
2814                 rte_free(dev->data->mac_addrs);
2815                 dev->data->mac_addrs = NULL;
2816         }
2817
2818         memset(hw, 0, sizeof(*hw));
2819
2820         return 0;
2821 }
2822
2823 /*
2824  * The set of PCI devices this driver supports. This driver will enable both PF
2825  * and SRIOV-VF devices.
2826  */
2827 static const struct rte_pci_id pci_id_fm10k_map[] = {
2828 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2829 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2830 #include "rte_pci_dev_ids.h"
2831         { .vendor_id = 0, /* sentinel */ },
2832 };
2833
2834 static struct eth_driver rte_pmd_fm10k = {
2835         .pci_drv = {
2836                 .name = "rte_pmd_fm10k",
2837                 .id_table = pci_id_fm10k_map,
2838                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
2839                         RTE_PCI_DRV_DETACHABLE,
2840         },
2841         .eth_dev_init = eth_fm10k_dev_init,
2842         .eth_dev_uninit = eth_fm10k_dev_uninit,
2843         .dev_private_size = sizeof(struct fm10k_adapter),
2844 };
2845
2846 /*
2847  * Driver initialization routine.
2848  * Invoked once at EAL init time.
2849  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2850  */
2851 static int
2852 rte_pmd_fm10k_init(__rte_unused const char *name,
2853         __rte_unused const char *params)
2854 {
2855         PMD_INIT_FUNC_TRACE();
2856         rte_eth_driver_register(&rte_pmd_fm10k);
2857         return 0;
2858 }
2859
2860 static struct rte_driver rte_fm10k_driver = {
2861         .type = PMD_PDEV,
2862         .init = rte_pmd_fm10k_init,
2863 };
2864
2865 PMD_REGISTER_DRIVER(rte_fm10k_driver);