drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40 #include <rte_kvargs.h>
  41
  42 #include "fm10k.h"
  43 #include "base/fm10k_api.h"
  44
  45 /* Default delay to acquire mailbox lock */
  46 #define FM10K_MBXLOCK_DELAY_US 20
  47 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  48
  49 #define MAIN_VSI_POOL_NUMBER 0
  50
  51 /* Max try times to acquire switch status */
  52 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  53 /* Wait interval to get switch status */
  54 #define WAIT_SWITCH_MSG_US    100000
  55 /* Number of chars per uint32 type */
  56 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  57 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  58
  59 /* default 1:1 map from queue ID to interrupt vector ID */
  60 #define Q2V(dev, queue_id) (dev->pci_dev->intr_handle.intr_vec[queue_id])
  61
  62 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  63 #define MAX_LPORT_NUM    128
  64 #define GLORT_FD_Q_BASE  0x40
  65 #define GLORT_PF_MASK    0xFFC0
  66 #define GLORT_FD_MASK    GLORT_PF_MASK
  67 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  68
  69 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  70 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  71 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  72 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  73 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  74 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  75 static int
  76 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  77 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  78         const u8 *mac, bool add, uint32_t pool);
  79 static void fm10k_tx_queue_release(void *queue);
  80 static void fm10k_rx_queue_release(void *queue);
  81 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  82 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  83 static int fm10k_check_ftag(struct rte_devargs *devargs);
  84
  85 struct fm10k_xstats_name_off {
  86         char name[RTE_ETH_XSTATS_NAME_SIZE];
  87         unsigned offset;
  88 };
  89
  90 struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  91         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  92         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  93         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  94         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  95         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  96         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  97         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  98         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  99                 nodesc_drop)},
 100 };
 101
 102 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
 103                 sizeof(fm10k_hw_stats_strings[0]))
 104
 105 struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
 106         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
 107         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
 108         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
 109 };
 110
 111 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
 112                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
 113
 114 struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 115         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 116         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 117 };
 118
 119 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 120                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 121
 122 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 123                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 124 static int
 125 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 126
 127 static void
 128 fm10k_mbx_initlock(struct fm10k_hw *hw)
 129 {
 130         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 131 }
 132
 133 static void
 134 fm10k_mbx_lock(struct fm10k_hw *hw)
 135 {
 136         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 137                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 138 }
 139
 140 static void
 141 fm10k_mbx_unlock(struct fm10k_hw *hw)
 142 {
 143         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 144 }
 145
 146 /* Stubs needed for linkage when vPMD is disabled */
 147 int __attribute__((weak))
 148 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 149 {
 150         return -1;
 151 }
 152
 153 uint16_t __attribute__((weak))
 154 fm10k_recv_pkts_vec(
 155         __rte_unused void *rx_queue,
 156         __rte_unused struct rte_mbuf **rx_pkts,
 157         __rte_unused uint16_t nb_pkts)
 158 {
 159         return 0;
 160 }
 161
 162 uint16_t __attribute__((weak))
 163 fm10k_recv_scattered_pkts_vec(
 164                 __rte_unused void *rx_queue,
 165                 __rte_unused struct rte_mbuf **rx_pkts,
 166                 __rte_unused uint16_t nb_pkts)
 167 {
 168         return 0;
 169 }
 170
 171 int __attribute__((weak))
 172 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 173
 174 {
 175         return -1;
 176 }
 177
 178 void __attribute__((weak))
 179 fm10k_rx_queue_release_mbufs_vec(
 180                 __rte_unused struct fm10k_rx_queue *rxq)
 181 {
 182         return;
 183 }
 184
 185 void __attribute__((weak))
 186 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 187 {
 188         return;
 189 }
 190
 191 int __attribute__((weak))
 192 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 193 {
 194         return -1;
 195 }
 196
 197 uint16_t __attribute__((weak))
 198 fm10k_xmit_pkts_vec(__rte_unused void *tx_queue,
 199                 __rte_unused struct rte_mbuf **tx_pkts,
 200                 __rte_unused uint16_t nb_pkts)
 201 {
 202         return 0;
 203 }
 204
 205 /*
 206  * reset queue to initial state, allocate software buffers used when starting
 207  * device.
 208  * return 0 on success
 209  * return -ENOMEM if buffers cannot be allocated
 210  * return -EINVAL if buffers do not satisfy alignment condition
 211  */
 212 static inline int
 213 rx_queue_reset(struct fm10k_rx_queue *q)
 214 {
 215         static const union fm10k_rx_desc zero = {{0} };
 216         uint64_t dma_addr;
 217         int i, diag;
 218         PMD_INIT_FUNC_TRACE();
 219
 220         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 221         if (diag != 0)
 222                 return -ENOMEM;
 223
 224         for (i = 0; i < q->nb_desc; ++i) {
 225                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 226                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 227                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 228                                                 q->nb_desc);
 229                         return -EINVAL;
 230                 }
 231                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 232                 q->hw_ring[i].q.pkt_addr = dma_addr;
 233                 q->hw_ring[i].q.hdr_addr = dma_addr;
 234         }
 235
 236         /* initialize extra software ring entries. Space for these extra
 237          * entries is always allocated.
 238          */
 239         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 240         for (i = 0; i < q->nb_fake_desc; ++i) {
 241                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 242                 q->hw_ring[q->nb_desc + i] = zero;
 243         }
 244
 245         q->next_dd = 0;
 246         q->next_alloc = 0;
 247         q->next_trigger = q->alloc_thresh - 1;
 248         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 249         q->rxrearm_start = 0;
 250         q->rxrearm_nb = 0;
 251
 252         return 0;
 253 }
 254
 255 /*
 256  * clean queue, descriptor rings, free software buffers used when stopping
 257  * device.
 258  */
 259 static inline void
 260 rx_queue_clean(struct fm10k_rx_queue *q)
 261 {
 262         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 263         uint32_t i;
 264         PMD_INIT_FUNC_TRACE();
 265
 266         /* zero descriptor rings */
 267         for (i = 0; i < q->nb_desc; ++i)
 268                 q->hw_ring[i] = zero;
 269
 270         /* zero faked descriptors */
 271         for (i = 0; i < q->nb_fake_desc; ++i)
 272                 q->hw_ring[q->nb_desc + i] = zero;
 273
 274         /* vPMD driver has a different way of releasing mbufs. */
 275         if (q->rx_using_sse) {
 276                 fm10k_rx_queue_release_mbufs_vec(q);
 277                 return;
 278         }
 279
 280         /* free software buffers */
 281         for (i = 0; i < q->nb_desc; ++i) {
 282                 if (q->sw_ring[i]) {
 283                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 284                         q->sw_ring[i] = NULL;
 285                 }
 286         }
 287 }
 288
 289 /*
 290  * free all queue memory used when releasing the queue (i.e. configure)
 291  */
 292 static inline void
 293 rx_queue_free(struct fm10k_rx_queue *q)
 294 {
 295         PMD_INIT_FUNC_TRACE();
 296         if (q) {
 297                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 298                 rx_queue_clean(q);
 299                 if (q->sw_ring) {
 300                         rte_free(q->sw_ring);
 301                         q->sw_ring = NULL;
 302                 }
 303                 rte_free(q);
 304                 q = NULL;
 305         }
 306 }
 307
 308 /*
 309  * disable RX queue, wait unitl HW finished necessary flush operation
 310  */
 311 static inline int
 312 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 313 {
 314         uint32_t reg, i;
 315
 316         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 317         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 318                         reg & ~FM10K_RXQCTL_ENABLE);
 319
 320         /* Wait 100us at most */
 321         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 322                 rte_delay_us(1);
 323                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 324                 if (!(reg & FM10K_RXQCTL_ENABLE))
 325                         break;
 326         }
 327
 328         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 329                 return -1;
 330
 331         return 0;
 332 }
 333
 334 /*
 335  * reset queue to initial state, allocate software buffers used when starting
 336  * device
 337  */
 338 static inline void
 339 tx_queue_reset(struct fm10k_tx_queue *q)
 340 {
 341         PMD_INIT_FUNC_TRACE();
 342         q->last_free = 0;
 343         q->next_free = 0;
 344         q->nb_used = 0;
 345         q->nb_free = q->nb_desc - 1;
 346         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 347         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 348 }
 349
 350 /*
 351  * clean queue, descriptor rings, free software buffers used when stopping
 352  * device
 353  */
 354 static inline void
 355 tx_queue_clean(struct fm10k_tx_queue *q)
 356 {
 357         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 358         uint32_t i;
 359         PMD_INIT_FUNC_TRACE();
 360
 361         /* zero descriptor rings */
 362         for (i = 0; i < q->nb_desc; ++i)
 363                 q->hw_ring[i] = zero;
 364
 365         /* free software buffers */
 366         for (i = 0; i < q->nb_desc; ++i) {
 367                 if (q->sw_ring[i]) {
 368                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 369                         q->sw_ring[i] = NULL;
 370                 }
 371         }
 372 }
 373
 374 /*
 375  * free all queue memory used when releasing the queue (i.e. configure)
 376  */
 377 static inline void
 378 tx_queue_free(struct fm10k_tx_queue *q)
 379 {
 380         PMD_INIT_FUNC_TRACE();
 381         if (q) {
 382                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 383                 tx_queue_clean(q);
 384                 if (q->rs_tracker.list) {
 385                         rte_free(q->rs_tracker.list);
 386                         q->rs_tracker.list = NULL;
 387                 }
 388                 if (q->sw_ring) {
 389                         rte_free(q->sw_ring);
 390                         q->sw_ring = NULL;
 391                 }
 392                 rte_free(q);
 393                 q = NULL;
 394         }
 395 }
 396
 397 /*
 398  * disable TX queue, wait unitl HW finished necessary flush operation
 399  */
 400 static inline int
 401 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 402 {
 403         uint32_t reg, i;
 404
 405         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 406         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 407                         reg & ~FM10K_TXDCTL_ENABLE);
 408
 409         /* Wait 100us at most */
 410         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 411                 rte_delay_us(1);
 412                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 413                 if (!(reg & FM10K_TXDCTL_ENABLE))
 414                         break;
 415         }
 416
 417         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 418                 return -1;
 419
 420         return 0;
 421 }
 422
 423 static int
 424 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 425 {
 426         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 427         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 428         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 429         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 430
 431         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 432
 433         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 434                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 435                 return -EINVAL;
 436         }
 437
 438         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 439                 return 0;
 440
 441         if (hw->mac.type == fm10k_mac_vf) {
 442                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 443                 return -EINVAL;
 444         }
 445
 446         /* Check VMDQ queue pool number */
 447         if (vmdq_conf->nb_queue_pools >
 448                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 449                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 450                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 451                         vmdq_conf->nb_queue_pools);
 452                 return -EINVAL;
 453         }
 454
 455         return 0;
 456 }
 457
 458 static const struct fm10k_txq_ops def_txq_ops = {
 459         .reset = tx_queue_reset,
 460 };
 461
 462 static int
 463 fm10k_dev_configure(struct rte_eth_dev *dev)
 464 {
 465         int ret;
 466
 467         PMD_INIT_FUNC_TRACE();
 468
 469         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 470                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 471         /* multipe queue mode checking */
 472         ret  = fm10k_check_mq_mode(dev);
 473         if (ret != 0) {
 474                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 475                             ret);
 476                 return ret;
 477         }
 478
 479         return 0;
 480 }
 481
 482 /* fls = find last set bit = 32 minus the number of leading zeros */
 483 #ifndef fls
 484 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 485 #endif
 486
 487 static void
 488 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 489 {
 490         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 491         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 492         uint32_t i;
 493
 494         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 495
 496         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 497                 if (!vmdq_conf->pool_map[i].pools)
 498                         continue;
 499                 fm10k_mbx_lock(hw);
 500                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 501                 fm10k_mbx_unlock(hw);
 502         }
 503 }
 504
 505 static void
 506 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 507 {
 508         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 509
 510         /* Add default mac address */
 511         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 512                 MAIN_VSI_POOL_NUMBER);
 513 }
 514
 515 static void
 516 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 517 {
 518         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 519         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 520         uint32_t mrqc, *key, i, reta, j;
 521         uint64_t hf;
 522
 523 #define RSS_KEY_SIZE 40
 524         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 525                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 526                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 527                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 528                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 529                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 530         };
 531
 532         if (dev->data->nb_rx_queues == 1 ||
 533             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 534             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 535                 return;
 536
 537         /* random key is rss_intel_key (default) or user provided (rss_key) */
 538         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 539                 key = (uint32_t *)rss_intel_key;
 540         else
 541                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 542
 543         /* Now fill our hash function seeds, 4 bytes at a time */
 544         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 545                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 546
 547         /*
 548          * Fill in redirection table
 549          * The byte-swap is needed because NIC registers are in
 550          * little-endian order.
 551          */
 552         reta = 0;
 553         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 554                 if (j == dev->data->nb_rx_queues)
 555                         j = 0;
 556                 reta = (reta << CHAR_BIT) | j;
 557                 if ((i & 3) == 3)
 558                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 559                                         rte_bswap32(reta));
 560         }
 561
 562         /*
 563          * Generate RSS hash based on packet types, TCP/UDP
 564          * port numbers and/or IPv4/v6 src and dst addresses
 565          */
 566         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 567         mrqc = 0;
 568         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 569         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 570         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 571         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 572         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 573         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 574         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 575         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 576         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 577
 578         if (mrqc == 0) {
 579                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 580                         "supported", hf);
 581                 return;
 582         }
 583
 584         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 585 }
 586
 587 static void
 588 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 589 {
 590         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 591         uint32_t i;
 592
 593         for (i = 0; i < nb_lport_new; i++) {
 594                 /* Set unicast mode by default. App can change
 595                  * to other mode in other API func.
 596                  */
 597                 fm10k_mbx_lock(hw);
 598                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 599                         FM10K_XCAST_MODE_NONE);
 600                 fm10k_mbx_unlock(hw);
 601         }
 602 }
 603
 604 static void
 605 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 606 {
 607         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 608         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 609         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 610         struct fm10k_macvlan_filter_info *macvlan;
 611         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 612         uint16_t nb_lport_new;
 613
 614         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 615         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 616
 617         fm10k_dev_rss_configure(dev);
 618
 619         /* only PF supports VMDQ */
 620         if (hw->mac.type != fm10k_mac_pf)
 621                 return;
 622
 623         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 624                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 625
 626         /* no pool number change, no need to update logic port and VLAN/MAC */
 627         if (macvlan->nb_queue_pools == nb_queue_pools)
 628                 return;
 629
 630         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 631         fm10k_dev_logic_port_update(dev, nb_lport_new);
 632
 633         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 634         memset(dev->data->mac_addrs, 0,
 635                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 636         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 637                 &dev->data->mac_addrs[0]);
 638         memset(macvlan, 0, sizeof(*macvlan));
 639         macvlan->nb_queue_pools = nb_queue_pools;
 640
 641         if (nb_queue_pools)
 642                 fm10k_dev_vmdq_rx_configure(dev);
 643         else
 644                 fm10k_dev_pf_main_vsi_reset(dev);
 645 }
 646
 647 static int
 648 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 649 {
 650         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 651         int i, ret;
 652         struct fm10k_tx_queue *txq;
 653         uint64_t base_addr;
 654         uint32_t size;
 655
 656         /* Disable TXINT to avoid possible interrupt */
 657         for (i = 0; i < hw->mac.max_queues; i++)
 658                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 659                                 3 << FM10K_TXINT_TIMER_SHIFT);
 660
 661         /* Setup TX queue */
 662         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 663                 txq = dev->data->tx_queues[i];
 664                 base_addr = txq->hw_ring_phys_addr;
 665                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 666
 667                 /* disable queue to avoid issues while updating state */
 668                 ret = tx_queue_disable(hw, i);
 669                 if (ret) {
 670                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 671                         return -1;
 672                 }
 673                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 674                  * register is read-only for VF.
 675                  */
 676                 if (fm10k_check_ftag(dev->pci_dev->devargs)) {
 677                         if (hw->mac.type == fm10k_mac_pf) {
 678                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 679                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 680                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 681                         } else {
 682                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 683                                 return -ENOTSUP;
 684                         }
 685                 }
 686
 687                 /* set location and size for descriptor ring */
 688                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 689                                 base_addr & UINT64_LOWER_32BITS_MASK);
 690                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 691                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 692                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 693
 694                 /* assign default SGLORT for each TX queue */
 695                 FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 696         }
 697
 698         /* set up vector or scalar TX function as appropriate */
 699         fm10k_set_tx_function(dev);
 700
 701         return 0;
 702 }
 703
 704 static int
 705 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 706 {
 707         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 708         struct fm10k_macvlan_filter_info *macvlan;
 709         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
 710         int i, ret;
 711         struct fm10k_rx_queue *rxq;
 712         uint64_t base_addr;
 713         uint32_t size;
 714         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 715         uint32_t logic_port = hw->mac.dglort_map;
 716         uint16_t buf_size;
 717         uint16_t queue_stride = 0;
 718
 719         /* enable RXINT for interrupt mode */
 720         i = 0;
 721         if (rte_intr_dp_is_en(intr_handle)) {
 722                 for (; i < dev->data->nb_rx_queues; i++) {
 723                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(dev, i));
 724                         if (hw->mac.type == fm10k_mac_pf)
 725                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
 726                                         FM10K_ITR_AUTOMASK |
 727                                         FM10K_ITR_MASK_CLEAR);
 728                         else
 729                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
 730                                         FM10K_ITR_AUTOMASK |
 731                                         FM10K_ITR_MASK_CLEAR);
 732                 }
 733         }
 734         /* Disable other RXINT to avoid possible interrupt */
 735         for (; i < hw->mac.max_queues; i++)
 736                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 737                         3 << FM10K_RXINT_TIMER_SHIFT);
 738
 739         /* Setup RX queues */
 740         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 741                 rxq = dev->data->rx_queues[i];
 742                 base_addr = rxq->hw_ring_phys_addr;
 743                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 744
 745                 /* disable queue to avoid issues while updating state */
 746                 ret = rx_queue_disable(hw, i);
 747                 if (ret) {
 748                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 749                         return -1;
 750                 }
 751
 752                 /* Setup the Base and Length of the Rx Descriptor Ring */
 753                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 754                                 base_addr & UINT64_LOWER_32BITS_MASK);
 755                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 756                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 757                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 758
 759                 /* Configure the Rx buffer size for one buff without split */
 760                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 761                         RTE_PKTMBUF_HEADROOM);
 762                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 763                  * reserved for this purpose, and the worst case could be 511B.
 764                  * But SRR reg assumes all buffers have the same size. In order
 765                  * to fill the gap, we'll have to consider the worst case and
 766                  * assume 512B is reserved. If we don't do so, it's possible
 767                  * for HW to overwrite data to next mbuf.
 768                  */
 769                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 770
 771                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 772                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 773                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 774
 775                 /* It adds dual VLAN length for supporting dual VLAN */
 776                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 777                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 778                         dev->data->dev_conf.rxmode.enable_scatter) {
 779                         uint32_t reg;
 780                         dev->data->scattered_rx = 1;
 781                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 782                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 783                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 784                 }
 785
 786                 /* Enable drop on empty, it's RO for VF */
 787                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 788                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 789
 790                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 791                 FM10K_WRITE_FLUSH(hw);
 792         }
 793
 794         /* Configure VMDQ/RSS if applicable */
 795         fm10k_dev_mq_rx_configure(dev);
 796
 797         /* Decide the best RX function */
 798         fm10k_set_rx_function(dev);
 799
 800         /* update RX_SGLORT for loopback suppress*/
 801         if (hw->mac.type != fm10k_mac_pf)
 802                 return 0;
 803         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 804         if (macvlan->nb_queue_pools)
 805                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 806         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 807                 if (i && queue_stride && !(i % queue_stride))
 808                         logic_port++;
 809                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 810         }
 811
 812         return 0;
 813 }
 814
 815 static int
 816 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 817 {
 818         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 819         int err = -1;
 820         uint32_t reg;
 821         struct fm10k_rx_queue *rxq;
 822
 823         PMD_INIT_FUNC_TRACE();
 824
 825         if (rx_queue_id < dev->data->nb_rx_queues) {
 826                 rxq = dev->data->rx_queues[rx_queue_id];
 827                 err = rx_queue_reset(rxq);
 828                 if (err == -ENOMEM) {
 829                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 830                         return err;
 831                 } else if (err == -EINVAL) {
 832                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 833                                 " %d", err);
 834                         return err;
 835                 }
 836
 837                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 838                  * Note: this must be done AFTER the queue is enabled on real
 839                  * hardware, but BEFORE the queue is enabled when using the
 840                  * emulation platform. Do it in both places for now and remove
 841                  * this comment and the following two register writes when the
 842                  * emulation platform is no longer being used.
 843                  */
 844                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 845                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 846
 847                 /* Set PF ownership flag for PF devices */
 848                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 849                 if (hw->mac.type == fm10k_mac_pf)
 850                         reg |= FM10K_RXQCTL_PF;
 851                 reg |= FM10K_RXQCTL_ENABLE;
 852                 /* enable RX queue */
 853                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 854                 FM10K_WRITE_FLUSH(hw);
 855
 856                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 857                  * Note: this must be done AFTER the queue is enabled
 858                  */
 859                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 860                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 861                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 862         }
 863
 864         return err;
 865 }
 866
 867 static int
 868 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 869 {
 870         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 871
 872         PMD_INIT_FUNC_TRACE();
 873
 874         if (rx_queue_id < dev->data->nb_rx_queues) {
 875                 /* Disable RX queue */
 876                 rx_queue_disable(hw, rx_queue_id);
 877
 878                 /* Free mbuf and clean HW ring */
 879                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 880                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 881         }
 882
 883         return 0;
 884 }
 885
 886 static int
 887 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 888 {
 889         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 890         /** @todo - this should be defined in the shared code */
 891 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 892         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 893         int err = 0;
 894
 895         PMD_INIT_FUNC_TRACE();
 896
 897         if (tx_queue_id < dev->data->nb_tx_queues) {
 898                 struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 899
 900                 q->ops->reset(q);
 901
 902                 /* reset head and tail pointers */
 903                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 904                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 905
 906                 /* enable TX queue */
 907                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 908                                         FM10K_TXDCTL_ENABLE | txdctl);
 909                 FM10K_WRITE_FLUSH(hw);
 910                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 911         } else
 912                 err = -1;
 913
 914         return err;
 915 }
 916
 917 static int
 918 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 919 {
 920         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 921
 922         PMD_INIT_FUNC_TRACE();
 923
 924         if (tx_queue_id < dev->data->nb_tx_queues) {
 925                 tx_queue_disable(hw, tx_queue_id);
 926                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 927                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 928         }
 929
 930         return 0;
 931 }
 932
 933 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 934 {
 935         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 936                 != FM10K_DGLORTMAP_NONE);
 937 }
 938
 939 static void
 940 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 941 {
 942         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 943         int status;
 944
 945         PMD_INIT_FUNC_TRACE();
 946
 947         /* Return if it didn't acquire valid glort range */
 948         if (!fm10k_glort_valid(hw))
 949                 return;
 950
 951         fm10k_mbx_lock(hw);
 952         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 953                                 FM10K_XCAST_MODE_PROMISC);
 954         fm10k_mbx_unlock(hw);
 955
 956         if (status != FM10K_SUCCESS)
 957                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 958 }
 959
 960 static void
 961 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 962 {
 963         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 964         uint8_t mode;
 965         int status;
 966
 967         PMD_INIT_FUNC_TRACE();
 968
 969         /* Return if it didn't acquire valid glort range */
 970         if (!fm10k_glort_valid(hw))
 971                 return;
 972
 973         if (dev->data->all_multicast == 1)
 974                 mode = FM10K_XCAST_MODE_ALLMULTI;
 975         else
 976                 mode = FM10K_XCAST_MODE_NONE;
 977
 978         fm10k_mbx_lock(hw);
 979         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 980                                 mode);
 981         fm10k_mbx_unlock(hw);
 982
 983         if (status != FM10K_SUCCESS)
 984                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 985 }
 986
 987 static void
 988 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 989 {
 990         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 991         int status;
 992
 993         PMD_INIT_FUNC_TRACE();
 994
 995         /* Return if it didn't acquire valid glort range */
 996         if (!fm10k_glort_valid(hw))
 997                 return;
 998
 999         /* If promiscuous mode is enabled, it doesn't make sense to enable
1000          * allmulticast and disable promiscuous since fm10k only can select
1001          * one of the modes.
1002          */
1003         if (dev->data->promiscuous) {
1004                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
1005                         "needn't enable allmulticast");
1006                 return;
1007         }
1008
1009         fm10k_mbx_lock(hw);
1010         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1011                                 FM10K_XCAST_MODE_ALLMULTI);
1012         fm10k_mbx_unlock(hw);
1013
1014         if (status != FM10K_SUCCESS)
1015                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
1016 }
1017
1018 static void
1019 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1020 {
1021         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1022         int status;
1023
1024         PMD_INIT_FUNC_TRACE();
1025
1026         /* Return if it didn't acquire valid glort range */
1027         if (!fm10k_glort_valid(hw))
1028                 return;
1029
1030         if (dev->data->promiscuous) {
1031                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1032                         "since promisc mode is enabled");
1033                 return;
1034         }
1035
1036         fm10k_mbx_lock(hw);
1037         /* Change mode to unicast mode */
1038         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1039                                 FM10K_XCAST_MODE_NONE);
1040         fm10k_mbx_unlock(hw);
1041
1042         if (status != FM10K_SUCCESS)
1043                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1044 }
1045
1046 static void
1047 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1048 {
1049         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1050         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1051         uint16_t nb_queue_pools;
1052         struct fm10k_macvlan_filter_info *macvlan;
1053
1054         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1055         nb_queue_pools = macvlan->nb_queue_pools;
1056         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
1057         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
1058
1059         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1060         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1061         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1062                         hw->mac.dglort_map;
1063         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1064         /* Configure VMDQ/RSS DGlort Decoder */
1065         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1066
1067         /* Flow Director configurations, only queue number is valid. */
1068         dglortdec = fls(dev->data->nb_rx_queues - 1);
1069         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1070                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1071         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1072         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1073
1074         /* Invalidate all other GLORT entries */
1075         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1076                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1077                                 FM10K_DGLORTMAP_NONE);
1078 }
1079
1080 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1081 static int
1082 fm10k_dev_start(struct rte_eth_dev *dev)
1083 {
1084         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1085         int i, diag;
1086
1087         PMD_INIT_FUNC_TRACE();
1088
1089         /* stop, init, then start the hw */
1090         diag = fm10k_stop_hw(hw);
1091         if (diag != FM10K_SUCCESS) {
1092                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1093                 return -EIO;
1094         }
1095
1096         diag = fm10k_init_hw(hw);
1097         if (diag != FM10K_SUCCESS) {
1098                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1099                 return -EIO;
1100         }
1101
1102         diag = fm10k_start_hw(hw);
1103         if (diag != FM10K_SUCCESS) {
1104                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1105                 return -EIO;
1106         }
1107
1108         diag = fm10k_dev_tx_init(dev);
1109         if (diag) {
1110                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1111                 return diag;
1112         }
1113
1114         if (fm10k_dev_rxq_interrupt_setup(dev))
1115                 return -EIO;
1116
1117         diag = fm10k_dev_rx_init(dev);
1118         if (diag) {
1119                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1120                 return diag;
1121         }
1122
1123         if (hw->mac.type == fm10k_mac_pf)
1124                 fm10k_dev_dglort_map_configure(dev);
1125
1126         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1127                 struct fm10k_rx_queue *rxq;
1128                 rxq = dev->data->rx_queues[i];
1129
1130                 if (rxq->rx_deferred_start)
1131                         continue;
1132                 diag = fm10k_dev_rx_queue_start(dev, i);
1133                 if (diag != 0) {
1134                         int j;
1135                         for (j = 0; j < i; ++j)
1136                                 rx_queue_clean(dev->data->rx_queues[j]);
1137                         return diag;
1138                 }
1139         }
1140
1141         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1142                 struct fm10k_tx_queue *txq;
1143                 txq = dev->data->tx_queues[i];
1144
1145                 if (txq->tx_deferred_start)
1146                         continue;
1147                 diag = fm10k_dev_tx_queue_start(dev, i);
1148                 if (diag != 0) {
1149                         int j;
1150                         for (j = 0; j < i; ++j)
1151                                 tx_queue_clean(dev->data->tx_queues[j]);
1152                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1153                                 rx_queue_clean(dev->data->rx_queues[j]);
1154                         return diag;
1155                 }
1156         }
1157
1158         /* Update default vlan when not in VMDQ mode */
1159         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1160                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1161
1162         return 0;
1163 }
1164
1165 static void
1166 fm10k_dev_stop(struct rte_eth_dev *dev)
1167 {
1168         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1169         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1170         int i;
1171
1172         PMD_INIT_FUNC_TRACE();
1173
1174         if (dev->data->tx_queues)
1175                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1176                         fm10k_dev_tx_queue_stop(dev, i);
1177
1178         if (dev->data->rx_queues)
1179                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1180                         fm10k_dev_rx_queue_stop(dev, i);
1181
1182         /* Disable datapath event */
1183         if (rte_intr_dp_is_en(intr_handle)) {
1184                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1185                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1186                                 3 << FM10K_RXINT_TIMER_SHIFT);
1187                         if (hw->mac.type == fm10k_mac_pf)
1188                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
1189                                         FM10K_ITR_MASK_SET);
1190                         else
1191                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
1192                                         FM10K_ITR_MASK_SET);
1193                 }
1194         }
1195         /* Clean datapath event and queue/vec mapping */
1196         rte_intr_efd_disable(intr_handle);
1197         rte_free(intr_handle->intr_vec);
1198         intr_handle->intr_vec = NULL;
1199 }
1200
1201 static void
1202 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1203 {
1204         int i;
1205
1206         PMD_INIT_FUNC_TRACE();
1207
1208         if (dev->data->tx_queues) {
1209                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1210                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1211
1212                         tx_queue_free(txq);
1213                 }
1214         }
1215
1216         if (dev->data->rx_queues) {
1217                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1218                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1219         }
1220 }
1221
1222 static void
1223 fm10k_dev_close(struct rte_eth_dev *dev)
1224 {
1225         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1226
1227         PMD_INIT_FUNC_TRACE();
1228
1229         fm10k_mbx_lock(hw);
1230         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1231                 MAX_LPORT_NUM, false);
1232         fm10k_mbx_unlock(hw);
1233
1234         /* Stop mailbox service first */
1235         fm10k_close_mbx_service(hw);
1236         fm10k_dev_stop(dev);
1237         fm10k_dev_queue_release(dev);
1238         fm10k_stop_hw(hw);
1239 }
1240
1241 static int
1242 fm10k_link_update(struct rte_eth_dev *dev,
1243         __rte_unused int wait_to_complete)
1244 {
1245         PMD_INIT_FUNC_TRACE();
1246
1247         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
1248          * x8 PCIe interface. For now, we leave the speed undefined since there
1249          * is no 50Gbps Ethernet. */
1250         dev->data->dev_link.link_speed  = 0;
1251         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1252         dev->data->dev_link.link_status = 1;
1253
1254         return 0;
1255 }
1256
1257 static int
1258 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstats *xstats,
1259                  unsigned n)
1260 {
1261         struct fm10k_hw_stats *hw_stats =
1262                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1263         unsigned i, q, count = 0;
1264
1265         if (n < FM10K_NB_XSTATS)
1266                 return FM10K_NB_XSTATS;
1267
1268         /* Global stats */
1269         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1270                 snprintf(xstats[count].name, sizeof(xstats[count].name),
1271                          "%s", fm10k_hw_stats_strings[count].name);
1272                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1273                         fm10k_hw_stats_strings[count].offset);
1274                 count++;
1275         }
1276
1277         /* PF queue stats */
1278         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1279                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1280                         snprintf(xstats[count].name, sizeof(xstats[count].name),
1281                                  "rx_q%u_%s", q,
1282                                  fm10k_hw_stats_rx_q_strings[i].name);
1283                         xstats[count].value =
1284                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1285                                 fm10k_hw_stats_rx_q_strings[i].offset);
1286                         count++;
1287                 }
1288                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1289                         snprintf(xstats[count].name, sizeof(xstats[count].name),
1290                                  "tx_q%u_%s", q,
1291                                  fm10k_hw_stats_tx_q_strings[i].name);
1292                         xstats[count].value =
1293                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1294                                 fm10k_hw_stats_tx_q_strings[i].offset);
1295                         count++;
1296                 }
1297         }
1298
1299         return FM10K_NB_XSTATS;
1300 }
1301
1302 static void
1303 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1304 {
1305         uint64_t ipackets, opackets, ibytes, obytes;
1306         struct fm10k_hw *hw =
1307                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1308         struct fm10k_hw_stats *hw_stats =
1309                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1310         int i;
1311
1312         PMD_INIT_FUNC_TRACE();
1313
1314         fm10k_update_hw_stats(hw, hw_stats);
1315
1316         ipackets = opackets = ibytes = obytes = 0;
1317         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1318                 (i < hw->mac.max_queues); ++i) {
1319                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1320                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1321                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1322                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1323                 ipackets += stats->q_ipackets[i];
1324                 opackets += stats->q_opackets[i];
1325                 ibytes   += stats->q_ibytes[i];
1326                 obytes   += stats->q_obytes[i];
1327         }
1328         stats->ipackets = ipackets;
1329         stats->opackets = opackets;
1330         stats->ibytes = ibytes;
1331         stats->obytes = obytes;
1332 }
1333
1334 static void
1335 fm10k_stats_reset(struct rte_eth_dev *dev)
1336 {
1337         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1338         struct fm10k_hw_stats *hw_stats =
1339                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1340
1341         PMD_INIT_FUNC_TRACE();
1342
1343         memset(hw_stats, 0, sizeof(*hw_stats));
1344         fm10k_rebind_hw_stats(hw, hw_stats);
1345 }
1346
1347 static void
1348 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1349         struct rte_eth_dev_info *dev_info)
1350 {
1351         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1352
1353         PMD_INIT_FUNC_TRACE();
1354
1355         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1356         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1357         dev_info->max_rx_queues      = hw->mac.max_queues;
1358         dev_info->max_tx_queues      = hw->mac.max_queues;
1359         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1360         dev_info->max_hash_mac_addrs = 0;
1361         dev_info->max_vfs            = dev->pci_dev->max_vfs;
1362         dev_info->vmdq_pool_base     = 0;
1363         dev_info->vmdq_queue_base    = 0;
1364         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1365         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1366         dev_info->rx_offload_capa =
1367                 DEV_RX_OFFLOAD_VLAN_STRIP |
1368                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1369                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1370                 DEV_RX_OFFLOAD_TCP_CKSUM;
1371         dev_info->tx_offload_capa =
1372                 DEV_TX_OFFLOAD_VLAN_INSERT |
1373                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1374                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1375                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1376                 DEV_TX_OFFLOAD_TCP_TSO;
1377
1378         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1379         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1380
1381         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1382                 .rx_thresh = {
1383                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1384                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1385                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1386                 },
1387                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1388                 .rx_drop_en = 0,
1389         };
1390
1391         dev_info->default_txconf = (struct rte_eth_txconf) {
1392                 .tx_thresh = {
1393                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1394                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1395                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1396                 },
1397                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1398                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1399                 .txq_flags = FM10K_SIMPLE_TX_FLAG,
1400         };
1401
1402         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1403                 .nb_max = FM10K_MAX_RX_DESC,
1404                 .nb_min = FM10K_MIN_RX_DESC,
1405                 .nb_align = FM10K_MULT_RX_DESC,
1406         };
1407
1408         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1409                 .nb_max = FM10K_MAX_TX_DESC,
1410                 .nb_min = FM10K_MIN_TX_DESC,
1411                 .nb_align = FM10K_MULT_TX_DESC,
1412         };
1413 }
1414
1415 static int
1416 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1417 {
1418         s32 result;
1419         uint16_t mac_num = 0;
1420         uint32_t vid_idx, vid_bit, mac_index;
1421         struct fm10k_hw *hw;
1422         struct fm10k_macvlan_filter_info *macvlan;
1423         struct rte_eth_dev_data *data = dev->data;
1424
1425         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1426         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1427
1428         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1429                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1430                 return -EINVAL;
1431         }
1432
1433         if (vlan_id > ETH_VLAN_ID_MAX) {
1434                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1435                 return -EINVAL;
1436         }
1437
1438         vid_idx = FM10K_VFTA_IDX(vlan_id);
1439         vid_bit = FM10K_VFTA_BIT(vlan_id);
1440         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1441         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1442                 return 0;
1443         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1444         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1445                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1446                         "in the VLAN filter table");
1447                 return -EINVAL;
1448         }
1449
1450         fm10k_mbx_lock(hw);
1451         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1452         fm10k_mbx_unlock(hw);
1453         if (result != FM10K_SUCCESS) {
1454                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1455                 return -EIO;
1456         }
1457
1458         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1459                         (result == FM10K_SUCCESS); mac_index++) {
1460                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1461                         continue;
1462                 if (mac_num > macvlan->mac_num - 1) {
1463                         PMD_INIT_LOG(ERR, "MAC address number "
1464                                         "not match");
1465                         break;
1466                 }
1467                 fm10k_mbx_lock(hw);
1468                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1469                         data->mac_addrs[mac_index].addr_bytes,
1470                         vlan_id, on, 0);
1471                 fm10k_mbx_unlock(hw);
1472                 mac_num++;
1473         }
1474         if (result != FM10K_SUCCESS) {
1475                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1476                 return -EIO;
1477         }
1478
1479         if (on) {
1480                 macvlan->vlan_num++;
1481                 macvlan->vfta[vid_idx] |= vid_bit;
1482         } else {
1483                 macvlan->vlan_num--;
1484                 macvlan->vfta[vid_idx] &= ~vid_bit;
1485         }
1486         return 0;
1487 }
1488
1489 static void
1490 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1491 {
1492         if (mask & ETH_VLAN_STRIP_MASK) {
1493                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1494                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1495                                         "always on in fm10k");
1496         }
1497
1498         if (mask & ETH_VLAN_EXTEND_MASK) {
1499                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1500                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1501                                         "supported in fm10k");
1502         }
1503
1504         if (mask & ETH_VLAN_FILTER_MASK) {
1505                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1506                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1507         }
1508 }
1509
1510 /* Add/Remove a MAC address, and update filters to main VSI */
1511 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1512                 const u8 *mac, bool add, uint32_t pool)
1513 {
1514         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1515         struct fm10k_macvlan_filter_info *macvlan;
1516         uint32_t i, j, k;
1517
1518         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1519
1520         if (pool != MAIN_VSI_POOL_NUMBER) {
1521                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1522                         "mac to pool %u", pool);
1523                 return;
1524         }
1525         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1526                 if (!macvlan->vfta[j])
1527                         continue;
1528                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1529                         if (!(macvlan->vfta[j] & (1 << k)))
1530                                 continue;
1531                         if (i + 1 > macvlan->vlan_num) {
1532                                 PMD_INIT_LOG(ERR, "vlan number not match");
1533                                 return;
1534                         }
1535                         fm10k_mbx_lock(hw);
1536                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1537                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1538                         fm10k_mbx_unlock(hw);
1539                         i++;
1540                 }
1541         }
1542 }
1543
1544 /* Add/Remove a MAC address, and update filters to VMDQ */
1545 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1546                 const u8 *mac, bool add, uint32_t pool)
1547 {
1548         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1549         struct fm10k_macvlan_filter_info *macvlan;
1550         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1551         uint32_t i;
1552
1553         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1554         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1555
1556         if (pool > macvlan->nb_queue_pools) {
1557                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1558                         " Max pool is %u",
1559                         pool, macvlan->nb_queue_pools);
1560                 return;
1561         }
1562         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1563                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1564                         continue;
1565                 fm10k_mbx_lock(hw);
1566                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1567                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1568                 fm10k_mbx_unlock(hw);
1569         }
1570 }
1571
1572 /* Add/Remove a MAC address, and update filters */
1573 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1574                 const u8 *mac, bool add, uint32_t pool)
1575 {
1576         struct fm10k_macvlan_filter_info *macvlan;
1577
1578         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1579
1580         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1581                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1582         else
1583                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1584
1585         if (add)
1586                 macvlan->mac_num++;
1587         else
1588                 macvlan->mac_num--;
1589 }
1590
1591 /* Add a MAC address, and update filters */
1592 static void
1593 fm10k_macaddr_add(struct rte_eth_dev *dev,
1594                 struct ether_addr *mac_addr,
1595                 uint32_t index,
1596                 uint32_t pool)
1597 {
1598         struct fm10k_macvlan_filter_info *macvlan;
1599
1600         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1601         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1602         macvlan->mac_vmdq_id[index] = pool;
1603 }
1604
1605 /* Remove a MAC address, and update filters */
1606 static void
1607 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1608 {
1609         struct rte_eth_dev_data *data = dev->data;
1610         struct fm10k_macvlan_filter_info *macvlan;
1611
1612         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1613         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1614                         FALSE, macvlan->mac_vmdq_id[index]);
1615         macvlan->mac_vmdq_id[index] = 0;
1616 }
1617
1618 static inline int
1619 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1620 {
1621         if ((request < min) || (request > max) || ((request % mult) != 0))
1622                 return -1;
1623         else
1624                 return 0;
1625 }
1626
1627
1628 static inline int
1629 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1630 {
1631         if ((request < min) || (request > max) || ((div % request) != 0))
1632                 return -1;
1633         else
1634                 return 0;
1635 }
1636
1637 static inline int
1638 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1639 {
1640         uint16_t rx_free_thresh;
1641
1642         if (conf->rx_free_thresh == 0)
1643                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1644         else
1645                 rx_free_thresh = conf->rx_free_thresh;
1646
1647         /* make sure the requested threshold satisfies the constraints */
1648         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1649                         FM10K_RX_FREE_THRESH_MAX(q),
1650                         FM10K_RX_FREE_THRESH_DIV(q),
1651                         rx_free_thresh)) {
1652                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1653                         "less than or equal to %u, "
1654                         "greater than or equal to %u, "
1655                         "and a divisor of %u",
1656                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1657                         FM10K_RX_FREE_THRESH_MIN(q),
1658                         FM10K_RX_FREE_THRESH_DIV(q));
1659                 return -EINVAL;
1660         }
1661
1662         q->alloc_thresh = rx_free_thresh;
1663         q->drop_en = conf->rx_drop_en;
1664         q->rx_deferred_start = conf->rx_deferred_start;
1665
1666         return 0;
1667 }
1668
1669 /*
1670  * Hardware requires specific alignment for Rx packet buffers. At
1671  * least one of the following two conditions must be satisfied.
1672  *  1. Address is 512B aligned
1673  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1674  *
1675  * As such, the driver may need to adjust the DMA address within the
1676  * buffer by up to 512B.
1677  *
1678  * return 1 if the element size is valid, otherwise return 0.
1679  */
1680 static int
1681 mempool_element_size_valid(struct rte_mempool *mp)
1682 {
1683         uint32_t min_size;
1684
1685         /* elt_size includes mbuf header and headroom */
1686         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1687                         RTE_PKTMBUF_HEADROOM;
1688
1689         /* account for up to 512B of alignment */
1690         min_size -= FM10K_RX_DATABUF_ALIGN;
1691
1692         /* sanity check for overflow */
1693         if (min_size > mp->elt_size)
1694                 return 0;
1695
1696         /* size is valid */
1697         return 1;
1698 }
1699
1700 static int
1701 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1702         uint16_t nb_desc, unsigned int socket_id,
1703         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1704 {
1705         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1706         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
1707         struct fm10k_rx_queue *q;
1708         const struct rte_memzone *mz;
1709
1710         PMD_INIT_FUNC_TRACE();
1711
1712         /* make sure the mempool element size can account for alignment. */
1713         if (!mempool_element_size_valid(mp)) {
1714                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1715                 return -EINVAL;
1716         }
1717
1718         /* make sure a valid number of descriptors have been requested */
1719         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1720                                 FM10K_MULT_RX_DESC, nb_desc)) {
1721                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1722                         "less than or equal to %"PRIu32", "
1723                         "greater than or equal to %u, "
1724                         "and a multiple of %u",
1725                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1726                         FM10K_MULT_RX_DESC);
1727                 return -EINVAL;
1728         }
1729
1730         /*
1731          * if this queue existed already, free the associated memory. The
1732          * queue cannot be reused in case we need to allocate memory on
1733          * different socket than was previously used.
1734          */
1735         if (dev->data->rx_queues[queue_id] != NULL) {
1736                 rx_queue_free(dev->data->rx_queues[queue_id]);
1737                 dev->data->rx_queues[queue_id] = NULL;
1738         }
1739
1740         /* allocate memory for the queue structure */
1741         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1742                                 socket_id);
1743         if (q == NULL) {
1744                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1745                 return -ENOMEM;
1746         }
1747
1748         /* setup queue */
1749         q->mp = mp;
1750         q->nb_desc = nb_desc;
1751         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1752         q->port_id = dev->data->port_id;
1753         q->queue_id = queue_id;
1754         q->tail_ptr = (volatile uint32_t *)
1755                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1756         if (handle_rxconf(q, conf))
1757                 return -EINVAL;
1758
1759         /* allocate memory for the software ring */
1760         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1761                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1762                         RTE_CACHE_LINE_SIZE, socket_id);
1763         if (q->sw_ring == NULL) {
1764                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1765                 rte_free(q);
1766                 return -ENOMEM;
1767         }
1768
1769         /*
1770          * allocate memory for the hardware descriptor ring. A memzone large
1771          * enough to hold the maximum ring size is requested to allow for
1772          * resizing in later calls to the queue setup function.
1773          */
1774         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1775                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1776                                       socket_id);
1777         if (mz == NULL) {
1778                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1779                 rte_free(q->sw_ring);
1780                 rte_free(q);
1781                 return -ENOMEM;
1782         }
1783         q->hw_ring = mz->addr;
1784         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1785
1786         /* Check if number of descs satisfied Vector requirement */
1787         if (!rte_is_power_of_2(nb_desc)) {
1788                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1789                                     "preconditions - canceling the feature for "
1790                                     "the whole port[%d]",
1791                              q->queue_id, q->port_id);
1792                 dev_info->rx_vec_allowed = false;
1793         } else
1794                 fm10k_rxq_vec_setup(q);
1795
1796         dev->data->rx_queues[queue_id] = q;
1797         return 0;
1798 }
1799
1800 static void
1801 fm10k_rx_queue_release(void *queue)
1802 {
1803         PMD_INIT_FUNC_TRACE();
1804
1805         rx_queue_free(queue);
1806 }
1807
1808 static inline int
1809 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1810 {
1811         uint16_t tx_free_thresh;
1812         uint16_t tx_rs_thresh;
1813
1814         /* constraint MACROs require that tx_free_thresh is configured
1815          * before tx_rs_thresh */
1816         if (conf->tx_free_thresh == 0)
1817                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1818         else
1819                 tx_free_thresh = conf->tx_free_thresh;
1820
1821         /* make sure the requested threshold satisfies the constraints */
1822         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1823                         FM10K_TX_FREE_THRESH_MAX(q),
1824                         FM10K_TX_FREE_THRESH_DIV(q),
1825                         tx_free_thresh)) {
1826                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1827                         "less than or equal to %u, "
1828                         "greater than or equal to %u, "
1829                         "and a divisor of %u",
1830                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1831                         FM10K_TX_FREE_THRESH_MIN(q),
1832                         FM10K_TX_FREE_THRESH_DIV(q));
1833                 return -EINVAL;
1834         }
1835
1836         q->free_thresh = tx_free_thresh;
1837
1838         if (conf->tx_rs_thresh == 0)
1839                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1840         else
1841                 tx_rs_thresh = conf->tx_rs_thresh;
1842
1843         q->tx_deferred_start = conf->tx_deferred_start;
1844
1845         /* make sure the requested threshold satisfies the constraints */
1846         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1847                         FM10K_TX_RS_THRESH_MAX(q),
1848                         FM10K_TX_RS_THRESH_DIV(q),
1849                         tx_rs_thresh)) {
1850                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1851                         "less than or equal to %u, "
1852                         "greater than or equal to %u, "
1853                         "and a divisor of %u",
1854                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1855                         FM10K_TX_RS_THRESH_MIN(q),
1856                         FM10K_TX_RS_THRESH_DIV(q));
1857                 return -EINVAL;
1858         }
1859
1860         q->rs_thresh = tx_rs_thresh;
1861
1862         return 0;
1863 }
1864
1865 static int
1866 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1867         uint16_t nb_desc, unsigned int socket_id,
1868         const struct rte_eth_txconf *conf)
1869 {
1870         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1871         struct fm10k_tx_queue *q;
1872         const struct rte_memzone *mz;
1873
1874         PMD_INIT_FUNC_TRACE();
1875
1876         /* make sure a valid number of descriptors have been requested */
1877         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1878                                 FM10K_MULT_TX_DESC, nb_desc)) {
1879                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1880                         "less than or equal to %"PRIu32", "
1881                         "greater than or equal to %u, "
1882                         "and a multiple of %u",
1883                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1884                         FM10K_MULT_TX_DESC);
1885                 return -EINVAL;
1886         }
1887
1888         /*
1889          * if this queue existed already, free the associated memory. The
1890          * queue cannot be reused in case we need to allocate memory on
1891          * different socket than was previously used.
1892          */
1893         if (dev->data->tx_queues[queue_id] != NULL) {
1894                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
1895
1896                 tx_queue_free(txq);
1897                 dev->data->tx_queues[queue_id] = NULL;
1898         }
1899
1900         /* allocate memory for the queue structure */
1901         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1902                                 socket_id);
1903         if (q == NULL) {
1904                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1905                 return -ENOMEM;
1906         }
1907
1908         /* setup queue */
1909         q->nb_desc = nb_desc;
1910         q->port_id = dev->data->port_id;
1911         q->queue_id = queue_id;
1912         q->txq_flags = conf->txq_flags;
1913         q->ops = &def_txq_ops;
1914         q->tail_ptr = (volatile uint32_t *)
1915                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1916         if (handle_txconf(q, conf))
1917                 return -EINVAL;
1918
1919         /* allocate memory for the software ring */
1920         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1921                                         nb_desc * sizeof(struct rte_mbuf *),
1922                                         RTE_CACHE_LINE_SIZE, socket_id);
1923         if (q->sw_ring == NULL) {
1924                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1925                 rte_free(q);
1926                 return -ENOMEM;
1927         }
1928
1929         /*
1930          * allocate memory for the hardware descriptor ring. A memzone large
1931          * enough to hold the maximum ring size is requested to allow for
1932          * resizing in later calls to the queue setup function.
1933          */
1934         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
1935                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
1936                                       socket_id);
1937         if (mz == NULL) {
1938                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1939                 rte_free(q->sw_ring);
1940                 rte_free(q);
1941                 return -ENOMEM;
1942         }
1943         q->hw_ring = mz->addr;
1944         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1945
1946         /*
1947          * allocate memory for the RS bit tracker. Enough slots to hold the
1948          * descriptor index for each RS bit needing to be set are required.
1949          */
1950         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1951                                 ((nb_desc + 1) / q->rs_thresh) *
1952                                 sizeof(uint16_t),
1953                                 RTE_CACHE_LINE_SIZE, socket_id);
1954         if (q->rs_tracker.list == NULL) {
1955                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1956                 rte_free(q->sw_ring);
1957                 rte_free(q);
1958                 return -ENOMEM;
1959         }
1960
1961         dev->data->tx_queues[queue_id] = q;
1962         return 0;
1963 }
1964
1965 static void
1966 fm10k_tx_queue_release(void *queue)
1967 {
1968         struct fm10k_tx_queue *q = queue;
1969         PMD_INIT_FUNC_TRACE();
1970
1971         tx_queue_free(q);
1972 }
1973
1974 static int
1975 fm10k_reta_update(struct rte_eth_dev *dev,
1976                         struct rte_eth_rss_reta_entry64 *reta_conf,
1977                         uint16_t reta_size)
1978 {
1979         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1980         uint16_t i, j, idx, shift;
1981         uint8_t mask;
1982         uint32_t reta;
1983
1984         PMD_INIT_FUNC_TRACE();
1985
1986         if (reta_size > FM10K_MAX_RSS_INDICES) {
1987                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1988                         "(%d) doesn't match the number hardware can supported "
1989                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1990                 return -EINVAL;
1991         }
1992
1993         /*
1994          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1995          * 128-entries in 32 registers
1996          */
1997         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1998                 idx = i / RTE_RETA_GROUP_SIZE;
1999                 shift = i % RTE_RETA_GROUP_SIZE;
2000                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2001                                 BIT_MASK_PER_UINT32);
2002                 if (mask == 0)
2003                         continue;
2004
2005                 reta = 0;
2006                 if (mask != BIT_MASK_PER_UINT32)
2007                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2008
2009                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2010                         if (mask & (0x1 << j)) {
2011                                 if (mask != 0xF)
2012                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2013                                 reta |= reta_conf[idx].reta[shift + j] <<
2014                                                 (CHAR_BIT * j);
2015                         }
2016                 }
2017                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2018         }
2019
2020         return 0;
2021 }
2022
2023 static int
2024 fm10k_reta_query(struct rte_eth_dev *dev,
2025                         struct rte_eth_rss_reta_entry64 *reta_conf,
2026                         uint16_t reta_size)
2027 {
2028         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2029         uint16_t i, j, idx, shift;
2030         uint8_t mask;
2031         uint32_t reta;
2032
2033         PMD_INIT_FUNC_TRACE();
2034
2035         if (reta_size < FM10K_MAX_RSS_INDICES) {
2036                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2037                         "(%d) doesn't match the number hardware can supported "
2038                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2039                 return -EINVAL;
2040         }
2041
2042         /*
2043          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2044          * 128-entries in 32 registers
2045          */
2046         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2047                 idx = i / RTE_RETA_GROUP_SIZE;
2048                 shift = i % RTE_RETA_GROUP_SIZE;
2049                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2050                                 BIT_MASK_PER_UINT32);
2051                 if (mask == 0)
2052                         continue;
2053
2054                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2055                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2056                         if (mask & (0x1 << j))
2057                                 reta_conf[idx].reta[shift + j] = ((reta >>
2058                                         CHAR_BIT * j) & UINT8_MAX);
2059                 }
2060         }
2061
2062         return 0;
2063 }
2064
2065 static int
2066 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2067         struct rte_eth_rss_conf *rss_conf)
2068 {
2069         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2070         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2071         uint32_t mrqc;
2072         uint64_t hf = rss_conf->rss_hf;
2073         int i;
2074
2075         PMD_INIT_FUNC_TRACE();
2076
2077         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2078                 FM10K_RSSRK_ENTRIES_PER_REG)
2079                 return -EINVAL;
2080
2081         if (hf == 0)
2082                 return -EINVAL;
2083
2084         mrqc = 0;
2085         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2086         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2087         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2088         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2089         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2090         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2091         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2092         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2093         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2094
2095         /* If the mapping doesn't fit any supported, return */
2096         if (mrqc == 0)
2097                 return -EINVAL;
2098
2099         if (key != NULL)
2100                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2101                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2102
2103         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2104
2105         return 0;
2106 }
2107
2108 static int
2109 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2110         struct rte_eth_rss_conf *rss_conf)
2111 {
2112         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2113         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2114         uint32_t mrqc;
2115         uint64_t hf;
2116         int i;
2117
2118         PMD_INIT_FUNC_TRACE();
2119
2120         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2121                                 FM10K_RSSRK_ENTRIES_PER_REG)
2122                 return -EINVAL;
2123
2124         if (key != NULL)
2125                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2126                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2127
2128         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2129         hf = 0;
2130         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2131         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2132         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2133         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2134         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2135         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2136         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2137         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2138         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2139
2140         rss_conf->rss_hf = hf;
2141
2142         return 0;
2143 }
2144
2145 static void
2146 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2147 {
2148         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2149         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2150
2151         /* Bind all local non-queue interrupt to vector 0 */
2152         int_map |= FM10K_MISC_VEC_ID;
2153
2154         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2155         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2156         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2157         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2158         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2159         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2160
2161         /* Enable misc causes */
2162         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2163                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2164                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2165                                 FM10K_EIMR_ENABLE(MAILBOX) |
2166                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2167                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2168                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2169                                 FM10K_EIMR_ENABLE(VFLR));
2170
2171         /* Enable ITR 0 */
2172         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2173                                         FM10K_ITR_MASK_CLEAR);
2174         FM10K_WRITE_FLUSH(hw);
2175 }
2176
2177 static void
2178 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2179 {
2180         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2181         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2182
2183         int_map |= FM10K_MISC_VEC_ID;
2184
2185         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2186         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2187         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2188         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2189         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2190         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2191
2192         /* Disable misc causes */
2193         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2194                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2195                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2196                                 FM10K_EIMR_DISABLE(MAILBOX) |
2197                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2198                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2199                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2200                                 FM10K_EIMR_DISABLE(VFLR));
2201
2202         /* Disable ITR 0 */
2203         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2204         FM10K_WRITE_FLUSH(hw);
2205 }
2206
2207 static void
2208 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2209 {
2210         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2211         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2212
2213         /* Bind all local non-queue interrupt to vector 0 */
2214         int_map |= FM10K_MISC_VEC_ID;
2215
2216         /* Only INT 0 available, other 15 are reserved. */
2217         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2218
2219         /* Enable ITR 0 */
2220         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2221                                         FM10K_ITR_MASK_CLEAR);
2222         FM10K_WRITE_FLUSH(hw);
2223 }
2224
2225 static void
2226 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2227 {
2228         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2229         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2230
2231         int_map |= FM10K_MISC_VEC_ID;
2232
2233         /* Only INT 0 available, other 15 are reserved. */
2234         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2235
2236         /* Disable ITR 0 */
2237         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2238         FM10K_WRITE_FLUSH(hw);
2239 }
2240
2241 static int
2242 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2243 {
2244         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2245
2246         /* Enable ITR */
2247         if (hw->mac.type == fm10k_mac_pf)
2248                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
2249                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2250         else
2251                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
2252                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2253         rte_intr_enable(&dev->pci_dev->intr_handle);
2254         return 0;
2255 }
2256
2257 static int
2258 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2259 {
2260         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2261
2262         /* Disable ITR */
2263         if (hw->mac.type == fm10k_mac_pf)
2264                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
2265                         FM10K_ITR_MASK_SET);
2266         else
2267                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
2268                         FM10K_ITR_MASK_SET);
2269         return 0;
2270 }
2271
2272 static int
2273 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2274 {
2275         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2276         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
2277         uint32_t intr_vector, vec;
2278         uint16_t queue_id;
2279         int result = 0;
2280
2281         /* fm10k needs one separate interrupt for mailbox,
2282          * so only drivers which support multiple interrupt vectors
2283          * e.g. vfio-pci can work for fm10k interrupt mode
2284          */
2285         if (!rte_intr_cap_multiple(intr_handle) ||
2286                         dev->data->dev_conf.intr_conf.rxq == 0)
2287                 return result;
2288
2289         intr_vector = dev->data->nb_rx_queues;
2290
2291         /* disable interrupt first */
2292         rte_intr_disable(&dev->pci_dev->intr_handle);
2293         if (hw->mac.type == fm10k_mac_pf)
2294                 fm10k_dev_disable_intr_pf(dev);
2295         else
2296                 fm10k_dev_disable_intr_vf(dev);
2297
2298         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2299                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2300                 result = -EIO;
2301         }
2302
2303         if (rte_intr_dp_is_en(intr_handle) && !result) {
2304                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2305                         dev->data->nb_rx_queues * sizeof(int), 0);
2306                 if (intr_handle->intr_vec) {
2307                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2308                                         queue_id < dev->data->nb_rx_queues;
2309                                         queue_id++) {
2310                                 intr_handle->intr_vec[queue_id] = vec;
2311                                 if (vec < intr_handle->nb_efd - 1
2312                                                 + FM10K_RX_VEC_START)
2313                                         vec++;
2314                         }
2315                 } else {
2316                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2317                                 " intr_vec", dev->data->nb_rx_queues);
2318                         rte_intr_efd_disable(intr_handle);
2319                         result = -ENOMEM;
2320                 }
2321         }
2322
2323         if (hw->mac.type == fm10k_mac_pf)
2324                 fm10k_dev_enable_intr_pf(dev);
2325         else
2326                 fm10k_dev_enable_intr_vf(dev);
2327         rte_intr_enable(&dev->pci_dev->intr_handle);
2328         hw->mac.ops.update_int_moderator(hw);
2329         return result;
2330 }
2331
2332 static int
2333 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2334 {
2335         struct fm10k_fault fault;
2336         int err;
2337         const char *estr = "Unknown error";
2338
2339         /* Process PCA fault */
2340         if (eicr & FM10K_EICR_PCA_FAULT) {
2341                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2342                 if (err)
2343                         goto error;
2344                 switch (fault.type) {
2345                 case PCA_NO_FAULT:
2346                         estr = "PCA_NO_FAULT"; break;
2347                 case PCA_UNMAPPED_ADDR:
2348                         estr = "PCA_UNMAPPED_ADDR"; break;
2349                 case PCA_BAD_QACCESS_PF:
2350                         estr = "PCA_BAD_QACCESS_PF"; break;
2351                 case PCA_BAD_QACCESS_VF:
2352                         estr = "PCA_BAD_QACCESS_VF"; break;
2353                 case PCA_MALICIOUS_REQ:
2354                         estr = "PCA_MALICIOUS_REQ"; break;
2355                 case PCA_POISONED_TLP:
2356                         estr = "PCA_POISONED_TLP"; break;
2357                 case PCA_TLP_ABORT:
2358                         estr = "PCA_TLP_ABORT"; break;
2359                 default:
2360                         goto error;
2361                 }
2362                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2363                         estr, fault.func ? "VF" : "PF", fault.func,
2364                         fault.address, fault.specinfo);
2365         }
2366
2367         /* Process THI fault */
2368         if (eicr & FM10K_EICR_THI_FAULT) {
2369                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2370                 if (err)
2371                         goto error;
2372                 switch (fault.type) {
2373                 case THI_NO_FAULT:
2374                         estr = "THI_NO_FAULT"; break;
2375                 case THI_MAL_DIS_Q_FAULT:
2376                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2377                 default:
2378                         goto error;
2379                 }
2380                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2381                         estr, fault.func ? "VF" : "PF", fault.func,
2382                         fault.address, fault.specinfo);
2383         }
2384
2385         /* Process FUM fault */
2386         if (eicr & FM10K_EICR_FUM_FAULT) {
2387                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2388                 if (err)
2389                         goto error;
2390                 switch (fault.type) {
2391                 case FUM_NO_FAULT:
2392                         estr = "FUM_NO_FAULT"; break;
2393                 case FUM_UNMAPPED_ADDR:
2394                         estr = "FUM_UNMAPPED_ADDR"; break;
2395                 case FUM_POISONED_TLP:
2396                         estr = "FUM_POISONED_TLP"; break;
2397                 case FUM_BAD_VF_QACCESS:
2398                         estr = "FUM_BAD_VF_QACCESS"; break;
2399                 case FUM_ADD_DECODE_ERR:
2400                         estr = "FUM_ADD_DECODE_ERR"; break;
2401                 case FUM_RO_ERROR:
2402                         estr = "FUM_RO_ERROR"; break;
2403                 case FUM_QPRC_CRC_ERROR:
2404                         estr = "FUM_QPRC_CRC_ERROR"; break;
2405                 case FUM_CSR_TIMEOUT:
2406                         estr = "FUM_CSR_TIMEOUT"; break;
2407                 case FUM_INVALID_TYPE:
2408                         estr = "FUM_INVALID_TYPE"; break;
2409                 case FUM_INVALID_LENGTH:
2410                         estr = "FUM_INVALID_LENGTH"; break;
2411                 case FUM_INVALID_BE:
2412                         estr = "FUM_INVALID_BE"; break;
2413                 case FUM_INVALID_ALIGN:
2414                         estr = "FUM_INVALID_ALIGN"; break;
2415                 default:
2416                         goto error;
2417                 }
2418                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2419                         estr, fault.func ? "VF" : "PF", fault.func,
2420                         fault.address, fault.specinfo);
2421         }
2422
2423         return 0;
2424 error:
2425         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2426         return err;
2427 }
2428
2429 /**
2430  * PF interrupt handler triggered by NIC for handling specific interrupt.
2431  *
2432  * @param handle
2433  *  Pointer to interrupt handle.
2434  * @param param
2435  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2436  *
2437  * @return
2438  *  void
2439  */
2440 static void
2441 fm10k_dev_interrupt_handler_pf(
2442                         __rte_unused struct rte_intr_handle *handle,
2443                         void *param)
2444 {
2445         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2446         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2447         uint32_t cause, status;
2448
2449         if (hw->mac.type != fm10k_mac_pf)
2450                 return;
2451
2452         cause = FM10K_READ_REG(hw, FM10K_EICR);
2453
2454         /* Handle PCI fault cases */
2455         if (cause & FM10K_EICR_FAULT_MASK) {
2456                 PMD_INIT_LOG(ERR, "INT: find fault!");
2457                 fm10k_dev_handle_fault(hw, cause);
2458         }
2459
2460         /* Handle switch up/down */
2461         if (cause & FM10K_EICR_SWITCHNOTREADY)
2462                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2463
2464         if (cause & FM10K_EICR_SWITCHREADY)
2465                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2466
2467         /* Handle mailbox message */
2468         fm10k_mbx_lock(hw);
2469         hw->mbx.ops.process(hw, &hw->mbx);
2470         fm10k_mbx_unlock(hw);
2471
2472         /* Handle SRAM error */
2473         if (cause & FM10K_EICR_SRAMERROR) {
2474                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2475
2476                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2477                 /* Write to clear pending bits */
2478                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2479
2480                 /* Todo: print out error message after shared code  updates */
2481         }
2482
2483         /* Clear these 3 events if having any */
2484         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2485                  FM10K_EICR_SWITCHREADY;
2486         if (cause)
2487                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2488
2489         /* Re-enable interrupt from device side */
2490         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2491                                         FM10K_ITR_MASK_CLEAR);
2492         /* Re-enable interrupt from host side */
2493         rte_intr_enable(&(dev->pci_dev->intr_handle));
2494 }
2495
2496 /**
2497  * VF interrupt handler triggered by NIC for handling specific interrupt.
2498  *
2499  * @param handle
2500  *  Pointer to interrupt handle.
2501  * @param param
2502  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2503  *
2504  * @return
2505  *  void
2506  */
2507 static void
2508 fm10k_dev_interrupt_handler_vf(
2509                         __rte_unused struct rte_intr_handle *handle,
2510                         void *param)
2511 {
2512         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2513         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2514
2515         if (hw->mac.type != fm10k_mac_vf)
2516                 return;
2517
2518         /* Handle mailbox message if lock is acquired */
2519         fm10k_mbx_lock(hw);
2520         hw->mbx.ops.process(hw, &hw->mbx);
2521         fm10k_mbx_unlock(hw);
2522
2523         /* Re-enable interrupt from device side */
2524         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2525                                         FM10K_ITR_MASK_CLEAR);
2526         /* Re-enable interrupt from host side */
2527         rte_intr_enable(&(dev->pci_dev->intr_handle));
2528 }
2529
2530 /* Mailbox message handler in VF */
2531 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2532         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2533         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2534         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2535         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2536 };
2537
2538 static int
2539 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2540 {
2541         int err = 0;
2542
2543         /* Initialize mailbox lock */
2544         fm10k_mbx_initlock(hw);
2545
2546         /* Replace default message handler with new ones */
2547         if (hw->mac.type == fm10k_mac_vf)
2548                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2549
2550         if (err) {
2551                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2552                                 err);
2553                 return err;
2554         }
2555         /* Connect to SM for PF device or PF for VF device */
2556         return hw->mbx.ops.connect(hw, &hw->mbx);
2557 }
2558
2559 static void
2560 fm10k_close_mbx_service(struct fm10k_hw *hw)
2561 {
2562         /* Disconnect from SM for PF device or PF for VF device */
2563         hw->mbx.ops.disconnect(hw, &hw->mbx);
2564 }
2565
2566 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2567         .dev_configure          = fm10k_dev_configure,
2568         .dev_start              = fm10k_dev_start,
2569         .dev_stop               = fm10k_dev_stop,
2570         .dev_close              = fm10k_dev_close,
2571         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2572         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2573         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2574         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2575         .stats_get              = fm10k_stats_get,
2576         .xstats_get             = fm10k_xstats_get,
2577         .stats_reset            = fm10k_stats_reset,
2578         .xstats_reset           = fm10k_stats_reset,
2579         .link_update            = fm10k_link_update,
2580         .dev_infos_get          = fm10k_dev_infos_get,
2581         .vlan_filter_set        = fm10k_vlan_filter_set,
2582         .vlan_offload_set       = fm10k_vlan_offload_set,
2583         .mac_addr_add           = fm10k_macaddr_add,
2584         .mac_addr_remove        = fm10k_macaddr_remove,
2585         .rx_queue_start         = fm10k_dev_rx_queue_start,
2586         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2587         .tx_queue_start         = fm10k_dev_tx_queue_start,
2588         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2589         .rx_queue_setup         = fm10k_rx_queue_setup,
2590         .rx_queue_release       = fm10k_rx_queue_release,
2591         .tx_queue_setup         = fm10k_tx_queue_setup,
2592         .tx_queue_release       = fm10k_tx_queue_release,
2593         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2594         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2595         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2596         .reta_update            = fm10k_reta_update,
2597         .reta_query             = fm10k_reta_query,
2598         .rss_hash_update        = fm10k_rss_hash_update,
2599         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2600 };
2601
2602 static int ftag_check_handler(__rte_unused const char *key,
2603                 const char *value, __rte_unused void *opaque)
2604 {
2605         if (strcmp(value, "1"))
2606                 return -1;
2607
2608         return 0;
2609 }
2610
2611 static int
2612 fm10k_check_ftag(struct rte_devargs *devargs)
2613 {
2614         struct rte_kvargs *kvlist;
2615         const char *ftag_key = "enable_ftag";
2616
2617         if (devargs == NULL)
2618                 return 0;
2619
2620         kvlist = rte_kvargs_parse(devargs->args, NULL);
2621         if (kvlist == NULL)
2622                 return 0;
2623
2624         if (!rte_kvargs_count(kvlist, ftag_key)) {
2625                 rte_kvargs_free(kvlist);
2626                 return 0;
2627         }
2628         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2629         if (rte_kvargs_process(kvlist, ftag_key,
2630                                 ftag_check_handler, NULL) < 0) {
2631                 rte_kvargs_free(kvlist);
2632                 return 0;
2633         }
2634         rte_kvargs_free(kvlist);
2635
2636         return 1;
2637 }
2638
2639 static void __attribute__((cold))
2640 fm10k_set_tx_function(struct rte_eth_dev *dev)
2641 {
2642         struct fm10k_tx_queue *txq;
2643         int i;
2644         int use_sse = 1;
2645         uint16_t tx_ftag_en = 0;
2646
2647         if (fm10k_check_ftag(dev->pci_dev->devargs))
2648                 tx_ftag_en = 1;
2649
2650         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2651                 txq = dev->data->tx_queues[i];
2652                 txq->tx_ftag_en = tx_ftag_en;
2653                 /* Check if Vector Tx is satisfied */
2654                 if (fm10k_tx_vec_condition_check(txq)) {
2655                         use_sse = 0;
2656                         break;
2657                 }
2658         }
2659
2660         if (use_sse) {
2661                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2662                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2663                         txq = dev->data->tx_queues[i];
2664                         fm10k_txq_vec_setup(txq);
2665                 }
2666                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2667         } else {
2668                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2669                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2670         }
2671 }
2672
2673 static void __attribute__((cold))
2674 fm10k_set_rx_function(struct rte_eth_dev *dev)
2675 {
2676         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2677         uint16_t i, rx_using_sse;
2678         uint16_t rx_ftag_en = 0;
2679
2680         if (fm10k_check_ftag(dev->pci_dev->devargs))
2681                 rx_ftag_en = 1;
2682
2683         /* In order to allow Vector Rx there are a few configuration
2684          * conditions to be met.
2685          */
2686         if (!fm10k_rx_vec_condition_check(dev) &&
2687                         dev_info->rx_vec_allowed && !rx_ftag_en) {
2688                 if (dev->data->scattered_rx)
2689                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2690                 else
2691                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2692         } else if (dev->data->scattered_rx)
2693                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2694         else
2695                 dev->rx_pkt_burst = fm10k_recv_pkts;
2696
2697         rx_using_sse =
2698                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2699                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2700
2701         if (rx_using_sse)
2702                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
2703         else
2704                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
2705
2706         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2707                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2708
2709                 rxq->rx_using_sse = rx_using_sse;
2710                 rxq->rx_ftag_en = rx_ftag_en;
2711         }
2712 }
2713
2714 static void
2715 fm10k_params_init(struct rte_eth_dev *dev)
2716 {
2717         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2718         struct fm10k_dev_info *info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2719
2720         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2721          * there is no way to get link status without reading BAR4.  Until this
2722          * works, assume we have maximum bandwidth.
2723          * @todo - fix bus info
2724          */
2725         hw->bus_caps.speed = fm10k_bus_speed_8000;
2726         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2727         hw->bus_caps.payload = fm10k_bus_payload_512;
2728         hw->bus.speed = fm10k_bus_speed_8000;
2729         hw->bus.width = fm10k_bus_width_pcie_x8;
2730         hw->bus.payload = fm10k_bus_payload_256;
2731
2732         info->rx_vec_allowed = true;
2733 }
2734
2735 static int
2736 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2737 {
2738         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2739         int diag, i;
2740         struct fm10k_macvlan_filter_info *macvlan;
2741
2742         PMD_INIT_FUNC_TRACE();
2743
2744         dev->dev_ops = &fm10k_eth_dev_ops;
2745         dev->rx_pkt_burst = &fm10k_recv_pkts;
2746         dev->tx_pkt_burst = &fm10k_xmit_pkts;
2747
2748         /* only initialize in the primary process */
2749         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2750                 return 0;
2751
2752         rte_eth_copy_pci_info(dev, dev->pci_dev);
2753
2754         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2755         memset(macvlan, 0, sizeof(*macvlan));
2756         /* Vendor and Device ID need to be set before init of shared code */
2757         memset(hw, 0, sizeof(*hw));
2758         hw->device_id = dev->pci_dev->id.device_id;
2759         hw->vendor_id = dev->pci_dev->id.vendor_id;
2760         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
2761         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
2762         hw->revision_id = 0;
2763         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2764         if (hw->hw_addr == NULL) {
2765                 PMD_INIT_LOG(ERR, "Bad mem resource."
2766                         " Try to blacklist unused devices.");
2767                 return -EIO;
2768         }
2769
2770         /* Store fm10k_adapter pointer */
2771         hw->back = dev->data->dev_private;
2772
2773         /* Initialize the shared code */
2774         diag = fm10k_init_shared_code(hw);
2775         if (diag != FM10K_SUCCESS) {
2776                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2777                 return -EIO;
2778         }
2779
2780         /* Initialize parameters */
2781         fm10k_params_init(dev);
2782
2783         /* Initialize the hw */
2784         diag = fm10k_init_hw(hw);
2785         if (diag != FM10K_SUCCESS) {
2786                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2787                 return -EIO;
2788         }
2789
2790         /* Initialize MAC address(es) */
2791         dev->data->mac_addrs = rte_zmalloc("fm10k",
2792                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2793         if (dev->data->mac_addrs == NULL) {
2794                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2795                 return -ENOMEM;
2796         }
2797
2798         diag = fm10k_read_mac_addr(hw);
2799
2800         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2801                         &dev->data->mac_addrs[0]);
2802
2803         if (diag != FM10K_SUCCESS ||
2804                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2805
2806                 /* Generate a random addr */
2807                 eth_random_addr(hw->mac.addr);
2808                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2809                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2810                 &dev->data->mac_addrs[0]);
2811         }
2812
2813         /* Reset the hw statistics */
2814         fm10k_stats_reset(dev);
2815
2816         /* Reset the hw */
2817         diag = fm10k_reset_hw(hw);
2818         if (diag != FM10K_SUCCESS) {
2819                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2820                 return -EIO;
2821         }
2822
2823         /* Setup mailbox service */
2824         diag = fm10k_setup_mbx_service(hw);
2825         if (diag != FM10K_SUCCESS) {
2826                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2827                 return -EIO;
2828         }
2829
2830         /*PF/VF has different interrupt handling mechanism */
2831         if (hw->mac.type == fm10k_mac_pf) {
2832                 /* register callback func to eal lib */
2833                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2834                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2835
2836                 /* enable MISC interrupt */
2837                 fm10k_dev_enable_intr_pf(dev);
2838         } else { /* VF */
2839                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2840                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2841
2842                 fm10k_dev_enable_intr_vf(dev);
2843         }
2844
2845         /* Enable intr after callback registered */
2846         rte_intr_enable(&(dev->pci_dev->intr_handle));
2847
2848         hw->mac.ops.update_int_moderator(hw);
2849
2850         /* Make sure Switch Manager is ready before going forward. */
2851         if (hw->mac.type == fm10k_mac_pf) {
2852                 int switch_ready = 0;
2853
2854                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2855                         fm10k_mbx_lock(hw);
2856                         hw->mac.ops.get_host_state(hw, &switch_ready);
2857                         fm10k_mbx_unlock(hw);
2858                         if (switch_ready)
2859                                 break;
2860                         /* Delay some time to acquire async LPORT_MAP info. */
2861                         rte_delay_us(WAIT_SWITCH_MSG_US);
2862                 }
2863
2864                 if (switch_ready == 0) {
2865                         PMD_INIT_LOG(ERR, "switch is not ready");
2866                         return -1;
2867                 }
2868         }
2869
2870         /*
2871          * Below function will trigger operations on mailbox, acquire lock to
2872          * avoid race condition from interrupt handler. Operations on mailbox
2873          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2874          * will handle and generate an interrupt to our side. Then,  FIFO in
2875          * mailbox will be touched.
2876          */
2877         fm10k_mbx_lock(hw);
2878         /* Enable port first */
2879         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2880                                         MAX_LPORT_NUM, 1);
2881
2882         /* Set unicast mode by default. App can change to other mode in other
2883          * API func.
2884          */
2885         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2886                                         FM10K_XCAST_MODE_NONE);
2887
2888         fm10k_mbx_unlock(hw);
2889
2890         /* Make sure default VID is ready before going forward. */
2891         if (hw->mac.type == fm10k_mac_pf) {
2892                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2893                         if (hw->mac.default_vid)
2894                                 break;
2895                         /* Delay some time to acquire async port VLAN info. */
2896                         rte_delay_us(WAIT_SWITCH_MSG_US);
2897                 }
2898
2899                 if (!hw->mac.default_vid) {
2900                         PMD_INIT_LOG(ERR, "default VID is not ready");
2901                         return -1;
2902                 }
2903         }
2904
2905         /* Add default mac address */
2906         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2907                 MAIN_VSI_POOL_NUMBER);
2908
2909         return 0;
2910 }
2911
2912 static int
2913 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
2914 {
2915         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2916
2917         PMD_INIT_FUNC_TRACE();
2918
2919         /* only uninitialize in the primary process */
2920         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2921                 return 0;
2922
2923         /* safe to close dev here */
2924         fm10k_dev_close(dev);
2925
2926         dev->dev_ops = NULL;
2927         dev->rx_pkt_burst = NULL;
2928         dev->tx_pkt_burst = NULL;
2929
2930         /* disable uio/vfio intr */
2931         rte_intr_disable(&(dev->pci_dev->intr_handle));
2932
2933         /*PF/VF has different interrupt handling mechanism */
2934         if (hw->mac.type == fm10k_mac_pf) {
2935                 /* disable interrupt */
2936                 fm10k_dev_disable_intr_pf(dev);
2937
2938                 /* unregister callback func to eal lib */
2939                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2940                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2941         } else {
2942                 /* disable interrupt */
2943                 fm10k_dev_disable_intr_vf(dev);
2944
2945                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2946                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2947         }
2948
2949         /* free mac memory */
2950         if (dev->data->mac_addrs) {
2951                 rte_free(dev->data->mac_addrs);
2952                 dev->data->mac_addrs = NULL;
2953         }
2954
2955         memset(hw, 0, sizeof(*hw));
2956
2957         return 0;
2958 }
2959
2960 /*
2961  * The set of PCI devices this driver supports. This driver will enable both PF
2962  * and SRIOV-VF devices.
2963  */
2964 static const struct rte_pci_id pci_id_fm10k_map[] = {
2965 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2966 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2967 #include "rte_pci_dev_ids.h"
2968         { .vendor_id = 0, /* sentinel */ },
2969 };
2970
2971 static struct eth_driver rte_pmd_fm10k = {
2972         .pci_drv = {
2973                 .name = "rte_pmd_fm10k",
2974                 .id_table = pci_id_fm10k_map,
2975                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
2976                         RTE_PCI_DRV_DETACHABLE,
2977         },
2978         .eth_dev_init = eth_fm10k_dev_init,
2979         .eth_dev_uninit = eth_fm10k_dev_uninit,
2980         .dev_private_size = sizeof(struct fm10k_adapter),
2981 };
2982
2983 /*
2984  * Driver initialization routine.
2985  * Invoked once at EAL init time.
2986  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2987  */
2988 static int
2989 rte_pmd_fm10k_init(__rte_unused const char *name,
2990         __rte_unused const char *params)
2991 {
2992         PMD_INIT_FUNC_TRACE();
2993         rte_eth_driver_register(&rte_pmd_fm10k);
2994         return 0;
2995 }
2996
2997 static struct rte_driver rte_fm10k_driver = {
2998         .type = PMD_PDEV,
2999         .init = rte_pmd_fm10k_init,
3000 };
3001
3002 PMD_REGISTER_DRIVER(rte_fm10k_driver);