drivers/net/fm10k/fm10k_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2013-2016 Intel Corporation
   3  */
   4
   5 #include <rte_ethdev_driver.h>
   6 #include <rte_ethdev_pci.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_string_fns.h>
  10 #include <rte_dev.h>
  11 #include <rte_spinlock.h>
  12 #include <rte_kvargs.h>
  13
  14 #include "fm10k.h"
  15 #include "base/fm10k_api.h"
  16
  17 /* Default delay to acquire mailbox lock */
  18 #define FM10K_MBXLOCK_DELAY_US 20
  19 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  20
  21 #define MAIN_VSI_POOL_NUMBER 0
  22
  23 /* Max try times to acquire switch status */
  24 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  25 /* Wait interval to get switch status */
  26 #define WAIT_SWITCH_MSG_US    100000
  27 /* A period of quiescence for switch */
  28 #define FM10K_SWITCH_QUIESCE_US 100000
  29 /* Number of chars per uint32 type */
  30 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  31 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  32
  33 /* default 1:1 map from queue ID to interrupt vector ID */
  34 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  35
  36 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  37 #define MAX_LPORT_NUM    128
  38 #define GLORT_FD_Q_BASE  0x40
  39 #define GLORT_PF_MASK    0xFFC0
  40 #define GLORT_FD_MASK    GLORT_PF_MASK
  41 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  42
  43 int fm10k_logtype_init;
  44 int fm10k_logtype_driver;
  45
  46 #ifdef RTE_LIBRTE_FM10K_DEBUG_RX
  47 int fm10k_logtype_rx;
  48 #endif
  49 #ifdef RTE_LIBRTE_FM10K_DEBUG_TX
  50 int fm10k_logtype_tx;
  51 #endif
  52 #ifdef RTE_LIBRTE_FM10K_DEBUG_TX_FREE
  53 int fm10k_logtype_tx_free;
  54 #endif
  55
  56 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  57 static int fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  58 static int fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  59 static int fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  60 static int fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  61 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  62 static int
  63 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  64 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  65         const u8 *mac, bool add, uint32_t pool);
  66 static void fm10k_tx_queue_release(void *queue);
  67 static void fm10k_rx_queue_release(void *queue);
  68 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  69 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  70 static int fm10k_check_ftag(struct rte_devargs *devargs);
  71 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  72
  73 static int fm10k_dev_infos_get(struct rte_eth_dev *dev,
  74                                struct rte_eth_dev_info *dev_info);
  75 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev);
  76 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev);
  77 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev);
  78 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev);
  79
  80 struct fm10k_xstats_name_off {
  81         char name[RTE_ETH_XSTATS_NAME_SIZE];
  82         unsigned offset;
  83 };
  84
  85 static const struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  86         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  87         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  88         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  89         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  90         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  91         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  92         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  93         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  94                 nodesc_drop)},
  95 };
  96
  97 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  98                 sizeof(fm10k_hw_stats_strings[0]))
  99
 100 static const struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
 101         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
 102         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
 103         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
 104 };
 105
 106 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
 107                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
 108
 109 static const struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 110         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 111         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 112 };
 113
 114 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 115                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 116
 117 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 118                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 119 static int
 120 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 121
 122 static void
 123 fm10k_mbx_initlock(struct fm10k_hw *hw)
 124 {
 125         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 126 }
 127
 128 static void
 129 fm10k_mbx_lock(struct fm10k_hw *hw)
 130 {
 131         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 132                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 133 }
 134
 135 static void
 136 fm10k_mbx_unlock(struct fm10k_hw *hw)
 137 {
 138         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 139 }
 140
 141 /* Stubs needed for linkage when vPMD is disabled */
 142 __rte_weak int
 143 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 144 {
 145         return -1;
 146 }
 147
 148 __rte_weak uint16_t
 149 fm10k_recv_pkts_vec(
 150         __rte_unused void *rx_queue,
 151         __rte_unused struct rte_mbuf **rx_pkts,
 152         __rte_unused uint16_t nb_pkts)
 153 {
 154         return 0;
 155 }
 156
 157 __rte_weak uint16_t
 158 fm10k_recv_scattered_pkts_vec(
 159                 __rte_unused void *rx_queue,
 160                 __rte_unused struct rte_mbuf **rx_pkts,
 161                 __rte_unused uint16_t nb_pkts)
 162 {
 163         return 0;
 164 }
 165
 166 __rte_weak int
 167 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 168
 169 {
 170         return -1;
 171 }
 172
 173 __rte_weak void
 174 fm10k_rx_queue_release_mbufs_vec(
 175                 __rte_unused struct fm10k_rx_queue *rxq)
 176 {
 177         return;
 178 }
 179
 180 __rte_weak void
 181 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 182 {
 183         return;
 184 }
 185
 186 __rte_weak int
 187 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 188 {
 189         return -1;
 190 }
 191
 192 __rte_weak uint16_t
 193 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 194                            __rte_unused struct rte_mbuf **tx_pkts,
 195                            __rte_unused uint16_t nb_pkts)
 196 {
 197         return 0;
 198 }
 199
 200 /*
 201  * reset queue to initial state, allocate software buffers used when starting
 202  * device.
 203  * return 0 on success
 204  * return -ENOMEM if buffers cannot be allocated
 205  * return -EINVAL if buffers do not satisfy alignment condition
 206  */
 207 static inline int
 208 rx_queue_reset(struct fm10k_rx_queue *q)
 209 {
 210         static const union fm10k_rx_desc zero = {{0} };
 211         uint64_t dma_addr;
 212         int i, diag;
 213         PMD_INIT_FUNC_TRACE();
 214
 215         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 216         if (diag != 0)
 217                 return -ENOMEM;
 218
 219         for (i = 0; i < q->nb_desc; ++i) {
 220                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 221                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 222                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 223                                                 q->nb_desc);
 224                         return -EINVAL;
 225                 }
 226                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 227                 q->hw_ring[i].q.pkt_addr = dma_addr;
 228                 q->hw_ring[i].q.hdr_addr = dma_addr;
 229         }
 230
 231         /* initialize extra software ring entries. Space for these extra
 232          * entries is always allocated.
 233          */
 234         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 235         for (i = 0; i < q->nb_fake_desc; ++i) {
 236                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 237                 q->hw_ring[q->nb_desc + i] = zero;
 238         }
 239
 240         q->next_dd = 0;
 241         q->next_alloc = 0;
 242         q->next_trigger = q->alloc_thresh - 1;
 243         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 244         q->rxrearm_start = 0;
 245         q->rxrearm_nb = 0;
 246
 247         return 0;
 248 }
 249
 250 /*
 251  * clean queue, descriptor rings, free software buffers used when stopping
 252  * device.
 253  */
 254 static inline void
 255 rx_queue_clean(struct fm10k_rx_queue *q)
 256 {
 257         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 258         uint32_t i;
 259         PMD_INIT_FUNC_TRACE();
 260
 261         /* zero descriptor rings */
 262         for (i = 0; i < q->nb_desc; ++i)
 263                 q->hw_ring[i] = zero;
 264
 265         /* zero faked descriptors */
 266         for (i = 0; i < q->nb_fake_desc; ++i)
 267                 q->hw_ring[q->nb_desc + i] = zero;
 268
 269         /* vPMD driver has a different way of releasing mbufs. */
 270         if (q->rx_using_sse) {
 271                 fm10k_rx_queue_release_mbufs_vec(q);
 272                 return;
 273         }
 274
 275         /* free software buffers */
 276         for (i = 0; i < q->nb_desc; ++i) {
 277                 if (q->sw_ring[i]) {
 278                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 279                         q->sw_ring[i] = NULL;
 280                 }
 281         }
 282 }
 283
 284 /*
 285  * free all queue memory used when releasing the queue (i.e. configure)
 286  */
 287 static inline void
 288 rx_queue_free(struct fm10k_rx_queue *q)
 289 {
 290         PMD_INIT_FUNC_TRACE();
 291         if (q) {
 292                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 293                 rx_queue_clean(q);
 294                 if (q->sw_ring) {
 295                         rte_free(q->sw_ring);
 296                         q->sw_ring = NULL;
 297                 }
 298                 rte_free(q);
 299                 q = NULL;
 300         }
 301 }
 302
 303 /*
 304  * disable RX queue, wait unitl HW finished necessary flush operation
 305  */
 306 static inline int
 307 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 308 {
 309         uint32_t reg, i;
 310
 311         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 312         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 313                         reg & ~FM10K_RXQCTL_ENABLE);
 314
 315         /* Wait 100us at most */
 316         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 317                 rte_delay_us(1);
 318                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 319                 if (!(reg & FM10K_RXQCTL_ENABLE))
 320                         break;
 321         }
 322
 323         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 324                 return -1;
 325
 326         return 0;
 327 }
 328
 329 /*
 330  * reset queue to initial state, allocate software buffers used when starting
 331  * device
 332  */
 333 static inline void
 334 tx_queue_reset(struct fm10k_tx_queue *q)
 335 {
 336         PMD_INIT_FUNC_TRACE();
 337         q->last_free = 0;
 338         q->next_free = 0;
 339         q->nb_used = 0;
 340         q->nb_free = q->nb_desc - 1;
 341         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 342         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 343 }
 344
 345 /*
 346  * clean queue, descriptor rings, free software buffers used when stopping
 347  * device
 348  */
 349 static inline void
 350 tx_queue_clean(struct fm10k_tx_queue *q)
 351 {
 352         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 353         uint32_t i;
 354         PMD_INIT_FUNC_TRACE();
 355
 356         /* zero descriptor rings */
 357         for (i = 0; i < q->nb_desc; ++i)
 358                 q->hw_ring[i] = zero;
 359
 360         /* free software buffers */
 361         for (i = 0; i < q->nb_desc; ++i) {
 362                 if (q->sw_ring[i]) {
 363                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 364                         q->sw_ring[i] = NULL;
 365                 }
 366         }
 367 }
 368
 369 /*
 370  * free all queue memory used when releasing the queue (i.e. configure)
 371  */
 372 static inline void
 373 tx_queue_free(struct fm10k_tx_queue *q)
 374 {
 375         PMD_INIT_FUNC_TRACE();
 376         if (q) {
 377                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 378                 tx_queue_clean(q);
 379                 if (q->rs_tracker.list) {
 380                         rte_free(q->rs_tracker.list);
 381                         q->rs_tracker.list = NULL;
 382                 }
 383                 if (q->sw_ring) {
 384                         rte_free(q->sw_ring);
 385                         q->sw_ring = NULL;
 386                 }
 387                 rte_free(q);
 388                 q = NULL;
 389         }
 390 }
 391
 392 /*
 393  * disable TX queue, wait unitl HW finished necessary flush operation
 394  */
 395 static inline int
 396 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 397 {
 398         uint32_t reg, i;
 399
 400         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 401         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 402                         reg & ~FM10K_TXDCTL_ENABLE);
 403
 404         /* Wait 100us at most */
 405         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 406                 rte_delay_us(1);
 407                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 408                 if (!(reg & FM10K_TXDCTL_ENABLE))
 409                         break;
 410         }
 411
 412         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 413                 return -1;
 414
 415         return 0;
 416 }
 417
 418 static int
 419 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 420 {
 421         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 422         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 423         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 424         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 425
 426         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 427
 428         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 429                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 430                 return -EINVAL;
 431         }
 432
 433         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 434                 return 0;
 435
 436         if (hw->mac.type == fm10k_mac_vf) {
 437                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 438                 return -EINVAL;
 439         }
 440
 441         /* Check VMDQ queue pool number */
 442         if (vmdq_conf->nb_queue_pools >
 443                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 444                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 445                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 446                         vmdq_conf->nb_queue_pools);
 447                 return -EINVAL;
 448         }
 449
 450         return 0;
 451 }
 452
 453 static const struct fm10k_txq_ops def_txq_ops = {
 454         .reset = tx_queue_reset,
 455 };
 456
 457 static int
 458 fm10k_dev_configure(struct rte_eth_dev *dev)
 459 {
 460         int ret;
 461
 462         PMD_INIT_FUNC_TRACE();
 463
 464         /* multipe queue mode checking */
 465         ret  = fm10k_check_mq_mode(dev);
 466         if (ret != 0) {
 467                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 468                             ret);
 469                 return ret;
 470         }
 471
 472         dev->data->scattered_rx = 0;
 473
 474         return 0;
 475 }
 476
 477 static void
 478 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 479 {
 480         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 481         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 482         uint32_t i;
 483
 484         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 485
 486         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 487                 if (!vmdq_conf->pool_map[i].pools)
 488                         continue;
 489                 fm10k_mbx_lock(hw);
 490                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 491                 fm10k_mbx_unlock(hw);
 492         }
 493 }
 494
 495 static void
 496 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 497 {
 498         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 499
 500         /* Add default mac address */
 501         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 502                 MAIN_VSI_POOL_NUMBER);
 503 }
 504
 505 static void
 506 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 507 {
 508         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 509         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 510         uint32_t mrqc, *key, i, reta, j;
 511         uint64_t hf;
 512
 513 #define RSS_KEY_SIZE 40
 514         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 515                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 516                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 517                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 518                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 519                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 520         };
 521
 522         if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 523                 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 524                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 525                 return;
 526         }
 527
 528         /* random key is rss_intel_key (default) or user provided (rss_key) */
 529         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 530                 key = (uint32_t *)rss_intel_key;
 531         else
 532                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 533
 534         /* Now fill our hash function seeds, 4 bytes at a time */
 535         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 536                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 537
 538         /*
 539          * Fill in redirection table
 540          * The byte-swap is needed because NIC registers are in
 541          * little-endian order.
 542          */
 543         reta = 0;
 544         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 545                 if (j == dev->data->nb_rx_queues)
 546                         j = 0;
 547                 reta = (reta << CHAR_BIT) | j;
 548                 if ((i & 3) == 3)
 549                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 550                                         rte_bswap32(reta));
 551         }
 552
 553         /*
 554          * Generate RSS hash based on packet types, TCP/UDP
 555          * port numbers and/or IPv4/v6 src and dst addresses
 556          */
 557         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 558         mrqc = 0;
 559         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 560         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 561         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 562         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 563         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 564         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 565         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 566         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 567         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 568
 569         if (mrqc == 0) {
 570                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 571                         "supported", hf);
 572                 return;
 573         }
 574
 575         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 576 }
 577
 578 static void
 579 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 580 {
 581         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 582         uint32_t i;
 583
 584         for (i = 0; i < nb_lport_new; i++) {
 585                 /* Set unicast mode by default. App can change
 586                  * to other mode in other API func.
 587                  */
 588                 fm10k_mbx_lock(hw);
 589                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 590                         FM10K_XCAST_MODE_NONE);
 591                 fm10k_mbx_unlock(hw);
 592         }
 593 }
 594
 595 static void
 596 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 597 {
 598         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 599         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 600         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 601         struct fm10k_macvlan_filter_info *macvlan;
 602         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 603         uint16_t nb_lport_new;
 604
 605         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 606         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 607
 608         fm10k_dev_rss_configure(dev);
 609
 610         /* only PF supports VMDQ */
 611         if (hw->mac.type != fm10k_mac_pf)
 612                 return;
 613
 614         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 615                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 616
 617         /* no pool number change, no need to update logic port and VLAN/MAC */
 618         if (macvlan->nb_queue_pools == nb_queue_pools)
 619                 return;
 620
 621         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 622         fm10k_dev_logic_port_update(dev, nb_lport_new);
 623
 624         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 625         memset(dev->data->mac_addrs, 0,
 626                 RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 627         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
 628                 &dev->data->mac_addrs[0]);
 629         memset(macvlan, 0, sizeof(*macvlan));
 630         macvlan->nb_queue_pools = nb_queue_pools;
 631
 632         if (nb_queue_pools)
 633                 fm10k_dev_vmdq_rx_configure(dev);
 634         else
 635                 fm10k_dev_pf_main_vsi_reset(dev);
 636 }
 637
 638 static int
 639 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 640 {
 641         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 642         int i, ret;
 643         struct fm10k_tx_queue *txq;
 644         uint64_t base_addr;
 645         uint32_t size;
 646
 647         /* Disable TXINT to avoid possible interrupt */
 648         for (i = 0; i < hw->mac.max_queues; i++)
 649                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 650                                 3 << FM10K_TXINT_TIMER_SHIFT);
 651
 652         /* Setup TX queue */
 653         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 654                 txq = dev->data->tx_queues[i];
 655                 base_addr = txq->hw_ring_phys_addr;
 656                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 657
 658                 /* disable queue to avoid issues while updating state */
 659                 ret = tx_queue_disable(hw, i);
 660                 if (ret) {
 661                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 662                         return -1;
 663                 }
 664                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 665                  * register is read-only for VF.
 666                  */
 667                 if (fm10k_check_ftag(dev->device->devargs)) {
 668                         if (hw->mac.type == fm10k_mac_pf) {
 669                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 670                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 671                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 672                         } else {
 673                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 674                                 return -ENOTSUP;
 675                         }
 676                 }
 677
 678                 /* set location and size for descriptor ring */
 679                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 680                                 base_addr & UINT64_LOWER_32BITS_MASK);
 681                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 682                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 683                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 684
 685                 /* assign default SGLORT for each TX queue by PF */
 686                 if (hw->mac.type == fm10k_mac_pf)
 687                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 688         }
 689
 690         /* set up vector or scalar TX function as appropriate */
 691         fm10k_set_tx_function(dev);
 692
 693         return 0;
 694 }
 695
 696 static int
 697 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 698 {
 699         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 700         struct fm10k_macvlan_filter_info *macvlan;
 701         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 702         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 703         int i, ret;
 704         struct fm10k_rx_queue *rxq;
 705         uint64_t base_addr;
 706         uint32_t size;
 707         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 708         uint32_t logic_port = hw->mac.dglort_map;
 709         uint16_t buf_size;
 710         uint16_t queue_stride = 0;
 711
 712         /* enable RXINT for interrupt mode */
 713         i = 0;
 714         if (rte_intr_dp_is_en(intr_handle)) {
 715                 for (; i < dev->data->nb_rx_queues; i++) {
 716                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 717                         if (hw->mac.type == fm10k_mac_pf)
 718                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 719                                         FM10K_ITR_AUTOMASK |
 720                                         FM10K_ITR_MASK_CLEAR);
 721                         else
 722                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 723                                         FM10K_ITR_AUTOMASK |
 724                                         FM10K_ITR_MASK_CLEAR);
 725                 }
 726         }
 727         /* Disable other RXINT to avoid possible interrupt */
 728         for (; i < hw->mac.max_queues; i++)
 729                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 730                         3 << FM10K_RXINT_TIMER_SHIFT);
 731
 732         /* Setup RX queues */
 733         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 734                 rxq = dev->data->rx_queues[i];
 735                 base_addr = rxq->hw_ring_phys_addr;
 736                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 737
 738                 /* disable queue to avoid issues while updating state */
 739                 ret = rx_queue_disable(hw, i);
 740                 if (ret) {
 741                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 742                         return -1;
 743                 }
 744
 745                 /* Setup the Base and Length of the Rx Descriptor Ring */
 746                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 747                                 base_addr & UINT64_LOWER_32BITS_MASK);
 748                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 749                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 750                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 751
 752                 /* Configure the Rx buffer size for one buff without split */
 753                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 754                         RTE_PKTMBUF_HEADROOM);
 755                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 756                  * reserved for this purpose, and the worst case could be 511B.
 757                  * But SRR reg assumes all buffers have the same size. In order
 758                  * to fill the gap, we'll have to consider the worst case and
 759                  * assume 512B is reserved. If we don't do so, it's possible
 760                  * for HW to overwrite data to next mbuf.
 761                  */
 762                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 763
 764                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 765                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 766                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 767
 768                 /* It adds dual VLAN length for supporting dual VLAN */
 769                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 770                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 771                         rxq->offloads & DEV_RX_OFFLOAD_SCATTER) {
 772                         uint32_t reg;
 773                         dev->data->scattered_rx = 1;
 774                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 775                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 776                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 777                 }
 778
 779                 /* Enable drop on empty, it's RO for VF */
 780                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 781                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 782
 783                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 784                 FM10K_WRITE_FLUSH(hw);
 785         }
 786
 787         /* Configure VMDQ/RSS if applicable */
 788         fm10k_dev_mq_rx_configure(dev);
 789
 790         /* Decide the best RX function */
 791         fm10k_set_rx_function(dev);
 792
 793         /* update RX_SGLORT for loopback suppress*/
 794         if (hw->mac.type != fm10k_mac_pf)
 795                 return 0;
 796         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 797         if (macvlan->nb_queue_pools)
 798                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 799         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 800                 if (i && queue_stride && !(i % queue_stride))
 801                         logic_port++;
 802                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 803         }
 804
 805         return 0;
 806 }
 807
 808 static int
 809 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 810 {
 811         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 812         int err;
 813         uint32_t reg;
 814         struct fm10k_rx_queue *rxq;
 815
 816         PMD_INIT_FUNC_TRACE();
 817
 818         rxq = dev->data->rx_queues[rx_queue_id];
 819         err = rx_queue_reset(rxq);
 820         if (err == -ENOMEM) {
 821                 PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 822                 return err;
 823         } else if (err == -EINVAL) {
 824                 PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 825                         " %d", err);
 826                 return err;
 827         }
 828
 829         /* Setup the HW Rx Head and Tail Descriptor Pointers
 830          * Note: this must be done AFTER the queue is enabled on real
 831          * hardware, but BEFORE the queue is enabled when using the
 832          * emulation platform. Do it in both places for now and remove
 833          * this comment and the following two register writes when the
 834          * emulation platform is no longer being used.
 835          */
 836         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 837         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 838
 839         /* Set PF ownership flag for PF devices */
 840         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 841         if (hw->mac.type == fm10k_mac_pf)
 842                 reg |= FM10K_RXQCTL_PF;
 843         reg |= FM10K_RXQCTL_ENABLE;
 844         /* enable RX queue */
 845         FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 846         FM10K_WRITE_FLUSH(hw);
 847
 848         /* Setup the HW Rx Head and Tail Descriptor Pointers
 849          * Note: this must be done AFTER the queue is enabled
 850          */
 851         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 852         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 853         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 854
 855         return 0;
 856 }
 857
 858 static int
 859 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 860 {
 861         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 862
 863         PMD_INIT_FUNC_TRACE();
 864
 865         /* Disable RX queue */
 866         rx_queue_disable(hw, rx_queue_id);
 867
 868         /* Free mbuf and clean HW ring */
 869         rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 870         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 871
 872         return 0;
 873 }
 874
 875 static int
 876 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 877 {
 878         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 879         /** @todo - this should be defined in the shared code */
 880 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 881         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 882         struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 883
 884         PMD_INIT_FUNC_TRACE();
 885
 886         q->ops->reset(q);
 887
 888         /* reset head and tail pointers */
 889         FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 890         FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 891
 892         /* enable TX queue */
 893         FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 894                                 FM10K_TXDCTL_ENABLE | txdctl);
 895         FM10K_WRITE_FLUSH(hw);
 896         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 897
 898         return 0;
 899 }
 900
 901 static int
 902 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 903 {
 904         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 905
 906         PMD_INIT_FUNC_TRACE();
 907
 908         tx_queue_disable(hw, tx_queue_id);
 909         tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 910         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 911
 912         return 0;
 913 }
 914
 915 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 916 {
 917         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 918                 != FM10K_DGLORTMAP_NONE);
 919 }
 920
 921 static int
 922 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 923 {
 924         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 925         int status;
 926
 927         PMD_INIT_FUNC_TRACE();
 928
 929         /* Return if it didn't acquire valid glort range */
 930         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 931                 return 0;
 932
 933         fm10k_mbx_lock(hw);
 934         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 935                                 FM10K_XCAST_MODE_PROMISC);
 936         fm10k_mbx_unlock(hw);
 937
 938         if (status != FM10K_SUCCESS) {
 939                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 940                 return -EAGAIN;
 941         }
 942
 943         return 0;
 944 }
 945
 946 static int
 947 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 948 {
 949         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 950         uint8_t mode;
 951         int status;
 952
 953         PMD_INIT_FUNC_TRACE();
 954
 955         /* Return if it didn't acquire valid glort range */
 956         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 957                 return 0;
 958
 959         if (dev->data->all_multicast == 1)
 960                 mode = FM10K_XCAST_MODE_ALLMULTI;
 961         else
 962                 mode = FM10K_XCAST_MODE_NONE;
 963
 964         fm10k_mbx_lock(hw);
 965         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 966                                 mode);
 967         fm10k_mbx_unlock(hw);
 968
 969         if (status != FM10K_SUCCESS) {
 970                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 971                 return -EAGAIN;
 972         }
 973
 974         return 0;
 975 }
 976
 977 static int
 978 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 979 {
 980         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 981         int status;
 982
 983         PMD_INIT_FUNC_TRACE();
 984
 985         /* Return if it didn't acquire valid glort range */
 986         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 987                 return 0;
 988
 989         /* If promiscuous mode is enabled, it doesn't make sense to enable
 990          * allmulticast and disable promiscuous since fm10k only can select
 991          * one of the modes.
 992          */
 993         if (dev->data->promiscuous) {
 994                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 995                         "needn't enable allmulticast");
 996                 return 0;
 997         }
 998
 999         fm10k_mbx_lock(hw);
1000         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1001                                 FM10K_XCAST_MODE_ALLMULTI);
1002         fm10k_mbx_unlock(hw);
1003
1004         if (status != FM10K_SUCCESS) {
1005                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
1006                 return -EAGAIN;
1007         }
1008
1009         return 0;
1010 }
1011
1012 static int
1013 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1014 {
1015         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1016         int status;
1017
1018         PMD_INIT_FUNC_TRACE();
1019
1020         /* Return if it didn't acquire valid glort range */
1021         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1022                 return 0;
1023
1024         if (dev->data->promiscuous) {
1025                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1026                         "since promisc mode is enabled");
1027                 return -EINVAL;
1028         }
1029
1030         fm10k_mbx_lock(hw);
1031         /* Change mode to unicast mode */
1032         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1033                                 FM10K_XCAST_MODE_NONE);
1034         fm10k_mbx_unlock(hw);
1035
1036         if (status != FM10K_SUCCESS) {
1037                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1038                 return -EAGAIN;
1039         }
1040
1041         return 0;
1042 }
1043
1044 static void
1045 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1046 {
1047         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1048         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1049         uint16_t nb_queue_pools;
1050         struct fm10k_macvlan_filter_info *macvlan;
1051
1052         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1053         nb_queue_pools = macvlan->nb_queue_pools;
1054         pool_len = nb_queue_pools ? rte_fls_u32(nb_queue_pools - 1) : 0;
1055         rss_len = rte_fls_u32(dev->data->nb_rx_queues - 1) - pool_len;
1056
1057         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1058         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1059         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1060                         hw->mac.dglort_map;
1061         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1062         /* Configure VMDQ/RSS DGlort Decoder */
1063         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1064
1065         /* Flow Director configurations, only queue number is valid. */
1066         dglortdec = rte_fls_u32(dev->data->nb_rx_queues - 1);
1067         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1068                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1069         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1070         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1071
1072         /* Invalidate all other GLORT entries */
1073         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1074                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1075                                 FM10K_DGLORTMAP_NONE);
1076 }
1077
1078 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1079 static int
1080 fm10k_dev_start(struct rte_eth_dev *dev)
1081 {
1082         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1083         int i, diag;
1084
1085         PMD_INIT_FUNC_TRACE();
1086
1087         /* stop, init, then start the hw */
1088         diag = fm10k_stop_hw(hw);
1089         if (diag != FM10K_SUCCESS) {
1090                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1091                 return -EIO;
1092         }
1093
1094         diag = fm10k_init_hw(hw);
1095         if (diag != FM10K_SUCCESS) {
1096                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1097                 return -EIO;
1098         }
1099
1100         diag = fm10k_start_hw(hw);
1101         if (diag != FM10K_SUCCESS) {
1102                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1103                 return -EIO;
1104         }
1105
1106         diag = fm10k_dev_tx_init(dev);
1107         if (diag) {
1108                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1109                 return diag;
1110         }
1111
1112         if (fm10k_dev_rxq_interrupt_setup(dev))
1113                 return -EIO;
1114
1115         diag = fm10k_dev_rx_init(dev);
1116         if (diag) {
1117                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1118                 return diag;
1119         }
1120
1121         if (hw->mac.type == fm10k_mac_pf)
1122                 fm10k_dev_dglort_map_configure(dev);
1123
1124         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1125                 struct fm10k_rx_queue *rxq;
1126                 rxq = dev->data->rx_queues[i];
1127
1128                 if (rxq->rx_deferred_start)
1129                         continue;
1130                 diag = fm10k_dev_rx_queue_start(dev, i);
1131                 if (diag != 0) {
1132                         int j;
1133                         for (j = 0; j < i; ++j)
1134                                 rx_queue_clean(dev->data->rx_queues[j]);
1135                         return diag;
1136                 }
1137         }
1138
1139         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1140                 struct fm10k_tx_queue *txq;
1141                 txq = dev->data->tx_queues[i];
1142
1143                 if (txq->tx_deferred_start)
1144                         continue;
1145                 diag = fm10k_dev_tx_queue_start(dev, i);
1146                 if (diag != 0) {
1147                         int j;
1148                         for (j = 0; j < i; ++j)
1149                                 tx_queue_clean(dev->data->tx_queues[j]);
1150                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1151                                 rx_queue_clean(dev->data->rx_queues[j]);
1152                         return diag;
1153                 }
1154         }
1155
1156         /* Update default vlan when not in VMDQ mode */
1157         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1158                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1159
1160         fm10k_link_update(dev, 0);
1161
1162         return 0;
1163 }
1164
1165 static void
1166 fm10k_dev_stop(struct rte_eth_dev *dev)
1167 {
1168         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1169         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1170         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1171         int i;
1172
1173         PMD_INIT_FUNC_TRACE();
1174
1175         if (dev->data->tx_queues)
1176                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1177                         fm10k_dev_tx_queue_stop(dev, i);
1178
1179         if (dev->data->rx_queues)
1180                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1181                         fm10k_dev_rx_queue_stop(dev, i);
1182
1183         /* Disable datapath event */
1184         if (rte_intr_dp_is_en(intr_handle)) {
1185                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1186                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1187                                 3 << FM10K_RXINT_TIMER_SHIFT);
1188                         if (hw->mac.type == fm10k_mac_pf)
1189                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1190                                         FM10K_ITR_MASK_SET);
1191                         else
1192                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1193                                         FM10K_ITR_MASK_SET);
1194                 }
1195         }
1196         /* Clean datapath event and queue/vec mapping */
1197         rte_intr_efd_disable(intr_handle);
1198         rte_free(intr_handle->intr_vec);
1199         intr_handle->intr_vec = NULL;
1200 }
1201
1202 static void
1203 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1204 {
1205         int i;
1206
1207         PMD_INIT_FUNC_TRACE();
1208
1209         if (dev->data->tx_queues) {
1210                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1211                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1212
1213                         tx_queue_free(txq);
1214                 }
1215         }
1216
1217         if (dev->data->rx_queues) {
1218                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1219                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1220         }
1221 }
1222
1223 static int
1224 fm10k_link_update(struct rte_eth_dev *dev,
1225         __rte_unused int wait_to_complete)
1226 {
1227         struct fm10k_dev_info *dev_info =
1228                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1229         PMD_INIT_FUNC_TRACE();
1230
1231         dev->data->dev_link.link_speed  = ETH_SPEED_NUM_50G;
1232         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1233         dev->data->dev_link.link_status =
1234                 dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP;
1235         dev->data->dev_link.link_autoneg = ETH_LINK_FIXED;
1236
1237         return 0;
1238 }
1239
1240 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1241         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1242 {
1243         unsigned i, q;
1244         unsigned count = 0;
1245
1246         if (xstats_names != NULL) {
1247                 /* Note: limit checked in rte_eth_xstats_names() */
1248
1249                 /* Global stats */
1250                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1251                         snprintf(xstats_names[count].name,
1252                                 sizeof(xstats_names[count].name),
1253                                 "%s", fm10k_hw_stats_strings[count].name);
1254                         count++;
1255                 }
1256
1257                 /* PF queue stats */
1258                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1259                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1260                                 snprintf(xstats_names[count].name,
1261                                         sizeof(xstats_names[count].name),
1262                                         "rx_q%u_%s", q,
1263                                         fm10k_hw_stats_rx_q_strings[i].name);
1264                                 count++;
1265                         }
1266                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1267                                 snprintf(xstats_names[count].name,
1268                                         sizeof(xstats_names[count].name),
1269                                         "tx_q%u_%s", q,
1270                                         fm10k_hw_stats_tx_q_strings[i].name);
1271                                 count++;
1272                         }
1273                 }
1274         }
1275         return FM10K_NB_XSTATS;
1276 }
1277
1278 static int
1279 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1280                  unsigned n)
1281 {
1282         struct fm10k_hw_stats *hw_stats =
1283                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1284         unsigned i, q, count = 0;
1285
1286         if (n < FM10K_NB_XSTATS)
1287                 return FM10K_NB_XSTATS;
1288
1289         /* Global stats */
1290         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1291                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1292                         fm10k_hw_stats_strings[count].offset);
1293                 xstats[count].id = count;
1294                 count++;
1295         }
1296
1297         /* PF queue stats */
1298         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1299                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1300                         xstats[count].value =
1301                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1302                                 fm10k_hw_stats_rx_q_strings[i].offset);
1303                         xstats[count].id = count;
1304                         count++;
1305                 }
1306                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1307                         xstats[count].value =
1308                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1309                                 fm10k_hw_stats_tx_q_strings[i].offset);
1310                         xstats[count].id = count;
1311                         count++;
1312                 }
1313         }
1314
1315         return FM10K_NB_XSTATS;
1316 }
1317
1318 static int
1319 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1320 {
1321         uint64_t ipackets, opackets, ibytes, obytes, imissed;
1322         struct fm10k_hw *hw =
1323                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1324         struct fm10k_hw_stats *hw_stats =
1325                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1326         int i;
1327
1328         PMD_INIT_FUNC_TRACE();
1329
1330         fm10k_update_hw_stats(hw, hw_stats);
1331
1332         ipackets = opackets = ibytes = obytes = imissed = 0;
1333         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1334                 (i < hw->mac.max_queues); ++i) {
1335                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1336                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1337                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1338                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1339                 stats->q_errors[i]   = hw_stats->q[i].rx_drops.count;
1340                 ipackets += stats->q_ipackets[i];
1341                 opackets += stats->q_opackets[i];
1342                 ibytes   += stats->q_ibytes[i];
1343                 obytes   += stats->q_obytes[i];
1344                 imissed  += stats->q_errors[i];
1345         }
1346         stats->ipackets = ipackets;
1347         stats->opackets = opackets;
1348         stats->ibytes = ibytes;
1349         stats->obytes = obytes;
1350         stats->imissed = imissed;
1351         return 0;
1352 }
1353
1354 static int
1355 fm10k_stats_reset(struct rte_eth_dev *dev)
1356 {
1357         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1358         struct fm10k_hw_stats *hw_stats =
1359                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1360
1361         PMD_INIT_FUNC_TRACE();
1362
1363         memset(hw_stats, 0, sizeof(*hw_stats));
1364         fm10k_rebind_hw_stats(hw, hw_stats);
1365
1366         return 0;
1367 }
1368
1369 static int
1370 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1371         struct rte_eth_dev_info *dev_info)
1372 {
1373         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1374         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1375
1376         PMD_INIT_FUNC_TRACE();
1377
1378         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1379         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1380         dev_info->max_rx_queues      = hw->mac.max_queues;
1381         dev_info->max_tx_queues      = hw->mac.max_queues;
1382         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1383         dev_info->max_hash_mac_addrs = 0;
1384         dev_info->max_vfs            = pdev->max_vfs;
1385         dev_info->vmdq_pool_base     = 0;
1386         dev_info->vmdq_queue_base    = 0;
1387         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1388         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1389         dev_info->rx_queue_offload_capa = fm10k_get_rx_queue_offloads_capa(dev);
1390         dev_info->rx_offload_capa = fm10k_get_rx_port_offloads_capa(dev) |
1391                                     dev_info->rx_queue_offload_capa;
1392         dev_info->tx_queue_offload_capa = fm10k_get_tx_queue_offloads_capa(dev);
1393         dev_info->tx_offload_capa = fm10k_get_tx_port_offloads_capa(dev) |
1394                                     dev_info->tx_queue_offload_capa;
1395
1396         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1397         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1398         dev_info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1399                                         ETH_RSS_IPV6 |
1400                                         ETH_RSS_IPV6_EX |
1401                                         ETH_RSS_NONFRAG_IPV4_TCP |
1402                                         ETH_RSS_NONFRAG_IPV6_TCP |
1403                                         ETH_RSS_IPV6_TCP_EX |
1404                                         ETH_RSS_NONFRAG_IPV4_UDP |
1405                                         ETH_RSS_NONFRAG_IPV6_UDP |
1406                                         ETH_RSS_IPV6_UDP_EX;
1407
1408         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1409                 .rx_thresh = {
1410                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1411                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1412                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1413                 },
1414                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1415                 .rx_drop_en = 0,
1416                 .offloads = 0,
1417         };
1418
1419         dev_info->default_txconf = (struct rte_eth_txconf) {
1420                 .tx_thresh = {
1421                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1422                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1423                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1424                 },
1425                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1426                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1427                 .offloads = 0,
1428         };
1429
1430         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1431                 .nb_max = FM10K_MAX_RX_DESC,
1432                 .nb_min = FM10K_MIN_RX_DESC,
1433                 .nb_align = FM10K_MULT_RX_DESC,
1434         };
1435
1436         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1437                 .nb_max = FM10K_MAX_TX_DESC,
1438                 .nb_min = FM10K_MIN_TX_DESC,
1439                 .nb_align = FM10K_MULT_TX_DESC,
1440                 .nb_seg_max = FM10K_TX_MAX_SEG,
1441                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1442         };
1443
1444         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1445                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1446                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1447
1448         return 0;
1449 }
1450
1451 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1452 static const uint32_t *
1453 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1454 {
1455         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1456             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1457                 static uint32_t ptypes[] = {
1458                         /* refers to rx_desc_to_ol_flags() */
1459                         RTE_PTYPE_L2_ETHER,
1460                         RTE_PTYPE_L3_IPV4,
1461                         RTE_PTYPE_L3_IPV4_EXT,
1462                         RTE_PTYPE_L3_IPV6,
1463                         RTE_PTYPE_L3_IPV6_EXT,
1464                         RTE_PTYPE_L4_TCP,
1465                         RTE_PTYPE_L4_UDP,
1466                         RTE_PTYPE_UNKNOWN
1467                 };
1468
1469                 return ptypes;
1470         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1471                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1472                 static uint32_t ptypes_vec[] = {
1473                         /* refers to fm10k_desc_to_pktype_v() */
1474                         RTE_PTYPE_L3_IPV4,
1475                         RTE_PTYPE_L3_IPV4_EXT,
1476                         RTE_PTYPE_L3_IPV6,
1477                         RTE_PTYPE_L3_IPV6_EXT,
1478                         RTE_PTYPE_L4_TCP,
1479                         RTE_PTYPE_L4_UDP,
1480                         RTE_PTYPE_TUNNEL_GENEVE,
1481                         RTE_PTYPE_TUNNEL_NVGRE,
1482                         RTE_PTYPE_TUNNEL_VXLAN,
1483                         RTE_PTYPE_TUNNEL_GRE,
1484                         RTE_PTYPE_UNKNOWN
1485                 };
1486
1487                 return ptypes_vec;
1488         }
1489
1490         return NULL;
1491 }
1492 #else
1493 static const uint32_t *
1494 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1495 {
1496         return NULL;
1497 }
1498 #endif
1499
1500 static int
1501 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1502 {
1503         s32 result;
1504         uint16_t mac_num = 0;
1505         uint32_t vid_idx, vid_bit, mac_index;
1506         struct fm10k_hw *hw;
1507         struct fm10k_macvlan_filter_info *macvlan;
1508         struct rte_eth_dev_data *data = dev->data;
1509
1510         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1511         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1512
1513         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1514                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1515                 return -EINVAL;
1516         }
1517
1518         if (vlan_id > ETH_VLAN_ID_MAX) {
1519                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1520                 return -EINVAL;
1521         }
1522
1523         vid_idx = FM10K_VFTA_IDX(vlan_id);
1524         vid_bit = FM10K_VFTA_BIT(vlan_id);
1525         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1526         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1527                 return 0;
1528         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1529         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1530                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1531                         "in the VLAN filter table");
1532                 return -EINVAL;
1533         }
1534
1535         fm10k_mbx_lock(hw);
1536         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1537         fm10k_mbx_unlock(hw);
1538         if (result != FM10K_SUCCESS) {
1539                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1540                 return -EIO;
1541         }
1542
1543         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1544                         (result == FM10K_SUCCESS); mac_index++) {
1545                 if (rte_is_zero_ether_addr(&data->mac_addrs[mac_index]))
1546                         continue;
1547                 if (mac_num > macvlan->mac_num - 1) {
1548                         PMD_INIT_LOG(ERR, "MAC address number "
1549                                         "not match");
1550                         break;
1551                 }
1552                 fm10k_mbx_lock(hw);
1553                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1554                         data->mac_addrs[mac_index].addr_bytes,
1555                         vlan_id, on, 0);
1556                 fm10k_mbx_unlock(hw);
1557                 mac_num++;
1558         }
1559         if (result != FM10K_SUCCESS) {
1560                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1561                 return -EIO;
1562         }
1563
1564         if (on) {
1565                 macvlan->vlan_num++;
1566                 macvlan->vfta[vid_idx] |= vid_bit;
1567         } else {
1568                 macvlan->vlan_num--;
1569                 macvlan->vfta[vid_idx] &= ~vid_bit;
1570         }
1571         return 0;
1572 }
1573
1574 static int
1575 fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1576 {
1577         if (mask & ETH_VLAN_STRIP_MASK) {
1578                 if (!(dev->data->dev_conf.rxmode.offloads &
1579                         DEV_RX_OFFLOAD_VLAN_STRIP))
1580                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1581                                         "always on in fm10k");
1582         }
1583
1584         if (mask & ETH_VLAN_EXTEND_MASK) {
1585                 if (dev->data->dev_conf.rxmode.offloads &
1586                         DEV_RX_OFFLOAD_VLAN_EXTEND)
1587                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1588                                         "supported in fm10k");
1589         }
1590
1591         if (mask & ETH_VLAN_FILTER_MASK) {
1592                 if (!(dev->data->dev_conf.rxmode.offloads &
1593                         DEV_RX_OFFLOAD_VLAN_FILTER))
1594                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1595         }
1596
1597         return 0;
1598 }
1599
1600 /* Add/Remove a MAC address, and update filters to main VSI */
1601 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1602                 const u8 *mac, bool add, uint32_t pool)
1603 {
1604         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1605         struct fm10k_macvlan_filter_info *macvlan;
1606         uint32_t i, j, k;
1607
1608         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1609
1610         if (pool != MAIN_VSI_POOL_NUMBER) {
1611                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1612                         "mac to pool %u", pool);
1613                 return;
1614         }
1615         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1616                 if (!macvlan->vfta[j])
1617                         continue;
1618                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1619                         if (!(macvlan->vfta[j] & (1 << k)))
1620                                 continue;
1621                         if (i + 1 > macvlan->vlan_num) {
1622                                 PMD_INIT_LOG(ERR, "vlan number not match");
1623                                 return;
1624                         }
1625                         fm10k_mbx_lock(hw);
1626                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1627                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1628                         fm10k_mbx_unlock(hw);
1629                         i++;
1630                 }
1631         }
1632 }
1633
1634 /* Add/Remove a MAC address, and update filters to VMDQ */
1635 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1636                 const u8 *mac, bool add, uint32_t pool)
1637 {
1638         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1639         struct fm10k_macvlan_filter_info *macvlan;
1640         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1641         uint32_t i;
1642
1643         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1644         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1645
1646         if (pool > macvlan->nb_queue_pools) {
1647                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1648                         " Max pool is %u",
1649                         pool, macvlan->nb_queue_pools);
1650                 return;
1651         }
1652         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1653                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1654                         continue;
1655                 fm10k_mbx_lock(hw);
1656                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1657                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1658                 fm10k_mbx_unlock(hw);
1659         }
1660 }
1661
1662 /* Add/Remove a MAC address, and update filters */
1663 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1664                 const u8 *mac, bool add, uint32_t pool)
1665 {
1666         struct fm10k_macvlan_filter_info *macvlan;
1667
1668         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1669
1670         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1671                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1672         else
1673                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1674
1675         if (add)
1676                 macvlan->mac_num++;
1677         else
1678                 macvlan->mac_num--;
1679 }
1680
1681 /* Add a MAC address, and update filters */
1682 static int
1683 fm10k_macaddr_add(struct rte_eth_dev *dev,
1684                 struct rte_ether_addr *mac_addr,
1685                 uint32_t index,
1686                 uint32_t pool)
1687 {
1688         struct fm10k_macvlan_filter_info *macvlan;
1689
1690         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1691         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1692         macvlan->mac_vmdq_id[index] = pool;
1693         return 0;
1694 }
1695
1696 /* Remove a MAC address, and update filters */
1697 static void
1698 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1699 {
1700         struct rte_eth_dev_data *data = dev->data;
1701         struct fm10k_macvlan_filter_info *macvlan;
1702
1703         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1704         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1705                         FALSE, macvlan->mac_vmdq_id[index]);
1706         macvlan->mac_vmdq_id[index] = 0;
1707 }
1708
1709 static inline int
1710 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1711 {
1712         if ((request < min) || (request > max) || ((request % mult) != 0))
1713                 return -1;
1714         else
1715                 return 0;
1716 }
1717
1718
1719 static inline int
1720 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1721 {
1722         if ((request < min) || (request > max) || ((div % request) != 0))
1723                 return -1;
1724         else
1725                 return 0;
1726 }
1727
1728 static inline int
1729 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1730 {
1731         uint16_t rx_free_thresh;
1732
1733         if (conf->rx_free_thresh == 0)
1734                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1735         else
1736                 rx_free_thresh = conf->rx_free_thresh;
1737
1738         /* make sure the requested threshold satisfies the constraints */
1739         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1740                         FM10K_RX_FREE_THRESH_MAX(q),
1741                         FM10K_RX_FREE_THRESH_DIV(q),
1742                         rx_free_thresh)) {
1743                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1744                         "less than or equal to %u, "
1745                         "greater than or equal to %u, "
1746                         "and a divisor of %u",
1747                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1748                         FM10K_RX_FREE_THRESH_MIN(q),
1749                         FM10K_RX_FREE_THRESH_DIV(q));
1750                 return -EINVAL;
1751         }
1752
1753         q->alloc_thresh = rx_free_thresh;
1754         q->drop_en = conf->rx_drop_en;
1755         q->rx_deferred_start = conf->rx_deferred_start;
1756
1757         return 0;
1758 }
1759
1760 /*
1761  * Hardware requires specific alignment for Rx packet buffers. At
1762  * least one of the following two conditions must be satisfied.
1763  *  1. Address is 512B aligned
1764  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1765  *
1766  * As such, the driver may need to adjust the DMA address within the
1767  * buffer by up to 512B.
1768  *
1769  * return 1 if the element size is valid, otherwise return 0.
1770  */
1771 static int
1772 mempool_element_size_valid(struct rte_mempool *mp)
1773 {
1774         uint32_t min_size;
1775
1776         /* elt_size includes mbuf header and headroom */
1777         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1778                         RTE_PKTMBUF_HEADROOM;
1779
1780         /* account for up to 512B of alignment */
1781         min_size -= FM10K_RX_DATABUF_ALIGN;
1782
1783         /* sanity check for overflow */
1784         if (min_size > mp->elt_size)
1785                 return 0;
1786
1787         /* size is valid */
1788         return 1;
1789 }
1790
1791 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev)
1792 {
1793         RTE_SET_USED(dev);
1794
1795         return (uint64_t)(DEV_RX_OFFLOAD_SCATTER);
1796 }
1797
1798 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
1799 {
1800         RTE_SET_USED(dev);
1801
1802         return  (uint64_t)(DEV_RX_OFFLOAD_VLAN_STRIP  |
1803                            DEV_RX_OFFLOAD_VLAN_FILTER |
1804                            DEV_RX_OFFLOAD_IPV4_CKSUM  |
1805                            DEV_RX_OFFLOAD_UDP_CKSUM   |
1806                            DEV_RX_OFFLOAD_TCP_CKSUM   |
1807                            DEV_RX_OFFLOAD_JUMBO_FRAME |
1808                            DEV_RX_OFFLOAD_HEADER_SPLIT);
1809 }
1810
1811 static int
1812 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1813         uint16_t nb_desc, unsigned int socket_id,
1814         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1815 {
1816         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1817         struct fm10k_dev_info *dev_info =
1818                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1819         struct fm10k_rx_queue *q;
1820         const struct rte_memzone *mz;
1821         uint64_t offloads;
1822
1823         PMD_INIT_FUNC_TRACE();
1824
1825         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1826
1827         /* make sure the mempool element size can account for alignment. */
1828         if (!mempool_element_size_valid(mp)) {
1829                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1830                 return -EINVAL;
1831         }
1832
1833         /* make sure a valid number of descriptors have been requested */
1834         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1835                                 FM10K_MULT_RX_DESC, nb_desc)) {
1836                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1837                         "less than or equal to %"PRIu32", "
1838                         "greater than or equal to %u, "
1839                         "and a multiple of %u",
1840                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1841                         FM10K_MULT_RX_DESC);
1842                 return -EINVAL;
1843         }
1844
1845         /*
1846          * if this queue existed already, free the associated memory. The
1847          * queue cannot be reused in case we need to allocate memory on
1848          * different socket than was previously used.
1849          */
1850         if (dev->data->rx_queues[queue_id] != NULL) {
1851                 rx_queue_free(dev->data->rx_queues[queue_id]);
1852                 dev->data->rx_queues[queue_id] = NULL;
1853         }
1854
1855         /* allocate memory for the queue structure */
1856         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1857                                 socket_id);
1858         if (q == NULL) {
1859                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1860                 return -ENOMEM;
1861         }
1862
1863         /* setup queue */
1864         q->mp = mp;
1865         q->nb_desc = nb_desc;
1866         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1867         q->port_id = dev->data->port_id;
1868         q->queue_id = queue_id;
1869         q->tail_ptr = (volatile uint32_t *)
1870                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1871         q->offloads = offloads;
1872         if (handle_rxconf(q, conf))
1873                 return -EINVAL;
1874
1875         /* allocate memory for the software ring */
1876         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1877                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1878                         RTE_CACHE_LINE_SIZE, socket_id);
1879         if (q->sw_ring == NULL) {
1880                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1881                 rte_free(q);
1882                 return -ENOMEM;
1883         }
1884
1885         /*
1886          * allocate memory for the hardware descriptor ring. A memzone large
1887          * enough to hold the maximum ring size is requested to allow for
1888          * resizing in later calls to the queue setup function.
1889          */
1890         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1891                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1892                                       socket_id);
1893         if (mz == NULL) {
1894                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1895                 rte_free(q->sw_ring);
1896                 rte_free(q);
1897                 return -ENOMEM;
1898         }
1899         q->hw_ring = mz->addr;
1900         q->hw_ring_phys_addr = mz->iova;
1901
1902         /* Check if number of descs satisfied Vector requirement */
1903         if (!rte_is_power_of_2(nb_desc)) {
1904                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1905                                     "preconditions - canceling the feature for "
1906                                     "the whole port[%d]",
1907                              q->queue_id, q->port_id);
1908                 dev_info->rx_vec_allowed = false;
1909         } else
1910                 fm10k_rxq_vec_setup(q);
1911
1912         dev->data->rx_queues[queue_id] = q;
1913         return 0;
1914 }
1915
1916 static void
1917 fm10k_rx_queue_release(void *queue)
1918 {
1919         PMD_INIT_FUNC_TRACE();
1920
1921         rx_queue_free(queue);
1922 }
1923
1924 static inline int
1925 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1926 {
1927         uint16_t tx_free_thresh;
1928         uint16_t tx_rs_thresh;
1929
1930         /* constraint MACROs require that tx_free_thresh is configured
1931          * before tx_rs_thresh */
1932         if (conf->tx_free_thresh == 0)
1933                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1934         else
1935                 tx_free_thresh = conf->tx_free_thresh;
1936
1937         /* make sure the requested threshold satisfies the constraints */
1938         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1939                         FM10K_TX_FREE_THRESH_MAX(q),
1940                         FM10K_TX_FREE_THRESH_DIV(q),
1941                         tx_free_thresh)) {
1942                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1943                         "less than or equal to %u, "
1944                         "greater than or equal to %u, "
1945                         "and a divisor of %u",
1946                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1947                         FM10K_TX_FREE_THRESH_MIN(q),
1948                         FM10K_TX_FREE_THRESH_DIV(q));
1949                 return -EINVAL;
1950         }
1951
1952         q->free_thresh = tx_free_thresh;
1953
1954         if (conf->tx_rs_thresh == 0)
1955                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1956         else
1957                 tx_rs_thresh = conf->tx_rs_thresh;
1958
1959         q->tx_deferred_start = conf->tx_deferred_start;
1960
1961         /* make sure the requested threshold satisfies the constraints */
1962         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1963                         FM10K_TX_RS_THRESH_MAX(q),
1964                         FM10K_TX_RS_THRESH_DIV(q),
1965                         tx_rs_thresh)) {
1966                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1967                         "less than or equal to %u, "
1968                         "greater than or equal to %u, "
1969                         "and a divisor of %u",
1970                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1971                         FM10K_TX_RS_THRESH_MIN(q),
1972                         FM10K_TX_RS_THRESH_DIV(q));
1973                 return -EINVAL;
1974         }
1975
1976         q->rs_thresh = tx_rs_thresh;
1977
1978         return 0;
1979 }
1980
1981 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev)
1982 {
1983         RTE_SET_USED(dev);
1984
1985         return 0;
1986 }
1987
1988 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
1989 {
1990         RTE_SET_USED(dev);
1991
1992         return (uint64_t)(DEV_TX_OFFLOAD_VLAN_INSERT |
1993                           DEV_TX_OFFLOAD_MULTI_SEGS  |
1994                           DEV_TX_OFFLOAD_IPV4_CKSUM  |
1995                           DEV_TX_OFFLOAD_UDP_CKSUM   |
1996                           DEV_TX_OFFLOAD_TCP_CKSUM   |
1997                           DEV_TX_OFFLOAD_TCP_TSO);
1998 }
1999
2000 static int
2001 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
2002         uint16_t nb_desc, unsigned int socket_id,
2003         const struct rte_eth_txconf *conf)
2004 {
2005         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2006         struct fm10k_tx_queue *q;
2007         const struct rte_memzone *mz;
2008         uint64_t offloads;
2009
2010         PMD_INIT_FUNC_TRACE();
2011
2012         offloads = conf->offloads | dev->data->dev_conf.txmode.offloads;
2013
2014         /* make sure a valid number of descriptors have been requested */
2015         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
2016                                 FM10K_MULT_TX_DESC, nb_desc)) {
2017                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
2018                         "less than or equal to %"PRIu32", "
2019                         "greater than or equal to %u, "
2020                         "and a multiple of %u",
2021                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
2022                         FM10K_MULT_TX_DESC);
2023                 return -EINVAL;
2024         }
2025
2026         /*
2027          * if this queue existed already, free the associated memory. The
2028          * queue cannot be reused in case we need to allocate memory on
2029          * different socket than was previously used.
2030          */
2031         if (dev->data->tx_queues[queue_id] != NULL) {
2032                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2033
2034                 tx_queue_free(txq);
2035                 dev->data->tx_queues[queue_id] = NULL;
2036         }
2037
2038         /* allocate memory for the queue structure */
2039         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2040                                 socket_id);
2041         if (q == NULL) {
2042                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2043                 return -ENOMEM;
2044         }
2045
2046         /* setup queue */
2047         q->nb_desc = nb_desc;
2048         q->port_id = dev->data->port_id;
2049         q->queue_id = queue_id;
2050         q->offloads = offloads;
2051         q->ops = &def_txq_ops;
2052         q->tail_ptr = (volatile uint32_t *)
2053                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2054         if (handle_txconf(q, conf))
2055                 return -EINVAL;
2056
2057         /* allocate memory for the software ring */
2058         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2059                                         nb_desc * sizeof(struct rte_mbuf *),
2060                                         RTE_CACHE_LINE_SIZE, socket_id);
2061         if (q->sw_ring == NULL) {
2062                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2063                 rte_free(q);
2064                 return -ENOMEM;
2065         }
2066
2067         /*
2068          * allocate memory for the hardware descriptor ring. A memzone large
2069          * enough to hold the maximum ring size is requested to allow for
2070          * resizing in later calls to the queue setup function.
2071          */
2072         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2073                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2074                                       socket_id);
2075         if (mz == NULL) {
2076                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2077                 rte_free(q->sw_ring);
2078                 rte_free(q);
2079                 return -ENOMEM;
2080         }
2081         q->hw_ring = mz->addr;
2082         q->hw_ring_phys_addr = mz->iova;
2083
2084         /*
2085          * allocate memory for the RS bit tracker. Enough slots to hold the
2086          * descriptor index for each RS bit needing to be set are required.
2087          */
2088         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2089                                 ((nb_desc + 1) / q->rs_thresh) *
2090                                 sizeof(uint16_t),
2091                                 RTE_CACHE_LINE_SIZE, socket_id);
2092         if (q->rs_tracker.list == NULL) {
2093                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2094                 rte_free(q->sw_ring);
2095                 rte_free(q);
2096                 return -ENOMEM;
2097         }
2098
2099         dev->data->tx_queues[queue_id] = q;
2100         return 0;
2101 }
2102
2103 static void
2104 fm10k_tx_queue_release(void *queue)
2105 {
2106         struct fm10k_tx_queue *q = queue;
2107         PMD_INIT_FUNC_TRACE();
2108
2109         tx_queue_free(q);
2110 }
2111
2112 static int
2113 fm10k_reta_update(struct rte_eth_dev *dev,
2114                         struct rte_eth_rss_reta_entry64 *reta_conf,
2115                         uint16_t reta_size)
2116 {
2117         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2118         uint16_t i, j, idx, shift;
2119         uint8_t mask;
2120         uint32_t reta;
2121
2122         PMD_INIT_FUNC_TRACE();
2123
2124         if (reta_size > FM10K_MAX_RSS_INDICES) {
2125                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2126                         "(%d) doesn't match the number hardware can supported "
2127                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2128                 return -EINVAL;
2129         }
2130
2131         /*
2132          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2133          * 128-entries in 32 registers
2134          */
2135         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2136                 idx = i / RTE_RETA_GROUP_SIZE;
2137                 shift = i % RTE_RETA_GROUP_SIZE;
2138                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2139                                 BIT_MASK_PER_UINT32);
2140                 if (mask == 0)
2141                         continue;
2142
2143                 reta = 0;
2144                 if (mask != BIT_MASK_PER_UINT32)
2145                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2146
2147                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2148                         if (mask & (0x1 << j)) {
2149                                 if (mask != 0xF)
2150                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2151                                 reta |= reta_conf[idx].reta[shift + j] <<
2152                                                 (CHAR_BIT * j);
2153                         }
2154                 }
2155                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2156         }
2157
2158         return 0;
2159 }
2160
2161 static int
2162 fm10k_reta_query(struct rte_eth_dev *dev,
2163                         struct rte_eth_rss_reta_entry64 *reta_conf,
2164                         uint16_t reta_size)
2165 {
2166         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2167         uint16_t i, j, idx, shift;
2168         uint8_t mask;
2169         uint32_t reta;
2170
2171         PMD_INIT_FUNC_TRACE();
2172
2173         if (reta_size < FM10K_MAX_RSS_INDICES) {
2174                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2175                         "(%d) doesn't match the number hardware can supported "
2176                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2177                 return -EINVAL;
2178         }
2179
2180         /*
2181          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2182          * 128-entries in 32 registers
2183          */
2184         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2185                 idx = i / RTE_RETA_GROUP_SIZE;
2186                 shift = i % RTE_RETA_GROUP_SIZE;
2187                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2188                                 BIT_MASK_PER_UINT32);
2189                 if (mask == 0)
2190                         continue;
2191
2192                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2193                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2194                         if (mask & (0x1 << j))
2195                                 reta_conf[idx].reta[shift + j] = ((reta >>
2196                                         CHAR_BIT * j) & UINT8_MAX);
2197                 }
2198         }
2199
2200         return 0;
2201 }
2202
2203 static int
2204 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2205         struct rte_eth_rss_conf *rss_conf)
2206 {
2207         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2208         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2209         uint32_t mrqc;
2210         uint64_t hf = rss_conf->rss_hf;
2211         int i;
2212
2213         PMD_INIT_FUNC_TRACE();
2214
2215         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2216                                 FM10K_RSSRK_ENTRIES_PER_REG))
2217                 return -EINVAL;
2218
2219         if (hf == 0)
2220                 return -EINVAL;
2221
2222         mrqc = 0;
2223         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2224         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2225         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2226         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2227         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2228         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2229         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2230         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2231         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2232
2233         /* If the mapping doesn't fit any supported, return */
2234         if (mrqc == 0)
2235                 return -EINVAL;
2236
2237         if (key != NULL)
2238                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2239                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2240
2241         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2242
2243         return 0;
2244 }
2245
2246 static int
2247 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2248         struct rte_eth_rss_conf *rss_conf)
2249 {
2250         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2251         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2252         uint32_t mrqc;
2253         uint64_t hf;
2254         int i;
2255
2256         PMD_INIT_FUNC_TRACE();
2257
2258         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2259                                 FM10K_RSSRK_ENTRIES_PER_REG))
2260                 return -EINVAL;
2261
2262         if (key != NULL)
2263                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2264                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2265
2266         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2267         hf = 0;
2268         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2269         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2270         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2271         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2272         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2273         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2274         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2275         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2276         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2277
2278         rss_conf->rss_hf = hf;
2279
2280         return 0;
2281 }
2282
2283 static void
2284 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2285 {
2286         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2287         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2288
2289         /* Bind all local non-queue interrupt to vector 0 */
2290         int_map |= FM10K_MISC_VEC_ID;
2291
2292         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2293         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2294         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2295         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2296         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2297         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2298
2299         /* Enable misc causes */
2300         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2301                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2302                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2303                                 FM10K_EIMR_ENABLE(MAILBOX) |
2304                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2305                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2306                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2307                                 FM10K_EIMR_ENABLE(VFLR));
2308
2309         /* Enable ITR 0 */
2310         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2311                                         FM10K_ITR_MASK_CLEAR);
2312         FM10K_WRITE_FLUSH(hw);
2313 }
2314
2315 static void
2316 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2317 {
2318         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2319         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2320
2321         int_map |= FM10K_MISC_VEC_ID;
2322
2323         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2324         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2325         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2326         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2327         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2328         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2329
2330         /* Disable misc causes */
2331         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2332                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2333                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2334                                 FM10K_EIMR_DISABLE(MAILBOX) |
2335                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2336                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2337                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2338                                 FM10K_EIMR_DISABLE(VFLR));
2339
2340         /* Disable ITR 0 */
2341         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2342         FM10K_WRITE_FLUSH(hw);
2343 }
2344
2345 static void
2346 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2347 {
2348         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2349         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2350
2351         /* Bind all local non-queue interrupt to vector 0 */
2352         int_map |= FM10K_MISC_VEC_ID;
2353
2354         /* Only INT 0 available, other 15 are reserved. */
2355         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2356
2357         /* Enable ITR 0 */
2358         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2359                                         FM10K_ITR_MASK_CLEAR);
2360         FM10K_WRITE_FLUSH(hw);
2361 }
2362
2363 static void
2364 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2365 {
2366         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2367         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2368
2369         int_map |= FM10K_MISC_VEC_ID;
2370
2371         /* Only INT 0 available, other 15 are reserved. */
2372         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2373
2374         /* Disable ITR 0 */
2375         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2376         FM10K_WRITE_FLUSH(hw);
2377 }
2378
2379 static int
2380 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2381 {
2382         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2383         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2384
2385         /* Enable ITR */
2386         if (hw->mac.type == fm10k_mac_pf)
2387                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2388                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2389         else
2390                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2391                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2392         rte_intr_ack(&pdev->intr_handle);
2393         return 0;
2394 }
2395
2396 static int
2397 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2398 {
2399         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2400         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2401
2402         /* Disable ITR */
2403         if (hw->mac.type == fm10k_mac_pf)
2404                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2405                         FM10K_ITR_MASK_SET);
2406         else
2407                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2408                         FM10K_ITR_MASK_SET);
2409         return 0;
2410 }
2411
2412 static int
2413 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2414 {
2415         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2416         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2417         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2418         uint32_t intr_vector, vec;
2419         uint16_t queue_id;
2420         int result = 0;
2421
2422         /* fm10k needs one separate interrupt for mailbox,
2423          * so only drivers which support multiple interrupt vectors
2424          * e.g. vfio-pci can work for fm10k interrupt mode
2425          */
2426         if (!rte_intr_cap_multiple(intr_handle) ||
2427                         dev->data->dev_conf.intr_conf.rxq == 0)
2428                 return result;
2429
2430         intr_vector = dev->data->nb_rx_queues;
2431
2432         /* disable interrupt first */
2433         rte_intr_disable(intr_handle);
2434         if (hw->mac.type == fm10k_mac_pf)
2435                 fm10k_dev_disable_intr_pf(dev);
2436         else
2437                 fm10k_dev_disable_intr_vf(dev);
2438
2439         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2440                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2441                 result = -EIO;
2442         }
2443
2444         if (rte_intr_dp_is_en(intr_handle) && !result) {
2445                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2446                         dev->data->nb_rx_queues * sizeof(int), 0);
2447                 if (intr_handle->intr_vec) {
2448                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2449                                         queue_id < dev->data->nb_rx_queues;
2450                                         queue_id++) {
2451                                 intr_handle->intr_vec[queue_id] = vec;
2452                                 if (vec < intr_handle->nb_efd - 1
2453                                                 + FM10K_RX_VEC_START)
2454                                         vec++;
2455                         }
2456                 } else {
2457                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2458                                 " intr_vec", dev->data->nb_rx_queues);
2459                         rte_intr_efd_disable(intr_handle);
2460                         result = -ENOMEM;
2461                 }
2462         }
2463
2464         if (hw->mac.type == fm10k_mac_pf)
2465                 fm10k_dev_enable_intr_pf(dev);
2466         else
2467                 fm10k_dev_enable_intr_vf(dev);
2468         rte_intr_enable(intr_handle);
2469         hw->mac.ops.update_int_moderator(hw);
2470         return result;
2471 }
2472
2473 static int
2474 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2475 {
2476         struct fm10k_fault fault;
2477         int err;
2478         const char *estr = "Unknown error";
2479
2480         /* Process PCA fault */
2481         if (eicr & FM10K_EICR_PCA_FAULT) {
2482                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2483                 if (err)
2484                         goto error;
2485                 switch (fault.type) {
2486                 case PCA_NO_FAULT:
2487                         estr = "PCA_NO_FAULT"; break;
2488                 case PCA_UNMAPPED_ADDR:
2489                         estr = "PCA_UNMAPPED_ADDR"; break;
2490                 case PCA_BAD_QACCESS_PF:
2491                         estr = "PCA_BAD_QACCESS_PF"; break;
2492                 case PCA_BAD_QACCESS_VF:
2493                         estr = "PCA_BAD_QACCESS_VF"; break;
2494                 case PCA_MALICIOUS_REQ:
2495                         estr = "PCA_MALICIOUS_REQ"; break;
2496                 case PCA_POISONED_TLP:
2497                         estr = "PCA_POISONED_TLP"; break;
2498                 case PCA_TLP_ABORT:
2499                         estr = "PCA_TLP_ABORT"; break;
2500                 default:
2501                         goto error;
2502                 }
2503                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2504                         estr, fault.func ? "VF" : "PF", fault.func,
2505                         fault.address, fault.specinfo);
2506         }
2507
2508         /* Process THI fault */
2509         if (eicr & FM10K_EICR_THI_FAULT) {
2510                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2511                 if (err)
2512                         goto error;
2513                 switch (fault.type) {
2514                 case THI_NO_FAULT:
2515                         estr = "THI_NO_FAULT"; break;
2516                 case THI_MAL_DIS_Q_FAULT:
2517                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2518                 default:
2519                         goto error;
2520                 }
2521                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2522                         estr, fault.func ? "VF" : "PF", fault.func,
2523                         fault.address, fault.specinfo);
2524         }
2525
2526         /* Process FUM fault */
2527         if (eicr & FM10K_EICR_FUM_FAULT) {
2528                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2529                 if (err)
2530                         goto error;
2531                 switch (fault.type) {
2532                 case FUM_NO_FAULT:
2533                         estr = "FUM_NO_FAULT"; break;
2534                 case FUM_UNMAPPED_ADDR:
2535                         estr = "FUM_UNMAPPED_ADDR"; break;
2536                 case FUM_POISONED_TLP:
2537                         estr = "FUM_POISONED_TLP"; break;
2538                 case FUM_BAD_VF_QACCESS:
2539                         estr = "FUM_BAD_VF_QACCESS"; break;
2540                 case FUM_ADD_DECODE_ERR:
2541                         estr = "FUM_ADD_DECODE_ERR"; break;
2542                 case FUM_RO_ERROR:
2543                         estr = "FUM_RO_ERROR"; break;
2544                 case FUM_QPRC_CRC_ERROR:
2545                         estr = "FUM_QPRC_CRC_ERROR"; break;
2546                 case FUM_CSR_TIMEOUT:
2547                         estr = "FUM_CSR_TIMEOUT"; break;
2548                 case FUM_INVALID_TYPE:
2549                         estr = "FUM_INVALID_TYPE"; break;
2550                 case FUM_INVALID_LENGTH:
2551                         estr = "FUM_INVALID_LENGTH"; break;
2552                 case FUM_INVALID_BE:
2553                         estr = "FUM_INVALID_BE"; break;
2554                 case FUM_INVALID_ALIGN:
2555                         estr = "FUM_INVALID_ALIGN"; break;
2556                 default:
2557                         goto error;
2558                 }
2559                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2560                         estr, fault.func ? "VF" : "PF", fault.func,
2561                         fault.address, fault.specinfo);
2562         }
2563
2564         return 0;
2565 error:
2566         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2567         return err;
2568 }
2569
2570 /**
2571  * PF interrupt handler triggered by NIC for handling specific interrupt.
2572  *
2573  * @param handle
2574  *  Pointer to interrupt handle.
2575  * @param param
2576  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2577  *
2578  * @return
2579  *  void
2580  */
2581 static void
2582 fm10k_dev_interrupt_handler_pf(void *param)
2583 {
2584         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2585         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2586         uint32_t cause, status;
2587         struct fm10k_dev_info *dev_info =
2588                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2589         int status_mbx;
2590         s32 err;
2591
2592         if (hw->mac.type != fm10k_mac_pf)
2593                 return;
2594
2595         cause = FM10K_READ_REG(hw, FM10K_EICR);
2596
2597         /* Handle PCI fault cases */
2598         if (cause & FM10K_EICR_FAULT_MASK) {
2599                 PMD_INIT_LOG(ERR, "INT: find fault!");
2600                 fm10k_dev_handle_fault(hw, cause);
2601         }
2602
2603         /* Handle switch up/down */
2604         if (cause & FM10K_EICR_SWITCHNOTREADY)
2605                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2606
2607         if (cause & FM10K_EICR_SWITCHREADY) {
2608                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2609                 if (dev_info->sm_down == 1) {
2610                         fm10k_mbx_lock(hw);
2611
2612                         /* For recreating logical ports */
2613                         status_mbx = hw->mac.ops.update_lport_state(hw,
2614                                         hw->mac.dglort_map, MAX_LPORT_NUM, 1);
2615                         if (status_mbx == FM10K_SUCCESS)
2616                                 PMD_INIT_LOG(INFO,
2617                                         "INT: Recreated Logical port");
2618                         else
2619                                 PMD_INIT_LOG(INFO,
2620                                         "INT: Logical ports weren't recreated");
2621
2622                         status_mbx = hw->mac.ops.update_xcast_mode(hw,
2623                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2624                         if (status_mbx != FM10K_SUCCESS)
2625                                 PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2626
2627                         fm10k_mbx_unlock(hw);
2628
2629                         /* first clear the internal SW recording structure */
2630                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2631                                                 ETH_MQ_RX_VMDQ_FLAG))
2632                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2633                                         false);
2634
2635                         fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2636                                         MAIN_VSI_POOL_NUMBER);
2637
2638                         /*
2639                          * Add default mac address and vlan for the logical
2640                          * ports that have been created, leave to the
2641                          * application to fully recover Rx filtering.
2642                          */
2643                         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2644                                         MAIN_VSI_POOL_NUMBER);
2645
2646                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2647                                                 ETH_MQ_RX_VMDQ_FLAG))
2648                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2649                                         true);
2650
2651                         dev_info->sm_down = 0;
2652                         _rte_eth_dev_callback_process(dev,
2653                                         RTE_ETH_EVENT_INTR_LSC,
2654                                         NULL);
2655                 }
2656         }
2657
2658         /* Handle mailbox message */
2659         fm10k_mbx_lock(hw);
2660         err = hw->mbx.ops.process(hw, &hw->mbx);
2661         fm10k_mbx_unlock(hw);
2662
2663         if (err == FM10K_ERR_RESET_REQUESTED) {
2664                 PMD_INIT_LOG(INFO, "INT: Switch is down");
2665                 dev_info->sm_down = 1;
2666                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2667                                 NULL);
2668         }
2669
2670         /* Handle SRAM error */
2671         if (cause & FM10K_EICR_SRAMERROR) {
2672                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2673
2674                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2675                 /* Write to clear pending bits */
2676                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2677
2678                 /* Todo: print out error message after shared code  updates */
2679         }
2680
2681         /* Clear these 3 events if having any */
2682         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2683                  FM10K_EICR_SWITCHREADY;
2684         if (cause)
2685                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2686
2687         /* Re-enable interrupt from device side */
2688         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2689                                         FM10K_ITR_MASK_CLEAR);
2690         /* Re-enable interrupt from host side */
2691         rte_intr_ack(dev->intr_handle);
2692 }
2693
2694 /**
2695  * VF interrupt handler triggered by NIC for handling specific interrupt.
2696  *
2697  * @param handle
2698  *  Pointer to interrupt handle.
2699  * @param param
2700  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2701  *
2702  * @return
2703  *  void
2704  */
2705 static void
2706 fm10k_dev_interrupt_handler_vf(void *param)
2707 {
2708         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2709         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2710         struct fm10k_mbx_info *mbx = &hw->mbx;
2711         struct fm10k_dev_info *dev_info =
2712                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2713         const enum fm10k_mbx_state state = mbx->state;
2714         int status_mbx;
2715
2716         if (hw->mac.type != fm10k_mac_vf)
2717                 return;
2718
2719         /* Handle mailbox message if lock is acquired */
2720         fm10k_mbx_lock(hw);
2721         hw->mbx.ops.process(hw, &hw->mbx);
2722         fm10k_mbx_unlock(hw);
2723
2724         if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) {
2725                 PMD_INIT_LOG(INFO, "INT: Switch has gone down");
2726
2727                 fm10k_mbx_lock(hw);
2728                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2729                                 MAX_LPORT_NUM, 1);
2730                 fm10k_mbx_unlock(hw);
2731
2732                 /* Setting reset flag */
2733                 dev_info->sm_down = 1;
2734                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2735                                 NULL);
2736         }
2737
2738         if (dev_info->sm_down == 1 &&
2739                         hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) {
2740                 PMD_INIT_LOG(INFO, "INT: Switch has gone up");
2741                 fm10k_mbx_lock(hw);
2742                 status_mbx = hw->mac.ops.update_xcast_mode(hw,
2743                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2744                 if (status_mbx != FM10K_SUCCESS)
2745                         PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2746                 fm10k_mbx_unlock(hw);
2747
2748                 /* first clear the internal SW recording structure */
2749                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false);
2750                 fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2751                                 MAIN_VSI_POOL_NUMBER);
2752
2753                 /*
2754                  * Add default mac address and vlan for the logical ports that
2755                  * have been created, leave to the application to fully recover
2756                  * Rx filtering.
2757                  */
2758                 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2759                                 MAIN_VSI_POOL_NUMBER);
2760                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
2761
2762                 dev_info->sm_down = 0;
2763                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2764                                 NULL);
2765         }
2766
2767         /* Re-enable interrupt from device side */
2768         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2769                                         FM10K_ITR_MASK_CLEAR);
2770         /* Re-enable interrupt from host side */
2771         rte_intr_ack(dev->intr_handle);
2772 }
2773
2774 /* Mailbox message handler in VF */
2775 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2776         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2777         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2778         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2779         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2780 };
2781
2782 static int
2783 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2784 {
2785         int err = 0;
2786
2787         /* Initialize mailbox lock */
2788         fm10k_mbx_initlock(hw);
2789
2790         /* Replace default message handler with new ones */
2791         if (hw->mac.type == fm10k_mac_vf)
2792                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2793
2794         if (err) {
2795                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2796                                 err);
2797                 return err;
2798         }
2799         /* Connect to SM for PF device or PF for VF device */
2800         return hw->mbx.ops.connect(hw, &hw->mbx);
2801 }
2802
2803 static void
2804 fm10k_close_mbx_service(struct fm10k_hw *hw)
2805 {
2806         /* Disconnect from SM for PF device or PF for VF device */
2807         hw->mbx.ops.disconnect(hw, &hw->mbx);
2808 }
2809
2810 static void
2811 fm10k_dev_close(struct rte_eth_dev *dev)
2812 {
2813         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2814         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2815         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2816
2817         PMD_INIT_FUNC_TRACE();
2818
2819         fm10k_mbx_lock(hw);
2820         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2821                 MAX_LPORT_NUM, false);
2822         fm10k_mbx_unlock(hw);
2823
2824         /* allow 100ms for device to quiesce */
2825         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
2826
2827         /* Stop mailbox service first */
2828         fm10k_close_mbx_service(hw);
2829         fm10k_dev_stop(dev);
2830         fm10k_dev_queue_release(dev);
2831         fm10k_stop_hw(hw);
2832
2833         dev->dev_ops = NULL;
2834         dev->rx_pkt_burst = NULL;
2835         dev->tx_pkt_burst = NULL;
2836
2837         /* disable uio/vfio intr */
2838         rte_intr_disable(intr_handle);
2839
2840         /*PF/VF has different interrupt handling mechanism */
2841         if (hw->mac.type == fm10k_mac_pf) {
2842                 /* disable interrupt */
2843                 fm10k_dev_disable_intr_pf(dev);
2844
2845                 /* unregister callback func to eal lib */
2846                 rte_intr_callback_unregister(intr_handle,
2847                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2848         } else {
2849                 /* disable interrupt */
2850                 fm10k_dev_disable_intr_vf(dev);
2851
2852                 rte_intr_callback_unregister(intr_handle,
2853                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2854         }
2855 }
2856
2857 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2858         .dev_configure          = fm10k_dev_configure,
2859         .dev_start              = fm10k_dev_start,
2860         .dev_stop               = fm10k_dev_stop,
2861         .dev_close              = fm10k_dev_close,
2862         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2863         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2864         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2865         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2866         .stats_get              = fm10k_stats_get,
2867         .xstats_get             = fm10k_xstats_get,
2868         .xstats_get_names       = fm10k_xstats_get_names,
2869         .stats_reset            = fm10k_stats_reset,
2870         .xstats_reset           = fm10k_stats_reset,
2871         .link_update            = fm10k_link_update,
2872         .dev_infos_get          = fm10k_dev_infos_get,
2873         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2874         .vlan_filter_set        = fm10k_vlan_filter_set,
2875         .vlan_offload_set       = fm10k_vlan_offload_set,
2876         .mac_addr_add           = fm10k_macaddr_add,
2877         .mac_addr_remove        = fm10k_macaddr_remove,
2878         .rx_queue_start         = fm10k_dev_rx_queue_start,
2879         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2880         .tx_queue_start         = fm10k_dev_tx_queue_start,
2881         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2882         .rx_queue_setup         = fm10k_rx_queue_setup,
2883         .rx_queue_release       = fm10k_rx_queue_release,
2884         .tx_queue_setup         = fm10k_tx_queue_setup,
2885         .tx_queue_release       = fm10k_tx_queue_release,
2886         .rx_queue_count         = fm10k_dev_rx_queue_count,
2887         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2888         .rx_descriptor_status = fm10k_dev_rx_descriptor_status,
2889         .tx_descriptor_status = fm10k_dev_tx_descriptor_status,
2890         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2891         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2892         .reta_update            = fm10k_reta_update,
2893         .reta_query             = fm10k_reta_query,
2894         .rss_hash_update        = fm10k_rss_hash_update,
2895         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2896 };
2897
2898 static int ftag_check_handler(__rte_unused const char *key,
2899                 const char *value, __rte_unused void *opaque)
2900 {
2901         if (strcmp(value, "1"))
2902                 return -1;
2903
2904         return 0;
2905 }
2906
2907 static int
2908 fm10k_check_ftag(struct rte_devargs *devargs)
2909 {
2910         struct rte_kvargs *kvlist;
2911         const char *ftag_key = "enable_ftag";
2912
2913         if (devargs == NULL)
2914                 return 0;
2915
2916         kvlist = rte_kvargs_parse(devargs->args, NULL);
2917         if (kvlist == NULL)
2918                 return 0;
2919
2920         if (!rte_kvargs_count(kvlist, ftag_key)) {
2921                 rte_kvargs_free(kvlist);
2922                 return 0;
2923         }
2924         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2925         if (rte_kvargs_process(kvlist, ftag_key,
2926                                 ftag_check_handler, NULL) < 0) {
2927                 rte_kvargs_free(kvlist);
2928                 return 0;
2929         }
2930         rte_kvargs_free(kvlist);
2931
2932         return 1;
2933 }
2934
2935 static uint16_t
2936 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2937                     uint16_t nb_pkts)
2938 {
2939         uint16_t nb_tx = 0;
2940         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2941
2942         while (nb_pkts) {
2943                 uint16_t ret, num;
2944
2945                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2946                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2947                                                  num);
2948                 nb_tx += ret;
2949                 nb_pkts -= ret;
2950                 if (ret < num)
2951                         break;
2952         }
2953
2954         return nb_tx;
2955 }
2956
2957 static void __attribute__((cold))
2958 fm10k_set_tx_function(struct rte_eth_dev *dev)
2959 {
2960         struct fm10k_tx_queue *txq;
2961         int i;
2962         int use_sse = 1;
2963         uint16_t tx_ftag_en = 0;
2964
2965         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2966                 /* primary process has set the ftag flag and offloads */
2967                 txq = dev->data->tx_queues[0];
2968                 if (fm10k_tx_vec_condition_check(txq)) {
2969                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2970                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2971                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2972                 } else {
2973                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2974                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2975                         dev->tx_pkt_prepare = NULL;
2976                 }
2977                 return;
2978         }
2979
2980         if (fm10k_check_ftag(dev->device->devargs))
2981                 tx_ftag_en = 1;
2982
2983         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2984                 txq = dev->data->tx_queues[i];
2985                 txq->tx_ftag_en = tx_ftag_en;
2986                 /* Check if Vector Tx is satisfied */
2987                 if (fm10k_tx_vec_condition_check(txq))
2988                         use_sse = 0;
2989         }
2990
2991         if (use_sse) {
2992                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2993                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2994                         txq = dev->data->tx_queues[i];
2995                         fm10k_txq_vec_setup(txq);
2996                 }
2997                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2998                 dev->tx_pkt_prepare = NULL;
2999         } else {
3000                 dev->tx_pkt_burst = fm10k_xmit_pkts;
3001                 dev->tx_pkt_prepare = fm10k_prep_pkts;
3002                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
3003         }
3004 }
3005
3006 static void __attribute__((cold))
3007 fm10k_set_rx_function(struct rte_eth_dev *dev)
3008 {
3009         struct fm10k_dev_info *dev_info =
3010                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3011         uint16_t i, rx_using_sse;
3012         uint16_t rx_ftag_en = 0;
3013
3014         if (fm10k_check_ftag(dev->device->devargs))
3015                 rx_ftag_en = 1;
3016
3017         /* In order to allow Vector Rx there are a few configuration
3018          * conditions to be met.
3019          */
3020         if (!fm10k_rx_vec_condition_check(dev) &&
3021                         dev_info->rx_vec_allowed && !rx_ftag_en) {
3022                 if (dev->data->scattered_rx)
3023                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
3024                 else
3025                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
3026         } else if (dev->data->scattered_rx)
3027                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
3028         else
3029                 dev->rx_pkt_burst = fm10k_recv_pkts;
3030
3031         rx_using_sse =
3032                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
3033                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
3034
3035         if (rx_using_sse)
3036                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
3037         else
3038                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
3039
3040         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3041                 return;
3042
3043         for (i = 0; i < dev->data->nb_rx_queues; i++) {
3044                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
3045
3046                 rxq->rx_using_sse = rx_using_sse;
3047                 rxq->rx_ftag_en = rx_ftag_en;
3048         }
3049 }
3050
3051 static void
3052 fm10k_params_init(struct rte_eth_dev *dev)
3053 {
3054         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3055         struct fm10k_dev_info *info =
3056                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3057
3058         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
3059          * there is no way to get link status without reading BAR4.  Until this
3060          * works, assume we have maximum bandwidth.
3061          * @todo - fix bus info
3062          */
3063         hw->bus_caps.speed = fm10k_bus_speed_8000;
3064         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
3065         hw->bus_caps.payload = fm10k_bus_payload_512;
3066         hw->bus.speed = fm10k_bus_speed_8000;
3067         hw->bus.width = fm10k_bus_width_pcie_x8;
3068         hw->bus.payload = fm10k_bus_payload_256;
3069
3070         info->rx_vec_allowed = true;
3071         info->sm_down = false;
3072 }
3073
3074 static int
3075 eth_fm10k_dev_init(struct rte_eth_dev *dev)
3076 {
3077         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3078         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3079         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3080         int diag, i;
3081         struct fm10k_macvlan_filter_info *macvlan;
3082
3083         PMD_INIT_FUNC_TRACE();
3084
3085         dev->dev_ops = &fm10k_eth_dev_ops;
3086         dev->rx_pkt_burst = &fm10k_recv_pkts;
3087         dev->tx_pkt_burst = &fm10k_xmit_pkts;
3088         dev->tx_pkt_prepare = &fm10k_prep_pkts;
3089
3090         /*
3091          * Primary process does the whole initialization, for secondary
3092          * processes, we just select the same Rx and Tx function as primary.
3093          */
3094         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3095                 fm10k_set_rx_function(dev);
3096                 fm10k_set_tx_function(dev);
3097                 return 0;
3098         }
3099
3100         rte_eth_copy_pci_info(dev, pdev);
3101
3102         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
3103         memset(macvlan, 0, sizeof(*macvlan));
3104         /* Vendor and Device ID need to be set before init of shared code */
3105         memset(hw, 0, sizeof(*hw));
3106         hw->device_id = pdev->id.device_id;
3107         hw->vendor_id = pdev->id.vendor_id;
3108         hw->subsystem_device_id = pdev->id.subsystem_device_id;
3109         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
3110         hw->revision_id = 0;
3111         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
3112         if (hw->hw_addr == NULL) {
3113                 PMD_INIT_LOG(ERR, "Bad mem resource."
3114                         " Try to blacklist unused devices.");
3115                 return -EIO;
3116         }
3117
3118         /* Store fm10k_adapter pointer */
3119         hw->back = dev->data->dev_private;
3120
3121         /* Initialize the shared code */
3122         diag = fm10k_init_shared_code(hw);
3123         if (diag != FM10K_SUCCESS) {
3124                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
3125                 return -EIO;
3126         }
3127
3128         /* Initialize parameters */
3129         fm10k_params_init(dev);
3130
3131         /* Initialize the hw */
3132         diag = fm10k_init_hw(hw);
3133         if (diag != FM10K_SUCCESS) {
3134                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
3135                 return -EIO;
3136         }
3137
3138         /* Initialize MAC address(es) */
3139         dev->data->mac_addrs = rte_zmalloc("fm10k",
3140                         RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
3141         if (dev->data->mac_addrs == NULL) {
3142                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
3143                 return -ENOMEM;
3144         }
3145
3146         diag = fm10k_read_mac_addr(hw);
3147
3148         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3149                         &dev->data->mac_addrs[0]);
3150
3151         if (diag != FM10K_SUCCESS ||
3152                 !rte_is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
3153
3154                 /* Generate a random addr */
3155                 rte_eth_random_addr(hw->mac.addr);
3156                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
3157                 rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3158                 &dev->data->mac_addrs[0]);
3159         }
3160
3161         /* Pass the information to the rte_eth_dev_close() that it should also
3162          * release the private port resources.
3163          */
3164         dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
3165
3166         /* Reset the hw statistics */
3167         diag = fm10k_stats_reset(dev);
3168         if (diag != 0) {
3169                 PMD_INIT_LOG(ERR, "Stats reset failed: %d", diag);
3170                 return diag;
3171         }
3172
3173         /* Reset the hw */
3174         diag = fm10k_reset_hw(hw);
3175         if (diag != FM10K_SUCCESS) {
3176                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
3177                 return -EIO;
3178         }
3179
3180         /* Setup mailbox service */
3181         diag = fm10k_setup_mbx_service(hw);
3182         if (diag != FM10K_SUCCESS) {
3183                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
3184                 return -EIO;
3185         }
3186
3187         /*PF/VF has different interrupt handling mechanism */
3188         if (hw->mac.type == fm10k_mac_pf) {
3189                 /* register callback func to eal lib */
3190                 rte_intr_callback_register(intr_handle,
3191                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3192
3193                 /* enable MISC interrupt */
3194                 fm10k_dev_enable_intr_pf(dev);
3195         } else { /* VF */
3196                 rte_intr_callback_register(intr_handle,
3197                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3198
3199                 fm10k_dev_enable_intr_vf(dev);
3200         }
3201
3202         /* Enable intr after callback registered */
3203         rte_intr_enable(intr_handle);
3204
3205         hw->mac.ops.update_int_moderator(hw);
3206
3207         /* Make sure Switch Manager is ready before going forward. */
3208         if (hw->mac.type == fm10k_mac_pf) {
3209                 int switch_ready = 0;
3210
3211                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3212                         fm10k_mbx_lock(hw);
3213                         hw->mac.ops.get_host_state(hw, &switch_ready);
3214                         fm10k_mbx_unlock(hw);
3215                         if (switch_ready)
3216                                 break;
3217                         /* Delay some time to acquire async LPORT_MAP info. */
3218                         rte_delay_us(WAIT_SWITCH_MSG_US);
3219                 }
3220
3221                 if (switch_ready == 0) {
3222                         PMD_INIT_LOG(ERR, "switch is not ready");
3223                         return -1;
3224                 }
3225         }
3226
3227         /*
3228          * Below function will trigger operations on mailbox, acquire lock to
3229          * avoid race condition from interrupt handler. Operations on mailbox
3230          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3231          * will handle and generate an interrupt to our side. Then,  FIFO in
3232          * mailbox will be touched.
3233          */
3234         fm10k_mbx_lock(hw);
3235         /* Enable port first */
3236         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3237                                         MAX_LPORT_NUM, 1);
3238
3239         /* Set unicast mode by default. App can change to other mode in other
3240          * API func.
3241          */
3242         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3243                                         FM10K_XCAST_MODE_NONE);
3244
3245         fm10k_mbx_unlock(hw);
3246
3247         /* Make sure default VID is ready before going forward. */
3248         if (hw->mac.type == fm10k_mac_pf) {
3249                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3250                         if (hw->mac.default_vid)
3251                                 break;
3252                         /* Delay some time to acquire async port VLAN info. */
3253                         rte_delay_us(WAIT_SWITCH_MSG_US);
3254                 }
3255
3256                 if (!hw->mac.default_vid) {
3257                         PMD_INIT_LOG(ERR, "default VID is not ready");
3258                         return -1;
3259                 }
3260         }
3261
3262         /* Add default mac address */
3263         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3264                 MAIN_VSI_POOL_NUMBER);
3265
3266         return 0;
3267 }
3268
3269 static int
3270 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3271 {
3272         PMD_INIT_FUNC_TRACE();
3273
3274         /* only uninitialize in the primary process */
3275         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3276                 return 0;
3277
3278         /* safe to close dev here */
3279         fm10k_dev_close(dev);
3280
3281         return 0;
3282 }
3283
3284 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3285         struct rte_pci_device *pci_dev)
3286 {
3287         return rte_eth_dev_pci_generic_probe(pci_dev,
3288                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3289 }
3290
3291 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3292 {
3293         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3294 }
3295
3296 /*
3297  * The set of PCI devices this driver supports. This driver will enable both PF
3298  * and SRIOV-VF devices.
3299  */
3300 static const struct rte_pci_id pci_id_fm10k_map[] = {
3301         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3302         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3303         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3304         { .vendor_id = 0, /* sentinel */ },
3305 };
3306
3307 static struct rte_pci_driver rte_pmd_fm10k = {
3308         .id_table = pci_id_fm10k_map,
3309         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
3310         .probe = eth_fm10k_pci_probe,
3311         .remove = eth_fm10k_pci_remove,
3312 };
3313
3314 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3315 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3316 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");
3317
3318 RTE_INIT(fm10k_init_log)
3319 {
3320         fm10k_logtype_init = rte_log_register("pmd.net.fm10k.init");
3321         if (fm10k_logtype_init >= 0)
3322                 rte_log_set_level(fm10k_logtype_init, RTE_LOG_NOTICE);
3323         fm10k_logtype_driver = rte_log_register("pmd.net.fm10k.driver");
3324         if (fm10k_logtype_driver >= 0)
3325                 rte_log_set_level(fm10k_logtype_driver, RTE_LOG_NOTICE);
3326
3327 #ifdef RTE_LIBRTE_FM10K_DEBUG_RX
3328         fm10k_logtype_rx = rte_log_register("pmd.net.fm10k.rx");
3329         if (fm10k_logtype_rx >= 0)
3330                 rte_log_set_level(fm10k_logtype_rx, RTE_LOG_DEBUG);
3331 #endif
3332
3333 #ifdef RTE_LIBRTE_FM10K_DEBUG_TX
3334         fm10k_logtype_tx = rte_log_register("pmd.net.fm10k.tx");
3335         if (fm10k_logtype_tx >= 0)
3336                 rte_log_set_level(fm10k_logtype_tx, RTE_LOG_DEBUG);
3337 #endif
3338
3339 #ifdef RTE_LIBRTE_FM10K_DEBUG_TX_FREE
3340         fm10k_logtype_tx_free = rte_log_register("pmd.net.fm10k.tx_free");
3341         if (fm10k_logtype_tx_free >= 0)
3342                 rte_log_set_level(fm10k_logtype_tx_free, RTE_LOG_DEBUG);
3343 #endif
3344 }