drivers/net/fm10k/fm10k_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2013-2016 Intel Corporation
   3  */
   4
   5 #include <rte_ethdev_driver.h>
   6 #include <rte_ethdev_pci.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_string_fns.h>
  10 #include <rte_dev.h>
  11 #include <rte_spinlock.h>
  12 #include <rte_kvargs.h>
  13
  14 #include "fm10k.h"
  15 #include "base/fm10k_api.h"
  16
  17 /* Default delay to acquire mailbox lock */
  18 #define FM10K_MBXLOCK_DELAY_US 20
  19 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  20
  21 #define MAIN_VSI_POOL_NUMBER 0
  22
  23 /* Max try times to acquire switch status */
  24 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  25 /* Wait interval to get switch status */
  26 #define WAIT_SWITCH_MSG_US    100000
  27 /* A period of quiescence for switch */
  28 #define FM10K_SWITCH_QUIESCE_US 100000
  29 /* Number of chars per uint32 type */
  30 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  31 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  32
  33 /* default 1:1 map from queue ID to interrupt vector ID */
  34 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  35
  36 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  37 #define MAX_LPORT_NUM    128
  38 #define GLORT_FD_Q_BASE  0x40
  39 #define GLORT_PF_MASK    0xFFC0
  40 #define GLORT_FD_MASK    GLORT_PF_MASK
  41 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  42
  43 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  44 static int fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  45 static int fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  46 static int fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  47 static int fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  48 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  49 static int
  50 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  51 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  52         const u8 *mac, bool add, uint32_t pool);
  53 static void fm10k_tx_queue_release(void *queue);
  54 static void fm10k_rx_queue_release(void *queue);
  55 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  56 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  57 static int fm10k_check_ftag(struct rte_devargs *devargs);
  58 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  59
  60 static int fm10k_dev_infos_get(struct rte_eth_dev *dev,
  61                                struct rte_eth_dev_info *dev_info);
  62 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev);
  63 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev);
  64 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev);
  65 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev);
  66
  67 struct fm10k_xstats_name_off {
  68         char name[RTE_ETH_XSTATS_NAME_SIZE];
  69         unsigned offset;
  70 };
  71
  72 static const struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  73         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  74         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  75         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  76         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  77         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  78         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  79         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  80         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  81                 nodesc_drop)},
  82 };
  83
  84 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  85                 sizeof(fm10k_hw_stats_strings[0]))
  86
  87 static const struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
  88         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
  89         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
  90         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
  91 };
  92
  93 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
  94                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
  95
  96 static const struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
  97         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
  98         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
  99 };
 100
 101 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 102                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 103
 104 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 105                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 106 static int
 107 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 108
 109 static void
 110 fm10k_mbx_initlock(struct fm10k_hw *hw)
 111 {
 112         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 113 }
 114
 115 static void
 116 fm10k_mbx_lock(struct fm10k_hw *hw)
 117 {
 118         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 119                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 120 }
 121
 122 static void
 123 fm10k_mbx_unlock(struct fm10k_hw *hw)
 124 {
 125         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 126 }
 127
 128 /* Stubs needed for linkage when vPMD is disabled */
 129 __rte_weak int
 130 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 131 {
 132         return -1;
 133 }
 134
 135 __rte_weak uint16_t
 136 fm10k_recv_pkts_vec(
 137         __rte_unused void *rx_queue,
 138         __rte_unused struct rte_mbuf **rx_pkts,
 139         __rte_unused uint16_t nb_pkts)
 140 {
 141         return 0;
 142 }
 143
 144 __rte_weak uint16_t
 145 fm10k_recv_scattered_pkts_vec(
 146                 __rte_unused void *rx_queue,
 147                 __rte_unused struct rte_mbuf **rx_pkts,
 148                 __rte_unused uint16_t nb_pkts)
 149 {
 150         return 0;
 151 }
 152
 153 __rte_weak int
 154 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 155
 156 {
 157         return -1;
 158 }
 159
 160 __rte_weak void
 161 fm10k_rx_queue_release_mbufs_vec(
 162                 __rte_unused struct fm10k_rx_queue *rxq)
 163 {
 164         return;
 165 }
 166
 167 __rte_weak void
 168 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 169 {
 170         return;
 171 }
 172
 173 __rte_weak int
 174 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 175 {
 176         return -1;
 177 }
 178
 179 __rte_weak uint16_t
 180 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 181                            __rte_unused struct rte_mbuf **tx_pkts,
 182                            __rte_unused uint16_t nb_pkts)
 183 {
 184         return 0;
 185 }
 186
 187 /*
 188  * reset queue to initial state, allocate software buffers used when starting
 189  * device.
 190  * return 0 on success
 191  * return -ENOMEM if buffers cannot be allocated
 192  * return -EINVAL if buffers do not satisfy alignment condition
 193  */
 194 static inline int
 195 rx_queue_reset(struct fm10k_rx_queue *q)
 196 {
 197         static const union fm10k_rx_desc zero = {{0} };
 198         uint64_t dma_addr;
 199         int i, diag;
 200         PMD_INIT_FUNC_TRACE();
 201
 202         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 203         if (diag != 0)
 204                 return -ENOMEM;
 205
 206         for (i = 0; i < q->nb_desc; ++i) {
 207                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 208                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 209                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 210                                                 q->nb_desc);
 211                         return -EINVAL;
 212                 }
 213                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 214                 q->hw_ring[i].q.pkt_addr = dma_addr;
 215                 q->hw_ring[i].q.hdr_addr = dma_addr;
 216         }
 217
 218         /* initialize extra software ring entries. Space for these extra
 219          * entries is always allocated.
 220          */
 221         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 222         for (i = 0; i < q->nb_fake_desc; ++i) {
 223                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 224                 q->hw_ring[q->nb_desc + i] = zero;
 225         }
 226
 227         q->next_dd = 0;
 228         q->next_alloc = 0;
 229         q->next_trigger = q->alloc_thresh - 1;
 230         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 231         q->rxrearm_start = 0;
 232         q->rxrearm_nb = 0;
 233
 234         return 0;
 235 }
 236
 237 /*
 238  * clean queue, descriptor rings, free software buffers used when stopping
 239  * device.
 240  */
 241 static inline void
 242 rx_queue_clean(struct fm10k_rx_queue *q)
 243 {
 244         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 245         uint32_t i;
 246         PMD_INIT_FUNC_TRACE();
 247
 248         /* zero descriptor rings */
 249         for (i = 0; i < q->nb_desc; ++i)
 250                 q->hw_ring[i] = zero;
 251
 252         /* zero faked descriptors */
 253         for (i = 0; i < q->nb_fake_desc; ++i)
 254                 q->hw_ring[q->nb_desc + i] = zero;
 255
 256         /* vPMD driver has a different way of releasing mbufs. */
 257         if (q->rx_using_sse) {
 258                 fm10k_rx_queue_release_mbufs_vec(q);
 259                 return;
 260         }
 261
 262         /* free software buffers */
 263         for (i = 0; i < q->nb_desc; ++i) {
 264                 if (q->sw_ring[i]) {
 265                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 266                         q->sw_ring[i] = NULL;
 267                 }
 268         }
 269 }
 270
 271 /*
 272  * free all queue memory used when releasing the queue (i.e. configure)
 273  */
 274 static inline void
 275 rx_queue_free(struct fm10k_rx_queue *q)
 276 {
 277         PMD_INIT_FUNC_TRACE();
 278         if (q) {
 279                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 280                 rx_queue_clean(q);
 281                 if (q->sw_ring) {
 282                         rte_free(q->sw_ring);
 283                         q->sw_ring = NULL;
 284                 }
 285                 rte_free(q);
 286                 q = NULL;
 287         }
 288 }
 289
 290 /*
 291  * disable RX queue, wait unitl HW finished necessary flush operation
 292  */
 293 static inline int
 294 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 295 {
 296         uint32_t reg, i;
 297
 298         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 299         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 300                         reg & ~FM10K_RXQCTL_ENABLE);
 301
 302         /* Wait 100us at most */
 303         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 304                 rte_delay_us(1);
 305                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 306                 if (!(reg & FM10K_RXQCTL_ENABLE))
 307                         break;
 308         }
 309
 310         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 311                 return -1;
 312
 313         return 0;
 314 }
 315
 316 /*
 317  * reset queue to initial state, allocate software buffers used when starting
 318  * device
 319  */
 320 static inline void
 321 tx_queue_reset(struct fm10k_tx_queue *q)
 322 {
 323         PMD_INIT_FUNC_TRACE();
 324         q->last_free = 0;
 325         q->next_free = 0;
 326         q->nb_used = 0;
 327         q->nb_free = q->nb_desc - 1;
 328         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 329         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 330 }
 331
 332 /*
 333  * clean queue, descriptor rings, free software buffers used when stopping
 334  * device
 335  */
 336 static inline void
 337 tx_queue_clean(struct fm10k_tx_queue *q)
 338 {
 339         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 340         uint32_t i;
 341         PMD_INIT_FUNC_TRACE();
 342
 343         /* zero descriptor rings */
 344         for (i = 0; i < q->nb_desc; ++i)
 345                 q->hw_ring[i] = zero;
 346
 347         /* free software buffers */
 348         for (i = 0; i < q->nb_desc; ++i) {
 349                 if (q->sw_ring[i]) {
 350                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 351                         q->sw_ring[i] = NULL;
 352                 }
 353         }
 354 }
 355
 356 /*
 357  * free all queue memory used when releasing the queue (i.e. configure)
 358  */
 359 static inline void
 360 tx_queue_free(struct fm10k_tx_queue *q)
 361 {
 362         PMD_INIT_FUNC_TRACE();
 363         if (q) {
 364                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 365                 tx_queue_clean(q);
 366                 if (q->rs_tracker.list) {
 367                         rte_free(q->rs_tracker.list);
 368                         q->rs_tracker.list = NULL;
 369                 }
 370                 if (q->sw_ring) {
 371                         rte_free(q->sw_ring);
 372                         q->sw_ring = NULL;
 373                 }
 374                 rte_free(q);
 375                 q = NULL;
 376         }
 377 }
 378
 379 /*
 380  * disable TX queue, wait unitl HW finished necessary flush operation
 381  */
 382 static inline int
 383 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 384 {
 385         uint32_t reg, i;
 386
 387         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 388         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 389                         reg & ~FM10K_TXDCTL_ENABLE);
 390
 391         /* Wait 100us at most */
 392         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 393                 rte_delay_us(1);
 394                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 395                 if (!(reg & FM10K_TXDCTL_ENABLE))
 396                         break;
 397         }
 398
 399         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 400                 return -1;
 401
 402         return 0;
 403 }
 404
 405 static int
 406 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 407 {
 408         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 409         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 410         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 411         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 412
 413         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 414
 415         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 416                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 417                 return -EINVAL;
 418         }
 419
 420         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 421                 return 0;
 422
 423         if (hw->mac.type == fm10k_mac_vf) {
 424                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 425                 return -EINVAL;
 426         }
 427
 428         /* Check VMDQ queue pool number */
 429         if (vmdq_conf->nb_queue_pools >
 430                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 431                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 432                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 433                         vmdq_conf->nb_queue_pools);
 434                 return -EINVAL;
 435         }
 436
 437         return 0;
 438 }
 439
 440 static const struct fm10k_txq_ops def_txq_ops = {
 441         .reset = tx_queue_reset,
 442 };
 443
 444 static int
 445 fm10k_dev_configure(struct rte_eth_dev *dev)
 446 {
 447         int ret;
 448
 449         PMD_INIT_FUNC_TRACE();
 450
 451         if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)
 452                 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH;
 453
 454         /* multipe queue mode checking */
 455         ret  = fm10k_check_mq_mode(dev);
 456         if (ret != 0) {
 457                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 458                             ret);
 459                 return ret;
 460         }
 461
 462         dev->data->scattered_rx = 0;
 463
 464         return 0;
 465 }
 466
 467 static void
 468 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 469 {
 470         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 471         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 472         uint32_t i;
 473
 474         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 475
 476         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 477                 if (!vmdq_conf->pool_map[i].pools)
 478                         continue;
 479                 fm10k_mbx_lock(hw);
 480                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 481                 fm10k_mbx_unlock(hw);
 482         }
 483 }
 484
 485 static void
 486 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 487 {
 488         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 489
 490         /* Add default mac address */
 491         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 492                 MAIN_VSI_POOL_NUMBER);
 493 }
 494
 495 static void
 496 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 497 {
 498         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 499         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 500         uint32_t mrqc, *key, i, reta, j;
 501         uint64_t hf;
 502
 503 #define RSS_KEY_SIZE 40
 504         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 505                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 506                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 507                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 508                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 509                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 510         };
 511
 512         if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 513                 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 514                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 515                 return;
 516         }
 517
 518         /* random key is rss_intel_key (default) or user provided (rss_key) */
 519         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 520                 key = (uint32_t *)rss_intel_key;
 521         else
 522                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 523
 524         /* Now fill our hash function seeds, 4 bytes at a time */
 525         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 526                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 527
 528         /*
 529          * Fill in redirection table
 530          * The byte-swap is needed because NIC registers are in
 531          * little-endian order.
 532          */
 533         reta = 0;
 534         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 535                 if (j == dev->data->nb_rx_queues)
 536                         j = 0;
 537                 reta = (reta << CHAR_BIT) | j;
 538                 if ((i & 3) == 3)
 539                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 540                                         rte_bswap32(reta));
 541         }
 542
 543         /*
 544          * Generate RSS hash based on packet types, TCP/UDP
 545          * port numbers and/or IPv4/v6 src and dst addresses
 546          */
 547         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 548         mrqc = 0;
 549         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 550         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 551         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 552         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 553         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 554         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 555         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 556         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 557         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 558
 559         if (mrqc == 0) {
 560                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 561                         "supported", hf);
 562                 return;
 563         }
 564
 565         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 566 }
 567
 568 static void
 569 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 570 {
 571         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 572         uint32_t i;
 573
 574         for (i = 0; i < nb_lport_new; i++) {
 575                 /* Set unicast mode by default. App can change
 576                  * to other mode in other API func.
 577                  */
 578                 fm10k_mbx_lock(hw);
 579                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 580                         FM10K_XCAST_MODE_NONE);
 581                 fm10k_mbx_unlock(hw);
 582         }
 583 }
 584
 585 static void
 586 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 587 {
 588         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 589         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 590         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 591         struct fm10k_macvlan_filter_info *macvlan;
 592         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 593         uint16_t nb_lport_new;
 594
 595         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 596         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 597
 598         fm10k_dev_rss_configure(dev);
 599
 600         /* only PF supports VMDQ */
 601         if (hw->mac.type != fm10k_mac_pf)
 602                 return;
 603
 604         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 605                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 606
 607         /* no pool number change, no need to update logic port and VLAN/MAC */
 608         if (macvlan->nb_queue_pools == nb_queue_pools)
 609                 return;
 610
 611         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 612         fm10k_dev_logic_port_update(dev, nb_lport_new);
 613
 614         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 615         memset(dev->data->mac_addrs, 0,
 616                 RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 617         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
 618                 &dev->data->mac_addrs[0]);
 619         memset(macvlan, 0, sizeof(*macvlan));
 620         macvlan->nb_queue_pools = nb_queue_pools;
 621
 622         if (nb_queue_pools)
 623                 fm10k_dev_vmdq_rx_configure(dev);
 624         else
 625                 fm10k_dev_pf_main_vsi_reset(dev);
 626 }
 627
 628 static int
 629 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 630 {
 631         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 632         int i, ret;
 633         struct fm10k_tx_queue *txq;
 634         uint64_t base_addr;
 635         uint32_t size;
 636
 637         /* Disable TXINT to avoid possible interrupt */
 638         for (i = 0; i < hw->mac.max_queues; i++)
 639                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 640                                 3 << FM10K_TXINT_TIMER_SHIFT);
 641
 642         /* Setup TX queue */
 643         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 644                 txq = dev->data->tx_queues[i];
 645                 base_addr = txq->hw_ring_phys_addr;
 646                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 647
 648                 /* disable queue to avoid issues while updating state */
 649                 ret = tx_queue_disable(hw, i);
 650                 if (ret) {
 651                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 652                         return -1;
 653                 }
 654                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 655                  * register is read-only for VF.
 656                  */
 657                 if (fm10k_check_ftag(dev->device->devargs)) {
 658                         if (hw->mac.type == fm10k_mac_pf) {
 659                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 660                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 661                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 662                         } else {
 663                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 664                                 return -ENOTSUP;
 665                         }
 666                 }
 667
 668                 /* set location and size for descriptor ring */
 669                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 670                                 base_addr & UINT64_LOWER_32BITS_MASK);
 671                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 672                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 673                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 674
 675                 /* assign default SGLORT for each TX queue by PF */
 676                 if (hw->mac.type == fm10k_mac_pf)
 677                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 678         }
 679
 680         /* set up vector or scalar TX function as appropriate */
 681         fm10k_set_tx_function(dev);
 682
 683         return 0;
 684 }
 685
 686 static int
 687 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 688 {
 689         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 690         struct fm10k_macvlan_filter_info *macvlan;
 691         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 692         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 693         int i, ret;
 694         struct fm10k_rx_queue *rxq;
 695         uint64_t base_addr;
 696         uint32_t size;
 697         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 698         uint32_t logic_port = hw->mac.dglort_map;
 699         uint16_t buf_size;
 700         uint16_t queue_stride = 0;
 701
 702         /* enable RXINT for interrupt mode */
 703         i = 0;
 704         if (rte_intr_dp_is_en(intr_handle)) {
 705                 for (; i < dev->data->nb_rx_queues; i++) {
 706                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 707                         if (hw->mac.type == fm10k_mac_pf)
 708                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 709                                         FM10K_ITR_AUTOMASK |
 710                                         FM10K_ITR_MASK_CLEAR);
 711                         else
 712                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 713                                         FM10K_ITR_AUTOMASK |
 714                                         FM10K_ITR_MASK_CLEAR);
 715                 }
 716         }
 717         /* Disable other RXINT to avoid possible interrupt */
 718         for (; i < hw->mac.max_queues; i++)
 719                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 720                         3 << FM10K_RXINT_TIMER_SHIFT);
 721
 722         /* Setup RX queues */
 723         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 724                 rxq = dev->data->rx_queues[i];
 725                 base_addr = rxq->hw_ring_phys_addr;
 726                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 727
 728                 /* disable queue to avoid issues while updating state */
 729                 ret = rx_queue_disable(hw, i);
 730                 if (ret) {
 731                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 732                         return -1;
 733                 }
 734
 735                 /* Setup the Base and Length of the Rx Descriptor Ring */
 736                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 737                                 base_addr & UINT64_LOWER_32BITS_MASK);
 738                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 739                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 740                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 741
 742                 /* Configure the Rx buffer size for one buff without split */
 743                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 744                         RTE_PKTMBUF_HEADROOM);
 745                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 746                  * reserved for this purpose, and the worst case could be 511B.
 747                  * But SRR reg assumes all buffers have the same size. In order
 748                  * to fill the gap, we'll have to consider the worst case and
 749                  * assume 512B is reserved. If we don't do so, it's possible
 750                  * for HW to overwrite data to next mbuf.
 751                  */
 752                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 753
 754                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 755                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 756                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 757
 758                 /* It adds dual VLAN length for supporting dual VLAN */
 759                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 760                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 761                         rxq->offloads & DEV_RX_OFFLOAD_SCATTER) {
 762                         uint32_t reg;
 763                         dev->data->scattered_rx = 1;
 764                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 765                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 766                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 767                 }
 768
 769                 /* Enable drop on empty, it's RO for VF */
 770                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 771                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 772
 773                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 774                 FM10K_WRITE_FLUSH(hw);
 775         }
 776
 777         /* Configure VMDQ/RSS if applicable */
 778         fm10k_dev_mq_rx_configure(dev);
 779
 780         /* Decide the best RX function */
 781         fm10k_set_rx_function(dev);
 782
 783         /* update RX_SGLORT for loopback suppress*/
 784         if (hw->mac.type != fm10k_mac_pf)
 785                 return 0;
 786         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 787         if (macvlan->nb_queue_pools)
 788                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 789         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 790                 if (i && queue_stride && !(i % queue_stride))
 791                         logic_port++;
 792                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 793         }
 794
 795         return 0;
 796 }
 797
 798 static int
 799 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 800 {
 801         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 802         int err;
 803         uint32_t reg;
 804         struct fm10k_rx_queue *rxq;
 805
 806         PMD_INIT_FUNC_TRACE();
 807
 808         rxq = dev->data->rx_queues[rx_queue_id];
 809         err = rx_queue_reset(rxq);
 810         if (err == -ENOMEM) {
 811                 PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 812                 return err;
 813         } else if (err == -EINVAL) {
 814                 PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 815                         " %d", err);
 816                 return err;
 817         }
 818
 819         /* Setup the HW Rx Head and Tail Descriptor Pointers
 820          * Note: this must be done AFTER the queue is enabled on real
 821          * hardware, but BEFORE the queue is enabled when using the
 822          * emulation platform. Do it in both places for now and remove
 823          * this comment and the following two register writes when the
 824          * emulation platform is no longer being used.
 825          */
 826         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 827         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 828
 829         /* Set PF ownership flag for PF devices */
 830         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 831         if (hw->mac.type == fm10k_mac_pf)
 832                 reg |= FM10K_RXQCTL_PF;
 833         reg |= FM10K_RXQCTL_ENABLE;
 834         /* enable RX queue */
 835         FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 836         FM10K_WRITE_FLUSH(hw);
 837
 838         /* Setup the HW Rx Head and Tail Descriptor Pointers
 839          * Note: this must be done AFTER the queue is enabled
 840          */
 841         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 842         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 843         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 844
 845         return 0;
 846 }
 847
 848 static int
 849 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 850 {
 851         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 852
 853         PMD_INIT_FUNC_TRACE();
 854
 855         /* Disable RX queue */
 856         rx_queue_disable(hw, rx_queue_id);
 857
 858         /* Free mbuf and clean HW ring */
 859         rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 860         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 861
 862         return 0;
 863 }
 864
 865 static int
 866 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 867 {
 868         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 869         /** @todo - this should be defined in the shared code */
 870 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 871         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 872         struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 873
 874         PMD_INIT_FUNC_TRACE();
 875
 876         q->ops->reset(q);
 877
 878         /* reset head and tail pointers */
 879         FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 880         FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 881
 882         /* enable TX queue */
 883         FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 884                                 FM10K_TXDCTL_ENABLE | txdctl);
 885         FM10K_WRITE_FLUSH(hw);
 886         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 887
 888         return 0;
 889 }
 890
 891 static int
 892 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 893 {
 894         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 895
 896         PMD_INIT_FUNC_TRACE();
 897
 898         tx_queue_disable(hw, tx_queue_id);
 899         tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 900         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 901
 902         return 0;
 903 }
 904
 905 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 906 {
 907         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 908                 != FM10K_DGLORTMAP_NONE);
 909 }
 910
 911 static int
 912 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 913 {
 914         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 915         int status;
 916
 917         PMD_INIT_FUNC_TRACE();
 918
 919         /* Return if it didn't acquire valid glort range */
 920         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 921                 return 0;
 922
 923         fm10k_mbx_lock(hw);
 924         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 925                                 FM10K_XCAST_MODE_PROMISC);
 926         fm10k_mbx_unlock(hw);
 927
 928         if (status != FM10K_SUCCESS) {
 929                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 930                 return -EAGAIN;
 931         }
 932
 933         return 0;
 934 }
 935
 936 static int
 937 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 938 {
 939         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 940         uint8_t mode;
 941         int status;
 942
 943         PMD_INIT_FUNC_TRACE();
 944
 945         /* Return if it didn't acquire valid glort range */
 946         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 947                 return 0;
 948
 949         if (dev->data->all_multicast == 1)
 950                 mode = FM10K_XCAST_MODE_ALLMULTI;
 951         else
 952                 mode = FM10K_XCAST_MODE_NONE;
 953
 954         fm10k_mbx_lock(hw);
 955         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 956                                 mode);
 957         fm10k_mbx_unlock(hw);
 958
 959         if (status != FM10K_SUCCESS) {
 960                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 961                 return -EAGAIN;
 962         }
 963
 964         return 0;
 965 }
 966
 967 static int
 968 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 969 {
 970         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 971         int status;
 972
 973         PMD_INIT_FUNC_TRACE();
 974
 975         /* Return if it didn't acquire valid glort range */
 976         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 977                 return 0;
 978
 979         /* If promiscuous mode is enabled, it doesn't make sense to enable
 980          * allmulticast and disable promiscuous since fm10k only can select
 981          * one of the modes.
 982          */
 983         if (dev->data->promiscuous) {
 984                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 985                         "needn't enable allmulticast");
 986                 return 0;
 987         }
 988
 989         fm10k_mbx_lock(hw);
 990         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 991                                 FM10K_XCAST_MODE_ALLMULTI);
 992         fm10k_mbx_unlock(hw);
 993
 994         if (status != FM10K_SUCCESS) {
 995                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 996                 return -EAGAIN;
 997         }
 998
 999         return 0;
1000 }
1001
1002 static int
1003 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1004 {
1005         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1006         int status;
1007
1008         PMD_INIT_FUNC_TRACE();
1009
1010         /* Return if it didn't acquire valid glort range */
1011         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1012                 return 0;
1013
1014         if (dev->data->promiscuous) {
1015                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1016                         "since promisc mode is enabled");
1017                 return -EINVAL;
1018         }
1019
1020         fm10k_mbx_lock(hw);
1021         /* Change mode to unicast mode */
1022         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1023                                 FM10K_XCAST_MODE_NONE);
1024         fm10k_mbx_unlock(hw);
1025
1026         if (status != FM10K_SUCCESS) {
1027                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1028                 return -EAGAIN;
1029         }
1030
1031         return 0;
1032 }
1033
1034 static void
1035 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1036 {
1037         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1038         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1039         uint16_t nb_queue_pools;
1040         struct fm10k_macvlan_filter_info *macvlan;
1041
1042         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1043         nb_queue_pools = macvlan->nb_queue_pools;
1044         pool_len = nb_queue_pools ? rte_fls_u32(nb_queue_pools - 1) : 0;
1045         rss_len = rte_fls_u32(dev->data->nb_rx_queues - 1) - pool_len;
1046
1047         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1048         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1049         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1050                         hw->mac.dglort_map;
1051         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1052         /* Configure VMDQ/RSS DGlort Decoder */
1053         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1054
1055         /* Flow Director configurations, only queue number is valid. */
1056         dglortdec = rte_fls_u32(dev->data->nb_rx_queues - 1);
1057         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1058                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1059         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1060         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1061
1062         /* Invalidate all other GLORT entries */
1063         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1064                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1065                                 FM10K_DGLORTMAP_NONE);
1066 }
1067
1068 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1069 static int
1070 fm10k_dev_start(struct rte_eth_dev *dev)
1071 {
1072         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1073         int i, diag;
1074
1075         PMD_INIT_FUNC_TRACE();
1076
1077         /* stop, init, then start the hw */
1078         diag = fm10k_stop_hw(hw);
1079         if (diag != FM10K_SUCCESS) {
1080                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1081                 return -EIO;
1082         }
1083
1084         diag = fm10k_init_hw(hw);
1085         if (diag != FM10K_SUCCESS) {
1086                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1087                 return -EIO;
1088         }
1089
1090         diag = fm10k_start_hw(hw);
1091         if (diag != FM10K_SUCCESS) {
1092                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1093                 return -EIO;
1094         }
1095
1096         diag = fm10k_dev_tx_init(dev);
1097         if (diag) {
1098                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1099                 return diag;
1100         }
1101
1102         if (fm10k_dev_rxq_interrupt_setup(dev))
1103                 return -EIO;
1104
1105         diag = fm10k_dev_rx_init(dev);
1106         if (diag) {
1107                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1108                 return diag;
1109         }
1110
1111         if (hw->mac.type == fm10k_mac_pf)
1112                 fm10k_dev_dglort_map_configure(dev);
1113
1114         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1115                 struct fm10k_rx_queue *rxq;
1116                 rxq = dev->data->rx_queues[i];
1117
1118                 if (rxq->rx_deferred_start)
1119                         continue;
1120                 diag = fm10k_dev_rx_queue_start(dev, i);
1121                 if (diag != 0) {
1122                         int j;
1123                         for (j = 0; j < i; ++j)
1124                                 rx_queue_clean(dev->data->rx_queues[j]);
1125                         return diag;
1126                 }
1127         }
1128
1129         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1130                 struct fm10k_tx_queue *txq;
1131                 txq = dev->data->tx_queues[i];
1132
1133                 if (txq->tx_deferred_start)
1134                         continue;
1135                 diag = fm10k_dev_tx_queue_start(dev, i);
1136                 if (diag != 0) {
1137                         int j;
1138                         for (j = 0; j < i; ++j)
1139                                 tx_queue_clean(dev->data->tx_queues[j]);
1140                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1141                                 rx_queue_clean(dev->data->rx_queues[j]);
1142                         return diag;
1143                 }
1144         }
1145
1146         /* Update default vlan when not in VMDQ mode */
1147         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1148                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1149
1150         fm10k_link_update(dev, 0);
1151
1152         return 0;
1153 }
1154
1155 static void
1156 fm10k_dev_stop(struct rte_eth_dev *dev)
1157 {
1158         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1159         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1160         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1161         int i;
1162
1163         PMD_INIT_FUNC_TRACE();
1164
1165         if (dev->data->tx_queues)
1166                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1167                         fm10k_dev_tx_queue_stop(dev, i);
1168
1169         if (dev->data->rx_queues)
1170                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1171                         fm10k_dev_rx_queue_stop(dev, i);
1172
1173         /* Disable datapath event */
1174         if (rte_intr_dp_is_en(intr_handle)) {
1175                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1176                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1177                                 3 << FM10K_RXINT_TIMER_SHIFT);
1178                         if (hw->mac.type == fm10k_mac_pf)
1179                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1180                                         FM10K_ITR_MASK_SET);
1181                         else
1182                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1183                                         FM10K_ITR_MASK_SET);
1184                 }
1185         }
1186         /* Clean datapath event and queue/vec mapping */
1187         rte_intr_efd_disable(intr_handle);
1188         rte_free(intr_handle->intr_vec);
1189         intr_handle->intr_vec = NULL;
1190 }
1191
1192 static void
1193 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1194 {
1195         int i;
1196
1197         PMD_INIT_FUNC_TRACE();
1198
1199         if (dev->data->tx_queues) {
1200                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1201                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1202
1203                         tx_queue_free(txq);
1204                 }
1205         }
1206
1207         if (dev->data->rx_queues) {
1208                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1209                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1210         }
1211 }
1212
1213 static int
1214 fm10k_link_update(struct rte_eth_dev *dev,
1215         __rte_unused int wait_to_complete)
1216 {
1217         struct fm10k_dev_info *dev_info =
1218                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1219         PMD_INIT_FUNC_TRACE();
1220
1221         dev->data->dev_link.link_speed  = ETH_SPEED_NUM_50G;
1222         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1223         dev->data->dev_link.link_status =
1224                 dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP;
1225         dev->data->dev_link.link_autoneg = ETH_LINK_FIXED;
1226
1227         return 0;
1228 }
1229
1230 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1231         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1232 {
1233         unsigned i, q;
1234         unsigned count = 0;
1235
1236         if (xstats_names != NULL) {
1237                 /* Note: limit checked in rte_eth_xstats_names() */
1238
1239                 /* Global stats */
1240                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1241                         snprintf(xstats_names[count].name,
1242                                 sizeof(xstats_names[count].name),
1243                                 "%s", fm10k_hw_stats_strings[count].name);
1244                         count++;
1245                 }
1246
1247                 /* PF queue stats */
1248                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1249                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1250                                 snprintf(xstats_names[count].name,
1251                                         sizeof(xstats_names[count].name),
1252                                         "rx_q%u_%s", q,
1253                                         fm10k_hw_stats_rx_q_strings[i].name);
1254                                 count++;
1255                         }
1256                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1257                                 snprintf(xstats_names[count].name,
1258                                         sizeof(xstats_names[count].name),
1259                                         "tx_q%u_%s", q,
1260                                         fm10k_hw_stats_tx_q_strings[i].name);
1261                                 count++;
1262                         }
1263                 }
1264         }
1265         return FM10K_NB_XSTATS;
1266 }
1267
1268 static int
1269 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1270                  unsigned n)
1271 {
1272         struct fm10k_hw_stats *hw_stats =
1273                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1274         unsigned i, q, count = 0;
1275
1276         if (n < FM10K_NB_XSTATS)
1277                 return FM10K_NB_XSTATS;
1278
1279         /* Global stats */
1280         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1281                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1282                         fm10k_hw_stats_strings[count].offset);
1283                 xstats[count].id = count;
1284                 count++;
1285         }
1286
1287         /* PF queue stats */
1288         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1289                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1290                         xstats[count].value =
1291                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1292                                 fm10k_hw_stats_rx_q_strings[i].offset);
1293                         xstats[count].id = count;
1294                         count++;
1295                 }
1296                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1297                         xstats[count].value =
1298                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1299                                 fm10k_hw_stats_tx_q_strings[i].offset);
1300                         xstats[count].id = count;
1301                         count++;
1302                 }
1303         }
1304
1305         return FM10K_NB_XSTATS;
1306 }
1307
1308 static int
1309 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1310 {
1311         uint64_t ipackets, opackets, ibytes, obytes, imissed;
1312         struct fm10k_hw *hw =
1313                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1314         struct fm10k_hw_stats *hw_stats =
1315                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1316         int i;
1317
1318         PMD_INIT_FUNC_TRACE();
1319
1320         fm10k_update_hw_stats(hw, hw_stats);
1321
1322         ipackets = opackets = ibytes = obytes = imissed = 0;
1323         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1324                 (i < hw->mac.max_queues); ++i) {
1325                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1326                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1327                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1328                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1329                 stats->q_errors[i]   = hw_stats->q[i].rx_drops.count;
1330                 ipackets += stats->q_ipackets[i];
1331                 opackets += stats->q_opackets[i];
1332                 ibytes   += stats->q_ibytes[i];
1333                 obytes   += stats->q_obytes[i];
1334                 imissed  += stats->q_errors[i];
1335         }
1336         stats->ipackets = ipackets;
1337         stats->opackets = opackets;
1338         stats->ibytes = ibytes;
1339         stats->obytes = obytes;
1340         stats->imissed = imissed;
1341         return 0;
1342 }
1343
1344 static int
1345 fm10k_stats_reset(struct rte_eth_dev *dev)
1346 {
1347         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1348         struct fm10k_hw_stats *hw_stats =
1349                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1350
1351         PMD_INIT_FUNC_TRACE();
1352
1353         memset(hw_stats, 0, sizeof(*hw_stats));
1354         fm10k_rebind_hw_stats(hw, hw_stats);
1355
1356         return 0;
1357 }
1358
1359 static int
1360 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1361         struct rte_eth_dev_info *dev_info)
1362 {
1363         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1364         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1365
1366         PMD_INIT_FUNC_TRACE();
1367
1368         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1369         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1370         dev_info->max_rx_queues      = hw->mac.max_queues;
1371         dev_info->max_tx_queues      = hw->mac.max_queues;
1372         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1373         dev_info->max_hash_mac_addrs = 0;
1374         dev_info->max_vfs            = pdev->max_vfs;
1375         dev_info->vmdq_pool_base     = 0;
1376         dev_info->vmdq_queue_base    = 0;
1377         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1378         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1379         dev_info->rx_queue_offload_capa = fm10k_get_rx_queue_offloads_capa(dev);
1380         dev_info->rx_offload_capa = fm10k_get_rx_port_offloads_capa(dev) |
1381                                     dev_info->rx_queue_offload_capa;
1382         dev_info->tx_queue_offload_capa = fm10k_get_tx_queue_offloads_capa(dev);
1383         dev_info->tx_offload_capa = fm10k_get_tx_port_offloads_capa(dev) |
1384                                     dev_info->tx_queue_offload_capa;
1385
1386         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1387         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1388         dev_info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1389                                         ETH_RSS_IPV6 |
1390                                         ETH_RSS_IPV6_EX |
1391                                         ETH_RSS_NONFRAG_IPV4_TCP |
1392                                         ETH_RSS_NONFRAG_IPV6_TCP |
1393                                         ETH_RSS_IPV6_TCP_EX |
1394                                         ETH_RSS_NONFRAG_IPV4_UDP |
1395                                         ETH_RSS_NONFRAG_IPV6_UDP |
1396                                         ETH_RSS_IPV6_UDP_EX;
1397
1398         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1399                 .rx_thresh = {
1400                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1401                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1402                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1403                 },
1404                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1405                 .rx_drop_en = 0,
1406                 .offloads = 0,
1407         };
1408
1409         dev_info->default_txconf = (struct rte_eth_txconf) {
1410                 .tx_thresh = {
1411                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1412                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1413                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1414                 },
1415                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1416                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1417                 .offloads = 0,
1418         };
1419
1420         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1421                 .nb_max = FM10K_MAX_RX_DESC,
1422                 .nb_min = FM10K_MIN_RX_DESC,
1423                 .nb_align = FM10K_MULT_RX_DESC,
1424         };
1425
1426         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1427                 .nb_max = FM10K_MAX_TX_DESC,
1428                 .nb_min = FM10K_MIN_TX_DESC,
1429                 .nb_align = FM10K_MULT_TX_DESC,
1430                 .nb_seg_max = FM10K_TX_MAX_SEG,
1431                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1432         };
1433
1434         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1435                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1436                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1437
1438         return 0;
1439 }
1440
1441 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1442 static const uint32_t *
1443 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1444 {
1445         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1446             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1447                 static uint32_t ptypes[] = {
1448                         /* refers to rx_desc_to_ol_flags() */
1449                         RTE_PTYPE_L2_ETHER,
1450                         RTE_PTYPE_L3_IPV4,
1451                         RTE_PTYPE_L3_IPV4_EXT,
1452                         RTE_PTYPE_L3_IPV6,
1453                         RTE_PTYPE_L3_IPV6_EXT,
1454                         RTE_PTYPE_L4_TCP,
1455                         RTE_PTYPE_L4_UDP,
1456                         RTE_PTYPE_UNKNOWN
1457                 };
1458
1459                 return ptypes;
1460         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1461                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1462                 static uint32_t ptypes_vec[] = {
1463                         /* refers to fm10k_desc_to_pktype_v() */
1464                         RTE_PTYPE_L3_IPV4,
1465                         RTE_PTYPE_L3_IPV4_EXT,
1466                         RTE_PTYPE_L3_IPV6,
1467                         RTE_PTYPE_L3_IPV6_EXT,
1468                         RTE_PTYPE_L4_TCP,
1469                         RTE_PTYPE_L4_UDP,
1470                         RTE_PTYPE_TUNNEL_GENEVE,
1471                         RTE_PTYPE_TUNNEL_NVGRE,
1472                         RTE_PTYPE_TUNNEL_VXLAN,
1473                         RTE_PTYPE_TUNNEL_GRE,
1474                         RTE_PTYPE_UNKNOWN
1475                 };
1476
1477                 return ptypes_vec;
1478         }
1479
1480         return NULL;
1481 }
1482 #else
1483 static const uint32_t *
1484 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1485 {
1486         return NULL;
1487 }
1488 #endif
1489
1490 static int
1491 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1492 {
1493         s32 result;
1494         uint16_t mac_num = 0;
1495         uint32_t vid_idx, vid_bit, mac_index;
1496         struct fm10k_hw *hw;
1497         struct fm10k_macvlan_filter_info *macvlan;
1498         struct rte_eth_dev_data *data = dev->data;
1499
1500         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1501         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1502
1503         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1504                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1505                 return -EINVAL;
1506         }
1507
1508         if (vlan_id > ETH_VLAN_ID_MAX) {
1509                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1510                 return -EINVAL;
1511         }
1512
1513         vid_idx = FM10K_VFTA_IDX(vlan_id);
1514         vid_bit = FM10K_VFTA_BIT(vlan_id);
1515         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1516         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1517                 return 0;
1518         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1519         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1520                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1521                         "in the VLAN filter table");
1522                 return -EINVAL;
1523         }
1524
1525         fm10k_mbx_lock(hw);
1526         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1527         fm10k_mbx_unlock(hw);
1528         if (result != FM10K_SUCCESS) {
1529                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1530                 return -EIO;
1531         }
1532
1533         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1534                         (result == FM10K_SUCCESS); mac_index++) {
1535                 if (rte_is_zero_ether_addr(&data->mac_addrs[mac_index]))
1536                         continue;
1537                 if (mac_num > macvlan->mac_num - 1) {
1538                         PMD_INIT_LOG(ERR, "MAC address number "
1539                                         "not match");
1540                         break;
1541                 }
1542                 fm10k_mbx_lock(hw);
1543                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1544                         data->mac_addrs[mac_index].addr_bytes,
1545                         vlan_id, on, 0);
1546                 fm10k_mbx_unlock(hw);
1547                 mac_num++;
1548         }
1549         if (result != FM10K_SUCCESS) {
1550                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1551                 return -EIO;
1552         }
1553
1554         if (on) {
1555                 macvlan->vlan_num++;
1556                 macvlan->vfta[vid_idx] |= vid_bit;
1557         } else {
1558                 macvlan->vlan_num--;
1559                 macvlan->vfta[vid_idx] &= ~vid_bit;
1560         }
1561         return 0;
1562 }
1563
1564 static int
1565 fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1566 {
1567         if (mask & ETH_VLAN_STRIP_MASK) {
1568                 if (!(dev->data->dev_conf.rxmode.offloads &
1569                         DEV_RX_OFFLOAD_VLAN_STRIP))
1570                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1571                                         "always on in fm10k");
1572         }
1573
1574         if (mask & ETH_VLAN_EXTEND_MASK) {
1575                 if (dev->data->dev_conf.rxmode.offloads &
1576                         DEV_RX_OFFLOAD_VLAN_EXTEND)
1577                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1578                                         "supported in fm10k");
1579         }
1580
1581         if (mask & ETH_VLAN_FILTER_MASK) {
1582                 if (!(dev->data->dev_conf.rxmode.offloads &
1583                         DEV_RX_OFFLOAD_VLAN_FILTER))
1584                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1585         }
1586
1587         return 0;
1588 }
1589
1590 /* Add/Remove a MAC address, and update filters to main VSI */
1591 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1592                 const u8 *mac, bool add, uint32_t pool)
1593 {
1594         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1595         struct fm10k_macvlan_filter_info *macvlan;
1596         uint32_t i, j, k;
1597
1598         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1599
1600         if (pool != MAIN_VSI_POOL_NUMBER) {
1601                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1602                         "mac to pool %u", pool);
1603                 return;
1604         }
1605         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1606                 if (!macvlan->vfta[j])
1607                         continue;
1608                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1609                         if (!(macvlan->vfta[j] & (1 << k)))
1610                                 continue;
1611                         if (i + 1 > macvlan->vlan_num) {
1612                                 PMD_INIT_LOG(ERR, "vlan number not match");
1613                                 return;
1614                         }
1615                         fm10k_mbx_lock(hw);
1616                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1617                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1618                         fm10k_mbx_unlock(hw);
1619                         i++;
1620                 }
1621         }
1622 }
1623
1624 /* Add/Remove a MAC address, and update filters to VMDQ */
1625 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1626                 const u8 *mac, bool add, uint32_t pool)
1627 {
1628         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1629         struct fm10k_macvlan_filter_info *macvlan;
1630         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1631         uint32_t i;
1632
1633         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1634         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1635
1636         if (pool > macvlan->nb_queue_pools) {
1637                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1638                         " Max pool is %u",
1639                         pool, macvlan->nb_queue_pools);
1640                 return;
1641         }
1642         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1643                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1644                         continue;
1645                 fm10k_mbx_lock(hw);
1646                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1647                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1648                 fm10k_mbx_unlock(hw);
1649         }
1650 }
1651
1652 /* Add/Remove a MAC address, and update filters */
1653 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1654                 const u8 *mac, bool add, uint32_t pool)
1655 {
1656         struct fm10k_macvlan_filter_info *macvlan;
1657
1658         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1659
1660         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1661                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1662         else
1663                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1664
1665         if (add)
1666                 macvlan->mac_num++;
1667         else
1668                 macvlan->mac_num--;
1669 }
1670
1671 /* Add a MAC address, and update filters */
1672 static int
1673 fm10k_macaddr_add(struct rte_eth_dev *dev,
1674                 struct rte_ether_addr *mac_addr,
1675                 uint32_t index,
1676                 uint32_t pool)
1677 {
1678         struct fm10k_macvlan_filter_info *macvlan;
1679
1680         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1681         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1682         macvlan->mac_vmdq_id[index] = pool;
1683         return 0;
1684 }
1685
1686 /* Remove a MAC address, and update filters */
1687 static void
1688 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1689 {
1690         struct rte_eth_dev_data *data = dev->data;
1691         struct fm10k_macvlan_filter_info *macvlan;
1692
1693         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1694         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1695                         FALSE, macvlan->mac_vmdq_id[index]);
1696         macvlan->mac_vmdq_id[index] = 0;
1697 }
1698
1699 static inline int
1700 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1701 {
1702         if ((request < min) || (request > max) || ((request % mult) != 0))
1703                 return -1;
1704         else
1705                 return 0;
1706 }
1707
1708
1709 static inline int
1710 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1711 {
1712         if ((request < min) || (request > max) || ((div % request) != 0))
1713                 return -1;
1714         else
1715                 return 0;
1716 }
1717
1718 static inline int
1719 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1720 {
1721         uint16_t rx_free_thresh;
1722
1723         if (conf->rx_free_thresh == 0)
1724                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1725         else
1726                 rx_free_thresh = conf->rx_free_thresh;
1727
1728         /* make sure the requested threshold satisfies the constraints */
1729         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1730                         FM10K_RX_FREE_THRESH_MAX(q),
1731                         FM10K_RX_FREE_THRESH_DIV(q),
1732                         rx_free_thresh)) {
1733                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1734                         "less than or equal to %u, "
1735                         "greater than or equal to %u, "
1736                         "and a divisor of %u",
1737                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1738                         FM10K_RX_FREE_THRESH_MIN(q),
1739                         FM10K_RX_FREE_THRESH_DIV(q));
1740                 return -EINVAL;
1741         }
1742
1743         q->alloc_thresh = rx_free_thresh;
1744         q->drop_en = conf->rx_drop_en;
1745         q->rx_deferred_start = conf->rx_deferred_start;
1746
1747         return 0;
1748 }
1749
1750 /*
1751  * Hardware requires specific alignment for Rx packet buffers. At
1752  * least one of the following two conditions must be satisfied.
1753  *  1. Address is 512B aligned
1754  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1755  *
1756  * As such, the driver may need to adjust the DMA address within the
1757  * buffer by up to 512B.
1758  *
1759  * return 1 if the element size is valid, otherwise return 0.
1760  */
1761 static int
1762 mempool_element_size_valid(struct rte_mempool *mp)
1763 {
1764         uint32_t min_size;
1765
1766         /* elt_size includes mbuf header and headroom */
1767         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1768                         RTE_PKTMBUF_HEADROOM;
1769
1770         /* account for up to 512B of alignment */
1771         min_size -= FM10K_RX_DATABUF_ALIGN;
1772
1773         /* sanity check for overflow */
1774         if (min_size > mp->elt_size)
1775                 return 0;
1776
1777         /* size is valid */
1778         return 1;
1779 }
1780
1781 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev)
1782 {
1783         RTE_SET_USED(dev);
1784
1785         return (uint64_t)(DEV_RX_OFFLOAD_SCATTER);
1786 }
1787
1788 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
1789 {
1790         RTE_SET_USED(dev);
1791
1792         return  (uint64_t)(DEV_RX_OFFLOAD_VLAN_STRIP  |
1793                            DEV_RX_OFFLOAD_VLAN_FILTER |
1794                            DEV_RX_OFFLOAD_IPV4_CKSUM  |
1795                            DEV_RX_OFFLOAD_UDP_CKSUM   |
1796                            DEV_RX_OFFLOAD_TCP_CKSUM   |
1797                            DEV_RX_OFFLOAD_JUMBO_FRAME |
1798                            DEV_RX_OFFLOAD_HEADER_SPLIT |
1799                            DEV_RX_OFFLOAD_RSS_HASH);
1800 }
1801
1802 static int
1803 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1804         uint16_t nb_desc, unsigned int socket_id,
1805         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1806 {
1807         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1808         struct fm10k_dev_info *dev_info =
1809                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1810         struct fm10k_rx_queue *q;
1811         const struct rte_memzone *mz;
1812         uint64_t offloads;
1813
1814         PMD_INIT_FUNC_TRACE();
1815
1816         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1817
1818         /* make sure the mempool element size can account for alignment. */
1819         if (!mempool_element_size_valid(mp)) {
1820                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1821                 return -EINVAL;
1822         }
1823
1824         /* make sure a valid number of descriptors have been requested */
1825         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1826                                 FM10K_MULT_RX_DESC, nb_desc)) {
1827                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1828                         "less than or equal to %"PRIu32", "
1829                         "greater than or equal to %u, "
1830                         "and a multiple of %u",
1831                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1832                         FM10K_MULT_RX_DESC);
1833                 return -EINVAL;
1834         }
1835
1836         /*
1837          * if this queue existed already, free the associated memory. The
1838          * queue cannot be reused in case we need to allocate memory on
1839          * different socket than was previously used.
1840          */
1841         if (dev->data->rx_queues[queue_id] != NULL) {
1842                 rx_queue_free(dev->data->rx_queues[queue_id]);
1843                 dev->data->rx_queues[queue_id] = NULL;
1844         }
1845
1846         /* allocate memory for the queue structure */
1847         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1848                                 socket_id);
1849         if (q == NULL) {
1850                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1851                 return -ENOMEM;
1852         }
1853
1854         /* setup queue */
1855         q->mp = mp;
1856         q->nb_desc = nb_desc;
1857         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1858         q->port_id = dev->data->port_id;
1859         q->queue_id = queue_id;
1860         q->tail_ptr = (volatile uint32_t *)
1861                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1862         q->offloads = offloads;
1863         if (handle_rxconf(q, conf))
1864                 return -EINVAL;
1865
1866         /* allocate memory for the software ring */
1867         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1868                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1869                         RTE_CACHE_LINE_SIZE, socket_id);
1870         if (q->sw_ring == NULL) {
1871                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1872                 rte_free(q);
1873                 return -ENOMEM;
1874         }
1875
1876         /*
1877          * allocate memory for the hardware descriptor ring. A memzone large
1878          * enough to hold the maximum ring size is requested to allow for
1879          * resizing in later calls to the queue setup function.
1880          */
1881         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1882                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1883                                       socket_id);
1884         if (mz == NULL) {
1885                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1886                 rte_free(q->sw_ring);
1887                 rte_free(q);
1888                 return -ENOMEM;
1889         }
1890         q->hw_ring = mz->addr;
1891         q->hw_ring_phys_addr = mz->iova;
1892
1893         /* Check if number of descs satisfied Vector requirement */
1894         if (!rte_is_power_of_2(nb_desc)) {
1895                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1896                                     "preconditions - canceling the feature for "
1897                                     "the whole port[%d]",
1898                              q->queue_id, q->port_id);
1899                 dev_info->rx_vec_allowed = false;
1900         } else
1901                 fm10k_rxq_vec_setup(q);
1902
1903         dev->data->rx_queues[queue_id] = q;
1904         return 0;
1905 }
1906
1907 static void
1908 fm10k_rx_queue_release(void *queue)
1909 {
1910         PMD_INIT_FUNC_TRACE();
1911
1912         rx_queue_free(queue);
1913 }
1914
1915 static inline int
1916 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1917 {
1918         uint16_t tx_free_thresh;
1919         uint16_t tx_rs_thresh;
1920
1921         /* constraint MACROs require that tx_free_thresh is configured
1922          * before tx_rs_thresh */
1923         if (conf->tx_free_thresh == 0)
1924                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1925         else
1926                 tx_free_thresh = conf->tx_free_thresh;
1927
1928         /* make sure the requested threshold satisfies the constraints */
1929         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1930                         FM10K_TX_FREE_THRESH_MAX(q),
1931                         FM10K_TX_FREE_THRESH_DIV(q),
1932                         tx_free_thresh)) {
1933                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1934                         "less than or equal to %u, "
1935                         "greater than or equal to %u, "
1936                         "and a divisor of %u",
1937                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1938                         FM10K_TX_FREE_THRESH_MIN(q),
1939                         FM10K_TX_FREE_THRESH_DIV(q));
1940                 return -EINVAL;
1941         }
1942
1943         q->free_thresh = tx_free_thresh;
1944
1945         if (conf->tx_rs_thresh == 0)
1946                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1947         else
1948                 tx_rs_thresh = conf->tx_rs_thresh;
1949
1950         q->tx_deferred_start = conf->tx_deferred_start;
1951
1952         /* make sure the requested threshold satisfies the constraints */
1953         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1954                         FM10K_TX_RS_THRESH_MAX(q),
1955                         FM10K_TX_RS_THRESH_DIV(q),
1956                         tx_rs_thresh)) {
1957                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1958                         "less than or equal to %u, "
1959                         "greater than or equal to %u, "
1960                         "and a divisor of %u",
1961                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1962                         FM10K_TX_RS_THRESH_MIN(q),
1963                         FM10K_TX_RS_THRESH_DIV(q));
1964                 return -EINVAL;
1965         }
1966
1967         q->rs_thresh = tx_rs_thresh;
1968
1969         return 0;
1970 }
1971
1972 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev)
1973 {
1974         RTE_SET_USED(dev);
1975
1976         return 0;
1977 }
1978
1979 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
1980 {
1981         RTE_SET_USED(dev);
1982
1983         return (uint64_t)(DEV_TX_OFFLOAD_VLAN_INSERT |
1984                           DEV_TX_OFFLOAD_MULTI_SEGS  |
1985                           DEV_TX_OFFLOAD_IPV4_CKSUM  |
1986                           DEV_TX_OFFLOAD_UDP_CKSUM   |
1987                           DEV_TX_OFFLOAD_TCP_CKSUM   |
1988                           DEV_TX_OFFLOAD_TCP_TSO);
1989 }
1990
1991 static int
1992 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1993         uint16_t nb_desc, unsigned int socket_id,
1994         const struct rte_eth_txconf *conf)
1995 {
1996         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1997         struct fm10k_tx_queue *q;
1998         const struct rte_memzone *mz;
1999         uint64_t offloads;
2000
2001         PMD_INIT_FUNC_TRACE();
2002
2003         offloads = conf->offloads | dev->data->dev_conf.txmode.offloads;
2004
2005         /* make sure a valid number of descriptors have been requested */
2006         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
2007                                 FM10K_MULT_TX_DESC, nb_desc)) {
2008                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
2009                         "less than or equal to %"PRIu32", "
2010                         "greater than or equal to %u, "
2011                         "and a multiple of %u",
2012                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
2013                         FM10K_MULT_TX_DESC);
2014                 return -EINVAL;
2015         }
2016
2017         /*
2018          * if this queue existed already, free the associated memory. The
2019          * queue cannot be reused in case we need to allocate memory on
2020          * different socket than was previously used.
2021          */
2022         if (dev->data->tx_queues[queue_id] != NULL) {
2023                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2024
2025                 tx_queue_free(txq);
2026                 dev->data->tx_queues[queue_id] = NULL;
2027         }
2028
2029         /* allocate memory for the queue structure */
2030         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2031                                 socket_id);
2032         if (q == NULL) {
2033                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2034                 return -ENOMEM;
2035         }
2036
2037         /* setup queue */
2038         q->nb_desc = nb_desc;
2039         q->port_id = dev->data->port_id;
2040         q->queue_id = queue_id;
2041         q->offloads = offloads;
2042         q->ops = &def_txq_ops;
2043         q->tail_ptr = (volatile uint32_t *)
2044                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2045         if (handle_txconf(q, conf))
2046                 return -EINVAL;
2047
2048         /* allocate memory for the software ring */
2049         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2050                                         nb_desc * sizeof(struct rte_mbuf *),
2051                                         RTE_CACHE_LINE_SIZE, socket_id);
2052         if (q->sw_ring == NULL) {
2053                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2054                 rte_free(q);
2055                 return -ENOMEM;
2056         }
2057
2058         /*
2059          * allocate memory for the hardware descriptor ring. A memzone large
2060          * enough to hold the maximum ring size is requested to allow for
2061          * resizing in later calls to the queue setup function.
2062          */
2063         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2064                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2065                                       socket_id);
2066         if (mz == NULL) {
2067                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2068                 rte_free(q->sw_ring);
2069                 rte_free(q);
2070                 return -ENOMEM;
2071         }
2072         q->hw_ring = mz->addr;
2073         q->hw_ring_phys_addr = mz->iova;
2074
2075         /*
2076          * allocate memory for the RS bit tracker. Enough slots to hold the
2077          * descriptor index for each RS bit needing to be set are required.
2078          */
2079         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2080                                 ((nb_desc + 1) / q->rs_thresh) *
2081                                 sizeof(uint16_t),
2082                                 RTE_CACHE_LINE_SIZE, socket_id);
2083         if (q->rs_tracker.list == NULL) {
2084                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2085                 rte_free(q->sw_ring);
2086                 rte_free(q);
2087                 return -ENOMEM;
2088         }
2089
2090         dev->data->tx_queues[queue_id] = q;
2091         return 0;
2092 }
2093
2094 static void
2095 fm10k_tx_queue_release(void *queue)
2096 {
2097         struct fm10k_tx_queue *q = queue;
2098         PMD_INIT_FUNC_TRACE();
2099
2100         tx_queue_free(q);
2101 }
2102
2103 static int
2104 fm10k_reta_update(struct rte_eth_dev *dev,
2105                         struct rte_eth_rss_reta_entry64 *reta_conf,
2106                         uint16_t reta_size)
2107 {
2108         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2109         uint16_t i, j, idx, shift;
2110         uint8_t mask;
2111         uint32_t reta;
2112
2113         PMD_INIT_FUNC_TRACE();
2114
2115         if (reta_size > FM10K_MAX_RSS_INDICES) {
2116                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2117                         "(%d) doesn't match the number hardware can supported "
2118                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2119                 return -EINVAL;
2120         }
2121
2122         /*
2123          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2124          * 128-entries in 32 registers
2125          */
2126         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2127                 idx = i / RTE_RETA_GROUP_SIZE;
2128                 shift = i % RTE_RETA_GROUP_SIZE;
2129                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2130                                 BIT_MASK_PER_UINT32);
2131                 if (mask == 0)
2132                         continue;
2133
2134                 reta = 0;
2135                 if (mask != BIT_MASK_PER_UINT32)
2136                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2137
2138                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2139                         if (mask & (0x1 << j)) {
2140                                 if (mask != 0xF)
2141                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2142                                 reta |= reta_conf[idx].reta[shift + j] <<
2143                                                 (CHAR_BIT * j);
2144                         }
2145                 }
2146                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2147         }
2148
2149         return 0;
2150 }
2151
2152 static int
2153 fm10k_reta_query(struct rte_eth_dev *dev,
2154                         struct rte_eth_rss_reta_entry64 *reta_conf,
2155                         uint16_t reta_size)
2156 {
2157         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2158         uint16_t i, j, idx, shift;
2159         uint8_t mask;
2160         uint32_t reta;
2161
2162         PMD_INIT_FUNC_TRACE();
2163
2164         if (reta_size < FM10K_MAX_RSS_INDICES) {
2165                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2166                         "(%d) doesn't match the number hardware can supported "
2167                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2168                 return -EINVAL;
2169         }
2170
2171         /*
2172          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2173          * 128-entries in 32 registers
2174          */
2175         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2176                 idx = i / RTE_RETA_GROUP_SIZE;
2177                 shift = i % RTE_RETA_GROUP_SIZE;
2178                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2179                                 BIT_MASK_PER_UINT32);
2180                 if (mask == 0)
2181                         continue;
2182
2183                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2184                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2185                         if (mask & (0x1 << j))
2186                                 reta_conf[idx].reta[shift + j] = ((reta >>
2187                                         CHAR_BIT * j) & UINT8_MAX);
2188                 }
2189         }
2190
2191         return 0;
2192 }
2193
2194 static int
2195 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2196         struct rte_eth_rss_conf *rss_conf)
2197 {
2198         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2199         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2200         uint32_t mrqc;
2201         uint64_t hf = rss_conf->rss_hf;
2202         int i;
2203
2204         PMD_INIT_FUNC_TRACE();
2205
2206         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2207                                 FM10K_RSSRK_ENTRIES_PER_REG))
2208                 return -EINVAL;
2209
2210         if (hf == 0)
2211                 return -EINVAL;
2212
2213         mrqc = 0;
2214         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2215         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2216         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2217         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2218         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2219         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2220         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2221         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2222         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2223
2224         /* If the mapping doesn't fit any supported, return */
2225         if (mrqc == 0)
2226                 return -EINVAL;
2227
2228         if (key != NULL)
2229                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2230                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2231
2232         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2233
2234         return 0;
2235 }
2236
2237 static int
2238 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2239         struct rte_eth_rss_conf *rss_conf)
2240 {
2241         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2242         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2243         uint32_t mrqc;
2244         uint64_t hf;
2245         int i;
2246
2247         PMD_INIT_FUNC_TRACE();
2248
2249         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2250                                 FM10K_RSSRK_ENTRIES_PER_REG))
2251                 return -EINVAL;
2252
2253         if (key != NULL)
2254                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2255                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2256
2257         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2258         hf = 0;
2259         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2260         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2261         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2262         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2263         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2264         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2265         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2266         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2267         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2268
2269         rss_conf->rss_hf = hf;
2270
2271         return 0;
2272 }
2273
2274 static void
2275 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2276 {
2277         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2278         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2279
2280         /* Bind all local non-queue interrupt to vector 0 */
2281         int_map |= FM10K_MISC_VEC_ID;
2282
2283         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2284         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2285         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2286         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2287         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2288         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2289
2290         /* Enable misc causes */
2291         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2292                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2293                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2294                                 FM10K_EIMR_ENABLE(MAILBOX) |
2295                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2296                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2297                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2298                                 FM10K_EIMR_ENABLE(VFLR));
2299
2300         /* Enable ITR 0 */
2301         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2302                                         FM10K_ITR_MASK_CLEAR);
2303         FM10K_WRITE_FLUSH(hw);
2304 }
2305
2306 static void
2307 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2308 {
2309         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2310         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2311
2312         int_map |= FM10K_MISC_VEC_ID;
2313
2314         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2315         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2316         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2317         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2318         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2319         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2320
2321         /* Disable misc causes */
2322         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2323                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2324                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2325                                 FM10K_EIMR_DISABLE(MAILBOX) |
2326                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2327                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2328                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2329                                 FM10K_EIMR_DISABLE(VFLR));
2330
2331         /* Disable ITR 0 */
2332         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2333         FM10K_WRITE_FLUSH(hw);
2334 }
2335
2336 static void
2337 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2338 {
2339         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2340         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2341
2342         /* Bind all local non-queue interrupt to vector 0 */
2343         int_map |= FM10K_MISC_VEC_ID;
2344
2345         /* Only INT 0 available, other 15 are reserved. */
2346         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2347
2348         /* Enable ITR 0 */
2349         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2350                                         FM10K_ITR_MASK_CLEAR);
2351         FM10K_WRITE_FLUSH(hw);
2352 }
2353
2354 static void
2355 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2356 {
2357         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2358         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2359
2360         int_map |= FM10K_MISC_VEC_ID;
2361
2362         /* Only INT 0 available, other 15 are reserved. */
2363         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2364
2365         /* Disable ITR 0 */
2366         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2367         FM10K_WRITE_FLUSH(hw);
2368 }
2369
2370 static int
2371 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2372 {
2373         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2374         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2375
2376         /* Enable ITR */
2377         if (hw->mac.type == fm10k_mac_pf)
2378                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2379                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2380         else
2381                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2382                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2383         rte_intr_ack(&pdev->intr_handle);
2384         return 0;
2385 }
2386
2387 static int
2388 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2389 {
2390         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2391         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2392
2393         /* Disable ITR */
2394         if (hw->mac.type == fm10k_mac_pf)
2395                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2396                         FM10K_ITR_MASK_SET);
2397         else
2398                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2399                         FM10K_ITR_MASK_SET);
2400         return 0;
2401 }
2402
2403 static int
2404 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2405 {
2406         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2407         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2408         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2409         uint32_t intr_vector, vec;
2410         uint16_t queue_id;
2411         int result = 0;
2412
2413         /* fm10k needs one separate interrupt for mailbox,
2414          * so only drivers which support multiple interrupt vectors
2415          * e.g. vfio-pci can work for fm10k interrupt mode
2416          */
2417         if (!rte_intr_cap_multiple(intr_handle) ||
2418                         dev->data->dev_conf.intr_conf.rxq == 0)
2419                 return result;
2420
2421         intr_vector = dev->data->nb_rx_queues;
2422
2423         /* disable interrupt first */
2424         rte_intr_disable(intr_handle);
2425         if (hw->mac.type == fm10k_mac_pf)
2426                 fm10k_dev_disable_intr_pf(dev);
2427         else
2428                 fm10k_dev_disable_intr_vf(dev);
2429
2430         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2431                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2432                 result = -EIO;
2433         }
2434
2435         if (rte_intr_dp_is_en(intr_handle) && !result) {
2436                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2437                         dev->data->nb_rx_queues * sizeof(int), 0);
2438                 if (intr_handle->intr_vec) {
2439                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2440                                         queue_id < dev->data->nb_rx_queues;
2441                                         queue_id++) {
2442                                 intr_handle->intr_vec[queue_id] = vec;
2443                                 if (vec < intr_handle->nb_efd - 1
2444                                                 + FM10K_RX_VEC_START)
2445                                         vec++;
2446                         }
2447                 } else {
2448                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2449                                 " intr_vec", dev->data->nb_rx_queues);
2450                         rte_intr_efd_disable(intr_handle);
2451                         result = -ENOMEM;
2452                 }
2453         }
2454
2455         if (hw->mac.type == fm10k_mac_pf)
2456                 fm10k_dev_enable_intr_pf(dev);
2457         else
2458                 fm10k_dev_enable_intr_vf(dev);
2459         rte_intr_enable(intr_handle);
2460         hw->mac.ops.update_int_moderator(hw);
2461         return result;
2462 }
2463
2464 static int
2465 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2466 {
2467         struct fm10k_fault fault;
2468         int err;
2469         const char *estr = "Unknown error";
2470
2471         /* Process PCA fault */
2472         if (eicr & FM10K_EICR_PCA_FAULT) {
2473                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2474                 if (err)
2475                         goto error;
2476                 switch (fault.type) {
2477                 case PCA_NO_FAULT:
2478                         estr = "PCA_NO_FAULT"; break;
2479                 case PCA_UNMAPPED_ADDR:
2480                         estr = "PCA_UNMAPPED_ADDR"; break;
2481                 case PCA_BAD_QACCESS_PF:
2482                         estr = "PCA_BAD_QACCESS_PF"; break;
2483                 case PCA_BAD_QACCESS_VF:
2484                         estr = "PCA_BAD_QACCESS_VF"; break;
2485                 case PCA_MALICIOUS_REQ:
2486                         estr = "PCA_MALICIOUS_REQ"; break;
2487                 case PCA_POISONED_TLP:
2488                         estr = "PCA_POISONED_TLP"; break;
2489                 case PCA_TLP_ABORT:
2490                         estr = "PCA_TLP_ABORT"; break;
2491                 default:
2492                         goto error;
2493                 }
2494                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2495                         estr, fault.func ? "VF" : "PF", fault.func,
2496                         fault.address, fault.specinfo);
2497         }
2498
2499         /* Process THI fault */
2500         if (eicr & FM10K_EICR_THI_FAULT) {
2501                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2502                 if (err)
2503                         goto error;
2504                 switch (fault.type) {
2505                 case THI_NO_FAULT:
2506                         estr = "THI_NO_FAULT"; break;
2507                 case THI_MAL_DIS_Q_FAULT:
2508                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2509                 default:
2510                         goto error;
2511                 }
2512                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2513                         estr, fault.func ? "VF" : "PF", fault.func,
2514                         fault.address, fault.specinfo);
2515         }
2516
2517         /* Process FUM fault */
2518         if (eicr & FM10K_EICR_FUM_FAULT) {
2519                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2520                 if (err)
2521                         goto error;
2522                 switch (fault.type) {
2523                 case FUM_NO_FAULT:
2524                         estr = "FUM_NO_FAULT"; break;
2525                 case FUM_UNMAPPED_ADDR:
2526                         estr = "FUM_UNMAPPED_ADDR"; break;
2527                 case FUM_POISONED_TLP:
2528                         estr = "FUM_POISONED_TLP"; break;
2529                 case FUM_BAD_VF_QACCESS:
2530                         estr = "FUM_BAD_VF_QACCESS"; break;
2531                 case FUM_ADD_DECODE_ERR:
2532                         estr = "FUM_ADD_DECODE_ERR"; break;
2533                 case FUM_RO_ERROR:
2534                         estr = "FUM_RO_ERROR"; break;
2535                 case FUM_QPRC_CRC_ERROR:
2536                         estr = "FUM_QPRC_CRC_ERROR"; break;
2537                 case FUM_CSR_TIMEOUT:
2538                         estr = "FUM_CSR_TIMEOUT"; break;
2539                 case FUM_INVALID_TYPE:
2540                         estr = "FUM_INVALID_TYPE"; break;
2541                 case FUM_INVALID_LENGTH:
2542                         estr = "FUM_INVALID_LENGTH"; break;
2543                 case FUM_INVALID_BE:
2544                         estr = "FUM_INVALID_BE"; break;
2545                 case FUM_INVALID_ALIGN:
2546                         estr = "FUM_INVALID_ALIGN"; break;
2547                 default:
2548                         goto error;
2549                 }
2550                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2551                         estr, fault.func ? "VF" : "PF", fault.func,
2552                         fault.address, fault.specinfo);
2553         }
2554
2555         return 0;
2556 error:
2557         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2558         return err;
2559 }
2560
2561 /**
2562  * PF interrupt handler triggered by NIC for handling specific interrupt.
2563  *
2564  * @param handle
2565  *  Pointer to interrupt handle.
2566  * @param param
2567  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2568  *
2569  * @return
2570  *  void
2571  */
2572 static void
2573 fm10k_dev_interrupt_handler_pf(void *param)
2574 {
2575         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2576         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2577         uint32_t cause, status;
2578         struct fm10k_dev_info *dev_info =
2579                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2580         int status_mbx;
2581         s32 err;
2582
2583         if (hw->mac.type != fm10k_mac_pf)
2584                 return;
2585
2586         cause = FM10K_READ_REG(hw, FM10K_EICR);
2587
2588         /* Handle PCI fault cases */
2589         if (cause & FM10K_EICR_FAULT_MASK) {
2590                 PMD_INIT_LOG(ERR, "INT: find fault!");
2591                 fm10k_dev_handle_fault(hw, cause);
2592         }
2593
2594         /* Handle switch up/down */
2595         if (cause & FM10K_EICR_SWITCHNOTREADY)
2596                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2597
2598         if (cause & FM10K_EICR_SWITCHREADY) {
2599                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2600                 if (dev_info->sm_down == 1) {
2601                         fm10k_mbx_lock(hw);
2602
2603                         /* For recreating logical ports */
2604                         status_mbx = hw->mac.ops.update_lport_state(hw,
2605                                         hw->mac.dglort_map, MAX_LPORT_NUM, 1);
2606                         if (status_mbx == FM10K_SUCCESS)
2607                                 PMD_INIT_LOG(INFO,
2608                                         "INT: Recreated Logical port");
2609                         else
2610                                 PMD_INIT_LOG(INFO,
2611                                         "INT: Logical ports weren't recreated");
2612
2613                         status_mbx = hw->mac.ops.update_xcast_mode(hw,
2614                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2615                         if (status_mbx != FM10K_SUCCESS)
2616                                 PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2617
2618                         fm10k_mbx_unlock(hw);
2619
2620                         /* first clear the internal SW recording structure */
2621                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2622                                                 ETH_MQ_RX_VMDQ_FLAG))
2623                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2624                                         false);
2625
2626                         fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2627                                         MAIN_VSI_POOL_NUMBER);
2628
2629                         /*
2630                          * Add default mac address and vlan for the logical
2631                          * ports that have been created, leave to the
2632                          * application to fully recover Rx filtering.
2633                          */
2634                         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2635                                         MAIN_VSI_POOL_NUMBER);
2636
2637                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2638                                                 ETH_MQ_RX_VMDQ_FLAG))
2639                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2640                                         true);
2641
2642                         dev_info->sm_down = 0;
2643                         _rte_eth_dev_callback_process(dev,
2644                                         RTE_ETH_EVENT_INTR_LSC,
2645                                         NULL);
2646                 }
2647         }
2648
2649         /* Handle mailbox message */
2650         fm10k_mbx_lock(hw);
2651         err = hw->mbx.ops.process(hw, &hw->mbx);
2652         fm10k_mbx_unlock(hw);
2653
2654         if (err == FM10K_ERR_RESET_REQUESTED) {
2655                 PMD_INIT_LOG(INFO, "INT: Switch is down");
2656                 dev_info->sm_down = 1;
2657                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2658                                 NULL);
2659         }
2660
2661         /* Handle SRAM error */
2662         if (cause & FM10K_EICR_SRAMERROR) {
2663                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2664
2665                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2666                 /* Write to clear pending bits */
2667                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2668
2669                 /* Todo: print out error message after shared code  updates */
2670         }
2671
2672         /* Clear these 3 events if having any */
2673         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2674                  FM10K_EICR_SWITCHREADY;
2675         if (cause)
2676                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2677
2678         /* Re-enable interrupt from device side */
2679         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2680                                         FM10K_ITR_MASK_CLEAR);
2681         /* Re-enable interrupt from host side */
2682         rte_intr_ack(dev->intr_handle);
2683 }
2684
2685 /**
2686  * VF interrupt handler triggered by NIC for handling specific interrupt.
2687  *
2688  * @param handle
2689  *  Pointer to interrupt handle.
2690  * @param param
2691  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2692  *
2693  * @return
2694  *  void
2695  */
2696 static void
2697 fm10k_dev_interrupt_handler_vf(void *param)
2698 {
2699         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2700         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2701         struct fm10k_mbx_info *mbx = &hw->mbx;
2702         struct fm10k_dev_info *dev_info =
2703                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2704         const enum fm10k_mbx_state state = mbx->state;
2705         int status_mbx;
2706
2707         if (hw->mac.type != fm10k_mac_vf)
2708                 return;
2709
2710         /* Handle mailbox message if lock is acquired */
2711         fm10k_mbx_lock(hw);
2712         hw->mbx.ops.process(hw, &hw->mbx);
2713         fm10k_mbx_unlock(hw);
2714
2715         if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) {
2716                 PMD_INIT_LOG(INFO, "INT: Switch has gone down");
2717
2718                 fm10k_mbx_lock(hw);
2719                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2720                                 MAX_LPORT_NUM, 1);
2721                 fm10k_mbx_unlock(hw);
2722
2723                 /* Setting reset flag */
2724                 dev_info->sm_down = 1;
2725                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2726                                 NULL);
2727         }
2728
2729         if (dev_info->sm_down == 1 &&
2730                         hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) {
2731                 PMD_INIT_LOG(INFO, "INT: Switch has gone up");
2732                 fm10k_mbx_lock(hw);
2733                 status_mbx = hw->mac.ops.update_xcast_mode(hw,
2734                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2735                 if (status_mbx != FM10K_SUCCESS)
2736                         PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2737                 fm10k_mbx_unlock(hw);
2738
2739                 /* first clear the internal SW recording structure */
2740                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false);
2741                 fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2742                                 MAIN_VSI_POOL_NUMBER);
2743
2744                 /*
2745                  * Add default mac address and vlan for the logical ports that
2746                  * have been created, leave to the application to fully recover
2747                  * Rx filtering.
2748                  */
2749                 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2750                                 MAIN_VSI_POOL_NUMBER);
2751                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
2752
2753                 dev_info->sm_down = 0;
2754                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2755                                 NULL);
2756         }
2757
2758         /* Re-enable interrupt from device side */
2759         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2760                                         FM10K_ITR_MASK_CLEAR);
2761         /* Re-enable interrupt from host side */
2762         rte_intr_ack(dev->intr_handle);
2763 }
2764
2765 /* Mailbox message handler in VF */
2766 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2767         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2768         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2769         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2770         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2771 };
2772
2773 static int
2774 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2775 {
2776         int err = 0;
2777
2778         /* Initialize mailbox lock */
2779         fm10k_mbx_initlock(hw);
2780
2781         /* Replace default message handler with new ones */
2782         if (hw->mac.type == fm10k_mac_vf)
2783                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2784
2785         if (err) {
2786                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2787                                 err);
2788                 return err;
2789         }
2790         /* Connect to SM for PF device or PF for VF device */
2791         return hw->mbx.ops.connect(hw, &hw->mbx);
2792 }
2793
2794 static void
2795 fm10k_close_mbx_service(struct fm10k_hw *hw)
2796 {
2797         /* Disconnect from SM for PF device or PF for VF device */
2798         hw->mbx.ops.disconnect(hw, &hw->mbx);
2799 }
2800
2801 static void
2802 fm10k_dev_close(struct rte_eth_dev *dev)
2803 {
2804         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2805         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2806         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2807
2808         PMD_INIT_FUNC_TRACE();
2809
2810         fm10k_mbx_lock(hw);
2811         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2812                 MAX_LPORT_NUM, false);
2813         fm10k_mbx_unlock(hw);
2814
2815         /* allow 100ms for device to quiesce */
2816         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
2817
2818         /* Stop mailbox service first */
2819         fm10k_close_mbx_service(hw);
2820         fm10k_dev_stop(dev);
2821         fm10k_dev_queue_release(dev);
2822         fm10k_stop_hw(hw);
2823
2824         dev->dev_ops = NULL;
2825         dev->rx_pkt_burst = NULL;
2826         dev->tx_pkt_burst = NULL;
2827
2828         /* disable uio/vfio intr */
2829         rte_intr_disable(intr_handle);
2830
2831         /*PF/VF has different interrupt handling mechanism */
2832         if (hw->mac.type == fm10k_mac_pf) {
2833                 /* disable interrupt */
2834                 fm10k_dev_disable_intr_pf(dev);
2835
2836                 /* unregister callback func to eal lib */
2837                 rte_intr_callback_unregister(intr_handle,
2838                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2839         } else {
2840                 /* disable interrupt */
2841                 fm10k_dev_disable_intr_vf(dev);
2842
2843                 rte_intr_callback_unregister(intr_handle,
2844                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2845         }
2846 }
2847
2848 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2849         .dev_configure          = fm10k_dev_configure,
2850         .dev_start              = fm10k_dev_start,
2851         .dev_stop               = fm10k_dev_stop,
2852         .dev_close              = fm10k_dev_close,
2853         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2854         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2855         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2856         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2857         .stats_get              = fm10k_stats_get,
2858         .xstats_get             = fm10k_xstats_get,
2859         .xstats_get_names       = fm10k_xstats_get_names,
2860         .stats_reset            = fm10k_stats_reset,
2861         .xstats_reset           = fm10k_stats_reset,
2862         .link_update            = fm10k_link_update,
2863         .dev_infos_get          = fm10k_dev_infos_get,
2864         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2865         .vlan_filter_set        = fm10k_vlan_filter_set,
2866         .vlan_offload_set       = fm10k_vlan_offload_set,
2867         .mac_addr_add           = fm10k_macaddr_add,
2868         .mac_addr_remove        = fm10k_macaddr_remove,
2869         .rx_queue_start         = fm10k_dev_rx_queue_start,
2870         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2871         .tx_queue_start         = fm10k_dev_tx_queue_start,
2872         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2873         .rx_queue_setup         = fm10k_rx_queue_setup,
2874         .rx_queue_release       = fm10k_rx_queue_release,
2875         .tx_queue_setup         = fm10k_tx_queue_setup,
2876         .tx_queue_release       = fm10k_tx_queue_release,
2877         .rx_queue_count         = fm10k_dev_rx_queue_count,
2878         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2879         .rx_descriptor_status = fm10k_dev_rx_descriptor_status,
2880         .tx_descriptor_status = fm10k_dev_tx_descriptor_status,
2881         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2882         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2883         .reta_update            = fm10k_reta_update,
2884         .reta_query             = fm10k_reta_query,
2885         .rss_hash_update        = fm10k_rss_hash_update,
2886         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2887 };
2888
2889 static int ftag_check_handler(__rte_unused const char *key,
2890                 const char *value, __rte_unused void *opaque)
2891 {
2892         if (strcmp(value, "1"))
2893                 return -1;
2894
2895         return 0;
2896 }
2897
2898 static int
2899 fm10k_check_ftag(struct rte_devargs *devargs)
2900 {
2901         struct rte_kvargs *kvlist;
2902         const char *ftag_key = "enable_ftag";
2903
2904         if (devargs == NULL)
2905                 return 0;
2906
2907         kvlist = rte_kvargs_parse(devargs->args, NULL);
2908         if (kvlist == NULL)
2909                 return 0;
2910
2911         if (!rte_kvargs_count(kvlist, ftag_key)) {
2912                 rte_kvargs_free(kvlist);
2913                 return 0;
2914         }
2915         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2916         if (rte_kvargs_process(kvlist, ftag_key,
2917                                 ftag_check_handler, NULL) < 0) {
2918                 rte_kvargs_free(kvlist);
2919                 return 0;
2920         }
2921         rte_kvargs_free(kvlist);
2922
2923         return 1;
2924 }
2925
2926 static uint16_t
2927 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2928                     uint16_t nb_pkts)
2929 {
2930         uint16_t nb_tx = 0;
2931         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2932
2933         while (nb_pkts) {
2934                 uint16_t ret, num;
2935
2936                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2937                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2938                                                  num);
2939                 nb_tx += ret;
2940                 nb_pkts -= ret;
2941                 if (ret < num)
2942                         break;
2943         }
2944
2945         return nb_tx;
2946 }
2947
2948 static void __rte_cold
2949 fm10k_set_tx_function(struct rte_eth_dev *dev)
2950 {
2951         struct fm10k_tx_queue *txq;
2952         int i;
2953         int use_sse = 1;
2954         uint16_t tx_ftag_en = 0;
2955
2956         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2957                 /* primary process has set the ftag flag and offloads */
2958                 txq = dev->data->tx_queues[0];
2959                 if (fm10k_tx_vec_condition_check(txq)) {
2960                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2961                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2962                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2963                 } else {
2964                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2965                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2966                         dev->tx_pkt_prepare = NULL;
2967                 }
2968                 return;
2969         }
2970
2971         if (fm10k_check_ftag(dev->device->devargs))
2972                 tx_ftag_en = 1;
2973
2974         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2975                 txq = dev->data->tx_queues[i];
2976                 txq->tx_ftag_en = tx_ftag_en;
2977                 /* Check if Vector Tx is satisfied */
2978                 if (fm10k_tx_vec_condition_check(txq))
2979                         use_sse = 0;
2980         }
2981
2982         if (use_sse) {
2983                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2984                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2985                         txq = dev->data->tx_queues[i];
2986                         fm10k_txq_vec_setup(txq);
2987                 }
2988                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2989                 dev->tx_pkt_prepare = NULL;
2990         } else {
2991                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2992                 dev->tx_pkt_prepare = fm10k_prep_pkts;
2993                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2994         }
2995 }
2996
2997 static void __rte_cold
2998 fm10k_set_rx_function(struct rte_eth_dev *dev)
2999 {
3000         struct fm10k_dev_info *dev_info =
3001                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3002         uint16_t i, rx_using_sse;
3003         uint16_t rx_ftag_en = 0;
3004
3005         if (fm10k_check_ftag(dev->device->devargs))
3006                 rx_ftag_en = 1;
3007
3008         /* In order to allow Vector Rx there are a few configuration
3009          * conditions to be met.
3010          */
3011         if (!fm10k_rx_vec_condition_check(dev) &&
3012                         dev_info->rx_vec_allowed && !rx_ftag_en) {
3013                 if (dev->data->scattered_rx)
3014                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
3015                 else
3016                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
3017         } else if (dev->data->scattered_rx)
3018                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
3019         else
3020                 dev->rx_pkt_burst = fm10k_recv_pkts;
3021
3022         rx_using_sse =
3023                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
3024                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
3025
3026         if (rx_using_sse)
3027                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
3028         else
3029                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
3030
3031         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3032                 return;
3033
3034         for (i = 0; i < dev->data->nb_rx_queues; i++) {
3035                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
3036
3037                 rxq->rx_using_sse = rx_using_sse;
3038                 rxq->rx_ftag_en = rx_ftag_en;
3039         }
3040 }
3041
3042 static void
3043 fm10k_params_init(struct rte_eth_dev *dev)
3044 {
3045         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3046         struct fm10k_dev_info *info =
3047                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3048
3049         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
3050          * there is no way to get link status without reading BAR4.  Until this
3051          * works, assume we have maximum bandwidth.
3052          * @todo - fix bus info
3053          */
3054         hw->bus_caps.speed = fm10k_bus_speed_8000;
3055         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
3056         hw->bus_caps.payload = fm10k_bus_payload_512;
3057         hw->bus.speed = fm10k_bus_speed_8000;
3058         hw->bus.width = fm10k_bus_width_pcie_x8;
3059         hw->bus.payload = fm10k_bus_payload_256;
3060
3061         info->rx_vec_allowed = true;
3062         info->sm_down = false;
3063 }
3064
3065 static int
3066 eth_fm10k_dev_init(struct rte_eth_dev *dev)
3067 {
3068         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3069         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3070         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3071         int diag, i;
3072         struct fm10k_macvlan_filter_info *macvlan;
3073
3074         PMD_INIT_FUNC_TRACE();
3075
3076         dev->dev_ops = &fm10k_eth_dev_ops;
3077         dev->rx_pkt_burst = &fm10k_recv_pkts;
3078         dev->tx_pkt_burst = &fm10k_xmit_pkts;
3079         dev->tx_pkt_prepare = &fm10k_prep_pkts;
3080
3081         /*
3082          * Primary process does the whole initialization, for secondary
3083          * processes, we just select the same Rx and Tx function as primary.
3084          */
3085         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3086                 fm10k_set_rx_function(dev);
3087                 fm10k_set_tx_function(dev);
3088                 return 0;
3089         }
3090
3091         rte_eth_copy_pci_info(dev, pdev);
3092
3093         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
3094         memset(macvlan, 0, sizeof(*macvlan));
3095         /* Vendor and Device ID need to be set before init of shared code */
3096         memset(hw, 0, sizeof(*hw));
3097         hw->device_id = pdev->id.device_id;
3098         hw->vendor_id = pdev->id.vendor_id;
3099         hw->subsystem_device_id = pdev->id.subsystem_device_id;
3100         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
3101         hw->revision_id = 0;
3102         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
3103         if (hw->hw_addr == NULL) {
3104                 PMD_INIT_LOG(ERR, "Bad mem resource."
3105                         " Try to blacklist unused devices.");
3106                 return -EIO;
3107         }
3108
3109         /* Store fm10k_adapter pointer */
3110         hw->back = dev->data->dev_private;
3111
3112         /* Initialize the shared code */
3113         diag = fm10k_init_shared_code(hw);
3114         if (diag != FM10K_SUCCESS) {
3115                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
3116                 return -EIO;
3117         }
3118
3119         /* Initialize parameters */
3120         fm10k_params_init(dev);
3121
3122         /* Initialize the hw */
3123         diag = fm10k_init_hw(hw);
3124         if (diag != FM10K_SUCCESS) {
3125                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
3126                 return -EIO;
3127         }
3128
3129         /* Initialize MAC address(es) */
3130         dev->data->mac_addrs = rte_zmalloc("fm10k",
3131                         RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
3132         if (dev->data->mac_addrs == NULL) {
3133                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
3134                 return -ENOMEM;
3135         }
3136
3137         diag = fm10k_read_mac_addr(hw);
3138
3139         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3140                         &dev->data->mac_addrs[0]);
3141
3142         if (diag != FM10K_SUCCESS ||
3143                 !rte_is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
3144
3145                 /* Generate a random addr */
3146                 rte_eth_random_addr(hw->mac.addr);
3147                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
3148                 rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3149                 &dev->data->mac_addrs[0]);
3150         }
3151
3152         /* Pass the information to the rte_eth_dev_close() that it should also
3153          * release the private port resources.
3154          */
3155         dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
3156
3157         /* Reset the hw statistics */
3158         diag = fm10k_stats_reset(dev);
3159         if (diag != 0) {
3160                 PMD_INIT_LOG(ERR, "Stats reset failed: %d", diag);
3161                 return diag;
3162         }
3163
3164         /* Reset the hw */
3165         diag = fm10k_reset_hw(hw);
3166         if (diag != FM10K_SUCCESS) {
3167                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
3168                 return -EIO;
3169         }
3170
3171         /* Setup mailbox service */
3172         diag = fm10k_setup_mbx_service(hw);
3173         if (diag != FM10K_SUCCESS) {
3174                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
3175                 return -EIO;
3176         }
3177
3178         /*PF/VF has different interrupt handling mechanism */
3179         if (hw->mac.type == fm10k_mac_pf) {
3180                 /* register callback func to eal lib */
3181                 rte_intr_callback_register(intr_handle,
3182                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3183
3184                 /* enable MISC interrupt */
3185                 fm10k_dev_enable_intr_pf(dev);
3186         } else { /* VF */
3187                 rte_intr_callback_register(intr_handle,
3188                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3189
3190                 fm10k_dev_enable_intr_vf(dev);
3191         }
3192
3193         /* Enable intr after callback registered */
3194         rte_intr_enable(intr_handle);
3195
3196         hw->mac.ops.update_int_moderator(hw);
3197
3198         /* Make sure Switch Manager is ready before going forward. */
3199         if (hw->mac.type == fm10k_mac_pf) {
3200                 bool switch_ready = false;
3201
3202                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3203                         fm10k_mbx_lock(hw);
3204                         hw->mac.ops.get_host_state(hw, &switch_ready);
3205                         fm10k_mbx_unlock(hw);
3206                         if (switch_ready == true)
3207                                 break;
3208                         /* Delay some time to acquire async LPORT_MAP info. */
3209                         rte_delay_us(WAIT_SWITCH_MSG_US);
3210                 }
3211
3212                 if (switch_ready == false) {
3213                         PMD_INIT_LOG(ERR, "switch is not ready");
3214                         return -1;
3215                 }
3216         }
3217
3218         /*
3219          * Below function will trigger operations on mailbox, acquire lock to
3220          * avoid race condition from interrupt handler. Operations on mailbox
3221          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3222          * will handle and generate an interrupt to our side. Then,  FIFO in
3223          * mailbox will be touched.
3224          */
3225         fm10k_mbx_lock(hw);
3226         /* Enable port first */
3227         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3228                                         MAX_LPORT_NUM, 1);
3229
3230         /* Set unicast mode by default. App can change to other mode in other
3231          * API func.
3232          */
3233         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3234                                         FM10K_XCAST_MODE_NONE);
3235
3236         fm10k_mbx_unlock(hw);
3237
3238         /* Make sure default VID is ready before going forward. */
3239         if (hw->mac.type == fm10k_mac_pf) {
3240                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3241                         if (hw->mac.default_vid)
3242                                 break;
3243                         /* Delay some time to acquire async port VLAN info. */
3244                         rte_delay_us(WAIT_SWITCH_MSG_US);
3245                 }
3246
3247                 if (!hw->mac.default_vid) {
3248                         PMD_INIT_LOG(ERR, "default VID is not ready");
3249                         return -1;
3250                 }
3251         }
3252
3253         /* Add default mac address */
3254         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3255                 MAIN_VSI_POOL_NUMBER);
3256
3257         return 0;
3258 }
3259
3260 static int
3261 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3262 {
3263         PMD_INIT_FUNC_TRACE();
3264
3265         /* only uninitialize in the primary process */
3266         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3267                 return 0;
3268
3269         /* safe to close dev here */
3270         fm10k_dev_close(dev);
3271
3272         return 0;
3273 }
3274
3275 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3276         struct rte_pci_device *pci_dev)
3277 {
3278         return rte_eth_dev_pci_generic_probe(pci_dev,
3279                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3280 }
3281
3282 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3283 {
3284         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3285 }
3286
3287 /*
3288  * The set of PCI devices this driver supports. This driver will enable both PF
3289  * and SRIOV-VF devices.
3290  */
3291 static const struct rte_pci_id pci_id_fm10k_map[] = {
3292         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3293         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3294         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3295         { .vendor_id = 0, /* sentinel */ },
3296 };
3297
3298 static struct rte_pci_driver rte_pmd_fm10k = {
3299         .id_table = pci_id_fm10k_map,
3300         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
3301         .probe = eth_fm10k_pci_probe,
3302         .remove = eth_fm10k_pci_remove,
3303 };
3304
3305 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3306 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3307 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");
3308 RTE_LOG_REGISTER(fm10k_logtype_init, pmd.net.fm10k.init, NOTICE);
3309 RTE_LOG_REGISTER(fm10k_logtype_driver, pmd.net.fm10k.driver, NOTICE);
3310 #ifdef RTE_LIBRTE_FM10K_DEBUG_RX
3311 RTE_LOG_REGISTER(fm10k_logtype_rx, pmd.net.fm10k.rx, DEBUG);
3312 #endif
3313 #ifdef RTE_LIBRTE_FM10K_DEBUG_TX
3314 RTE_LOG_REGISTER(fm10k_logtype_tx, pmd.net.fm10k.tx, DEBUG);
3315 #endif
3316 #ifdef RTE_LIBRTE_FM10K_DEBUG_TX_FREE
3317 RTE_LOG_REGISTER(fm10k_logtype_tx_free, pmd.net.fm10k.tx_free, DEBUG);
3318 #endif