drivers/net/fm10k/fm10k_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2013-2016 Intel Corporation
   3  */
   4
   5 #include <ethdev_driver.h>
   6 #include <ethdev_pci.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_string_fns.h>
  10 #include <rte_dev.h>
  11 #include <rte_spinlock.h>
  12 #include <rte_kvargs.h>
  13 #include <rte_vect.h>
  14
  15 #include "fm10k.h"
  16 #include "base/fm10k_api.h"
  17
  18 /* Default delay to acquire mailbox lock */
  19 #define FM10K_MBXLOCK_DELAY_US 20
  20 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  21
  22 #define MAIN_VSI_POOL_NUMBER 0
  23
  24 /* Max try times to acquire switch status */
  25 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  26 /* Wait interval to get switch status */
  27 #define WAIT_SWITCH_MSG_US    100000
  28 /* A period of quiescence for switch */
  29 #define FM10K_SWITCH_QUIESCE_US 100000
  30 /* Number of chars per uint32 type */
  31 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  32 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  33
  34 /* default 1:1 map from queue ID to interrupt vector ID */
  35 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  36
  37 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  38 #define MAX_LPORT_NUM    128
  39 #define GLORT_FD_Q_BASE  0x40
  40 #define GLORT_PF_MASK    0xFFC0
  41 #define GLORT_FD_MASK    GLORT_PF_MASK
  42 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  43
  44 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  45 static int fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  46 static int fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  47 static int fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  48 static int fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  49 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  50 static int
  51 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  52 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  53         const u8 *mac, bool add, uint32_t pool);
  54 static void fm10k_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid);
  55 static void fm10k_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid);
  56 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  57 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  58 static int fm10k_check_ftag(struct rte_devargs *devargs);
  59 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  60
  61 static int fm10k_dev_infos_get(struct rte_eth_dev *dev,
  62                                struct rte_eth_dev_info *dev_info);
  63 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev);
  64 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev);
  65 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev);
  66 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev);
  67
  68 struct fm10k_xstats_name_off {
  69         char name[RTE_ETH_XSTATS_NAME_SIZE];
  70         unsigned offset;
  71 };
  72
  73 static const struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  74         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  75         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  76         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  77         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  78         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  79         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  80         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  81         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  82                 nodesc_drop)},
  83 };
  84
  85 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  86                 sizeof(fm10k_hw_stats_strings[0]))
  87
  88 static const struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
  89         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
  90         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
  91         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
  92 };
  93
  94 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
  95                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
  96
  97 static const struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
  98         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
  99         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 100 };
 101
 102 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 103                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 104
 105 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 106                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 107 static int
 108 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 109
 110 static void
 111 fm10k_mbx_initlock(struct fm10k_hw *hw)
 112 {
 113         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 114 }
 115
 116 static void
 117 fm10k_mbx_lock(struct fm10k_hw *hw)
 118 {
 119         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 120                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 121 }
 122
 123 static void
 124 fm10k_mbx_unlock(struct fm10k_hw *hw)
 125 {
 126         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 127 }
 128
 129 /* Stubs needed for linkage when vPMD is disabled */
 130 __rte_weak int
 131 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 132 {
 133         return -1;
 134 }
 135
 136 __rte_weak uint16_t
 137 fm10k_recv_pkts_vec(
 138         __rte_unused void *rx_queue,
 139         __rte_unused struct rte_mbuf **rx_pkts,
 140         __rte_unused uint16_t nb_pkts)
 141 {
 142         return 0;
 143 }
 144
 145 __rte_weak uint16_t
 146 fm10k_recv_scattered_pkts_vec(
 147                 __rte_unused void *rx_queue,
 148                 __rte_unused struct rte_mbuf **rx_pkts,
 149                 __rte_unused uint16_t nb_pkts)
 150 {
 151         return 0;
 152 }
 153
 154 __rte_weak int
 155 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 156
 157 {
 158         return -1;
 159 }
 160
 161 __rte_weak void
 162 fm10k_rx_queue_release_mbufs_vec(
 163                 __rte_unused struct fm10k_rx_queue *rxq)
 164 {
 165         return;
 166 }
 167
 168 __rte_weak void
 169 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 170 {
 171         return;
 172 }
 173
 174 __rte_weak int
 175 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 176 {
 177         return -1;
 178 }
 179
 180 __rte_weak uint16_t
 181 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 182                            __rte_unused struct rte_mbuf **tx_pkts,
 183                            __rte_unused uint16_t nb_pkts)
 184 {
 185         return 0;
 186 }
 187
 188 /*
 189  * reset queue to initial state, allocate software buffers used when starting
 190  * device.
 191  * return 0 on success
 192  * return -ENOMEM if buffers cannot be allocated
 193  * return -EINVAL if buffers do not satisfy alignment condition
 194  */
 195 static inline int
 196 rx_queue_reset(struct fm10k_rx_queue *q)
 197 {
 198         static const union fm10k_rx_desc zero = {{0} };
 199         uint64_t dma_addr;
 200         int i, diag;
 201         PMD_INIT_FUNC_TRACE();
 202
 203         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 204         if (diag != 0)
 205                 return -ENOMEM;
 206
 207         for (i = 0; i < q->nb_desc; ++i) {
 208                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 209                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 210                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 211                                                 q->nb_desc);
 212                         return -EINVAL;
 213                 }
 214                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 215                 q->hw_ring[i].q.pkt_addr = dma_addr;
 216                 q->hw_ring[i].q.hdr_addr = dma_addr;
 217         }
 218
 219         /* initialize extra software ring entries. Space for these extra
 220          * entries is always allocated.
 221          */
 222         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 223         for (i = 0; i < q->nb_fake_desc; ++i) {
 224                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 225                 q->hw_ring[q->nb_desc + i] = zero;
 226         }
 227
 228         q->next_dd = 0;
 229         q->next_alloc = 0;
 230         q->next_trigger = q->alloc_thresh - 1;
 231         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 232         q->rxrearm_start = 0;
 233         q->rxrearm_nb = 0;
 234
 235         return 0;
 236 }
 237
 238 /*
 239  * clean queue, descriptor rings, free software buffers used when stopping
 240  * device.
 241  */
 242 static inline void
 243 rx_queue_clean(struct fm10k_rx_queue *q)
 244 {
 245         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 246         uint32_t i;
 247         PMD_INIT_FUNC_TRACE();
 248
 249         /* zero descriptor rings */
 250         for (i = 0; i < q->nb_desc; ++i)
 251                 q->hw_ring[i] = zero;
 252
 253         /* zero faked descriptors */
 254         for (i = 0; i < q->nb_fake_desc; ++i)
 255                 q->hw_ring[q->nb_desc + i] = zero;
 256
 257         /* vPMD driver has a different way of releasing mbufs. */
 258         if (q->rx_using_sse) {
 259                 fm10k_rx_queue_release_mbufs_vec(q);
 260                 return;
 261         }
 262
 263         /* free software buffers */
 264         for (i = 0; i < q->nb_desc; ++i) {
 265                 if (q->sw_ring[i]) {
 266                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 267                         q->sw_ring[i] = NULL;
 268                 }
 269         }
 270 }
 271
 272 /*
 273  * free all queue memory used when releasing the queue (i.e. configure)
 274  */
 275 static inline void
 276 rx_queue_free(struct fm10k_rx_queue *q)
 277 {
 278         PMD_INIT_FUNC_TRACE();
 279         if (q) {
 280                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 281                 rx_queue_clean(q);
 282                 if (q->sw_ring) {
 283                         rte_free(q->sw_ring);
 284                         q->sw_ring = NULL;
 285                 }
 286                 rte_free(q);
 287                 q = NULL;
 288         }
 289 }
 290
 291 /*
 292  * disable RX queue, wait unitl HW finished necessary flush operation
 293  */
 294 static inline int
 295 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 296 {
 297         uint32_t reg, i;
 298
 299         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 300         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 301                         reg & ~FM10K_RXQCTL_ENABLE);
 302
 303         /* Wait 100us at most */
 304         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 305                 rte_delay_us(1);
 306                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 307                 if (!(reg & FM10K_RXQCTL_ENABLE))
 308                         break;
 309         }
 310
 311         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 312                 return -1;
 313
 314         return 0;
 315 }
 316
 317 /*
 318  * reset queue to initial state, allocate software buffers used when starting
 319  * device
 320  */
 321 static inline void
 322 tx_queue_reset(struct fm10k_tx_queue *q)
 323 {
 324         PMD_INIT_FUNC_TRACE();
 325         q->last_free = 0;
 326         q->next_free = 0;
 327         q->nb_used = 0;
 328         q->nb_free = q->nb_desc - 1;
 329         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 330         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 331 }
 332
 333 /*
 334  * clean queue, descriptor rings, free software buffers used when stopping
 335  * device
 336  */
 337 static inline void
 338 tx_queue_clean(struct fm10k_tx_queue *q)
 339 {
 340         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 341         uint32_t i;
 342         PMD_INIT_FUNC_TRACE();
 343
 344         /* zero descriptor rings */
 345         for (i = 0; i < q->nb_desc; ++i)
 346                 q->hw_ring[i] = zero;
 347
 348         /* free software buffers */
 349         for (i = 0; i < q->nb_desc; ++i) {
 350                 if (q->sw_ring[i]) {
 351                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 352                         q->sw_ring[i] = NULL;
 353                 }
 354         }
 355 }
 356
 357 /*
 358  * free all queue memory used when releasing the queue (i.e. configure)
 359  */
 360 static inline void
 361 tx_queue_free(struct fm10k_tx_queue *q)
 362 {
 363         PMD_INIT_FUNC_TRACE();
 364         if (q) {
 365                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 366                 tx_queue_clean(q);
 367                 if (q->rs_tracker.list) {
 368                         rte_free(q->rs_tracker.list);
 369                         q->rs_tracker.list = NULL;
 370                 }
 371                 if (q->sw_ring) {
 372                         rte_free(q->sw_ring);
 373                         q->sw_ring = NULL;
 374                 }
 375                 rte_free(q);
 376                 q = NULL;
 377         }
 378 }
 379
 380 /*
 381  * disable TX queue, wait unitl HW finished necessary flush operation
 382  */
 383 static inline int
 384 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 385 {
 386         uint32_t reg, i;
 387
 388         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 389         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 390                         reg & ~FM10K_TXDCTL_ENABLE);
 391
 392         /* Wait 100us at most */
 393         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 394                 rte_delay_us(1);
 395                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 396                 if (!(reg & FM10K_TXDCTL_ENABLE))
 397                         break;
 398         }
 399
 400         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 401                 return -1;
 402
 403         return 0;
 404 }
 405
 406 static int
 407 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 408 {
 409         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 410         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 411         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 412         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 413
 414         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 415
 416         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 417                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 418                 return -EINVAL;
 419         }
 420
 421         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 422                 return 0;
 423
 424         if (hw->mac.type == fm10k_mac_vf) {
 425                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 426                 return -EINVAL;
 427         }
 428
 429         /* Check VMDQ queue pool number */
 430         if (vmdq_conf->nb_queue_pools >
 431                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 432                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 433                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 434                         vmdq_conf->nb_queue_pools);
 435                 return -EINVAL;
 436         }
 437
 438         return 0;
 439 }
 440
 441 static const struct fm10k_txq_ops def_txq_ops = {
 442         .reset = tx_queue_reset,
 443 };
 444
 445 static int
 446 fm10k_dev_configure(struct rte_eth_dev *dev)
 447 {
 448         int ret;
 449
 450         PMD_INIT_FUNC_TRACE();
 451
 452         if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)
 453                 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH;
 454
 455         /* multipe queue mode checking */
 456         ret  = fm10k_check_mq_mode(dev);
 457         if (ret != 0) {
 458                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 459                             ret);
 460                 return ret;
 461         }
 462
 463         dev->data->scattered_rx = 0;
 464
 465         return 0;
 466 }
 467
 468 static void
 469 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 470 {
 471         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 472         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 473         uint32_t i;
 474
 475         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 476
 477         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 478                 if (!vmdq_conf->pool_map[i].pools)
 479                         continue;
 480                 fm10k_mbx_lock(hw);
 481                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 482                 fm10k_mbx_unlock(hw);
 483         }
 484 }
 485
 486 static void
 487 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 488 {
 489         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 490
 491         /* Add default mac address */
 492         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 493                 MAIN_VSI_POOL_NUMBER);
 494 }
 495
 496 static void
 497 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 498 {
 499         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 500         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 501         uint32_t mrqc, *key, i, reta, j;
 502         uint64_t hf;
 503
 504 #define RSS_KEY_SIZE 40
 505         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 506                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 507                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 508                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 509                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 510                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 511         };
 512
 513         if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 514                 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 515                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 516                 return;
 517         }
 518
 519         /* random key is rss_intel_key (default) or user provided (rss_key) */
 520         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 521                 key = (uint32_t *)rss_intel_key;
 522         else
 523                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 524
 525         /* Now fill our hash function seeds, 4 bytes at a time */
 526         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 527                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 528
 529         /*
 530          * Fill in redirection table
 531          * The byte-swap is needed because NIC registers are in
 532          * little-endian order.
 533          */
 534         reta = 0;
 535         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 536                 if (j == dev->data->nb_rx_queues)
 537                         j = 0;
 538                 reta = (reta << CHAR_BIT) | j;
 539                 if ((i & 3) == 3)
 540                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 541                                         rte_bswap32(reta));
 542         }
 543
 544         /*
 545          * Generate RSS hash based on packet types, TCP/UDP
 546          * port numbers and/or IPv4/v6 src and dst addresses
 547          */
 548         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 549         mrqc = 0;
 550         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 551         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 552         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 553         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 554         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 555         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 556         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 557         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 558         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 559
 560         if (mrqc == 0) {
 561                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 562                         "supported", hf);
 563                 return;
 564         }
 565
 566         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 567 }
 568
 569 static void
 570 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 571 {
 572         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 573         uint32_t i;
 574
 575         for (i = 0; i < nb_lport_new; i++) {
 576                 /* Set unicast mode by default. App can change
 577                  * to other mode in other API func.
 578                  */
 579                 fm10k_mbx_lock(hw);
 580                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 581                         FM10K_XCAST_MODE_NONE);
 582                 fm10k_mbx_unlock(hw);
 583         }
 584 }
 585
 586 static void
 587 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 588 {
 589         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 590         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 591         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 592         struct fm10k_macvlan_filter_info *macvlan;
 593         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 594         uint16_t nb_lport_new;
 595
 596         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 597         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 598
 599         fm10k_dev_rss_configure(dev);
 600
 601         /* only PF supports VMDQ */
 602         if (hw->mac.type != fm10k_mac_pf)
 603                 return;
 604
 605         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 606                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 607
 608         /* no pool number change, no need to update logic port and VLAN/MAC */
 609         if (macvlan->nb_queue_pools == nb_queue_pools)
 610                 return;
 611
 612         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 613         fm10k_dev_logic_port_update(dev, nb_lport_new);
 614
 615         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 616         memset(dev->data->mac_addrs, 0,
 617                 RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 618         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
 619                 &dev->data->mac_addrs[0]);
 620         memset(macvlan, 0, sizeof(*macvlan));
 621         macvlan->nb_queue_pools = nb_queue_pools;
 622
 623         if (nb_queue_pools)
 624                 fm10k_dev_vmdq_rx_configure(dev);
 625         else
 626                 fm10k_dev_pf_main_vsi_reset(dev);
 627 }
 628
 629 static int
 630 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 631 {
 632         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 633         int i, ret;
 634         struct fm10k_tx_queue *txq;
 635         uint64_t base_addr;
 636         uint32_t size;
 637
 638         /* Disable TXINT to avoid possible interrupt */
 639         for (i = 0; i < hw->mac.max_queues; i++)
 640                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 641                                 3 << FM10K_TXINT_TIMER_SHIFT);
 642
 643         /* Setup TX queue */
 644         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 645                 txq = dev->data->tx_queues[i];
 646                 base_addr = txq->hw_ring_phys_addr;
 647                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 648
 649                 /* disable queue to avoid issues while updating state */
 650                 ret = tx_queue_disable(hw, i);
 651                 if (ret) {
 652                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 653                         return -1;
 654                 }
 655                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 656                  * register is read-only for VF.
 657                  */
 658                 if (fm10k_check_ftag(dev->device->devargs)) {
 659                         if (hw->mac.type == fm10k_mac_pf) {
 660                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 661                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 662                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 663                         } else {
 664                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 665                                 return -ENOTSUP;
 666                         }
 667                 }
 668
 669                 /* set location and size for descriptor ring */
 670                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 671                                 base_addr & UINT64_LOWER_32BITS_MASK);
 672                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 673                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 674                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 675
 676                 /* assign default SGLORT for each TX queue by PF */
 677                 if (hw->mac.type == fm10k_mac_pf)
 678                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 679         }
 680
 681         /* set up vector or scalar TX function as appropriate */
 682         fm10k_set_tx_function(dev);
 683
 684         return 0;
 685 }
 686
 687 static int
 688 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 689 {
 690         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 691         struct fm10k_macvlan_filter_info *macvlan;
 692         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 693         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 694         int i, ret;
 695         struct fm10k_rx_queue *rxq;
 696         uint64_t base_addr;
 697         uint32_t size;
 698         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 699         uint32_t logic_port = hw->mac.dglort_map;
 700         uint16_t buf_size;
 701         uint16_t queue_stride = 0;
 702
 703         /* enable RXINT for interrupt mode */
 704         i = 0;
 705         if (rte_intr_dp_is_en(intr_handle)) {
 706                 for (; i < dev->data->nb_rx_queues; i++) {
 707                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 708                         if (hw->mac.type == fm10k_mac_pf)
 709                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 710                                         FM10K_ITR_AUTOMASK |
 711                                         FM10K_ITR_MASK_CLEAR);
 712                         else
 713                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 714                                         FM10K_ITR_AUTOMASK |
 715                                         FM10K_ITR_MASK_CLEAR);
 716                 }
 717         }
 718         /* Disable other RXINT to avoid possible interrupt */
 719         for (; i < hw->mac.max_queues; i++)
 720                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 721                         3 << FM10K_RXINT_TIMER_SHIFT);
 722
 723         /* Setup RX queues */
 724         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 725                 rxq = dev->data->rx_queues[i];
 726                 base_addr = rxq->hw_ring_phys_addr;
 727                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 728
 729                 /* disable queue to avoid issues while updating state */
 730                 ret = rx_queue_disable(hw, i);
 731                 if (ret) {
 732                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 733                         return -1;
 734                 }
 735
 736                 /* Setup the Base and Length of the Rx Descriptor Ring */
 737                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 738                                 base_addr & UINT64_LOWER_32BITS_MASK);
 739                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 740                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 741                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 742
 743                 /* Configure the Rx buffer size for one buff without split */
 744                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 745                         RTE_PKTMBUF_HEADROOM);
 746                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 747                  * reserved for this purpose, and the worst case could be 511B.
 748                  * But SRR reg assumes all buffers have the same size. In order
 749                  * to fill the gap, we'll have to consider the worst case and
 750                  * assume 512B is reserved. If we don't do so, it's possible
 751                  * for HW to overwrite data to next mbuf.
 752                  */
 753                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 754
 755                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 756                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 757                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 758
 759                 /* It adds dual VLAN length for supporting dual VLAN */
 760                 if ((dev->data->mtu + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN +
 761                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 762                         rxq->offloads & DEV_RX_OFFLOAD_SCATTER) {
 763                         uint32_t reg;
 764                         dev->data->scattered_rx = 1;
 765                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 766                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 767                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 768                 }
 769
 770                 /* Enable drop on empty, it's RO for VF */
 771                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 772                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 773
 774                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 775                 FM10K_WRITE_FLUSH(hw);
 776         }
 777
 778         /* Configure VMDQ/RSS if applicable */
 779         fm10k_dev_mq_rx_configure(dev);
 780
 781         /* Decide the best RX function */
 782         fm10k_set_rx_function(dev);
 783
 784         /* update RX_SGLORT for loopback suppress*/
 785         if (hw->mac.type != fm10k_mac_pf)
 786                 return 0;
 787         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 788         if (macvlan->nb_queue_pools)
 789                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 790         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 791                 if (i && queue_stride && !(i % queue_stride))
 792                         logic_port++;
 793                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 794         }
 795
 796         return 0;
 797 }
 798
 799 static int
 800 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 801 {
 802         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 803         int err;
 804         uint32_t reg;
 805         struct fm10k_rx_queue *rxq;
 806
 807         PMD_INIT_FUNC_TRACE();
 808
 809         rxq = dev->data->rx_queues[rx_queue_id];
 810         err = rx_queue_reset(rxq);
 811         if (err == -ENOMEM) {
 812                 PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 813                 return err;
 814         } else if (err == -EINVAL) {
 815                 PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 816                         " %d", err);
 817                 return err;
 818         }
 819
 820         /* Setup the HW Rx Head and Tail Descriptor Pointers
 821          * Note: this must be done AFTER the queue is enabled on real
 822          * hardware, but BEFORE the queue is enabled when using the
 823          * emulation platform. Do it in both places for now and remove
 824          * this comment and the following two register writes when the
 825          * emulation platform is no longer being used.
 826          */
 827         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 828         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 829
 830         /* Set PF ownership flag for PF devices */
 831         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 832         if (hw->mac.type == fm10k_mac_pf)
 833                 reg |= FM10K_RXQCTL_PF;
 834         reg |= FM10K_RXQCTL_ENABLE;
 835         /* enable RX queue */
 836         FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 837         FM10K_WRITE_FLUSH(hw);
 838
 839         /* Setup the HW Rx Head and Tail Descriptor Pointers
 840          * Note: this must be done AFTER the queue is enabled
 841          */
 842         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 843         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 844         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 845
 846         return 0;
 847 }
 848
 849 static int
 850 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 851 {
 852         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 853
 854         PMD_INIT_FUNC_TRACE();
 855
 856         /* Disable RX queue */
 857         rx_queue_disable(hw, rx_queue_id);
 858
 859         /* Free mbuf and clean HW ring */
 860         rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 861         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 862
 863         return 0;
 864 }
 865
 866 static int
 867 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 868 {
 869         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 870         /** @todo - this should be defined in the shared code */
 871 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 872         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 873         struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 874
 875         PMD_INIT_FUNC_TRACE();
 876
 877         q->ops->reset(q);
 878
 879         /* reset head and tail pointers */
 880         FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 881         FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 882
 883         /* enable TX queue */
 884         FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 885                                 FM10K_TXDCTL_ENABLE | txdctl);
 886         FM10K_WRITE_FLUSH(hw);
 887         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 888
 889         return 0;
 890 }
 891
 892 static int
 893 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 894 {
 895         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 896
 897         PMD_INIT_FUNC_TRACE();
 898
 899         tx_queue_disable(hw, tx_queue_id);
 900         tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 901         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 902
 903         return 0;
 904 }
 905
 906 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 907 {
 908         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 909                 != FM10K_DGLORTMAP_NONE);
 910 }
 911
 912 static int
 913 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 914 {
 915         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 916         int status;
 917
 918         PMD_INIT_FUNC_TRACE();
 919
 920         /* Return if it didn't acquire valid glort range */
 921         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 922                 return 0;
 923
 924         fm10k_mbx_lock(hw);
 925         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 926                                 FM10K_XCAST_MODE_PROMISC);
 927         fm10k_mbx_unlock(hw);
 928
 929         if (status != FM10K_SUCCESS) {
 930                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 931                 return -EAGAIN;
 932         }
 933
 934         return 0;
 935 }
 936
 937 static int
 938 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 939 {
 940         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 941         uint8_t mode;
 942         int status;
 943
 944         PMD_INIT_FUNC_TRACE();
 945
 946         /* Return if it didn't acquire valid glort range */
 947         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 948                 return 0;
 949
 950         if (dev->data->all_multicast == 1)
 951                 mode = FM10K_XCAST_MODE_ALLMULTI;
 952         else
 953                 mode = FM10K_XCAST_MODE_NONE;
 954
 955         fm10k_mbx_lock(hw);
 956         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 957                                 mode);
 958         fm10k_mbx_unlock(hw);
 959
 960         if (status != FM10K_SUCCESS) {
 961                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 962                 return -EAGAIN;
 963         }
 964
 965         return 0;
 966 }
 967
 968 static int
 969 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 970 {
 971         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 972         int status;
 973
 974         PMD_INIT_FUNC_TRACE();
 975
 976         /* Return if it didn't acquire valid glort range */
 977         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 978                 return 0;
 979
 980         /* If promiscuous mode is enabled, it doesn't make sense to enable
 981          * allmulticast and disable promiscuous since fm10k only can select
 982          * one of the modes.
 983          */
 984         if (dev->data->promiscuous) {
 985                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 986                         "needn't enable allmulticast");
 987                 return 0;
 988         }
 989
 990         fm10k_mbx_lock(hw);
 991         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 992                                 FM10K_XCAST_MODE_ALLMULTI);
 993         fm10k_mbx_unlock(hw);
 994
 995         if (status != FM10K_SUCCESS) {
 996                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 997                 return -EAGAIN;
 998         }
 999
1000         return 0;
1001 }
1002
1003 static int
1004 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1005 {
1006         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1007         int status;
1008
1009         PMD_INIT_FUNC_TRACE();
1010
1011         /* Return if it didn't acquire valid glort range */
1012         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1013                 return 0;
1014
1015         if (dev->data->promiscuous) {
1016                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1017                         "since promisc mode is enabled");
1018                 return -EINVAL;
1019         }
1020
1021         fm10k_mbx_lock(hw);
1022         /* Change mode to unicast mode */
1023         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1024                                 FM10K_XCAST_MODE_NONE);
1025         fm10k_mbx_unlock(hw);
1026
1027         if (status != FM10K_SUCCESS) {
1028                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1029                 return -EAGAIN;
1030         }
1031
1032         return 0;
1033 }
1034
1035 static void
1036 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1037 {
1038         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1039         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1040         uint16_t nb_queue_pools;
1041         struct fm10k_macvlan_filter_info *macvlan;
1042
1043         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1044         nb_queue_pools = macvlan->nb_queue_pools;
1045         pool_len = nb_queue_pools ? rte_fls_u32(nb_queue_pools - 1) : 0;
1046         rss_len = rte_fls_u32(dev->data->nb_rx_queues - 1) - pool_len;
1047
1048         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1049         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1050         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1051                         hw->mac.dglort_map;
1052         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1053         /* Configure VMDQ/RSS DGlort Decoder */
1054         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1055
1056         /* Flow Director configurations, only queue number is valid. */
1057         dglortdec = rte_fls_u32(dev->data->nb_rx_queues - 1);
1058         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1059                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1060         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1061         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1062
1063         /* Invalidate all other GLORT entries */
1064         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1065                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1066                                 FM10K_DGLORTMAP_NONE);
1067 }
1068
1069 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1070 static int
1071 fm10k_dev_start(struct rte_eth_dev *dev)
1072 {
1073         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1074         int i, diag;
1075
1076         PMD_INIT_FUNC_TRACE();
1077
1078         /* stop, init, then start the hw */
1079         diag = fm10k_stop_hw(hw);
1080         if (diag != FM10K_SUCCESS) {
1081                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1082                 return -EIO;
1083         }
1084
1085         diag = fm10k_init_hw(hw);
1086         if (diag != FM10K_SUCCESS) {
1087                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1088                 return -EIO;
1089         }
1090
1091         diag = fm10k_start_hw(hw);
1092         if (diag != FM10K_SUCCESS) {
1093                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1094                 return -EIO;
1095         }
1096
1097         diag = fm10k_dev_tx_init(dev);
1098         if (diag) {
1099                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1100                 return diag;
1101         }
1102
1103         if (fm10k_dev_rxq_interrupt_setup(dev))
1104                 return -EIO;
1105
1106         diag = fm10k_dev_rx_init(dev);
1107         if (diag) {
1108                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1109                 return diag;
1110         }
1111
1112         if (hw->mac.type == fm10k_mac_pf)
1113                 fm10k_dev_dglort_map_configure(dev);
1114
1115         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1116                 struct fm10k_rx_queue *rxq;
1117                 rxq = dev->data->rx_queues[i];
1118
1119                 if (rxq->rx_deferred_start)
1120                         continue;
1121                 diag = fm10k_dev_rx_queue_start(dev, i);
1122                 if (diag != 0) {
1123                         int j;
1124                         for (j = 0; j < i; ++j)
1125                                 rx_queue_clean(dev->data->rx_queues[j]);
1126                         return diag;
1127                 }
1128         }
1129
1130         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1131                 struct fm10k_tx_queue *txq;
1132                 txq = dev->data->tx_queues[i];
1133
1134                 if (txq->tx_deferred_start)
1135                         continue;
1136                 diag = fm10k_dev_tx_queue_start(dev, i);
1137                 if (diag != 0) {
1138                         int j;
1139                         for (j = 0; j < i; ++j)
1140                                 tx_queue_clean(dev->data->tx_queues[j]);
1141                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1142                                 rx_queue_clean(dev->data->rx_queues[j]);
1143                         return diag;
1144                 }
1145         }
1146
1147         /* Update default vlan when not in VMDQ mode */
1148         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1149                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1150
1151         fm10k_link_update(dev, 0);
1152
1153         return 0;
1154 }
1155
1156 static int
1157 fm10k_dev_stop(struct rte_eth_dev *dev)
1158 {
1159         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1160         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1161         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1162         int i;
1163
1164         PMD_INIT_FUNC_TRACE();
1165         dev->data->dev_started = 0;
1166
1167         if (dev->data->tx_queues)
1168                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1169                         fm10k_dev_tx_queue_stop(dev, i);
1170
1171         if (dev->data->rx_queues)
1172                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1173                         fm10k_dev_rx_queue_stop(dev, i);
1174
1175         /* Disable datapath event */
1176         if (rte_intr_dp_is_en(intr_handle)) {
1177                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1178                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1179                                 3 << FM10K_RXINT_TIMER_SHIFT);
1180                         if (hw->mac.type == fm10k_mac_pf)
1181                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1182                                         FM10K_ITR_MASK_SET);
1183                         else
1184                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1185                                         FM10K_ITR_MASK_SET);
1186                 }
1187         }
1188         /* Clean datapath event and queue/vec mapping */
1189         rte_intr_efd_disable(intr_handle);
1190         rte_free(intr_handle->intr_vec);
1191         intr_handle->intr_vec = NULL;
1192
1193         return 0;
1194 }
1195
1196 static void
1197 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1198 {
1199         int i;
1200
1201         PMD_INIT_FUNC_TRACE();
1202
1203         if (dev->data->tx_queues) {
1204                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1205                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1206
1207                         tx_queue_free(txq);
1208                 }
1209         }
1210
1211         if (dev->data->rx_queues) {
1212                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1213                         fm10k_rx_queue_release(dev, i);
1214         }
1215 }
1216
1217 static int
1218 fm10k_link_update(struct rte_eth_dev *dev,
1219         __rte_unused int wait_to_complete)
1220 {
1221         struct fm10k_dev_info *dev_info =
1222                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1223         PMD_INIT_FUNC_TRACE();
1224
1225         dev->data->dev_link.link_speed  = ETH_SPEED_NUM_50G;
1226         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1227         dev->data->dev_link.link_status =
1228                 dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP;
1229         dev->data->dev_link.link_autoneg = ETH_LINK_FIXED;
1230
1231         return 0;
1232 }
1233
1234 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1235         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1236 {
1237         unsigned i, q;
1238         unsigned count = 0;
1239
1240         if (xstats_names != NULL) {
1241                 /* Note: limit checked in rte_eth_xstats_names() */
1242
1243                 /* Global stats */
1244                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1245                         snprintf(xstats_names[count].name,
1246                                 sizeof(xstats_names[count].name),
1247                                 "%s", fm10k_hw_stats_strings[count].name);
1248                         count++;
1249                 }
1250
1251                 /* PF queue stats */
1252                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1253                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1254                                 snprintf(xstats_names[count].name,
1255                                         sizeof(xstats_names[count].name),
1256                                         "rx_q%u_%s", q,
1257                                         fm10k_hw_stats_rx_q_strings[i].name);
1258                                 count++;
1259                         }
1260                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1261                                 snprintf(xstats_names[count].name,
1262                                         sizeof(xstats_names[count].name),
1263                                         "tx_q%u_%s", q,
1264                                         fm10k_hw_stats_tx_q_strings[i].name);
1265                                 count++;
1266                         }
1267                 }
1268         }
1269         return FM10K_NB_XSTATS;
1270 }
1271
1272 static int
1273 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1274                  unsigned n)
1275 {
1276         struct fm10k_hw_stats *hw_stats =
1277                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1278         unsigned i, q, count = 0;
1279
1280         if (n < FM10K_NB_XSTATS)
1281                 return FM10K_NB_XSTATS;
1282
1283         /* Global stats */
1284         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1285                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1286                         fm10k_hw_stats_strings[count].offset);
1287                 xstats[count].id = count;
1288                 count++;
1289         }
1290
1291         /* PF queue stats */
1292         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1293                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1294                         xstats[count].value =
1295                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1296                                 fm10k_hw_stats_rx_q_strings[i].offset);
1297                         xstats[count].id = count;
1298                         count++;
1299                 }
1300                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1301                         xstats[count].value =
1302                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1303                                 fm10k_hw_stats_tx_q_strings[i].offset);
1304                         xstats[count].id = count;
1305                         count++;
1306                 }
1307         }
1308
1309         return FM10K_NB_XSTATS;
1310 }
1311
1312 static int
1313 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1314 {
1315         uint64_t ipackets, opackets, ibytes, obytes, imissed;
1316         struct fm10k_hw *hw =
1317                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1318         struct fm10k_hw_stats *hw_stats =
1319                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1320         int i;
1321
1322         PMD_INIT_FUNC_TRACE();
1323
1324         fm10k_update_hw_stats(hw, hw_stats);
1325
1326         ipackets = opackets = ibytes = obytes = imissed = 0;
1327         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1328                 (i < hw->mac.max_queues); ++i) {
1329                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1330                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1331                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1332                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1333                 stats->q_errors[i]   = hw_stats->q[i].rx_drops.count;
1334                 ipackets += stats->q_ipackets[i];
1335                 opackets += stats->q_opackets[i];
1336                 ibytes   += stats->q_ibytes[i];
1337                 obytes   += stats->q_obytes[i];
1338                 imissed  += stats->q_errors[i];
1339         }
1340         stats->ipackets = ipackets;
1341         stats->opackets = opackets;
1342         stats->ibytes = ibytes;
1343         stats->obytes = obytes;
1344         stats->imissed = imissed;
1345         return 0;
1346 }
1347
1348 static int
1349 fm10k_stats_reset(struct rte_eth_dev *dev)
1350 {
1351         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1352         struct fm10k_hw_stats *hw_stats =
1353                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1354
1355         PMD_INIT_FUNC_TRACE();
1356
1357         memset(hw_stats, 0, sizeof(*hw_stats));
1358         fm10k_rebind_hw_stats(hw, hw_stats);
1359
1360         return 0;
1361 }
1362
1363 static int
1364 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1365         struct rte_eth_dev_info *dev_info)
1366 {
1367         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1368         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1369
1370         PMD_INIT_FUNC_TRACE();
1371
1372         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1373         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1374         dev_info->max_rx_queues      = hw->mac.max_queues;
1375         dev_info->max_tx_queues      = hw->mac.max_queues;
1376         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1377         dev_info->max_hash_mac_addrs = 0;
1378         dev_info->max_vfs            = pdev->max_vfs;
1379         dev_info->vmdq_pool_base     = 0;
1380         dev_info->vmdq_queue_base    = 0;
1381         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1382         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1383         dev_info->rx_queue_offload_capa = fm10k_get_rx_queue_offloads_capa(dev);
1384         dev_info->rx_offload_capa = fm10k_get_rx_port_offloads_capa(dev) |
1385                                     dev_info->rx_queue_offload_capa;
1386         dev_info->tx_queue_offload_capa = fm10k_get_tx_queue_offloads_capa(dev);
1387         dev_info->tx_offload_capa = fm10k_get_tx_port_offloads_capa(dev) |
1388                                     dev_info->tx_queue_offload_capa;
1389
1390         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1391         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1392         dev_info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1393                                         ETH_RSS_IPV6 |
1394                                         ETH_RSS_IPV6_EX |
1395                                         ETH_RSS_NONFRAG_IPV4_TCP |
1396                                         ETH_RSS_NONFRAG_IPV6_TCP |
1397                                         ETH_RSS_IPV6_TCP_EX |
1398                                         ETH_RSS_NONFRAG_IPV4_UDP |
1399                                         ETH_RSS_NONFRAG_IPV6_UDP |
1400                                         ETH_RSS_IPV6_UDP_EX;
1401
1402         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1403                 .rx_thresh = {
1404                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1405                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1406                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1407                 },
1408                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1409                 .rx_drop_en = 0,
1410                 .offloads = 0,
1411         };
1412
1413         dev_info->default_txconf = (struct rte_eth_txconf) {
1414                 .tx_thresh = {
1415                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1416                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1417                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1418                 },
1419                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1420                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1421                 .offloads = 0,
1422         };
1423
1424         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1425                 .nb_max = FM10K_MAX_RX_DESC,
1426                 .nb_min = FM10K_MIN_RX_DESC,
1427                 .nb_align = FM10K_MULT_RX_DESC,
1428         };
1429
1430         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1431                 .nb_max = FM10K_MAX_TX_DESC,
1432                 .nb_min = FM10K_MIN_TX_DESC,
1433                 .nb_align = FM10K_MULT_TX_DESC,
1434                 .nb_seg_max = FM10K_TX_MAX_SEG,
1435                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1436         };
1437
1438         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1439                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1440                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1441
1442         return 0;
1443 }
1444
1445 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1446 static const uint32_t *
1447 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1448 {
1449         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1450             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1451                 static uint32_t ptypes[] = {
1452                         /* refers to rx_desc_to_ol_flags() */
1453                         RTE_PTYPE_L2_ETHER,
1454                         RTE_PTYPE_L3_IPV4,
1455                         RTE_PTYPE_L3_IPV4_EXT,
1456                         RTE_PTYPE_L3_IPV6,
1457                         RTE_PTYPE_L3_IPV6_EXT,
1458                         RTE_PTYPE_L4_TCP,
1459                         RTE_PTYPE_L4_UDP,
1460                         RTE_PTYPE_UNKNOWN
1461                 };
1462
1463                 return ptypes;
1464         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1465                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1466                 static uint32_t ptypes_vec[] = {
1467                         /* refers to fm10k_desc_to_pktype_v() */
1468                         RTE_PTYPE_L3_IPV4,
1469                         RTE_PTYPE_L3_IPV4_EXT,
1470                         RTE_PTYPE_L3_IPV6,
1471                         RTE_PTYPE_L3_IPV6_EXT,
1472                         RTE_PTYPE_L4_TCP,
1473                         RTE_PTYPE_L4_UDP,
1474                         RTE_PTYPE_TUNNEL_GENEVE,
1475                         RTE_PTYPE_TUNNEL_NVGRE,
1476                         RTE_PTYPE_TUNNEL_VXLAN,
1477                         RTE_PTYPE_TUNNEL_GRE,
1478                         RTE_PTYPE_UNKNOWN
1479                 };
1480
1481                 return ptypes_vec;
1482         }
1483
1484         return NULL;
1485 }
1486 #else
1487 static const uint32_t *
1488 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1489 {
1490         return NULL;
1491 }
1492 #endif
1493
1494 static int
1495 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1496 {
1497         s32 result;
1498         uint16_t mac_num = 0;
1499         uint32_t vid_idx, vid_bit, mac_index;
1500         struct fm10k_hw *hw;
1501         struct fm10k_macvlan_filter_info *macvlan;
1502         struct rte_eth_dev_data *data = dev->data;
1503
1504         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1505         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1506
1507         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1508                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1509                 return -EINVAL;
1510         }
1511
1512         if (vlan_id > ETH_VLAN_ID_MAX) {
1513                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1514                 return -EINVAL;
1515         }
1516
1517         vid_idx = FM10K_VFTA_IDX(vlan_id);
1518         vid_bit = FM10K_VFTA_BIT(vlan_id);
1519         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1520         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1521                 return 0;
1522         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1523         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1524                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1525                         "in the VLAN filter table");
1526                 return -EINVAL;
1527         }
1528
1529         fm10k_mbx_lock(hw);
1530         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1531         fm10k_mbx_unlock(hw);
1532         if (result != FM10K_SUCCESS) {
1533                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1534                 return -EIO;
1535         }
1536
1537         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1538                         (result == FM10K_SUCCESS); mac_index++) {
1539                 if (rte_is_zero_ether_addr(&data->mac_addrs[mac_index]))
1540                         continue;
1541                 if (mac_num > macvlan->mac_num - 1) {
1542                         PMD_INIT_LOG(ERR, "MAC address number "
1543                                         "not match");
1544                         break;
1545                 }
1546                 fm10k_mbx_lock(hw);
1547                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1548                         data->mac_addrs[mac_index].addr_bytes,
1549                         vlan_id, on, 0);
1550                 fm10k_mbx_unlock(hw);
1551                 mac_num++;
1552         }
1553         if (result != FM10K_SUCCESS) {
1554                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1555                 return -EIO;
1556         }
1557
1558         if (on) {
1559                 macvlan->vlan_num++;
1560                 macvlan->vfta[vid_idx] |= vid_bit;
1561         } else {
1562                 macvlan->vlan_num--;
1563                 macvlan->vfta[vid_idx] &= ~vid_bit;
1564         }
1565         return 0;
1566 }
1567
1568 static int
1569 fm10k_vlan_offload_set(struct rte_eth_dev *dev __rte_unused,
1570                        int mask __rte_unused)
1571 {
1572         return 0;
1573 }
1574
1575 /* Add/Remove a MAC address, and update filters to main VSI */
1576 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1577                 const u8 *mac, bool add, uint32_t pool)
1578 {
1579         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1580         struct fm10k_macvlan_filter_info *macvlan;
1581         uint32_t i, j, k;
1582
1583         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1584
1585         if (pool != MAIN_VSI_POOL_NUMBER) {
1586                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1587                         "mac to pool %u", pool);
1588                 return;
1589         }
1590         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1591                 if (!macvlan->vfta[j])
1592                         continue;
1593                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1594                         if (!(macvlan->vfta[j] & (1 << k)))
1595                                 continue;
1596                         if (i + 1 > macvlan->vlan_num) {
1597                                 PMD_INIT_LOG(ERR, "vlan number not match");
1598                                 return;
1599                         }
1600                         fm10k_mbx_lock(hw);
1601                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1602                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1603                         fm10k_mbx_unlock(hw);
1604                         i++;
1605                 }
1606         }
1607 }
1608
1609 /* Add/Remove a MAC address, and update filters to VMDQ */
1610 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1611                 const u8 *mac, bool add, uint32_t pool)
1612 {
1613         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1614         struct fm10k_macvlan_filter_info *macvlan;
1615         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1616         uint32_t i;
1617
1618         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1619         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1620
1621         if (pool > macvlan->nb_queue_pools) {
1622                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1623                         " Max pool is %u",
1624                         pool, macvlan->nb_queue_pools);
1625                 return;
1626         }
1627         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1628                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1629                         continue;
1630                 fm10k_mbx_lock(hw);
1631                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1632                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1633                 fm10k_mbx_unlock(hw);
1634         }
1635 }
1636
1637 /* Add/Remove a MAC address, and update filters */
1638 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1639                 const u8 *mac, bool add, uint32_t pool)
1640 {
1641         struct fm10k_macvlan_filter_info *macvlan;
1642
1643         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1644
1645         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1646                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1647         else
1648                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1649
1650         if (add)
1651                 macvlan->mac_num++;
1652         else
1653                 macvlan->mac_num--;
1654 }
1655
1656 /* Add a MAC address, and update filters */
1657 static int
1658 fm10k_macaddr_add(struct rte_eth_dev *dev,
1659                 struct rte_ether_addr *mac_addr,
1660                 uint32_t index,
1661                 uint32_t pool)
1662 {
1663         struct fm10k_macvlan_filter_info *macvlan;
1664
1665         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1666         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1667         macvlan->mac_vmdq_id[index] = pool;
1668         return 0;
1669 }
1670
1671 /* Remove a MAC address, and update filters */
1672 static void
1673 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1674 {
1675         struct rte_eth_dev_data *data = dev->data;
1676         struct fm10k_macvlan_filter_info *macvlan;
1677
1678         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1679         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1680                         FALSE, macvlan->mac_vmdq_id[index]);
1681         macvlan->mac_vmdq_id[index] = 0;
1682 }
1683
1684 static inline int
1685 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1686 {
1687         if ((request < min) || (request > max) || ((request % mult) != 0))
1688                 return -1;
1689         else
1690                 return 0;
1691 }
1692
1693
1694 static inline int
1695 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1696 {
1697         if ((request < min) || (request > max) || ((div % request) != 0))
1698                 return -1;
1699         else
1700                 return 0;
1701 }
1702
1703 static inline int
1704 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1705 {
1706         uint16_t rx_free_thresh;
1707
1708         if (conf->rx_free_thresh == 0)
1709                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1710         else
1711                 rx_free_thresh = conf->rx_free_thresh;
1712
1713         /* make sure the requested threshold satisfies the constraints */
1714         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1715                         FM10K_RX_FREE_THRESH_MAX(q),
1716                         FM10K_RX_FREE_THRESH_DIV(q),
1717                         rx_free_thresh)) {
1718                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1719                         "less than or equal to %u, "
1720                         "greater than or equal to %u, "
1721                         "and a divisor of %u",
1722                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1723                         FM10K_RX_FREE_THRESH_MIN(q),
1724                         FM10K_RX_FREE_THRESH_DIV(q));
1725                 return -EINVAL;
1726         }
1727
1728         q->alloc_thresh = rx_free_thresh;
1729         q->drop_en = conf->rx_drop_en;
1730         q->rx_deferred_start = conf->rx_deferred_start;
1731
1732         return 0;
1733 }
1734
1735 /*
1736  * Hardware requires specific alignment for Rx packet buffers. At
1737  * least one of the following two conditions must be satisfied.
1738  *  1. Address is 512B aligned
1739  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1740  *
1741  * As such, the driver may need to adjust the DMA address within the
1742  * buffer by up to 512B.
1743  *
1744  * return 1 if the element size is valid, otherwise return 0.
1745  */
1746 static int
1747 mempool_element_size_valid(struct rte_mempool *mp)
1748 {
1749         uint32_t min_size;
1750
1751         /* elt_size includes mbuf header and headroom */
1752         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1753                         RTE_PKTMBUF_HEADROOM;
1754
1755         /* account for up to 512B of alignment */
1756         min_size -= FM10K_RX_DATABUF_ALIGN;
1757
1758         /* sanity check for overflow */
1759         if (min_size > mp->elt_size)
1760                 return 0;
1761
1762         /* size is valid */
1763         return 1;
1764 }
1765
1766 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev)
1767 {
1768         RTE_SET_USED(dev);
1769
1770         return (uint64_t)(DEV_RX_OFFLOAD_SCATTER);
1771 }
1772
1773 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
1774 {
1775         RTE_SET_USED(dev);
1776
1777         return  (uint64_t)(DEV_RX_OFFLOAD_VLAN_STRIP  |
1778                            DEV_RX_OFFLOAD_VLAN_FILTER |
1779                            DEV_RX_OFFLOAD_IPV4_CKSUM  |
1780                            DEV_RX_OFFLOAD_UDP_CKSUM   |
1781                            DEV_RX_OFFLOAD_TCP_CKSUM   |
1782                            DEV_RX_OFFLOAD_HEADER_SPLIT |
1783                            DEV_RX_OFFLOAD_RSS_HASH);
1784 }
1785
1786 static int
1787 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1788         uint16_t nb_desc, unsigned int socket_id,
1789         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1790 {
1791         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1792         struct fm10k_dev_info *dev_info =
1793                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1794         struct fm10k_rx_queue *q;
1795         const struct rte_memzone *mz;
1796         uint64_t offloads;
1797
1798         PMD_INIT_FUNC_TRACE();
1799
1800         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1801
1802         /* make sure the mempool element size can account for alignment. */
1803         if (!mempool_element_size_valid(mp)) {
1804                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1805                 return -EINVAL;
1806         }
1807
1808         /* make sure a valid number of descriptors have been requested */
1809         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1810                                 FM10K_MULT_RX_DESC, nb_desc)) {
1811                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1812                         "less than or equal to %"PRIu32", "
1813                         "greater than or equal to %u, "
1814                         "and a multiple of %u",
1815                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1816                         FM10K_MULT_RX_DESC);
1817                 return -EINVAL;
1818         }
1819
1820         /*
1821          * if this queue existed already, free the associated memory. The
1822          * queue cannot be reused in case we need to allocate memory on
1823          * different socket than was previously used.
1824          */
1825         if (dev->data->rx_queues[queue_id] != NULL) {
1826                 rx_queue_free(dev->data->rx_queues[queue_id]);
1827                 dev->data->rx_queues[queue_id] = NULL;
1828         }
1829
1830         /* allocate memory for the queue structure */
1831         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1832                                 socket_id);
1833         if (q == NULL) {
1834                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1835                 return -ENOMEM;
1836         }
1837
1838         /* setup queue */
1839         q->mp = mp;
1840         q->nb_desc = nb_desc;
1841         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1842         q->port_id = dev->data->port_id;
1843         q->queue_id = queue_id;
1844         q->tail_ptr = (volatile uint32_t *)
1845                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1846         q->offloads = offloads;
1847         if (handle_rxconf(q, conf)) {
1848                 rte_free(q);
1849                 return -EINVAL;
1850         }
1851         /* allocate memory for the software ring */
1852         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1853                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1854                         RTE_CACHE_LINE_SIZE, socket_id);
1855         if (q->sw_ring == NULL) {
1856                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1857                 rte_free(q);
1858                 return -ENOMEM;
1859         }
1860
1861         /*
1862          * allocate memory for the hardware descriptor ring. A memzone large
1863          * enough to hold the maximum ring size is requested to allow for
1864          * resizing in later calls to the queue setup function.
1865          */
1866         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1867                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1868                                       socket_id);
1869         if (mz == NULL) {
1870                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1871                 rte_free(q->sw_ring);
1872                 rte_free(q);
1873                 return -ENOMEM;
1874         }
1875         q->hw_ring = mz->addr;
1876         q->hw_ring_phys_addr = mz->iova;
1877
1878         /* Check if number of descs satisfied Vector requirement */
1879         if (!rte_is_power_of_2(nb_desc)) {
1880                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1881                                     "preconditions - canceling the feature for "
1882                                     "the whole port[%d]",
1883                              q->queue_id, q->port_id);
1884                 dev_info->rx_vec_allowed = false;
1885         } else
1886                 fm10k_rxq_vec_setup(q);
1887
1888         dev->data->rx_queues[queue_id] = q;
1889         return 0;
1890 }
1891
1892 static void
1893 fm10k_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
1894 {
1895         PMD_INIT_FUNC_TRACE();
1896
1897         rx_queue_free(dev->data->rx_queues[qid]);
1898 }
1899
1900 static inline int
1901 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1902 {
1903         uint16_t tx_free_thresh;
1904         uint16_t tx_rs_thresh;
1905
1906         /* constraint MACROs require that tx_free_thresh is configured
1907          * before tx_rs_thresh */
1908         if (conf->tx_free_thresh == 0)
1909                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1910         else
1911                 tx_free_thresh = conf->tx_free_thresh;
1912
1913         /* make sure the requested threshold satisfies the constraints */
1914         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1915                         FM10K_TX_FREE_THRESH_MAX(q),
1916                         FM10K_TX_FREE_THRESH_DIV(q),
1917                         tx_free_thresh)) {
1918                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1919                         "less than or equal to %u, "
1920                         "greater than or equal to %u, "
1921                         "and a divisor of %u",
1922                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1923                         FM10K_TX_FREE_THRESH_MIN(q),
1924                         FM10K_TX_FREE_THRESH_DIV(q));
1925                 return -EINVAL;
1926         }
1927
1928         q->free_thresh = tx_free_thresh;
1929
1930         if (conf->tx_rs_thresh == 0)
1931                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1932         else
1933                 tx_rs_thresh = conf->tx_rs_thresh;
1934
1935         q->tx_deferred_start = conf->tx_deferred_start;
1936
1937         /* make sure the requested threshold satisfies the constraints */
1938         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1939                         FM10K_TX_RS_THRESH_MAX(q),
1940                         FM10K_TX_RS_THRESH_DIV(q),
1941                         tx_rs_thresh)) {
1942                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1943                         "less than or equal to %u, "
1944                         "greater than or equal to %u, "
1945                         "and a divisor of %u",
1946                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1947                         FM10K_TX_RS_THRESH_MIN(q),
1948                         FM10K_TX_RS_THRESH_DIV(q));
1949                 return -EINVAL;
1950         }
1951
1952         q->rs_thresh = tx_rs_thresh;
1953
1954         return 0;
1955 }
1956
1957 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev)
1958 {
1959         RTE_SET_USED(dev);
1960
1961         return 0;
1962 }
1963
1964 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
1965 {
1966         RTE_SET_USED(dev);
1967
1968         return (uint64_t)(DEV_TX_OFFLOAD_VLAN_INSERT |
1969                           DEV_TX_OFFLOAD_MULTI_SEGS  |
1970                           DEV_TX_OFFLOAD_IPV4_CKSUM  |
1971                           DEV_TX_OFFLOAD_UDP_CKSUM   |
1972                           DEV_TX_OFFLOAD_TCP_CKSUM   |
1973                           DEV_TX_OFFLOAD_TCP_TSO);
1974 }
1975
1976 static int
1977 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1978         uint16_t nb_desc, unsigned int socket_id,
1979         const struct rte_eth_txconf *conf)
1980 {
1981         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1982         struct fm10k_tx_queue *q;
1983         const struct rte_memzone *mz;
1984         uint64_t offloads;
1985
1986         PMD_INIT_FUNC_TRACE();
1987
1988         offloads = conf->offloads | dev->data->dev_conf.txmode.offloads;
1989
1990         /* make sure a valid number of descriptors have been requested */
1991         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1992                                 FM10K_MULT_TX_DESC, nb_desc)) {
1993                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1994                         "less than or equal to %"PRIu32", "
1995                         "greater than or equal to %u, "
1996                         "and a multiple of %u",
1997                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1998                         FM10K_MULT_TX_DESC);
1999                 return -EINVAL;
2000         }
2001
2002         /*
2003          * if this queue existed already, free the associated memory. The
2004          * queue cannot be reused in case we need to allocate memory on
2005          * different socket than was previously used.
2006          */
2007         if (dev->data->tx_queues[queue_id] != NULL) {
2008                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2009
2010                 tx_queue_free(txq);
2011                 dev->data->tx_queues[queue_id] = NULL;
2012         }
2013
2014         /* allocate memory for the queue structure */
2015         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2016                                 socket_id);
2017         if (q == NULL) {
2018                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2019                 return -ENOMEM;
2020         }
2021
2022         /* setup queue */
2023         q->nb_desc = nb_desc;
2024         q->port_id = dev->data->port_id;
2025         q->queue_id = queue_id;
2026         q->offloads = offloads;
2027         q->ops = &def_txq_ops;
2028         q->tail_ptr = (volatile uint32_t *)
2029                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2030         if (handle_txconf(q, conf)) {
2031                 rte_free(q);
2032                 return -EINVAL;
2033         }
2034
2035         /* allocate memory for the software ring */
2036         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2037                                         nb_desc * sizeof(struct rte_mbuf *),
2038                                         RTE_CACHE_LINE_SIZE, socket_id);
2039         if (q->sw_ring == NULL) {
2040                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2041                 rte_free(q);
2042                 return -ENOMEM;
2043         }
2044
2045         /*
2046          * allocate memory for the hardware descriptor ring. A memzone large
2047          * enough to hold the maximum ring size is requested to allow for
2048          * resizing in later calls to the queue setup function.
2049          */
2050         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2051                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2052                                       socket_id);
2053         if (mz == NULL) {
2054                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2055                 rte_free(q->sw_ring);
2056                 rte_free(q);
2057                 return -ENOMEM;
2058         }
2059         q->hw_ring = mz->addr;
2060         q->hw_ring_phys_addr = mz->iova;
2061
2062         /*
2063          * allocate memory for the RS bit tracker. Enough slots to hold the
2064          * descriptor index for each RS bit needing to be set are required.
2065          */
2066         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2067                                 ((nb_desc + 1) / q->rs_thresh) *
2068                                 sizeof(uint16_t),
2069                                 RTE_CACHE_LINE_SIZE, socket_id);
2070         if (q->rs_tracker.list == NULL) {
2071                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2072                 rte_free(q->sw_ring);
2073                 rte_free(q);
2074                 return -ENOMEM;
2075         }
2076
2077         dev->data->tx_queues[queue_id] = q;
2078         return 0;
2079 }
2080
2081 static void
2082 fm10k_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
2083 {
2084         struct fm10k_tx_queue *q = dev->data->tx_queues[qid];
2085         PMD_INIT_FUNC_TRACE();
2086
2087         tx_queue_free(q);
2088 }
2089
2090 static int
2091 fm10k_reta_update(struct rte_eth_dev *dev,
2092                         struct rte_eth_rss_reta_entry64 *reta_conf,
2093                         uint16_t reta_size)
2094 {
2095         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2096         uint16_t i, j, idx, shift;
2097         uint8_t mask;
2098         uint32_t reta;
2099
2100         PMD_INIT_FUNC_TRACE();
2101
2102         if (reta_size > FM10K_MAX_RSS_INDICES) {
2103                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2104                         "(%d) doesn't match the number hardware can supported "
2105                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2106                 return -EINVAL;
2107         }
2108
2109         /*
2110          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2111          * 128-entries in 32 registers
2112          */
2113         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2114                 idx = i / RTE_RETA_GROUP_SIZE;
2115                 shift = i % RTE_RETA_GROUP_SIZE;
2116                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2117                                 BIT_MASK_PER_UINT32);
2118                 if (mask == 0)
2119                         continue;
2120
2121                 reta = 0;
2122                 if (mask != BIT_MASK_PER_UINT32)
2123                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2124
2125                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2126                         if (mask & (0x1 << j)) {
2127                                 if (mask != 0xF)
2128                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2129                                 reta |= reta_conf[idx].reta[shift + j] <<
2130                                                 (CHAR_BIT * j);
2131                         }
2132                 }
2133                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2134         }
2135
2136         return 0;
2137 }
2138
2139 static int
2140 fm10k_reta_query(struct rte_eth_dev *dev,
2141                         struct rte_eth_rss_reta_entry64 *reta_conf,
2142                         uint16_t reta_size)
2143 {
2144         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2145         uint16_t i, j, idx, shift;
2146         uint8_t mask;
2147         uint32_t reta;
2148
2149         PMD_INIT_FUNC_TRACE();
2150
2151         if (reta_size < FM10K_MAX_RSS_INDICES) {
2152                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2153                         "(%d) doesn't match the number hardware can supported "
2154                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2155                 return -EINVAL;
2156         }
2157
2158         /*
2159          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2160          * 128-entries in 32 registers
2161          */
2162         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2163                 idx = i / RTE_RETA_GROUP_SIZE;
2164                 shift = i % RTE_RETA_GROUP_SIZE;
2165                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2166                                 BIT_MASK_PER_UINT32);
2167                 if (mask == 0)
2168                         continue;
2169
2170                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2171                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2172                         if (mask & (0x1 << j))
2173                                 reta_conf[idx].reta[shift + j] = ((reta >>
2174                                         CHAR_BIT * j) & UINT8_MAX);
2175                 }
2176         }
2177
2178         return 0;
2179 }
2180
2181 static int
2182 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2183         struct rte_eth_rss_conf *rss_conf)
2184 {
2185         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2186         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2187         uint32_t mrqc;
2188         uint64_t hf = rss_conf->rss_hf;
2189         int i;
2190
2191         PMD_INIT_FUNC_TRACE();
2192
2193         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2194                                 FM10K_RSSRK_ENTRIES_PER_REG))
2195                 return -EINVAL;
2196
2197         if (hf == 0)
2198                 return -EINVAL;
2199
2200         mrqc = 0;
2201         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2202         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2203         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2204         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2205         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2206         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2207         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2208         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2209         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2210
2211         /* If the mapping doesn't fit any supported, return */
2212         if (mrqc == 0)
2213                 return -EINVAL;
2214
2215         if (key != NULL)
2216                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2217                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2218
2219         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2220
2221         return 0;
2222 }
2223
2224 static int
2225 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2226         struct rte_eth_rss_conf *rss_conf)
2227 {
2228         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2229         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2230         uint32_t mrqc;
2231         uint64_t hf;
2232         int i;
2233
2234         PMD_INIT_FUNC_TRACE();
2235
2236         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2237                                 FM10K_RSSRK_ENTRIES_PER_REG))
2238                 return -EINVAL;
2239
2240         if (key != NULL)
2241                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2242                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2243
2244         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2245         hf = 0;
2246         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2247         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2248         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2249         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2250         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2251         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2252         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2253         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2254         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2255
2256         rss_conf->rss_hf = hf;
2257
2258         return 0;
2259 }
2260
2261 static void
2262 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2263 {
2264         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2265         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2266
2267         /* Bind all local non-queue interrupt to vector 0 */
2268         int_map |= FM10K_MISC_VEC_ID;
2269
2270         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2271         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2272         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2273         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2274         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2275         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2276
2277         /* Enable misc causes */
2278         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2279                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2280                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2281                                 FM10K_EIMR_ENABLE(MAILBOX) |
2282                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2283                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2284                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2285                                 FM10K_EIMR_ENABLE(VFLR));
2286
2287         /* Enable ITR 0 */
2288         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2289                                         FM10K_ITR_MASK_CLEAR);
2290         FM10K_WRITE_FLUSH(hw);
2291 }
2292
2293 static void
2294 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2295 {
2296         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2297         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2298
2299         int_map |= FM10K_MISC_VEC_ID;
2300
2301         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2302         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2303         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2304         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2305         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2306         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2307
2308         /* Disable misc causes */
2309         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2310                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2311                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2312                                 FM10K_EIMR_DISABLE(MAILBOX) |
2313                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2314                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2315                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2316                                 FM10K_EIMR_DISABLE(VFLR));
2317
2318         /* Disable ITR 0 */
2319         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2320         FM10K_WRITE_FLUSH(hw);
2321 }
2322
2323 static void
2324 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2325 {
2326         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2327         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2328
2329         /* Bind all local non-queue interrupt to vector 0 */
2330         int_map |= FM10K_MISC_VEC_ID;
2331
2332         /* Only INT 0 available, other 15 are reserved. */
2333         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2334
2335         /* Enable ITR 0 */
2336         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2337                                         FM10K_ITR_MASK_CLEAR);
2338         FM10K_WRITE_FLUSH(hw);
2339 }
2340
2341 static void
2342 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2343 {
2344         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2345         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2346
2347         int_map |= FM10K_MISC_VEC_ID;
2348
2349         /* Only INT 0 available, other 15 are reserved. */
2350         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2351
2352         /* Disable ITR 0 */
2353         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2354         FM10K_WRITE_FLUSH(hw);
2355 }
2356
2357 static int
2358 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2359 {
2360         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2361         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2362
2363         /* Enable ITR */
2364         if (hw->mac.type == fm10k_mac_pf)
2365                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2366                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2367         else
2368                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2369                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2370         rte_intr_ack(&pdev->intr_handle);
2371         return 0;
2372 }
2373
2374 static int
2375 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2376 {
2377         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2378         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2379
2380         /* Disable ITR */
2381         if (hw->mac.type == fm10k_mac_pf)
2382                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2383                         FM10K_ITR_MASK_SET);
2384         else
2385                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2386                         FM10K_ITR_MASK_SET);
2387         return 0;
2388 }
2389
2390 static int
2391 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2392 {
2393         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2394         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2395         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2396         uint32_t intr_vector, vec;
2397         uint16_t queue_id;
2398         int result = 0;
2399
2400         /* fm10k needs one separate interrupt for mailbox,
2401          * so only drivers which support multiple interrupt vectors
2402          * e.g. vfio-pci can work for fm10k interrupt mode
2403          */
2404         if (!rte_intr_cap_multiple(intr_handle) ||
2405                         dev->data->dev_conf.intr_conf.rxq == 0)
2406                 return result;
2407
2408         intr_vector = dev->data->nb_rx_queues;
2409
2410         /* disable interrupt first */
2411         rte_intr_disable(intr_handle);
2412         if (hw->mac.type == fm10k_mac_pf)
2413                 fm10k_dev_disable_intr_pf(dev);
2414         else
2415                 fm10k_dev_disable_intr_vf(dev);
2416
2417         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2418                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2419                 result = -EIO;
2420         }
2421
2422         if (rte_intr_dp_is_en(intr_handle) && !result) {
2423                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2424                         dev->data->nb_rx_queues * sizeof(int), 0);
2425                 if (intr_handle->intr_vec) {
2426                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2427                                         queue_id < dev->data->nb_rx_queues;
2428                                         queue_id++) {
2429                                 intr_handle->intr_vec[queue_id] = vec;
2430                                 if (vec < intr_handle->nb_efd - 1
2431                                                 + FM10K_RX_VEC_START)
2432                                         vec++;
2433                         }
2434                 } else {
2435                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2436                                 " intr_vec", dev->data->nb_rx_queues);
2437                         rte_intr_efd_disable(intr_handle);
2438                         result = -ENOMEM;
2439                 }
2440         }
2441
2442         if (hw->mac.type == fm10k_mac_pf)
2443                 fm10k_dev_enable_intr_pf(dev);
2444         else
2445                 fm10k_dev_enable_intr_vf(dev);
2446         rte_intr_enable(intr_handle);
2447         hw->mac.ops.update_int_moderator(hw);
2448         return result;
2449 }
2450
2451 static int
2452 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2453 {
2454         struct fm10k_fault fault;
2455         int err;
2456         const char *estr = "Unknown error";
2457
2458         /* Process PCA fault */
2459         if (eicr & FM10K_EICR_PCA_FAULT) {
2460                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2461                 if (err)
2462                         goto error;
2463                 switch (fault.type) {
2464                 case PCA_NO_FAULT:
2465                         estr = "PCA_NO_FAULT"; break;
2466                 case PCA_UNMAPPED_ADDR:
2467                         estr = "PCA_UNMAPPED_ADDR"; break;
2468                 case PCA_BAD_QACCESS_PF:
2469                         estr = "PCA_BAD_QACCESS_PF"; break;
2470                 case PCA_BAD_QACCESS_VF:
2471                         estr = "PCA_BAD_QACCESS_VF"; break;
2472                 case PCA_MALICIOUS_REQ:
2473                         estr = "PCA_MALICIOUS_REQ"; break;
2474                 case PCA_POISONED_TLP:
2475                         estr = "PCA_POISONED_TLP"; break;
2476                 case PCA_TLP_ABORT:
2477                         estr = "PCA_TLP_ABORT"; break;
2478                 default:
2479                         goto error;
2480                 }
2481                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2482                         estr, fault.func ? "VF" : "PF", fault.func,
2483                         fault.address, fault.specinfo);
2484         }
2485
2486         /* Process THI fault */
2487         if (eicr & FM10K_EICR_THI_FAULT) {
2488                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2489                 if (err)
2490                         goto error;
2491                 switch (fault.type) {
2492                 case THI_NO_FAULT:
2493                         estr = "THI_NO_FAULT"; break;
2494                 case THI_MAL_DIS_Q_FAULT:
2495                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2496                 default:
2497                         goto error;
2498                 }
2499                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2500                         estr, fault.func ? "VF" : "PF", fault.func,
2501                         fault.address, fault.specinfo);
2502         }
2503
2504         /* Process FUM fault */
2505         if (eicr & FM10K_EICR_FUM_FAULT) {
2506                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2507                 if (err)
2508                         goto error;
2509                 switch (fault.type) {
2510                 case FUM_NO_FAULT:
2511                         estr = "FUM_NO_FAULT"; break;
2512                 case FUM_UNMAPPED_ADDR:
2513                         estr = "FUM_UNMAPPED_ADDR"; break;
2514                 case FUM_POISONED_TLP:
2515                         estr = "FUM_POISONED_TLP"; break;
2516                 case FUM_BAD_VF_QACCESS:
2517                         estr = "FUM_BAD_VF_QACCESS"; break;
2518                 case FUM_ADD_DECODE_ERR:
2519                         estr = "FUM_ADD_DECODE_ERR"; break;
2520                 case FUM_RO_ERROR:
2521                         estr = "FUM_RO_ERROR"; break;
2522                 case FUM_QPRC_CRC_ERROR:
2523                         estr = "FUM_QPRC_CRC_ERROR"; break;
2524                 case FUM_CSR_TIMEOUT:
2525                         estr = "FUM_CSR_TIMEOUT"; break;
2526                 case FUM_INVALID_TYPE:
2527                         estr = "FUM_INVALID_TYPE"; break;
2528                 case FUM_INVALID_LENGTH:
2529                         estr = "FUM_INVALID_LENGTH"; break;
2530                 case FUM_INVALID_BE:
2531                         estr = "FUM_INVALID_BE"; break;
2532                 case FUM_INVALID_ALIGN:
2533                         estr = "FUM_INVALID_ALIGN"; break;
2534                 default:
2535                         goto error;
2536                 }
2537                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2538                         estr, fault.func ? "VF" : "PF", fault.func,
2539                         fault.address, fault.specinfo);
2540         }
2541
2542         return 0;
2543 error:
2544         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2545         return err;
2546 }
2547
2548 /**
2549  * PF interrupt handler triggered by NIC for handling specific interrupt.
2550  *
2551  * @param handle
2552  *  Pointer to interrupt handle.
2553  * @param param
2554  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2555  *
2556  * @return
2557  *  void
2558  */
2559 static void
2560 fm10k_dev_interrupt_handler_pf(void *param)
2561 {
2562         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2563         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2564         uint32_t cause, status;
2565         struct fm10k_dev_info *dev_info =
2566                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2567         int status_mbx;
2568         s32 err;
2569
2570         if (hw->mac.type != fm10k_mac_pf)
2571                 return;
2572
2573         cause = FM10K_READ_REG(hw, FM10K_EICR);
2574
2575         /* Handle PCI fault cases */
2576         if (cause & FM10K_EICR_FAULT_MASK) {
2577                 PMD_INIT_LOG(ERR, "INT: find fault!");
2578                 fm10k_dev_handle_fault(hw, cause);
2579         }
2580
2581         /* Handle switch up/down */
2582         if (cause & FM10K_EICR_SWITCHNOTREADY)
2583                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2584
2585         if (cause & FM10K_EICR_SWITCHREADY) {
2586                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2587                 if (dev_info->sm_down == 1) {
2588                         fm10k_mbx_lock(hw);
2589
2590                         /* For recreating logical ports */
2591                         status_mbx = hw->mac.ops.update_lport_state(hw,
2592                                         hw->mac.dglort_map, MAX_LPORT_NUM, 1);
2593                         if (status_mbx == FM10K_SUCCESS)
2594                                 PMD_INIT_LOG(INFO,
2595                                         "INT: Recreated Logical port");
2596                         else
2597                                 PMD_INIT_LOG(INFO,
2598                                         "INT: Logical ports weren't recreated");
2599
2600                         status_mbx = hw->mac.ops.update_xcast_mode(hw,
2601                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2602                         if (status_mbx != FM10K_SUCCESS)
2603                                 PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2604
2605                         fm10k_mbx_unlock(hw);
2606
2607                         /* first clear the internal SW recording structure */
2608                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2609                                                 ETH_MQ_RX_VMDQ_FLAG))
2610                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2611                                         false);
2612
2613                         fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2614                                         MAIN_VSI_POOL_NUMBER);
2615
2616                         /*
2617                          * Add default mac address and vlan for the logical
2618                          * ports that have been created, leave to the
2619                          * application to fully recover Rx filtering.
2620                          */
2621                         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2622                                         MAIN_VSI_POOL_NUMBER);
2623
2624                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2625                                                 ETH_MQ_RX_VMDQ_FLAG))
2626                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2627                                         true);
2628
2629                         dev_info->sm_down = 0;
2630                         rte_eth_dev_callback_process(dev,
2631                                         RTE_ETH_EVENT_INTR_LSC,
2632                                         NULL);
2633                 }
2634         }
2635
2636         /* Handle mailbox message */
2637         fm10k_mbx_lock(hw);
2638         err = hw->mbx.ops.process(hw, &hw->mbx);
2639         fm10k_mbx_unlock(hw);
2640
2641         if (err == FM10K_ERR_RESET_REQUESTED) {
2642                 PMD_INIT_LOG(INFO, "INT: Switch is down");
2643                 dev_info->sm_down = 1;
2644                 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
2645         }
2646
2647         /* Handle SRAM error */
2648         if (cause & FM10K_EICR_SRAMERROR) {
2649                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2650
2651                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2652                 /* Write to clear pending bits */
2653                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2654
2655                 /* Todo: print out error message after shared code  updates */
2656         }
2657
2658         /* Clear these 3 events if having any */
2659         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2660                  FM10K_EICR_SWITCHREADY;
2661         if (cause)
2662                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2663
2664         /* Re-enable interrupt from device side */
2665         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2666                                         FM10K_ITR_MASK_CLEAR);
2667         /* Re-enable interrupt from host side */
2668         rte_intr_ack(dev->intr_handle);
2669 }
2670
2671 /**
2672  * VF interrupt handler triggered by NIC for handling specific interrupt.
2673  *
2674  * @param handle
2675  *  Pointer to interrupt handle.
2676  * @param param
2677  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2678  *
2679  * @return
2680  *  void
2681  */
2682 static void
2683 fm10k_dev_interrupt_handler_vf(void *param)
2684 {
2685         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2686         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2687         struct fm10k_mbx_info *mbx = &hw->mbx;
2688         struct fm10k_dev_info *dev_info =
2689                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2690         const enum fm10k_mbx_state state = mbx->state;
2691         int status_mbx;
2692
2693         if (hw->mac.type != fm10k_mac_vf)
2694                 return;
2695
2696         /* Handle mailbox message if lock is acquired */
2697         fm10k_mbx_lock(hw);
2698         hw->mbx.ops.process(hw, &hw->mbx);
2699         fm10k_mbx_unlock(hw);
2700
2701         if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) {
2702                 PMD_INIT_LOG(INFO, "INT: Switch has gone down");
2703
2704                 fm10k_mbx_lock(hw);
2705                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2706                                 MAX_LPORT_NUM, 1);
2707                 fm10k_mbx_unlock(hw);
2708
2709                 /* Setting reset flag */
2710                 dev_info->sm_down = 1;
2711                 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
2712         }
2713
2714         if (dev_info->sm_down == 1 &&
2715                         hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) {
2716                 PMD_INIT_LOG(INFO, "INT: Switch has gone up");
2717                 fm10k_mbx_lock(hw);
2718                 status_mbx = hw->mac.ops.update_xcast_mode(hw,
2719                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2720                 if (status_mbx != FM10K_SUCCESS)
2721                         PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2722                 fm10k_mbx_unlock(hw);
2723
2724                 /* first clear the internal SW recording structure */
2725                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false);
2726                 fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2727                                 MAIN_VSI_POOL_NUMBER);
2728
2729                 /*
2730                  * Add default mac address and vlan for the logical ports that
2731                  * have been created, leave to the application to fully recover
2732                  * Rx filtering.
2733                  */
2734                 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2735                                 MAIN_VSI_POOL_NUMBER);
2736                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
2737
2738                 dev_info->sm_down = 0;
2739                 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
2740         }
2741
2742         /* Re-enable interrupt from device side */
2743         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2744                                         FM10K_ITR_MASK_CLEAR);
2745         /* Re-enable interrupt from host side */
2746         rte_intr_ack(dev->intr_handle);
2747 }
2748
2749 /* Mailbox message handler in VF */
2750 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2751         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2752         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2753         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2754         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2755 };
2756
2757 static int
2758 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2759 {
2760         int err = 0;
2761
2762         /* Initialize mailbox lock */
2763         fm10k_mbx_initlock(hw);
2764
2765         /* Replace default message handler with new ones */
2766         if (hw->mac.type == fm10k_mac_vf)
2767                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2768
2769         if (err) {
2770                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2771                                 err);
2772                 return err;
2773         }
2774         /* Connect to SM for PF device or PF for VF device */
2775         return hw->mbx.ops.connect(hw, &hw->mbx);
2776 }
2777
2778 static void
2779 fm10k_close_mbx_service(struct fm10k_hw *hw)
2780 {
2781         /* Disconnect from SM for PF device or PF for VF device */
2782         hw->mbx.ops.disconnect(hw, &hw->mbx);
2783 }
2784
2785 static int
2786 fm10k_dev_close(struct rte_eth_dev *dev)
2787 {
2788         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2789         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2790         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2791         int ret;
2792
2793         PMD_INIT_FUNC_TRACE();
2794         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2795                 return 0;
2796
2797         fm10k_mbx_lock(hw);
2798         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2799                 MAX_LPORT_NUM, false);
2800         fm10k_mbx_unlock(hw);
2801
2802         /* allow 100ms for device to quiesce */
2803         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
2804
2805         /* Stop mailbox service first */
2806         fm10k_close_mbx_service(hw);
2807
2808         ret = fm10k_dev_stop(dev);
2809
2810         fm10k_dev_queue_release(dev);
2811         fm10k_stop_hw(hw);
2812
2813         /* disable uio/vfio intr */
2814         rte_intr_disable(intr_handle);
2815
2816         /*PF/VF has different interrupt handling mechanism */
2817         if (hw->mac.type == fm10k_mac_pf) {
2818                 /* disable interrupt */
2819                 fm10k_dev_disable_intr_pf(dev);
2820
2821                 /* unregister callback func to eal lib */
2822                 rte_intr_callback_unregister(intr_handle,
2823                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2824         } else {
2825                 /* disable interrupt */
2826                 fm10k_dev_disable_intr_vf(dev);
2827
2828                 rte_intr_callback_unregister(intr_handle,
2829                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2830         }
2831
2832         return ret;
2833 }
2834
2835 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2836         .dev_configure          = fm10k_dev_configure,
2837         .dev_start              = fm10k_dev_start,
2838         .dev_stop               = fm10k_dev_stop,
2839         .dev_close              = fm10k_dev_close,
2840         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2841         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2842         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2843         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2844         .stats_get              = fm10k_stats_get,
2845         .xstats_get             = fm10k_xstats_get,
2846         .xstats_get_names       = fm10k_xstats_get_names,
2847         .stats_reset            = fm10k_stats_reset,
2848         .xstats_reset           = fm10k_stats_reset,
2849         .link_update            = fm10k_link_update,
2850         .dev_infos_get          = fm10k_dev_infos_get,
2851         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2852         .vlan_filter_set        = fm10k_vlan_filter_set,
2853         .vlan_offload_set       = fm10k_vlan_offload_set,
2854         .mac_addr_add           = fm10k_macaddr_add,
2855         .mac_addr_remove        = fm10k_macaddr_remove,
2856         .rx_queue_start         = fm10k_dev_rx_queue_start,
2857         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2858         .tx_queue_start         = fm10k_dev_tx_queue_start,
2859         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2860         .rx_queue_setup         = fm10k_rx_queue_setup,
2861         .rx_queue_release       = fm10k_rx_queue_release,
2862         .tx_queue_setup         = fm10k_tx_queue_setup,
2863         .tx_queue_release       = fm10k_tx_queue_release,
2864         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2865         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2866         .reta_update            = fm10k_reta_update,
2867         .reta_query             = fm10k_reta_query,
2868         .rss_hash_update        = fm10k_rss_hash_update,
2869         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2870 };
2871
2872 static int ftag_check_handler(__rte_unused const char *key,
2873                 const char *value, __rte_unused void *opaque)
2874 {
2875         if (strcmp(value, "1"))
2876                 return -1;
2877
2878         return 0;
2879 }
2880
2881 static int
2882 fm10k_check_ftag(struct rte_devargs *devargs)
2883 {
2884         struct rte_kvargs *kvlist;
2885         const char *ftag_key = "enable_ftag";
2886
2887         if (devargs == NULL)
2888                 return 0;
2889
2890         kvlist = rte_kvargs_parse(devargs->args, NULL);
2891         if (kvlist == NULL)
2892                 return 0;
2893
2894         if (!rte_kvargs_count(kvlist, ftag_key)) {
2895                 rte_kvargs_free(kvlist);
2896                 return 0;
2897         }
2898         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2899         if (rte_kvargs_process(kvlist, ftag_key,
2900                                 ftag_check_handler, NULL) < 0) {
2901                 rte_kvargs_free(kvlist);
2902                 return 0;
2903         }
2904         rte_kvargs_free(kvlist);
2905
2906         return 1;
2907 }
2908
2909 static uint16_t
2910 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2911                     uint16_t nb_pkts)
2912 {
2913         uint16_t nb_tx = 0;
2914         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2915
2916         while (nb_pkts) {
2917                 uint16_t ret, num;
2918
2919                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2920                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2921                                                  num);
2922                 nb_tx += ret;
2923                 nb_pkts -= ret;
2924                 if (ret < num)
2925                         break;
2926         }
2927
2928         return nb_tx;
2929 }
2930
2931 static void __rte_cold
2932 fm10k_set_tx_function(struct rte_eth_dev *dev)
2933 {
2934         struct fm10k_tx_queue *txq;
2935         int i;
2936         int use_sse = 1;
2937         uint16_t tx_ftag_en = 0;
2938
2939         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2940                 /* primary process has set the ftag flag and offloads */
2941                 txq = dev->data->tx_queues[0];
2942                 if (fm10k_tx_vec_condition_check(txq) ||
2943                                 rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128) {
2944                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2945                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2946                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2947                 } else {
2948                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2949                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2950                         dev->tx_pkt_prepare = NULL;
2951                 }
2952                 return;
2953         }
2954
2955         if (fm10k_check_ftag(dev->device->devargs))
2956                 tx_ftag_en = 1;
2957
2958         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2959                 txq = dev->data->tx_queues[i];
2960                 txq->tx_ftag_en = tx_ftag_en;
2961                 /* Check if Vector Tx is satisfied */
2962                 if (fm10k_tx_vec_condition_check(txq) ||
2963                                 rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128)
2964                         use_sse = 0;
2965         }
2966
2967         if (use_sse) {
2968                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2969                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2970                         txq = dev->data->tx_queues[i];
2971                         fm10k_txq_vec_setup(txq);
2972                 }
2973                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2974                 dev->tx_pkt_prepare = NULL;
2975         } else {
2976                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2977                 dev->tx_pkt_prepare = fm10k_prep_pkts;
2978                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2979         }
2980 }
2981
2982 static void __rte_cold
2983 fm10k_set_rx_function(struct rte_eth_dev *dev)
2984 {
2985         struct fm10k_dev_info *dev_info =
2986                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2987         uint16_t i, rx_using_sse;
2988         uint16_t rx_ftag_en = 0;
2989
2990         if (fm10k_check_ftag(dev->device->devargs))
2991                 rx_ftag_en = 1;
2992
2993         /* In order to allow Vector Rx there are a few configuration
2994          * conditions to be met.
2995          */
2996         if (!fm10k_rx_vec_condition_check(dev) &&
2997                         dev_info->rx_vec_allowed && !rx_ftag_en &&
2998                         rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
2999                 if (dev->data->scattered_rx)
3000                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
3001                 else
3002                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
3003         } else if (dev->data->scattered_rx)
3004                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
3005         else
3006                 dev->rx_pkt_burst = fm10k_recv_pkts;
3007
3008         rx_using_sse =
3009                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
3010                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
3011
3012         if (rx_using_sse)
3013                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
3014         else
3015                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
3016
3017         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3018                 return;
3019
3020         for (i = 0; i < dev->data->nb_rx_queues; i++) {
3021                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
3022
3023                 rxq->rx_using_sse = rx_using_sse;
3024                 rxq->rx_ftag_en = rx_ftag_en;
3025         }
3026 }
3027
3028 static void
3029 fm10k_params_init(struct rte_eth_dev *dev)
3030 {
3031         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3032         struct fm10k_dev_info *info =
3033                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3034
3035         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
3036          * there is no way to get link status without reading BAR4.  Until this
3037          * works, assume we have maximum bandwidth.
3038          * @todo - fix bus info
3039          */
3040         hw->bus_caps.speed = fm10k_bus_speed_8000;
3041         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
3042         hw->bus_caps.payload = fm10k_bus_payload_512;
3043         hw->bus.speed = fm10k_bus_speed_8000;
3044         hw->bus.width = fm10k_bus_width_pcie_x8;
3045         hw->bus.payload = fm10k_bus_payload_256;
3046
3047         info->rx_vec_allowed = true;
3048         info->sm_down = false;
3049 }
3050
3051 static int
3052 eth_fm10k_dev_init(struct rte_eth_dev *dev)
3053 {
3054         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3055         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3056         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3057         int diag, i;
3058         struct fm10k_macvlan_filter_info *macvlan;
3059
3060         PMD_INIT_FUNC_TRACE();
3061
3062         dev->dev_ops = &fm10k_eth_dev_ops;
3063         dev->rx_queue_count = fm10k_dev_rx_queue_count;
3064         dev->rx_descriptor_status = fm10k_dev_rx_descriptor_status;
3065         dev->tx_descriptor_status = fm10k_dev_tx_descriptor_status;
3066         dev->rx_pkt_burst = &fm10k_recv_pkts;
3067         dev->tx_pkt_burst = &fm10k_xmit_pkts;
3068         dev->tx_pkt_prepare = &fm10k_prep_pkts;
3069
3070         /*
3071          * Primary process does the whole initialization, for secondary
3072          * processes, we just select the same Rx and Tx function as primary.
3073          */
3074         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3075                 fm10k_set_rx_function(dev);
3076                 fm10k_set_tx_function(dev);
3077                 return 0;
3078         }
3079
3080         rte_eth_copy_pci_info(dev, pdev);
3081         dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
3082
3083         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
3084         memset(macvlan, 0, sizeof(*macvlan));
3085         /* Vendor and Device ID need to be set before init of shared code */
3086         memset(hw, 0, sizeof(*hw));
3087         hw->device_id = pdev->id.device_id;
3088         hw->vendor_id = pdev->id.vendor_id;
3089         hw->subsystem_device_id = pdev->id.subsystem_device_id;
3090         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
3091         hw->revision_id = 0;
3092         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
3093         if (hw->hw_addr == NULL) {
3094                 PMD_INIT_LOG(ERR, "Bad mem resource."
3095                         " Try to refuse unused devices.");
3096                 return -EIO;
3097         }
3098
3099         /* Store fm10k_adapter pointer */
3100         hw->back = dev->data->dev_private;
3101
3102         /* Initialize the shared code */
3103         diag = fm10k_init_shared_code(hw);
3104         if (diag != FM10K_SUCCESS) {
3105                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
3106                 return -EIO;
3107         }
3108
3109         /* Initialize parameters */
3110         fm10k_params_init(dev);
3111
3112         /* Initialize the hw */
3113         diag = fm10k_init_hw(hw);
3114         if (diag != FM10K_SUCCESS) {
3115                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
3116                 return -EIO;
3117         }
3118
3119         /* Initialize MAC address(es) */
3120         dev->data->mac_addrs = rte_zmalloc("fm10k",
3121                         RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
3122         if (dev->data->mac_addrs == NULL) {
3123                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
3124                 return -ENOMEM;
3125         }
3126
3127         diag = fm10k_read_mac_addr(hw);
3128
3129         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3130                         &dev->data->mac_addrs[0]);
3131
3132         if (diag != FM10K_SUCCESS ||
3133                 !rte_is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
3134
3135                 /* Generate a random addr */
3136                 rte_eth_random_addr(hw->mac.addr);
3137                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
3138                 rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3139                 &dev->data->mac_addrs[0]);
3140         }
3141
3142         /* Reset the hw statistics */
3143         diag = fm10k_stats_reset(dev);
3144         if (diag != 0) {
3145                 PMD_INIT_LOG(ERR, "Stats reset failed: %d", diag);
3146                 return diag;
3147         }
3148
3149         /* Reset the hw */
3150         diag = fm10k_reset_hw(hw);
3151         if (diag != FM10K_SUCCESS) {
3152                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
3153                 return -EIO;
3154         }
3155
3156         /* Setup mailbox service */
3157         diag = fm10k_setup_mbx_service(hw);
3158         if (diag != FM10K_SUCCESS) {
3159                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
3160                 return -EIO;
3161         }
3162
3163         /*PF/VF has different interrupt handling mechanism */
3164         if (hw->mac.type == fm10k_mac_pf) {
3165                 /* register callback func to eal lib */
3166                 rte_intr_callback_register(intr_handle,
3167                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3168
3169                 /* enable MISC interrupt */
3170                 fm10k_dev_enable_intr_pf(dev);
3171         } else { /* VF */
3172                 rte_intr_callback_register(intr_handle,
3173                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3174
3175                 fm10k_dev_enable_intr_vf(dev);
3176         }
3177
3178         /* Enable intr after callback registered */
3179         rte_intr_enable(intr_handle);
3180
3181         hw->mac.ops.update_int_moderator(hw);
3182
3183         /* Make sure Switch Manager is ready before going forward. */
3184         if (hw->mac.type == fm10k_mac_pf) {
3185                 bool switch_ready = false;
3186
3187                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3188                         fm10k_mbx_lock(hw);
3189                         hw->mac.ops.get_host_state(hw, &switch_ready);
3190                         fm10k_mbx_unlock(hw);
3191                         if (switch_ready == true)
3192                                 break;
3193                         /* Delay some time to acquire async LPORT_MAP info. */
3194                         rte_delay_us(WAIT_SWITCH_MSG_US);
3195                 }
3196
3197                 if (switch_ready == false) {
3198                         PMD_INIT_LOG(ERR, "switch is not ready");
3199                         return -1;
3200                 }
3201         }
3202
3203         /*
3204          * Below function will trigger operations on mailbox, acquire lock to
3205          * avoid race condition from interrupt handler. Operations on mailbox
3206          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3207          * will handle and generate an interrupt to our side. Then,  FIFO in
3208          * mailbox will be touched.
3209          */
3210         fm10k_mbx_lock(hw);
3211         /* Enable port first */
3212         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3213                                         MAX_LPORT_NUM, 1);
3214
3215         /* Set unicast mode by default. App can change to other mode in other
3216          * API func.
3217          */
3218         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3219                                         FM10K_XCAST_MODE_NONE);
3220
3221         fm10k_mbx_unlock(hw);
3222
3223         /* Make sure default VID is ready before going forward. */
3224         if (hw->mac.type == fm10k_mac_pf) {
3225                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3226                         if (hw->mac.default_vid)
3227                                 break;
3228                         /* Delay some time to acquire async port VLAN info. */
3229                         rte_delay_us(WAIT_SWITCH_MSG_US);
3230                 }
3231
3232                 if (!hw->mac.default_vid) {
3233                         PMD_INIT_LOG(ERR, "default VID is not ready");
3234                         return -1;
3235                 }
3236         }
3237
3238         /* Add default mac address */
3239         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3240                 MAIN_VSI_POOL_NUMBER);
3241
3242         return 0;
3243 }
3244
3245 static int
3246 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3247 {
3248         PMD_INIT_FUNC_TRACE();
3249         fm10k_dev_close(dev);
3250         return 0;
3251 }
3252
3253 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3254         struct rte_pci_device *pci_dev)
3255 {
3256         return rte_eth_dev_pci_generic_probe(pci_dev,
3257                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3258 }
3259
3260 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3261 {
3262         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3263 }
3264
3265 /*
3266  * The set of PCI devices this driver supports. This driver will enable both PF
3267  * and SRIOV-VF devices.
3268  */
3269 static const struct rte_pci_id pci_id_fm10k_map[] = {
3270         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3271         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3272         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3273         { .vendor_id = 0, /* sentinel */ },
3274 };
3275
3276 static struct rte_pci_driver rte_pmd_fm10k = {
3277         .id_table = pci_id_fm10k_map,
3278         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
3279         .probe = eth_fm10k_pci_probe,
3280         .remove = eth_fm10k_pci_remove,
3281 };
3282
3283 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3284 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3285 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");
3286 RTE_LOG_REGISTER_SUFFIX(fm10k_logtype_init, init, NOTICE);
3287 RTE_LOG_REGISTER_SUFFIX(fm10k_logtype_driver, driver, NOTICE);
3288 #ifdef RTE_ETHDEV_DEBUG_RX
3289 RTE_LOG_REGISTER_SUFFIX(fm10k_logtype_rx, rx, DEBUG);
3290 #endif
3291 #ifdef RTE_ETHDEV_DEBUG_TX
3292 RTE_LOG_REGISTER_SUFFIX(fm10k_logtype_tx, tx, DEBUG);
3293 #endif