drivers/net/fm10k/fm10k_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2013-2016 Intel Corporation
   3  */
   4
   5 #include <rte_ethdev_driver.h>
   6 #include <rte_ethdev_pci.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_string_fns.h>
  10 #include <rte_dev.h>
  11 #include <rte_spinlock.h>
  12 #include <rte_kvargs.h>
  13
  14 #include "fm10k.h"
  15 #include "base/fm10k_api.h"
  16
  17 /* Default delay to acquire mailbox lock */
  18 #define FM10K_MBXLOCK_DELAY_US 20
  19 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  20
  21 #define MAIN_VSI_POOL_NUMBER 0
  22
  23 /* Max try times to acquire switch status */
  24 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  25 /* Wait interval to get switch status */
  26 #define WAIT_SWITCH_MSG_US    100000
  27 /* A period of quiescence for switch */
  28 #define FM10K_SWITCH_QUIESCE_US 100000
  29 /* Number of chars per uint32 type */
  30 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  31 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  32
  33 /* default 1:1 map from queue ID to interrupt vector ID */
  34 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  35
  36 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  37 #define MAX_LPORT_NUM    128
  38 #define GLORT_FD_Q_BASE  0x40
  39 #define GLORT_PF_MASK    0xFFC0
  40 #define GLORT_FD_MASK    GLORT_PF_MASK
  41 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  42
  43 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  44 static int fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  45 static int fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  46 static int fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  47 static int fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  48 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  49 static int
  50 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  51 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  52         const u8 *mac, bool add, uint32_t pool);
  53 static void fm10k_tx_queue_release(void *queue);
  54 static void fm10k_rx_queue_release(void *queue);
  55 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  56 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  57 static int fm10k_check_ftag(struct rte_devargs *devargs);
  58 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  59
  60 static int fm10k_dev_infos_get(struct rte_eth_dev *dev,
  61                                struct rte_eth_dev_info *dev_info);
  62 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev);
  63 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev);
  64 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev);
  65 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev);
  66
  67 struct fm10k_xstats_name_off {
  68         char name[RTE_ETH_XSTATS_NAME_SIZE];
  69         unsigned offset;
  70 };
  71
  72 static const struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  73         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  74         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  75         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  76         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  77         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  78         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  79         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  80         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  81                 nodesc_drop)},
  82 };
  83
  84 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  85                 sizeof(fm10k_hw_stats_strings[0]))
  86
  87 static const struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
  88         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
  89         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
  90         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
  91 };
  92
  93 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
  94                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
  95
  96 static const struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
  97         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
  98         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
  99 };
 100
 101 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 102                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 103
 104 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 105                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 106 static int
 107 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 108
 109 static void
 110 fm10k_mbx_initlock(struct fm10k_hw *hw)
 111 {
 112         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 113 }
 114
 115 static void
 116 fm10k_mbx_lock(struct fm10k_hw *hw)
 117 {
 118         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 119                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 120 }
 121
 122 static void
 123 fm10k_mbx_unlock(struct fm10k_hw *hw)
 124 {
 125         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 126 }
 127
 128 /* Stubs needed for linkage when vPMD is disabled */
 129 __rte_weak int
 130 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 131 {
 132         return -1;
 133 }
 134
 135 __rte_weak uint16_t
 136 fm10k_recv_pkts_vec(
 137         __rte_unused void *rx_queue,
 138         __rte_unused struct rte_mbuf **rx_pkts,
 139         __rte_unused uint16_t nb_pkts)
 140 {
 141         return 0;
 142 }
 143
 144 __rte_weak uint16_t
 145 fm10k_recv_scattered_pkts_vec(
 146                 __rte_unused void *rx_queue,
 147                 __rte_unused struct rte_mbuf **rx_pkts,
 148                 __rte_unused uint16_t nb_pkts)
 149 {
 150         return 0;
 151 }
 152
 153 __rte_weak int
 154 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 155
 156 {
 157         return -1;
 158 }
 159
 160 __rte_weak void
 161 fm10k_rx_queue_release_mbufs_vec(
 162                 __rte_unused struct fm10k_rx_queue *rxq)
 163 {
 164         return;
 165 }
 166
 167 __rte_weak void
 168 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 169 {
 170         return;
 171 }
 172
 173 __rte_weak int
 174 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 175 {
 176         return -1;
 177 }
 178
 179 __rte_weak uint16_t
 180 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 181                            __rte_unused struct rte_mbuf **tx_pkts,
 182                            __rte_unused uint16_t nb_pkts)
 183 {
 184         return 0;
 185 }
 186
 187 /*
 188  * reset queue to initial state, allocate software buffers used when starting
 189  * device.
 190  * return 0 on success
 191  * return -ENOMEM if buffers cannot be allocated
 192  * return -EINVAL if buffers do not satisfy alignment condition
 193  */
 194 static inline int
 195 rx_queue_reset(struct fm10k_rx_queue *q)
 196 {
 197         static const union fm10k_rx_desc zero = {{0} };
 198         uint64_t dma_addr;
 199         int i, diag;
 200         PMD_INIT_FUNC_TRACE();
 201
 202         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 203         if (diag != 0)
 204                 return -ENOMEM;
 205
 206         for (i = 0; i < q->nb_desc; ++i) {
 207                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 208                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 209                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 210                                                 q->nb_desc);
 211                         return -EINVAL;
 212                 }
 213                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 214                 q->hw_ring[i].q.pkt_addr = dma_addr;
 215                 q->hw_ring[i].q.hdr_addr = dma_addr;
 216         }
 217
 218         /* initialize extra software ring entries. Space for these extra
 219          * entries is always allocated.
 220          */
 221         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 222         for (i = 0; i < q->nb_fake_desc; ++i) {
 223                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 224                 q->hw_ring[q->nb_desc + i] = zero;
 225         }
 226
 227         q->next_dd = 0;
 228         q->next_alloc = 0;
 229         q->next_trigger = q->alloc_thresh - 1;
 230         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 231         q->rxrearm_start = 0;
 232         q->rxrearm_nb = 0;
 233
 234         return 0;
 235 }
 236
 237 /*
 238  * clean queue, descriptor rings, free software buffers used when stopping
 239  * device.
 240  */
 241 static inline void
 242 rx_queue_clean(struct fm10k_rx_queue *q)
 243 {
 244         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 245         uint32_t i;
 246         PMD_INIT_FUNC_TRACE();
 247
 248         /* zero descriptor rings */
 249         for (i = 0; i < q->nb_desc; ++i)
 250                 q->hw_ring[i] = zero;
 251
 252         /* zero faked descriptors */
 253         for (i = 0; i < q->nb_fake_desc; ++i)
 254                 q->hw_ring[q->nb_desc + i] = zero;
 255
 256         /* vPMD driver has a different way of releasing mbufs. */
 257         if (q->rx_using_sse) {
 258                 fm10k_rx_queue_release_mbufs_vec(q);
 259                 return;
 260         }
 261
 262         /* free software buffers */
 263         for (i = 0; i < q->nb_desc; ++i) {
 264                 if (q->sw_ring[i]) {
 265                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 266                         q->sw_ring[i] = NULL;
 267                 }
 268         }
 269 }
 270
 271 /*
 272  * free all queue memory used when releasing the queue (i.e. configure)
 273  */
 274 static inline void
 275 rx_queue_free(struct fm10k_rx_queue *q)
 276 {
 277         PMD_INIT_FUNC_TRACE();
 278         if (q) {
 279                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 280                 rx_queue_clean(q);
 281                 if (q->sw_ring) {
 282                         rte_free(q->sw_ring);
 283                         q->sw_ring = NULL;
 284                 }
 285                 rte_free(q);
 286                 q = NULL;
 287         }
 288 }
 289
 290 /*
 291  * disable RX queue, wait unitl HW finished necessary flush operation
 292  */
 293 static inline int
 294 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 295 {
 296         uint32_t reg, i;
 297
 298         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 299         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 300                         reg & ~FM10K_RXQCTL_ENABLE);
 301
 302         /* Wait 100us at most */
 303         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 304                 rte_delay_us(1);
 305                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 306                 if (!(reg & FM10K_RXQCTL_ENABLE))
 307                         break;
 308         }
 309
 310         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 311                 return -1;
 312
 313         return 0;
 314 }
 315
 316 /*
 317  * reset queue to initial state, allocate software buffers used when starting
 318  * device
 319  */
 320 static inline void
 321 tx_queue_reset(struct fm10k_tx_queue *q)
 322 {
 323         PMD_INIT_FUNC_TRACE();
 324         q->last_free = 0;
 325         q->next_free = 0;
 326         q->nb_used = 0;
 327         q->nb_free = q->nb_desc - 1;
 328         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 329         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 330 }
 331
 332 /*
 333  * clean queue, descriptor rings, free software buffers used when stopping
 334  * device
 335  */
 336 static inline void
 337 tx_queue_clean(struct fm10k_tx_queue *q)
 338 {
 339         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 340         uint32_t i;
 341         PMD_INIT_FUNC_TRACE();
 342
 343         /* zero descriptor rings */
 344         for (i = 0; i < q->nb_desc; ++i)
 345                 q->hw_ring[i] = zero;
 346
 347         /* free software buffers */
 348         for (i = 0; i < q->nb_desc; ++i) {
 349                 if (q->sw_ring[i]) {
 350                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 351                         q->sw_ring[i] = NULL;
 352                 }
 353         }
 354 }
 355
 356 /*
 357  * free all queue memory used when releasing the queue (i.e. configure)
 358  */
 359 static inline void
 360 tx_queue_free(struct fm10k_tx_queue *q)
 361 {
 362         PMD_INIT_FUNC_TRACE();
 363         if (q) {
 364                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 365                 tx_queue_clean(q);
 366                 if (q->rs_tracker.list) {
 367                         rte_free(q->rs_tracker.list);
 368                         q->rs_tracker.list = NULL;
 369                 }
 370                 if (q->sw_ring) {
 371                         rte_free(q->sw_ring);
 372                         q->sw_ring = NULL;
 373                 }
 374                 rte_free(q);
 375                 q = NULL;
 376         }
 377 }
 378
 379 /*
 380  * disable TX queue, wait unitl HW finished necessary flush operation
 381  */
 382 static inline int
 383 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 384 {
 385         uint32_t reg, i;
 386
 387         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 388         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 389                         reg & ~FM10K_TXDCTL_ENABLE);
 390
 391         /* Wait 100us at most */
 392         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 393                 rte_delay_us(1);
 394                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 395                 if (!(reg & FM10K_TXDCTL_ENABLE))
 396                         break;
 397         }
 398
 399         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 400                 return -1;
 401
 402         return 0;
 403 }
 404
 405 static int
 406 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 407 {
 408         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 409         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 410         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 411         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 412
 413         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 414
 415         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 416                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 417                 return -EINVAL;
 418         }
 419
 420         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 421                 return 0;
 422
 423         if (hw->mac.type == fm10k_mac_vf) {
 424                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 425                 return -EINVAL;
 426         }
 427
 428         /* Check VMDQ queue pool number */
 429         if (vmdq_conf->nb_queue_pools >
 430                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 431                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 432                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 433                         vmdq_conf->nb_queue_pools);
 434                 return -EINVAL;
 435         }
 436
 437         return 0;
 438 }
 439
 440 static const struct fm10k_txq_ops def_txq_ops = {
 441         .reset = tx_queue_reset,
 442 };
 443
 444 static int
 445 fm10k_dev_configure(struct rte_eth_dev *dev)
 446 {
 447         int ret;
 448
 449         PMD_INIT_FUNC_TRACE();
 450
 451         if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)
 452                 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH;
 453
 454         /* multipe queue mode checking */
 455         ret  = fm10k_check_mq_mode(dev);
 456         if (ret != 0) {
 457                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 458                             ret);
 459                 return ret;
 460         }
 461
 462         dev->data->scattered_rx = 0;
 463
 464         return 0;
 465 }
 466
 467 static void
 468 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 469 {
 470         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 471         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 472         uint32_t i;
 473
 474         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 475
 476         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 477                 if (!vmdq_conf->pool_map[i].pools)
 478                         continue;
 479                 fm10k_mbx_lock(hw);
 480                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 481                 fm10k_mbx_unlock(hw);
 482         }
 483 }
 484
 485 static void
 486 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 487 {
 488         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 489
 490         /* Add default mac address */
 491         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 492                 MAIN_VSI_POOL_NUMBER);
 493 }
 494
 495 static void
 496 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 497 {
 498         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 499         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 500         uint32_t mrqc, *key, i, reta, j;
 501         uint64_t hf;
 502
 503 #define RSS_KEY_SIZE 40
 504         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 505                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 506                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 507                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 508                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 509                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 510         };
 511
 512         if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 513                 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 514                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 515                 return;
 516         }
 517
 518         /* random key is rss_intel_key (default) or user provided (rss_key) */
 519         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 520                 key = (uint32_t *)rss_intel_key;
 521         else
 522                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 523
 524         /* Now fill our hash function seeds, 4 bytes at a time */
 525         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 526                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 527
 528         /*
 529          * Fill in redirection table
 530          * The byte-swap is needed because NIC registers are in
 531          * little-endian order.
 532          */
 533         reta = 0;
 534         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 535                 if (j == dev->data->nb_rx_queues)
 536                         j = 0;
 537                 reta = (reta << CHAR_BIT) | j;
 538                 if ((i & 3) == 3)
 539                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 540                                         rte_bswap32(reta));
 541         }
 542
 543         /*
 544          * Generate RSS hash based on packet types, TCP/UDP
 545          * port numbers and/or IPv4/v6 src and dst addresses
 546          */
 547         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 548         mrqc = 0;
 549         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 550         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 551         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 552         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 553         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 554         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 555         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 556         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 557         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 558
 559         if (mrqc == 0) {
 560                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 561                         "supported", hf);
 562                 return;
 563         }
 564
 565         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 566 }
 567
 568 static void
 569 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 570 {
 571         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 572         uint32_t i;
 573
 574         for (i = 0; i < nb_lport_new; i++) {
 575                 /* Set unicast mode by default. App can change
 576                  * to other mode in other API func.
 577                  */
 578                 fm10k_mbx_lock(hw);
 579                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 580                         FM10K_XCAST_MODE_NONE);
 581                 fm10k_mbx_unlock(hw);
 582         }
 583 }
 584
 585 static void
 586 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 587 {
 588         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 589         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 590         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 591         struct fm10k_macvlan_filter_info *macvlan;
 592         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 593         uint16_t nb_lport_new;
 594
 595         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 596         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 597
 598         fm10k_dev_rss_configure(dev);
 599
 600         /* only PF supports VMDQ */
 601         if (hw->mac.type != fm10k_mac_pf)
 602                 return;
 603
 604         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 605                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 606
 607         /* no pool number change, no need to update logic port and VLAN/MAC */
 608         if (macvlan->nb_queue_pools == nb_queue_pools)
 609                 return;
 610
 611         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 612         fm10k_dev_logic_port_update(dev, nb_lport_new);
 613
 614         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 615         memset(dev->data->mac_addrs, 0,
 616                 RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 617         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
 618                 &dev->data->mac_addrs[0]);
 619         memset(macvlan, 0, sizeof(*macvlan));
 620         macvlan->nb_queue_pools = nb_queue_pools;
 621
 622         if (nb_queue_pools)
 623                 fm10k_dev_vmdq_rx_configure(dev);
 624         else
 625                 fm10k_dev_pf_main_vsi_reset(dev);
 626 }
 627
 628 static int
 629 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 630 {
 631         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 632         int i, ret;
 633         struct fm10k_tx_queue *txq;
 634         uint64_t base_addr;
 635         uint32_t size;
 636
 637         /* Disable TXINT to avoid possible interrupt */
 638         for (i = 0; i < hw->mac.max_queues; i++)
 639                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 640                                 3 << FM10K_TXINT_TIMER_SHIFT);
 641
 642         /* Setup TX queue */
 643         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 644                 txq = dev->data->tx_queues[i];
 645                 base_addr = txq->hw_ring_phys_addr;
 646                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 647
 648                 /* disable queue to avoid issues while updating state */
 649                 ret = tx_queue_disable(hw, i);
 650                 if (ret) {
 651                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 652                         return -1;
 653                 }
 654                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 655                  * register is read-only for VF.
 656                  */
 657                 if (fm10k_check_ftag(dev->device->devargs)) {
 658                         if (hw->mac.type == fm10k_mac_pf) {
 659                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 660                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 661                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 662                         } else {
 663                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 664                                 return -ENOTSUP;
 665                         }
 666                 }
 667
 668                 /* set location and size for descriptor ring */
 669                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 670                                 base_addr & UINT64_LOWER_32BITS_MASK);
 671                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 672                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 673                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 674
 675                 /* assign default SGLORT for each TX queue by PF */
 676                 if (hw->mac.type == fm10k_mac_pf)
 677                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 678         }
 679
 680         /* set up vector or scalar TX function as appropriate */
 681         fm10k_set_tx_function(dev);
 682
 683         return 0;
 684 }
 685
 686 static int
 687 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 688 {
 689         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 690         struct fm10k_macvlan_filter_info *macvlan;
 691         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 692         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 693         int i, ret;
 694         struct fm10k_rx_queue *rxq;
 695         uint64_t base_addr;
 696         uint32_t size;
 697         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 698         uint32_t logic_port = hw->mac.dglort_map;
 699         uint16_t buf_size;
 700         uint16_t queue_stride = 0;
 701
 702         /* enable RXINT for interrupt mode */
 703         i = 0;
 704         if (rte_intr_dp_is_en(intr_handle)) {
 705                 for (; i < dev->data->nb_rx_queues; i++) {
 706                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 707                         if (hw->mac.type == fm10k_mac_pf)
 708                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 709                                         FM10K_ITR_AUTOMASK |
 710                                         FM10K_ITR_MASK_CLEAR);
 711                         else
 712                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 713                                         FM10K_ITR_AUTOMASK |
 714                                         FM10K_ITR_MASK_CLEAR);
 715                 }
 716         }
 717         /* Disable other RXINT to avoid possible interrupt */
 718         for (; i < hw->mac.max_queues; i++)
 719                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 720                         3 << FM10K_RXINT_TIMER_SHIFT);
 721
 722         /* Setup RX queues */
 723         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 724                 rxq = dev->data->rx_queues[i];
 725                 base_addr = rxq->hw_ring_phys_addr;
 726                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 727
 728                 /* disable queue to avoid issues while updating state */
 729                 ret = rx_queue_disable(hw, i);
 730                 if (ret) {
 731                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 732                         return -1;
 733                 }
 734
 735                 /* Setup the Base and Length of the Rx Descriptor Ring */
 736                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 737                                 base_addr & UINT64_LOWER_32BITS_MASK);
 738                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 739                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 740                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 741
 742                 /* Configure the Rx buffer size for one buff without split */
 743                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 744                         RTE_PKTMBUF_HEADROOM);
 745                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 746                  * reserved for this purpose, and the worst case could be 511B.
 747                  * But SRR reg assumes all buffers have the same size. In order
 748                  * to fill the gap, we'll have to consider the worst case and
 749                  * assume 512B is reserved. If we don't do so, it's possible
 750                  * for HW to overwrite data to next mbuf.
 751                  */
 752                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 753
 754                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 755                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 756                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 757
 758                 /* It adds dual VLAN length for supporting dual VLAN */
 759                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 760                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 761                         rxq->offloads & DEV_RX_OFFLOAD_SCATTER) {
 762                         uint32_t reg;
 763                         dev->data->scattered_rx = 1;
 764                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 765                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 766                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 767                 }
 768
 769                 /* Enable drop on empty, it's RO for VF */
 770                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 771                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 772
 773                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 774                 FM10K_WRITE_FLUSH(hw);
 775         }
 776
 777         /* Configure VMDQ/RSS if applicable */
 778         fm10k_dev_mq_rx_configure(dev);
 779
 780         /* Decide the best RX function */
 781         fm10k_set_rx_function(dev);
 782
 783         /* update RX_SGLORT for loopback suppress*/
 784         if (hw->mac.type != fm10k_mac_pf)
 785                 return 0;
 786         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 787         if (macvlan->nb_queue_pools)
 788                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 789         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 790                 if (i && queue_stride && !(i % queue_stride))
 791                         logic_port++;
 792                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 793         }
 794
 795         return 0;
 796 }
 797
 798 static int
 799 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 800 {
 801         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 802         int err;
 803         uint32_t reg;
 804         struct fm10k_rx_queue *rxq;
 805
 806         PMD_INIT_FUNC_TRACE();
 807
 808         rxq = dev->data->rx_queues[rx_queue_id];
 809         err = rx_queue_reset(rxq);
 810         if (err == -ENOMEM) {
 811                 PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 812                 return err;
 813         } else if (err == -EINVAL) {
 814                 PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 815                         " %d", err);
 816                 return err;
 817         }
 818
 819         /* Setup the HW Rx Head and Tail Descriptor Pointers
 820          * Note: this must be done AFTER the queue is enabled on real
 821          * hardware, but BEFORE the queue is enabled when using the
 822          * emulation platform. Do it in both places for now and remove
 823          * this comment and the following two register writes when the
 824          * emulation platform is no longer being used.
 825          */
 826         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 827         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 828
 829         /* Set PF ownership flag for PF devices */
 830         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 831         if (hw->mac.type == fm10k_mac_pf)
 832                 reg |= FM10K_RXQCTL_PF;
 833         reg |= FM10K_RXQCTL_ENABLE;
 834         /* enable RX queue */
 835         FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 836         FM10K_WRITE_FLUSH(hw);
 837
 838         /* Setup the HW Rx Head and Tail Descriptor Pointers
 839          * Note: this must be done AFTER the queue is enabled
 840          */
 841         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 842         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 843         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 844
 845         return 0;
 846 }
 847
 848 static int
 849 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 850 {
 851         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 852
 853         PMD_INIT_FUNC_TRACE();
 854
 855         /* Disable RX queue */
 856         rx_queue_disable(hw, rx_queue_id);
 857
 858         /* Free mbuf and clean HW ring */
 859         rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 860         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 861
 862         return 0;
 863 }
 864
 865 static int
 866 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 867 {
 868         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 869         /** @todo - this should be defined in the shared code */
 870 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 871         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 872         struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 873
 874         PMD_INIT_FUNC_TRACE();
 875
 876         q->ops->reset(q);
 877
 878         /* reset head and tail pointers */
 879         FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 880         FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 881
 882         /* enable TX queue */
 883         FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 884                                 FM10K_TXDCTL_ENABLE | txdctl);
 885         FM10K_WRITE_FLUSH(hw);
 886         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 887
 888         return 0;
 889 }
 890
 891 static int
 892 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 893 {
 894         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 895
 896         PMD_INIT_FUNC_TRACE();
 897
 898         tx_queue_disable(hw, tx_queue_id);
 899         tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 900         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 901
 902         return 0;
 903 }
 904
 905 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 906 {
 907         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 908                 != FM10K_DGLORTMAP_NONE);
 909 }
 910
 911 static int
 912 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 913 {
 914         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 915         int status;
 916
 917         PMD_INIT_FUNC_TRACE();
 918
 919         /* Return if it didn't acquire valid glort range */
 920         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 921                 return 0;
 922
 923         fm10k_mbx_lock(hw);
 924         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 925                                 FM10K_XCAST_MODE_PROMISC);
 926         fm10k_mbx_unlock(hw);
 927
 928         if (status != FM10K_SUCCESS) {
 929                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 930                 return -EAGAIN;
 931         }
 932
 933         return 0;
 934 }
 935
 936 static int
 937 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 938 {
 939         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 940         uint8_t mode;
 941         int status;
 942
 943         PMD_INIT_FUNC_TRACE();
 944
 945         /* Return if it didn't acquire valid glort range */
 946         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 947                 return 0;
 948
 949         if (dev->data->all_multicast == 1)
 950                 mode = FM10K_XCAST_MODE_ALLMULTI;
 951         else
 952                 mode = FM10K_XCAST_MODE_NONE;
 953
 954         fm10k_mbx_lock(hw);
 955         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 956                                 mode);
 957         fm10k_mbx_unlock(hw);
 958
 959         if (status != FM10K_SUCCESS) {
 960                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 961                 return -EAGAIN;
 962         }
 963
 964         return 0;
 965 }
 966
 967 static int
 968 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 969 {
 970         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 971         int status;
 972
 973         PMD_INIT_FUNC_TRACE();
 974
 975         /* Return if it didn't acquire valid glort range */
 976         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 977                 return 0;
 978
 979         /* If promiscuous mode is enabled, it doesn't make sense to enable
 980          * allmulticast and disable promiscuous since fm10k only can select
 981          * one of the modes.
 982          */
 983         if (dev->data->promiscuous) {
 984                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 985                         "needn't enable allmulticast");
 986                 return 0;
 987         }
 988
 989         fm10k_mbx_lock(hw);
 990         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 991                                 FM10K_XCAST_MODE_ALLMULTI);
 992         fm10k_mbx_unlock(hw);
 993
 994         if (status != FM10K_SUCCESS) {
 995                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 996                 return -EAGAIN;
 997         }
 998
 999         return 0;
1000 }
1001
1002 static int
1003 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1004 {
1005         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1006         int status;
1007
1008         PMD_INIT_FUNC_TRACE();
1009
1010         /* Return if it didn't acquire valid glort range */
1011         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1012                 return 0;
1013
1014         if (dev->data->promiscuous) {
1015                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1016                         "since promisc mode is enabled");
1017                 return -EINVAL;
1018         }
1019
1020         fm10k_mbx_lock(hw);
1021         /* Change mode to unicast mode */
1022         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1023                                 FM10K_XCAST_MODE_NONE);
1024         fm10k_mbx_unlock(hw);
1025
1026         if (status != FM10K_SUCCESS) {
1027                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1028                 return -EAGAIN;
1029         }
1030
1031         return 0;
1032 }
1033
1034 static void
1035 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1036 {
1037         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1038         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1039         uint16_t nb_queue_pools;
1040         struct fm10k_macvlan_filter_info *macvlan;
1041
1042         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1043         nb_queue_pools = macvlan->nb_queue_pools;
1044         pool_len = nb_queue_pools ? rte_fls_u32(nb_queue_pools - 1) : 0;
1045         rss_len = rte_fls_u32(dev->data->nb_rx_queues - 1) - pool_len;
1046
1047         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1048         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1049         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1050                         hw->mac.dglort_map;
1051         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1052         /* Configure VMDQ/RSS DGlort Decoder */
1053         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1054
1055         /* Flow Director configurations, only queue number is valid. */
1056         dglortdec = rte_fls_u32(dev->data->nb_rx_queues - 1);
1057         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1058                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1059         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1060         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1061
1062         /* Invalidate all other GLORT entries */
1063         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1064                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1065                                 FM10K_DGLORTMAP_NONE);
1066 }
1067
1068 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1069 static int
1070 fm10k_dev_start(struct rte_eth_dev *dev)
1071 {
1072         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1073         int i, diag;
1074
1075         PMD_INIT_FUNC_TRACE();
1076
1077         /* stop, init, then start the hw */
1078         diag = fm10k_stop_hw(hw);
1079         if (diag != FM10K_SUCCESS) {
1080                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1081                 return -EIO;
1082         }
1083
1084         diag = fm10k_init_hw(hw);
1085         if (diag != FM10K_SUCCESS) {
1086                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1087                 return -EIO;
1088         }
1089
1090         diag = fm10k_start_hw(hw);
1091         if (diag != FM10K_SUCCESS) {
1092                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1093                 return -EIO;
1094         }
1095
1096         diag = fm10k_dev_tx_init(dev);
1097         if (diag) {
1098                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1099                 return diag;
1100         }
1101
1102         if (fm10k_dev_rxq_interrupt_setup(dev))
1103                 return -EIO;
1104
1105         diag = fm10k_dev_rx_init(dev);
1106         if (diag) {
1107                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1108                 return diag;
1109         }
1110
1111         if (hw->mac.type == fm10k_mac_pf)
1112                 fm10k_dev_dglort_map_configure(dev);
1113
1114         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1115                 struct fm10k_rx_queue *rxq;
1116                 rxq = dev->data->rx_queues[i];
1117
1118                 if (rxq->rx_deferred_start)
1119                         continue;
1120                 diag = fm10k_dev_rx_queue_start(dev, i);
1121                 if (diag != 0) {
1122                         int j;
1123                         for (j = 0; j < i; ++j)
1124                                 rx_queue_clean(dev->data->rx_queues[j]);
1125                         return diag;
1126                 }
1127         }
1128
1129         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1130                 struct fm10k_tx_queue *txq;
1131                 txq = dev->data->tx_queues[i];
1132
1133                 if (txq->tx_deferred_start)
1134                         continue;
1135                 diag = fm10k_dev_tx_queue_start(dev, i);
1136                 if (diag != 0) {
1137                         int j;
1138                         for (j = 0; j < i; ++j)
1139                                 tx_queue_clean(dev->data->tx_queues[j]);
1140                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1141                                 rx_queue_clean(dev->data->rx_queues[j]);
1142                         return diag;
1143                 }
1144         }
1145
1146         /* Update default vlan when not in VMDQ mode */
1147         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1148                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1149
1150         fm10k_link_update(dev, 0);
1151
1152         return 0;
1153 }
1154
1155 static int
1156 fm10k_dev_stop(struct rte_eth_dev *dev)
1157 {
1158         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1159         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1160         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1161         int i;
1162
1163         PMD_INIT_FUNC_TRACE();
1164         dev->data->dev_started = 0;
1165
1166         if (dev->data->tx_queues)
1167                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1168                         fm10k_dev_tx_queue_stop(dev, i);
1169
1170         if (dev->data->rx_queues)
1171                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1172                         fm10k_dev_rx_queue_stop(dev, i);
1173
1174         /* Disable datapath event */
1175         if (rte_intr_dp_is_en(intr_handle)) {
1176                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1177                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1178                                 3 << FM10K_RXINT_TIMER_SHIFT);
1179                         if (hw->mac.type == fm10k_mac_pf)
1180                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1181                                         FM10K_ITR_MASK_SET);
1182                         else
1183                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1184                                         FM10K_ITR_MASK_SET);
1185                 }
1186         }
1187         /* Clean datapath event and queue/vec mapping */
1188         rte_intr_efd_disable(intr_handle);
1189         rte_free(intr_handle->intr_vec);
1190         intr_handle->intr_vec = NULL;
1191
1192         return 0;
1193 }
1194
1195 static void
1196 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1197 {
1198         int i;
1199
1200         PMD_INIT_FUNC_TRACE();
1201
1202         if (dev->data->tx_queues) {
1203                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1204                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1205
1206                         tx_queue_free(txq);
1207                 }
1208         }
1209
1210         if (dev->data->rx_queues) {
1211                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1212                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1213         }
1214 }
1215
1216 static int
1217 fm10k_link_update(struct rte_eth_dev *dev,
1218         __rte_unused int wait_to_complete)
1219 {
1220         struct fm10k_dev_info *dev_info =
1221                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1222         PMD_INIT_FUNC_TRACE();
1223
1224         dev->data->dev_link.link_speed  = ETH_SPEED_NUM_50G;
1225         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1226         dev->data->dev_link.link_status =
1227                 dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP;
1228         dev->data->dev_link.link_autoneg = ETH_LINK_FIXED;
1229
1230         return 0;
1231 }
1232
1233 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1234         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1235 {
1236         unsigned i, q;
1237         unsigned count = 0;
1238
1239         if (xstats_names != NULL) {
1240                 /* Note: limit checked in rte_eth_xstats_names() */
1241
1242                 /* Global stats */
1243                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1244                         snprintf(xstats_names[count].name,
1245                                 sizeof(xstats_names[count].name),
1246                                 "%s", fm10k_hw_stats_strings[count].name);
1247                         count++;
1248                 }
1249
1250                 /* PF queue stats */
1251                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1252                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1253                                 snprintf(xstats_names[count].name,
1254                                         sizeof(xstats_names[count].name),
1255                                         "rx_q%u_%s", q,
1256                                         fm10k_hw_stats_rx_q_strings[i].name);
1257                                 count++;
1258                         }
1259                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1260                                 snprintf(xstats_names[count].name,
1261                                         sizeof(xstats_names[count].name),
1262                                         "tx_q%u_%s", q,
1263                                         fm10k_hw_stats_tx_q_strings[i].name);
1264                                 count++;
1265                         }
1266                 }
1267         }
1268         return FM10K_NB_XSTATS;
1269 }
1270
1271 static int
1272 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1273                  unsigned n)
1274 {
1275         struct fm10k_hw_stats *hw_stats =
1276                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1277         unsigned i, q, count = 0;
1278
1279         if (n < FM10K_NB_XSTATS)
1280                 return FM10K_NB_XSTATS;
1281
1282         /* Global stats */
1283         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1284                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1285                         fm10k_hw_stats_strings[count].offset);
1286                 xstats[count].id = count;
1287                 count++;
1288         }
1289
1290         /* PF queue stats */
1291         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1292                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1293                         xstats[count].value =
1294                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1295                                 fm10k_hw_stats_rx_q_strings[i].offset);
1296                         xstats[count].id = count;
1297                         count++;
1298                 }
1299                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1300                         xstats[count].value =
1301                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1302                                 fm10k_hw_stats_tx_q_strings[i].offset);
1303                         xstats[count].id = count;
1304                         count++;
1305                 }
1306         }
1307
1308         return FM10K_NB_XSTATS;
1309 }
1310
1311 static int
1312 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1313 {
1314         uint64_t ipackets, opackets, ibytes, obytes, imissed;
1315         struct fm10k_hw *hw =
1316                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1317         struct fm10k_hw_stats *hw_stats =
1318                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1319         int i;
1320
1321         PMD_INIT_FUNC_TRACE();
1322
1323         fm10k_update_hw_stats(hw, hw_stats);
1324
1325         ipackets = opackets = ibytes = obytes = imissed = 0;
1326         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1327                 (i < hw->mac.max_queues); ++i) {
1328                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1329                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1330                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1331                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1332                 stats->q_errors[i]   = hw_stats->q[i].rx_drops.count;
1333                 ipackets += stats->q_ipackets[i];
1334                 opackets += stats->q_opackets[i];
1335                 ibytes   += stats->q_ibytes[i];
1336                 obytes   += stats->q_obytes[i];
1337                 imissed  += stats->q_errors[i];
1338         }
1339         stats->ipackets = ipackets;
1340         stats->opackets = opackets;
1341         stats->ibytes = ibytes;
1342         stats->obytes = obytes;
1343         stats->imissed = imissed;
1344         return 0;
1345 }
1346
1347 static int
1348 fm10k_stats_reset(struct rte_eth_dev *dev)
1349 {
1350         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1351         struct fm10k_hw_stats *hw_stats =
1352                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1353
1354         PMD_INIT_FUNC_TRACE();
1355
1356         memset(hw_stats, 0, sizeof(*hw_stats));
1357         fm10k_rebind_hw_stats(hw, hw_stats);
1358
1359         return 0;
1360 }
1361
1362 static int
1363 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1364         struct rte_eth_dev_info *dev_info)
1365 {
1366         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1367         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1368
1369         PMD_INIT_FUNC_TRACE();
1370
1371         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1372         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1373         dev_info->max_rx_queues      = hw->mac.max_queues;
1374         dev_info->max_tx_queues      = hw->mac.max_queues;
1375         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1376         dev_info->max_hash_mac_addrs = 0;
1377         dev_info->max_vfs            = pdev->max_vfs;
1378         dev_info->vmdq_pool_base     = 0;
1379         dev_info->vmdq_queue_base    = 0;
1380         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1381         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1382         dev_info->rx_queue_offload_capa = fm10k_get_rx_queue_offloads_capa(dev);
1383         dev_info->rx_offload_capa = fm10k_get_rx_port_offloads_capa(dev) |
1384                                     dev_info->rx_queue_offload_capa;
1385         dev_info->tx_queue_offload_capa = fm10k_get_tx_queue_offloads_capa(dev);
1386         dev_info->tx_offload_capa = fm10k_get_tx_port_offloads_capa(dev) |
1387                                     dev_info->tx_queue_offload_capa;
1388
1389         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1390         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1391         dev_info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1392                                         ETH_RSS_IPV6 |
1393                                         ETH_RSS_IPV6_EX |
1394                                         ETH_RSS_NONFRAG_IPV4_TCP |
1395                                         ETH_RSS_NONFRAG_IPV6_TCP |
1396                                         ETH_RSS_IPV6_TCP_EX |
1397                                         ETH_RSS_NONFRAG_IPV4_UDP |
1398                                         ETH_RSS_NONFRAG_IPV6_UDP |
1399                                         ETH_RSS_IPV6_UDP_EX;
1400
1401         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1402                 .rx_thresh = {
1403                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1404                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1405                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1406                 },
1407                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1408                 .rx_drop_en = 0,
1409                 .offloads = 0,
1410         };
1411
1412         dev_info->default_txconf = (struct rte_eth_txconf) {
1413                 .tx_thresh = {
1414                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1415                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1416                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1417                 },
1418                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1419                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1420                 .offloads = 0,
1421         };
1422
1423         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1424                 .nb_max = FM10K_MAX_RX_DESC,
1425                 .nb_min = FM10K_MIN_RX_DESC,
1426                 .nb_align = FM10K_MULT_RX_DESC,
1427         };
1428
1429         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1430                 .nb_max = FM10K_MAX_TX_DESC,
1431                 .nb_min = FM10K_MIN_TX_DESC,
1432                 .nb_align = FM10K_MULT_TX_DESC,
1433                 .nb_seg_max = FM10K_TX_MAX_SEG,
1434                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1435         };
1436
1437         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1438                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1439                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1440
1441         return 0;
1442 }
1443
1444 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1445 static const uint32_t *
1446 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1447 {
1448         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1449             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1450                 static uint32_t ptypes[] = {
1451                         /* refers to rx_desc_to_ol_flags() */
1452                         RTE_PTYPE_L2_ETHER,
1453                         RTE_PTYPE_L3_IPV4,
1454                         RTE_PTYPE_L3_IPV4_EXT,
1455                         RTE_PTYPE_L3_IPV6,
1456                         RTE_PTYPE_L3_IPV6_EXT,
1457                         RTE_PTYPE_L4_TCP,
1458                         RTE_PTYPE_L4_UDP,
1459                         RTE_PTYPE_UNKNOWN
1460                 };
1461
1462                 return ptypes;
1463         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1464                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1465                 static uint32_t ptypes_vec[] = {
1466                         /* refers to fm10k_desc_to_pktype_v() */
1467                         RTE_PTYPE_L3_IPV4,
1468                         RTE_PTYPE_L3_IPV4_EXT,
1469                         RTE_PTYPE_L3_IPV6,
1470                         RTE_PTYPE_L3_IPV6_EXT,
1471                         RTE_PTYPE_L4_TCP,
1472                         RTE_PTYPE_L4_UDP,
1473                         RTE_PTYPE_TUNNEL_GENEVE,
1474                         RTE_PTYPE_TUNNEL_NVGRE,
1475                         RTE_PTYPE_TUNNEL_VXLAN,
1476                         RTE_PTYPE_TUNNEL_GRE,
1477                         RTE_PTYPE_UNKNOWN
1478                 };
1479
1480                 return ptypes_vec;
1481         }
1482
1483         return NULL;
1484 }
1485 #else
1486 static const uint32_t *
1487 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1488 {
1489         return NULL;
1490 }
1491 #endif
1492
1493 static int
1494 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1495 {
1496         s32 result;
1497         uint16_t mac_num = 0;
1498         uint32_t vid_idx, vid_bit, mac_index;
1499         struct fm10k_hw *hw;
1500         struct fm10k_macvlan_filter_info *macvlan;
1501         struct rte_eth_dev_data *data = dev->data;
1502
1503         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1504         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1505
1506         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1507                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1508                 return -EINVAL;
1509         }
1510
1511         if (vlan_id > ETH_VLAN_ID_MAX) {
1512                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1513                 return -EINVAL;
1514         }
1515
1516         vid_idx = FM10K_VFTA_IDX(vlan_id);
1517         vid_bit = FM10K_VFTA_BIT(vlan_id);
1518         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1519         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1520                 return 0;
1521         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1522         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1523                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1524                         "in the VLAN filter table");
1525                 return -EINVAL;
1526         }
1527
1528         fm10k_mbx_lock(hw);
1529         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1530         fm10k_mbx_unlock(hw);
1531         if (result != FM10K_SUCCESS) {
1532                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1533                 return -EIO;
1534         }
1535
1536         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1537                         (result == FM10K_SUCCESS); mac_index++) {
1538                 if (rte_is_zero_ether_addr(&data->mac_addrs[mac_index]))
1539                         continue;
1540                 if (mac_num > macvlan->mac_num - 1) {
1541                         PMD_INIT_LOG(ERR, "MAC address number "
1542                                         "not match");
1543                         break;
1544                 }
1545                 fm10k_mbx_lock(hw);
1546                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1547                         data->mac_addrs[mac_index].addr_bytes,
1548                         vlan_id, on, 0);
1549                 fm10k_mbx_unlock(hw);
1550                 mac_num++;
1551         }
1552         if (result != FM10K_SUCCESS) {
1553                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1554                 return -EIO;
1555         }
1556
1557         if (on) {
1558                 macvlan->vlan_num++;
1559                 macvlan->vfta[vid_idx] |= vid_bit;
1560         } else {
1561                 macvlan->vlan_num--;
1562                 macvlan->vfta[vid_idx] &= ~vid_bit;
1563         }
1564         return 0;
1565 }
1566
1567 static int
1568 fm10k_vlan_offload_set(struct rte_eth_dev *dev __rte_unused,
1569                        int mask __rte_unused)
1570 {
1571         return 0;
1572 }
1573
1574 /* Add/Remove a MAC address, and update filters to main VSI */
1575 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1576                 const u8 *mac, bool add, uint32_t pool)
1577 {
1578         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1579         struct fm10k_macvlan_filter_info *macvlan;
1580         uint32_t i, j, k;
1581
1582         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1583
1584         if (pool != MAIN_VSI_POOL_NUMBER) {
1585                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1586                         "mac to pool %u", pool);
1587                 return;
1588         }
1589         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1590                 if (!macvlan->vfta[j])
1591                         continue;
1592                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1593                         if (!(macvlan->vfta[j] & (1 << k)))
1594                                 continue;
1595                         if (i + 1 > macvlan->vlan_num) {
1596                                 PMD_INIT_LOG(ERR, "vlan number not match");
1597                                 return;
1598                         }
1599                         fm10k_mbx_lock(hw);
1600                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1601                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1602                         fm10k_mbx_unlock(hw);
1603                         i++;
1604                 }
1605         }
1606 }
1607
1608 /* Add/Remove a MAC address, and update filters to VMDQ */
1609 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1610                 const u8 *mac, bool add, uint32_t pool)
1611 {
1612         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1613         struct fm10k_macvlan_filter_info *macvlan;
1614         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1615         uint32_t i;
1616
1617         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1618         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1619
1620         if (pool > macvlan->nb_queue_pools) {
1621                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1622                         " Max pool is %u",
1623                         pool, macvlan->nb_queue_pools);
1624                 return;
1625         }
1626         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1627                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1628                         continue;
1629                 fm10k_mbx_lock(hw);
1630                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1631                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1632                 fm10k_mbx_unlock(hw);
1633         }
1634 }
1635
1636 /* Add/Remove a MAC address, and update filters */
1637 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1638                 const u8 *mac, bool add, uint32_t pool)
1639 {
1640         struct fm10k_macvlan_filter_info *macvlan;
1641
1642         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1643
1644         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1645                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1646         else
1647                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1648
1649         if (add)
1650                 macvlan->mac_num++;
1651         else
1652                 macvlan->mac_num--;
1653 }
1654
1655 /* Add a MAC address, and update filters */
1656 static int
1657 fm10k_macaddr_add(struct rte_eth_dev *dev,
1658                 struct rte_ether_addr *mac_addr,
1659                 uint32_t index,
1660                 uint32_t pool)
1661 {
1662         struct fm10k_macvlan_filter_info *macvlan;
1663
1664         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1665         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1666         macvlan->mac_vmdq_id[index] = pool;
1667         return 0;
1668 }
1669
1670 /* Remove a MAC address, and update filters */
1671 static void
1672 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1673 {
1674         struct rte_eth_dev_data *data = dev->data;
1675         struct fm10k_macvlan_filter_info *macvlan;
1676
1677         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1678         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1679                         FALSE, macvlan->mac_vmdq_id[index]);
1680         macvlan->mac_vmdq_id[index] = 0;
1681 }
1682
1683 static inline int
1684 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1685 {
1686         if ((request < min) || (request > max) || ((request % mult) != 0))
1687                 return -1;
1688         else
1689                 return 0;
1690 }
1691
1692
1693 static inline int
1694 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1695 {
1696         if ((request < min) || (request > max) || ((div % request) != 0))
1697                 return -1;
1698         else
1699                 return 0;
1700 }
1701
1702 static inline int
1703 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1704 {
1705         uint16_t rx_free_thresh;
1706
1707         if (conf->rx_free_thresh == 0)
1708                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1709         else
1710                 rx_free_thresh = conf->rx_free_thresh;
1711
1712         /* make sure the requested threshold satisfies the constraints */
1713         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1714                         FM10K_RX_FREE_THRESH_MAX(q),
1715                         FM10K_RX_FREE_THRESH_DIV(q),
1716                         rx_free_thresh)) {
1717                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1718                         "less than or equal to %u, "
1719                         "greater than or equal to %u, "
1720                         "and a divisor of %u",
1721                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1722                         FM10K_RX_FREE_THRESH_MIN(q),
1723                         FM10K_RX_FREE_THRESH_DIV(q));
1724                 return -EINVAL;
1725         }
1726
1727         q->alloc_thresh = rx_free_thresh;
1728         q->drop_en = conf->rx_drop_en;
1729         q->rx_deferred_start = conf->rx_deferred_start;
1730
1731         return 0;
1732 }
1733
1734 /*
1735  * Hardware requires specific alignment for Rx packet buffers. At
1736  * least one of the following two conditions must be satisfied.
1737  *  1. Address is 512B aligned
1738  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1739  *
1740  * As such, the driver may need to adjust the DMA address within the
1741  * buffer by up to 512B.
1742  *
1743  * return 1 if the element size is valid, otherwise return 0.
1744  */
1745 static int
1746 mempool_element_size_valid(struct rte_mempool *mp)
1747 {
1748         uint32_t min_size;
1749
1750         /* elt_size includes mbuf header and headroom */
1751         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1752                         RTE_PKTMBUF_HEADROOM;
1753
1754         /* account for up to 512B of alignment */
1755         min_size -= FM10K_RX_DATABUF_ALIGN;
1756
1757         /* sanity check for overflow */
1758         if (min_size > mp->elt_size)
1759                 return 0;
1760
1761         /* size is valid */
1762         return 1;
1763 }
1764
1765 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev)
1766 {
1767         RTE_SET_USED(dev);
1768
1769         return (uint64_t)(DEV_RX_OFFLOAD_SCATTER);
1770 }
1771
1772 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
1773 {
1774         RTE_SET_USED(dev);
1775
1776         return  (uint64_t)(DEV_RX_OFFLOAD_VLAN_STRIP  |
1777                            DEV_RX_OFFLOAD_VLAN_FILTER |
1778                            DEV_RX_OFFLOAD_IPV4_CKSUM  |
1779                            DEV_RX_OFFLOAD_UDP_CKSUM   |
1780                            DEV_RX_OFFLOAD_TCP_CKSUM   |
1781                            DEV_RX_OFFLOAD_JUMBO_FRAME |
1782                            DEV_RX_OFFLOAD_HEADER_SPLIT |
1783                            DEV_RX_OFFLOAD_RSS_HASH);
1784 }
1785
1786 static int
1787 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1788         uint16_t nb_desc, unsigned int socket_id,
1789         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1790 {
1791         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1792         struct fm10k_dev_info *dev_info =
1793                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1794         struct fm10k_rx_queue *q;
1795         const struct rte_memzone *mz;
1796         uint64_t offloads;
1797
1798         PMD_INIT_FUNC_TRACE();
1799
1800         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1801
1802         /* make sure the mempool element size can account for alignment. */
1803         if (!mempool_element_size_valid(mp)) {
1804                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1805                 return -EINVAL;
1806         }
1807
1808         /* make sure a valid number of descriptors have been requested */
1809         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1810                                 FM10K_MULT_RX_DESC, nb_desc)) {
1811                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1812                         "less than or equal to %"PRIu32", "
1813                         "greater than or equal to %u, "
1814                         "and a multiple of %u",
1815                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1816                         FM10K_MULT_RX_DESC);
1817                 return -EINVAL;
1818         }
1819
1820         /*
1821          * if this queue existed already, free the associated memory. The
1822          * queue cannot be reused in case we need to allocate memory on
1823          * different socket than was previously used.
1824          */
1825         if (dev->data->rx_queues[queue_id] != NULL) {
1826                 rx_queue_free(dev->data->rx_queues[queue_id]);
1827                 dev->data->rx_queues[queue_id] = NULL;
1828         }
1829
1830         /* allocate memory for the queue structure */
1831         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1832                                 socket_id);
1833         if (q == NULL) {
1834                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1835                 return -ENOMEM;
1836         }
1837
1838         /* setup queue */
1839         q->mp = mp;
1840         q->nb_desc = nb_desc;
1841         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1842         q->port_id = dev->data->port_id;
1843         q->queue_id = queue_id;
1844         q->tail_ptr = (volatile uint32_t *)
1845                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1846         q->offloads = offloads;
1847         if (handle_rxconf(q, conf)) {
1848                 rte_free(q);
1849                 return -EINVAL;
1850         }
1851         /* allocate memory for the software ring */
1852         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1853                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1854                         RTE_CACHE_LINE_SIZE, socket_id);
1855         if (q->sw_ring == NULL) {
1856                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1857                 rte_free(q);
1858                 return -ENOMEM;
1859         }
1860
1861         /*
1862          * allocate memory for the hardware descriptor ring. A memzone large
1863          * enough to hold the maximum ring size is requested to allow for
1864          * resizing in later calls to the queue setup function.
1865          */
1866         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1867                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1868                                       socket_id);
1869         if (mz == NULL) {
1870                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1871                 rte_free(q->sw_ring);
1872                 rte_free(q);
1873                 return -ENOMEM;
1874         }
1875         q->hw_ring = mz->addr;
1876         q->hw_ring_phys_addr = mz->iova;
1877
1878         /* Check if number of descs satisfied Vector requirement */
1879         if (!rte_is_power_of_2(nb_desc)) {
1880                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1881                                     "preconditions - canceling the feature for "
1882                                     "the whole port[%d]",
1883                              q->queue_id, q->port_id);
1884                 dev_info->rx_vec_allowed = false;
1885         } else
1886                 fm10k_rxq_vec_setup(q);
1887
1888         dev->data->rx_queues[queue_id] = q;
1889         return 0;
1890 }
1891
1892 static void
1893 fm10k_rx_queue_release(void *queue)
1894 {
1895         PMD_INIT_FUNC_TRACE();
1896
1897         rx_queue_free(queue);
1898 }
1899
1900 static inline int
1901 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1902 {
1903         uint16_t tx_free_thresh;
1904         uint16_t tx_rs_thresh;
1905
1906         /* constraint MACROs require that tx_free_thresh is configured
1907          * before tx_rs_thresh */
1908         if (conf->tx_free_thresh == 0)
1909                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1910         else
1911                 tx_free_thresh = conf->tx_free_thresh;
1912
1913         /* make sure the requested threshold satisfies the constraints */
1914         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1915                         FM10K_TX_FREE_THRESH_MAX(q),
1916                         FM10K_TX_FREE_THRESH_DIV(q),
1917                         tx_free_thresh)) {
1918                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1919                         "less than or equal to %u, "
1920                         "greater than or equal to %u, "
1921                         "and a divisor of %u",
1922                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1923                         FM10K_TX_FREE_THRESH_MIN(q),
1924                         FM10K_TX_FREE_THRESH_DIV(q));
1925                 return -EINVAL;
1926         }
1927
1928         q->free_thresh = tx_free_thresh;
1929
1930         if (conf->tx_rs_thresh == 0)
1931                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1932         else
1933                 tx_rs_thresh = conf->tx_rs_thresh;
1934
1935         q->tx_deferred_start = conf->tx_deferred_start;
1936
1937         /* make sure the requested threshold satisfies the constraints */
1938         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1939                         FM10K_TX_RS_THRESH_MAX(q),
1940                         FM10K_TX_RS_THRESH_DIV(q),
1941                         tx_rs_thresh)) {
1942                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1943                         "less than or equal to %u, "
1944                         "greater than or equal to %u, "
1945                         "and a divisor of %u",
1946                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1947                         FM10K_TX_RS_THRESH_MIN(q),
1948                         FM10K_TX_RS_THRESH_DIV(q));
1949                 return -EINVAL;
1950         }
1951
1952         q->rs_thresh = tx_rs_thresh;
1953
1954         return 0;
1955 }
1956
1957 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev)
1958 {
1959         RTE_SET_USED(dev);
1960
1961         return 0;
1962 }
1963
1964 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
1965 {
1966         RTE_SET_USED(dev);
1967
1968         return (uint64_t)(DEV_TX_OFFLOAD_VLAN_INSERT |
1969                           DEV_TX_OFFLOAD_MULTI_SEGS  |
1970                           DEV_TX_OFFLOAD_IPV4_CKSUM  |
1971                           DEV_TX_OFFLOAD_UDP_CKSUM   |
1972                           DEV_TX_OFFLOAD_TCP_CKSUM   |
1973                           DEV_TX_OFFLOAD_TCP_TSO);
1974 }
1975
1976 static int
1977 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1978         uint16_t nb_desc, unsigned int socket_id,
1979         const struct rte_eth_txconf *conf)
1980 {
1981         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1982         struct fm10k_tx_queue *q;
1983         const struct rte_memzone *mz;
1984         uint64_t offloads;
1985
1986         PMD_INIT_FUNC_TRACE();
1987
1988         offloads = conf->offloads | dev->data->dev_conf.txmode.offloads;
1989
1990         /* make sure a valid number of descriptors have been requested */
1991         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1992                                 FM10K_MULT_TX_DESC, nb_desc)) {
1993                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1994                         "less than or equal to %"PRIu32", "
1995                         "greater than or equal to %u, "
1996                         "and a multiple of %u",
1997                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1998                         FM10K_MULT_TX_DESC);
1999                 return -EINVAL;
2000         }
2001
2002         /*
2003          * if this queue existed already, free the associated memory. The
2004          * queue cannot be reused in case we need to allocate memory on
2005          * different socket than was previously used.
2006          */
2007         if (dev->data->tx_queues[queue_id] != NULL) {
2008                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2009
2010                 tx_queue_free(txq);
2011                 dev->data->tx_queues[queue_id] = NULL;
2012         }
2013
2014         /* allocate memory for the queue structure */
2015         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2016                                 socket_id);
2017         if (q == NULL) {
2018                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2019                 return -ENOMEM;
2020         }
2021
2022         /* setup queue */
2023         q->nb_desc = nb_desc;
2024         q->port_id = dev->data->port_id;
2025         q->queue_id = queue_id;
2026         q->offloads = offloads;
2027         q->ops = &def_txq_ops;
2028         q->tail_ptr = (volatile uint32_t *)
2029                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2030         if (handle_txconf(q, conf)) {
2031                 rte_free(q);
2032                 return -EINVAL;
2033         }
2034
2035         /* allocate memory for the software ring */
2036         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2037                                         nb_desc * sizeof(struct rte_mbuf *),
2038                                         RTE_CACHE_LINE_SIZE, socket_id);
2039         if (q->sw_ring == NULL) {
2040                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2041                 rte_free(q);
2042                 return -ENOMEM;
2043         }
2044
2045         /*
2046          * allocate memory for the hardware descriptor ring. A memzone large
2047          * enough to hold the maximum ring size is requested to allow for
2048          * resizing in later calls to the queue setup function.
2049          */
2050         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2051                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2052                                       socket_id);
2053         if (mz == NULL) {
2054                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2055                 rte_free(q->sw_ring);
2056                 rte_free(q);
2057                 return -ENOMEM;
2058         }
2059         q->hw_ring = mz->addr;
2060         q->hw_ring_phys_addr = mz->iova;
2061
2062         /*
2063          * allocate memory for the RS bit tracker. Enough slots to hold the
2064          * descriptor index for each RS bit needing to be set are required.
2065          */
2066         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2067                                 ((nb_desc + 1) / q->rs_thresh) *
2068                                 sizeof(uint16_t),
2069                                 RTE_CACHE_LINE_SIZE, socket_id);
2070         if (q->rs_tracker.list == NULL) {
2071                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2072                 rte_free(q->sw_ring);
2073                 rte_free(q);
2074                 return -ENOMEM;
2075         }
2076
2077         dev->data->tx_queues[queue_id] = q;
2078         return 0;
2079 }
2080
2081 static void
2082 fm10k_tx_queue_release(void *queue)
2083 {
2084         struct fm10k_tx_queue *q = queue;
2085         PMD_INIT_FUNC_TRACE();
2086
2087         tx_queue_free(q);
2088 }
2089
2090 static int
2091 fm10k_reta_update(struct rte_eth_dev *dev,
2092                         struct rte_eth_rss_reta_entry64 *reta_conf,
2093                         uint16_t reta_size)
2094 {
2095         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2096         uint16_t i, j, idx, shift;
2097         uint8_t mask;
2098         uint32_t reta;
2099
2100         PMD_INIT_FUNC_TRACE();
2101
2102         if (reta_size > FM10K_MAX_RSS_INDICES) {
2103                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2104                         "(%d) doesn't match the number hardware can supported "
2105                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2106                 return -EINVAL;
2107         }
2108
2109         /*
2110          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2111          * 128-entries in 32 registers
2112          */
2113         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2114                 idx = i / RTE_RETA_GROUP_SIZE;
2115                 shift = i % RTE_RETA_GROUP_SIZE;
2116                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2117                                 BIT_MASK_PER_UINT32);
2118                 if (mask == 0)
2119                         continue;
2120
2121                 reta = 0;
2122                 if (mask != BIT_MASK_PER_UINT32)
2123                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2124
2125                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2126                         if (mask & (0x1 << j)) {
2127                                 if (mask != 0xF)
2128                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2129                                 reta |= reta_conf[idx].reta[shift + j] <<
2130                                                 (CHAR_BIT * j);
2131                         }
2132                 }
2133                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2134         }
2135
2136         return 0;
2137 }
2138
2139 static int
2140 fm10k_reta_query(struct rte_eth_dev *dev,
2141                         struct rte_eth_rss_reta_entry64 *reta_conf,
2142                         uint16_t reta_size)
2143 {
2144         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2145         uint16_t i, j, idx, shift;
2146         uint8_t mask;
2147         uint32_t reta;
2148
2149         PMD_INIT_FUNC_TRACE();
2150
2151         if (reta_size < FM10K_MAX_RSS_INDICES) {
2152                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2153                         "(%d) doesn't match the number hardware can supported "
2154                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2155                 return -EINVAL;
2156         }
2157
2158         /*
2159          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2160          * 128-entries in 32 registers
2161          */
2162         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2163                 idx = i / RTE_RETA_GROUP_SIZE;
2164                 shift = i % RTE_RETA_GROUP_SIZE;
2165                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2166                                 BIT_MASK_PER_UINT32);
2167                 if (mask == 0)
2168                         continue;
2169
2170                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2171                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2172                         if (mask & (0x1 << j))
2173                                 reta_conf[idx].reta[shift + j] = ((reta >>
2174                                         CHAR_BIT * j) & UINT8_MAX);
2175                 }
2176         }
2177
2178         return 0;
2179 }
2180
2181 static int
2182 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2183         struct rte_eth_rss_conf *rss_conf)
2184 {
2185         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2186         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2187         uint32_t mrqc;
2188         uint64_t hf = rss_conf->rss_hf;
2189         int i;
2190
2191         PMD_INIT_FUNC_TRACE();
2192
2193         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2194                                 FM10K_RSSRK_ENTRIES_PER_REG))
2195                 return -EINVAL;
2196
2197         if (hf == 0)
2198                 return -EINVAL;
2199
2200         mrqc = 0;
2201         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2202         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2203         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2204         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2205         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2206         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2207         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2208         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2209         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2210
2211         /* If the mapping doesn't fit any supported, return */
2212         if (mrqc == 0)
2213                 return -EINVAL;
2214
2215         if (key != NULL)
2216                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2217                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2218
2219         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2220
2221         return 0;
2222 }
2223
2224 static int
2225 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2226         struct rte_eth_rss_conf *rss_conf)
2227 {
2228         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2229         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2230         uint32_t mrqc;
2231         uint64_t hf;
2232         int i;
2233
2234         PMD_INIT_FUNC_TRACE();
2235
2236         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2237                                 FM10K_RSSRK_ENTRIES_PER_REG))
2238                 return -EINVAL;
2239
2240         if (key != NULL)
2241                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2242                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2243
2244         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2245         hf = 0;
2246         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2247         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2248         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2249         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2250         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2251         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2252         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2253         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2254         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2255
2256         rss_conf->rss_hf = hf;
2257
2258         return 0;
2259 }
2260
2261 static void
2262 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2263 {
2264         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2265         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2266
2267         /* Bind all local non-queue interrupt to vector 0 */
2268         int_map |= FM10K_MISC_VEC_ID;
2269
2270         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2271         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2272         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2273         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2274         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2275         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2276
2277         /* Enable misc causes */
2278         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2279                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2280                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2281                                 FM10K_EIMR_ENABLE(MAILBOX) |
2282                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2283                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2284                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2285                                 FM10K_EIMR_ENABLE(VFLR));
2286
2287         /* Enable ITR 0 */
2288         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2289                                         FM10K_ITR_MASK_CLEAR);
2290         FM10K_WRITE_FLUSH(hw);
2291 }
2292
2293 static void
2294 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2295 {
2296         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2297         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2298
2299         int_map |= FM10K_MISC_VEC_ID;
2300
2301         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2302         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2303         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2304         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2305         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2306         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2307
2308         /* Disable misc causes */
2309         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2310                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2311                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2312                                 FM10K_EIMR_DISABLE(MAILBOX) |
2313                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2314                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2315                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2316                                 FM10K_EIMR_DISABLE(VFLR));
2317
2318         /* Disable ITR 0 */
2319         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2320         FM10K_WRITE_FLUSH(hw);
2321 }
2322
2323 static void
2324 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2325 {
2326         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2327         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2328
2329         /* Bind all local non-queue interrupt to vector 0 */
2330         int_map |= FM10K_MISC_VEC_ID;
2331
2332         /* Only INT 0 available, other 15 are reserved. */
2333         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2334
2335         /* Enable ITR 0 */
2336         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2337                                         FM10K_ITR_MASK_CLEAR);
2338         FM10K_WRITE_FLUSH(hw);
2339 }
2340
2341 static void
2342 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2343 {
2344         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2345         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2346
2347         int_map |= FM10K_MISC_VEC_ID;
2348
2349         /* Only INT 0 available, other 15 are reserved. */
2350         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2351
2352         /* Disable ITR 0 */
2353         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2354         FM10K_WRITE_FLUSH(hw);
2355 }
2356
2357 static int
2358 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2359 {
2360         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2361         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2362
2363         /* Enable ITR */
2364         if (hw->mac.type == fm10k_mac_pf)
2365                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2366                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2367         else
2368                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2369                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2370         rte_intr_ack(&pdev->intr_handle);
2371         return 0;
2372 }
2373
2374 static int
2375 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2376 {
2377         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2378         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2379
2380         /* Disable ITR */
2381         if (hw->mac.type == fm10k_mac_pf)
2382                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2383                         FM10K_ITR_MASK_SET);
2384         else
2385                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2386                         FM10K_ITR_MASK_SET);
2387         return 0;
2388 }
2389
2390 static int
2391 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2392 {
2393         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2394         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2395         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2396         uint32_t intr_vector, vec;
2397         uint16_t queue_id;
2398         int result = 0;
2399
2400         /* fm10k needs one separate interrupt for mailbox,
2401          * so only drivers which support multiple interrupt vectors
2402          * e.g. vfio-pci can work for fm10k interrupt mode
2403          */
2404         if (!rte_intr_cap_multiple(intr_handle) ||
2405                         dev->data->dev_conf.intr_conf.rxq == 0)
2406                 return result;
2407
2408         intr_vector = dev->data->nb_rx_queues;
2409
2410         /* disable interrupt first */
2411         rte_intr_disable(intr_handle);
2412         if (hw->mac.type == fm10k_mac_pf)
2413                 fm10k_dev_disable_intr_pf(dev);
2414         else
2415                 fm10k_dev_disable_intr_vf(dev);
2416
2417         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2418                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2419                 result = -EIO;
2420         }
2421
2422         if (rte_intr_dp_is_en(intr_handle) && !result) {
2423                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2424                         dev->data->nb_rx_queues * sizeof(int), 0);
2425                 if (intr_handle->intr_vec) {
2426                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2427                                         queue_id < dev->data->nb_rx_queues;
2428                                         queue_id++) {
2429                                 intr_handle->intr_vec[queue_id] = vec;
2430                                 if (vec < intr_handle->nb_efd - 1
2431                                                 + FM10K_RX_VEC_START)
2432                                         vec++;
2433                         }
2434                 } else {
2435                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2436                                 " intr_vec", dev->data->nb_rx_queues);
2437                         rte_intr_efd_disable(intr_handle);
2438                         result = -ENOMEM;
2439                 }
2440         }
2441
2442         if (hw->mac.type == fm10k_mac_pf)
2443                 fm10k_dev_enable_intr_pf(dev);
2444         else
2445                 fm10k_dev_enable_intr_vf(dev);
2446         rte_intr_enable(intr_handle);
2447         hw->mac.ops.update_int_moderator(hw);
2448         return result;
2449 }
2450
2451 static int
2452 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2453 {
2454         struct fm10k_fault fault;
2455         int err;
2456         const char *estr = "Unknown error";
2457
2458         /* Process PCA fault */
2459         if (eicr & FM10K_EICR_PCA_FAULT) {
2460                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2461                 if (err)
2462                         goto error;
2463                 switch (fault.type) {
2464                 case PCA_NO_FAULT:
2465                         estr = "PCA_NO_FAULT"; break;
2466                 case PCA_UNMAPPED_ADDR:
2467                         estr = "PCA_UNMAPPED_ADDR"; break;
2468                 case PCA_BAD_QACCESS_PF:
2469                         estr = "PCA_BAD_QACCESS_PF"; break;
2470                 case PCA_BAD_QACCESS_VF:
2471                         estr = "PCA_BAD_QACCESS_VF"; break;
2472                 case PCA_MALICIOUS_REQ:
2473                         estr = "PCA_MALICIOUS_REQ"; break;
2474                 case PCA_POISONED_TLP:
2475                         estr = "PCA_POISONED_TLP"; break;
2476                 case PCA_TLP_ABORT:
2477                         estr = "PCA_TLP_ABORT"; break;
2478                 default:
2479                         goto error;
2480                 }
2481                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2482                         estr, fault.func ? "VF" : "PF", fault.func,
2483                         fault.address, fault.specinfo);
2484         }
2485
2486         /* Process THI fault */
2487         if (eicr & FM10K_EICR_THI_FAULT) {
2488                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2489                 if (err)
2490                         goto error;
2491                 switch (fault.type) {
2492                 case THI_NO_FAULT:
2493                         estr = "THI_NO_FAULT"; break;
2494                 case THI_MAL_DIS_Q_FAULT:
2495                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2496                 default:
2497                         goto error;
2498                 }
2499                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2500                         estr, fault.func ? "VF" : "PF", fault.func,
2501                         fault.address, fault.specinfo);
2502         }
2503
2504         /* Process FUM fault */
2505         if (eicr & FM10K_EICR_FUM_FAULT) {
2506                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2507                 if (err)
2508                         goto error;
2509                 switch (fault.type) {
2510                 case FUM_NO_FAULT:
2511                         estr = "FUM_NO_FAULT"; break;
2512                 case FUM_UNMAPPED_ADDR:
2513                         estr = "FUM_UNMAPPED_ADDR"; break;
2514                 case FUM_POISONED_TLP:
2515                         estr = "FUM_POISONED_TLP"; break;
2516                 case FUM_BAD_VF_QACCESS:
2517                         estr = "FUM_BAD_VF_QACCESS"; break;
2518                 case FUM_ADD_DECODE_ERR:
2519                         estr = "FUM_ADD_DECODE_ERR"; break;
2520                 case FUM_RO_ERROR:
2521                         estr = "FUM_RO_ERROR"; break;
2522                 case FUM_QPRC_CRC_ERROR:
2523                         estr = "FUM_QPRC_CRC_ERROR"; break;
2524                 case FUM_CSR_TIMEOUT:
2525                         estr = "FUM_CSR_TIMEOUT"; break;
2526                 case FUM_INVALID_TYPE:
2527                         estr = "FUM_INVALID_TYPE"; break;
2528                 case FUM_INVALID_LENGTH:
2529                         estr = "FUM_INVALID_LENGTH"; break;
2530                 case FUM_INVALID_BE:
2531                         estr = "FUM_INVALID_BE"; break;
2532                 case FUM_INVALID_ALIGN:
2533                         estr = "FUM_INVALID_ALIGN"; break;
2534                 default:
2535                         goto error;
2536                 }
2537                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2538                         estr, fault.func ? "VF" : "PF", fault.func,
2539                         fault.address, fault.specinfo);
2540         }
2541
2542         return 0;
2543 error:
2544         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2545         return err;
2546 }
2547
2548 /**
2549  * PF interrupt handler triggered by NIC for handling specific interrupt.
2550  *
2551  * @param handle
2552  *  Pointer to interrupt handle.
2553  * @param param
2554  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2555  *
2556  * @return
2557  *  void
2558  */
2559 static void
2560 fm10k_dev_interrupt_handler_pf(void *param)
2561 {
2562         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2563         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2564         uint32_t cause, status;
2565         struct fm10k_dev_info *dev_info =
2566                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2567         int status_mbx;
2568         s32 err;
2569
2570         if (hw->mac.type != fm10k_mac_pf)
2571                 return;
2572
2573         cause = FM10K_READ_REG(hw, FM10K_EICR);
2574
2575         /* Handle PCI fault cases */
2576         if (cause & FM10K_EICR_FAULT_MASK) {
2577                 PMD_INIT_LOG(ERR, "INT: find fault!");
2578                 fm10k_dev_handle_fault(hw, cause);
2579         }
2580
2581         /* Handle switch up/down */
2582         if (cause & FM10K_EICR_SWITCHNOTREADY)
2583                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2584
2585         if (cause & FM10K_EICR_SWITCHREADY) {
2586                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2587                 if (dev_info->sm_down == 1) {
2588                         fm10k_mbx_lock(hw);
2589
2590                         /* For recreating logical ports */
2591                         status_mbx = hw->mac.ops.update_lport_state(hw,
2592                                         hw->mac.dglort_map, MAX_LPORT_NUM, 1);
2593                         if (status_mbx == FM10K_SUCCESS)
2594                                 PMD_INIT_LOG(INFO,
2595                                         "INT: Recreated Logical port");
2596                         else
2597                                 PMD_INIT_LOG(INFO,
2598                                         "INT: Logical ports weren't recreated");
2599
2600                         status_mbx = hw->mac.ops.update_xcast_mode(hw,
2601                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2602                         if (status_mbx != FM10K_SUCCESS)
2603                                 PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2604
2605                         fm10k_mbx_unlock(hw);
2606
2607                         /* first clear the internal SW recording structure */
2608                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2609                                                 ETH_MQ_RX_VMDQ_FLAG))
2610                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2611                                         false);
2612
2613                         fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2614                                         MAIN_VSI_POOL_NUMBER);
2615
2616                         /*
2617                          * Add default mac address and vlan for the logical
2618                          * ports that have been created, leave to the
2619                          * application to fully recover Rx filtering.
2620                          */
2621                         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2622                                         MAIN_VSI_POOL_NUMBER);
2623
2624                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2625                                                 ETH_MQ_RX_VMDQ_FLAG))
2626                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2627                                         true);
2628
2629                         dev_info->sm_down = 0;
2630                         rte_eth_dev_callback_process(dev,
2631                                         RTE_ETH_EVENT_INTR_LSC,
2632                                         NULL);
2633                 }
2634         }
2635
2636         /* Handle mailbox message */
2637         fm10k_mbx_lock(hw);
2638         err = hw->mbx.ops.process(hw, &hw->mbx);
2639         fm10k_mbx_unlock(hw);
2640
2641         if (err == FM10K_ERR_RESET_REQUESTED) {
2642                 PMD_INIT_LOG(INFO, "INT: Switch is down");
2643                 dev_info->sm_down = 1;
2644                 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
2645         }
2646
2647         /* Handle SRAM error */
2648         if (cause & FM10K_EICR_SRAMERROR) {
2649                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2650
2651                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2652                 /* Write to clear pending bits */
2653                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2654
2655                 /* Todo: print out error message after shared code  updates */
2656         }
2657
2658         /* Clear these 3 events if having any */
2659         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2660                  FM10K_EICR_SWITCHREADY;
2661         if (cause)
2662                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2663
2664         /* Re-enable interrupt from device side */
2665         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2666                                         FM10K_ITR_MASK_CLEAR);
2667         /* Re-enable interrupt from host side */
2668         rte_intr_ack(dev->intr_handle);
2669 }
2670
2671 /**
2672  * VF interrupt handler triggered by NIC for handling specific interrupt.
2673  *
2674  * @param handle
2675  *  Pointer to interrupt handle.
2676  * @param param
2677  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2678  *
2679  * @return
2680  *  void
2681  */
2682 static void
2683 fm10k_dev_interrupt_handler_vf(void *param)
2684 {
2685         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2686         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2687         struct fm10k_mbx_info *mbx = &hw->mbx;
2688         struct fm10k_dev_info *dev_info =
2689                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2690         const enum fm10k_mbx_state state = mbx->state;
2691         int status_mbx;
2692
2693         if (hw->mac.type != fm10k_mac_vf)
2694                 return;
2695
2696         /* Handle mailbox message if lock is acquired */
2697         fm10k_mbx_lock(hw);
2698         hw->mbx.ops.process(hw, &hw->mbx);
2699         fm10k_mbx_unlock(hw);
2700
2701         if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) {
2702                 PMD_INIT_LOG(INFO, "INT: Switch has gone down");
2703
2704                 fm10k_mbx_lock(hw);
2705                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2706                                 MAX_LPORT_NUM, 1);
2707                 fm10k_mbx_unlock(hw);
2708
2709                 /* Setting reset flag */
2710                 dev_info->sm_down = 1;
2711                 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
2712         }
2713
2714         if (dev_info->sm_down == 1 &&
2715                         hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) {
2716                 PMD_INIT_LOG(INFO, "INT: Switch has gone up");
2717                 fm10k_mbx_lock(hw);
2718                 status_mbx = hw->mac.ops.update_xcast_mode(hw,
2719                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2720                 if (status_mbx != FM10K_SUCCESS)
2721                         PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2722                 fm10k_mbx_unlock(hw);
2723
2724                 /* first clear the internal SW recording structure */
2725                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false);
2726                 fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2727                                 MAIN_VSI_POOL_NUMBER);
2728
2729                 /*
2730                  * Add default mac address and vlan for the logical ports that
2731                  * have been created, leave to the application to fully recover
2732                  * Rx filtering.
2733                  */
2734                 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2735                                 MAIN_VSI_POOL_NUMBER);
2736                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
2737
2738                 dev_info->sm_down = 0;
2739                 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
2740         }
2741
2742         /* Re-enable interrupt from device side */
2743         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2744                                         FM10K_ITR_MASK_CLEAR);
2745         /* Re-enable interrupt from host side */
2746         rte_intr_ack(dev->intr_handle);
2747 }
2748
2749 /* Mailbox message handler in VF */
2750 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2751         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2752         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2753         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2754         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2755 };
2756
2757 static int
2758 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2759 {
2760         int err = 0;
2761
2762         /* Initialize mailbox lock */
2763         fm10k_mbx_initlock(hw);
2764
2765         /* Replace default message handler with new ones */
2766         if (hw->mac.type == fm10k_mac_vf)
2767                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2768
2769         if (err) {
2770                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2771                                 err);
2772                 return err;
2773         }
2774         /* Connect to SM for PF device or PF for VF device */
2775         return hw->mbx.ops.connect(hw, &hw->mbx);
2776 }
2777
2778 static void
2779 fm10k_close_mbx_service(struct fm10k_hw *hw)
2780 {
2781         /* Disconnect from SM for PF device or PF for VF device */
2782         hw->mbx.ops.disconnect(hw, &hw->mbx);
2783 }
2784
2785 static int
2786 fm10k_dev_close(struct rte_eth_dev *dev)
2787 {
2788         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2789         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2790         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2791         int ret;
2792
2793         PMD_INIT_FUNC_TRACE();
2794         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2795                 return 0;
2796
2797         fm10k_mbx_lock(hw);
2798         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2799                 MAX_LPORT_NUM, false);
2800         fm10k_mbx_unlock(hw);
2801
2802         /* allow 100ms for device to quiesce */
2803         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
2804
2805         /* Stop mailbox service first */
2806         fm10k_close_mbx_service(hw);
2807
2808         ret = fm10k_dev_stop(dev);
2809
2810         fm10k_dev_queue_release(dev);
2811         fm10k_stop_hw(hw);
2812
2813         /* disable uio/vfio intr */
2814         rte_intr_disable(intr_handle);
2815
2816         /*PF/VF has different interrupt handling mechanism */
2817         if (hw->mac.type == fm10k_mac_pf) {
2818                 /* disable interrupt */
2819                 fm10k_dev_disable_intr_pf(dev);
2820
2821                 /* unregister callback func to eal lib */
2822                 rte_intr_callback_unregister(intr_handle,
2823                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2824         } else {
2825                 /* disable interrupt */
2826                 fm10k_dev_disable_intr_vf(dev);
2827
2828                 rte_intr_callback_unregister(intr_handle,
2829                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2830         }
2831
2832         return ret;
2833 }
2834
2835 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2836         .dev_configure          = fm10k_dev_configure,
2837         .dev_start              = fm10k_dev_start,
2838         .dev_stop               = fm10k_dev_stop,
2839         .dev_close              = fm10k_dev_close,
2840         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2841         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2842         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2843         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2844         .stats_get              = fm10k_stats_get,
2845         .xstats_get             = fm10k_xstats_get,
2846         .xstats_get_names       = fm10k_xstats_get_names,
2847         .stats_reset            = fm10k_stats_reset,
2848         .xstats_reset           = fm10k_stats_reset,
2849         .link_update            = fm10k_link_update,
2850         .dev_infos_get          = fm10k_dev_infos_get,
2851         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2852         .vlan_filter_set        = fm10k_vlan_filter_set,
2853         .vlan_offload_set       = fm10k_vlan_offload_set,
2854         .mac_addr_add           = fm10k_macaddr_add,
2855         .mac_addr_remove        = fm10k_macaddr_remove,
2856         .rx_queue_start         = fm10k_dev_rx_queue_start,
2857         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2858         .tx_queue_start         = fm10k_dev_tx_queue_start,
2859         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2860         .rx_queue_setup         = fm10k_rx_queue_setup,
2861         .rx_queue_release       = fm10k_rx_queue_release,
2862         .tx_queue_setup         = fm10k_tx_queue_setup,
2863         .tx_queue_release       = fm10k_tx_queue_release,
2864         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2865         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2866         .reta_update            = fm10k_reta_update,
2867         .reta_query             = fm10k_reta_query,
2868         .rss_hash_update        = fm10k_rss_hash_update,
2869         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2870 };
2871
2872 static int ftag_check_handler(__rte_unused const char *key,
2873                 const char *value, __rte_unused void *opaque)
2874 {
2875         if (strcmp(value, "1"))
2876                 return -1;
2877
2878         return 0;
2879 }
2880
2881 static int
2882 fm10k_check_ftag(struct rte_devargs *devargs)
2883 {
2884         struct rte_kvargs *kvlist;
2885         const char *ftag_key = "enable_ftag";
2886
2887         if (devargs == NULL)
2888                 return 0;
2889
2890         kvlist = rte_kvargs_parse(devargs->args, NULL);
2891         if (kvlist == NULL)
2892                 return 0;
2893
2894         if (!rte_kvargs_count(kvlist, ftag_key)) {
2895                 rte_kvargs_free(kvlist);
2896                 return 0;
2897         }
2898         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2899         if (rte_kvargs_process(kvlist, ftag_key,
2900                                 ftag_check_handler, NULL) < 0) {
2901                 rte_kvargs_free(kvlist);
2902                 return 0;
2903         }
2904         rte_kvargs_free(kvlist);
2905
2906         return 1;
2907 }
2908
2909 static uint16_t
2910 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2911                     uint16_t nb_pkts)
2912 {
2913         uint16_t nb_tx = 0;
2914         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2915
2916         while (nb_pkts) {
2917                 uint16_t ret, num;
2918
2919                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2920                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2921                                                  num);
2922                 nb_tx += ret;
2923                 nb_pkts -= ret;
2924                 if (ret < num)
2925                         break;
2926         }
2927
2928         return nb_tx;
2929 }
2930
2931 static void __rte_cold
2932 fm10k_set_tx_function(struct rte_eth_dev *dev)
2933 {
2934         struct fm10k_tx_queue *txq;
2935         int i;
2936         int use_sse = 1;
2937         uint16_t tx_ftag_en = 0;
2938
2939         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2940                 /* primary process has set the ftag flag and offloads */
2941                 txq = dev->data->tx_queues[0];
2942                 if (fm10k_tx_vec_condition_check(txq)) {
2943                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2944                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2945                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2946                 } else {
2947                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2948                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2949                         dev->tx_pkt_prepare = NULL;
2950                 }
2951                 return;
2952         }
2953
2954         if (fm10k_check_ftag(dev->device->devargs))
2955                 tx_ftag_en = 1;
2956
2957         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2958                 txq = dev->data->tx_queues[i];
2959                 txq->tx_ftag_en = tx_ftag_en;
2960                 /* Check if Vector Tx is satisfied */
2961                 if (fm10k_tx_vec_condition_check(txq))
2962                         use_sse = 0;
2963         }
2964
2965         if (use_sse) {
2966                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2967                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2968                         txq = dev->data->tx_queues[i];
2969                         fm10k_txq_vec_setup(txq);
2970                 }
2971                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2972                 dev->tx_pkt_prepare = NULL;
2973         } else {
2974                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2975                 dev->tx_pkt_prepare = fm10k_prep_pkts;
2976                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2977         }
2978 }
2979
2980 static void __rte_cold
2981 fm10k_set_rx_function(struct rte_eth_dev *dev)
2982 {
2983         struct fm10k_dev_info *dev_info =
2984                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2985         uint16_t i, rx_using_sse;
2986         uint16_t rx_ftag_en = 0;
2987
2988         if (fm10k_check_ftag(dev->device->devargs))
2989                 rx_ftag_en = 1;
2990
2991         /* In order to allow Vector Rx there are a few configuration
2992          * conditions to be met.
2993          */
2994         if (!fm10k_rx_vec_condition_check(dev) &&
2995                         dev_info->rx_vec_allowed && !rx_ftag_en) {
2996                 if (dev->data->scattered_rx)
2997                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2998                 else
2999                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
3000         } else if (dev->data->scattered_rx)
3001                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
3002         else
3003                 dev->rx_pkt_burst = fm10k_recv_pkts;
3004
3005         rx_using_sse =
3006                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
3007                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
3008
3009         if (rx_using_sse)
3010                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
3011         else
3012                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
3013
3014         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3015                 return;
3016
3017         for (i = 0; i < dev->data->nb_rx_queues; i++) {
3018                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
3019
3020                 rxq->rx_using_sse = rx_using_sse;
3021                 rxq->rx_ftag_en = rx_ftag_en;
3022         }
3023 }
3024
3025 static void
3026 fm10k_params_init(struct rte_eth_dev *dev)
3027 {
3028         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3029         struct fm10k_dev_info *info =
3030                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3031
3032         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
3033          * there is no way to get link status without reading BAR4.  Until this
3034          * works, assume we have maximum bandwidth.
3035          * @todo - fix bus info
3036          */
3037         hw->bus_caps.speed = fm10k_bus_speed_8000;
3038         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
3039         hw->bus_caps.payload = fm10k_bus_payload_512;
3040         hw->bus.speed = fm10k_bus_speed_8000;
3041         hw->bus.width = fm10k_bus_width_pcie_x8;
3042         hw->bus.payload = fm10k_bus_payload_256;
3043
3044         info->rx_vec_allowed = true;
3045         info->sm_down = false;
3046 }
3047
3048 static int
3049 eth_fm10k_dev_init(struct rte_eth_dev *dev)
3050 {
3051         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3052         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3053         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3054         int diag, i;
3055         struct fm10k_macvlan_filter_info *macvlan;
3056
3057         PMD_INIT_FUNC_TRACE();
3058
3059         dev->dev_ops = &fm10k_eth_dev_ops;
3060         dev->rx_queue_count = fm10k_dev_rx_queue_count;
3061         dev->rx_descriptor_done = fm10k_dev_rx_descriptor_done;
3062         dev->rx_descriptor_status = fm10k_dev_rx_descriptor_status;
3063         dev->tx_descriptor_status = fm10k_dev_tx_descriptor_status;
3064         dev->rx_pkt_burst = &fm10k_recv_pkts;
3065         dev->tx_pkt_burst = &fm10k_xmit_pkts;
3066         dev->tx_pkt_prepare = &fm10k_prep_pkts;
3067
3068         /*
3069          * Primary process does the whole initialization, for secondary
3070          * processes, we just select the same Rx and Tx function as primary.
3071          */
3072         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3073                 fm10k_set_rx_function(dev);
3074                 fm10k_set_tx_function(dev);
3075                 return 0;
3076         }
3077
3078         rte_eth_copy_pci_info(dev, pdev);
3079         dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
3080
3081         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
3082         memset(macvlan, 0, sizeof(*macvlan));
3083         /* Vendor and Device ID need to be set before init of shared code */
3084         memset(hw, 0, sizeof(*hw));
3085         hw->device_id = pdev->id.device_id;
3086         hw->vendor_id = pdev->id.vendor_id;
3087         hw->subsystem_device_id = pdev->id.subsystem_device_id;
3088         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
3089         hw->revision_id = 0;
3090         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
3091         if (hw->hw_addr == NULL) {
3092                 PMD_INIT_LOG(ERR, "Bad mem resource."
3093                         " Try to refuse unused devices.");
3094                 return -EIO;
3095         }
3096
3097         /* Store fm10k_adapter pointer */
3098         hw->back = dev->data->dev_private;
3099
3100         /* Initialize the shared code */
3101         diag = fm10k_init_shared_code(hw);
3102         if (diag != FM10K_SUCCESS) {
3103                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
3104                 return -EIO;
3105         }
3106
3107         /* Initialize parameters */
3108         fm10k_params_init(dev);
3109
3110         /* Initialize the hw */
3111         diag = fm10k_init_hw(hw);
3112         if (diag != FM10K_SUCCESS) {
3113                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
3114                 return -EIO;
3115         }
3116
3117         /* Initialize MAC address(es) */
3118         dev->data->mac_addrs = rte_zmalloc("fm10k",
3119                         RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
3120         if (dev->data->mac_addrs == NULL) {
3121                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
3122                 return -ENOMEM;
3123         }
3124
3125         diag = fm10k_read_mac_addr(hw);
3126
3127         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3128                         &dev->data->mac_addrs[0]);
3129
3130         if (diag != FM10K_SUCCESS ||
3131                 !rte_is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
3132
3133                 /* Generate a random addr */
3134                 rte_eth_random_addr(hw->mac.addr);
3135                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
3136                 rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3137                 &dev->data->mac_addrs[0]);
3138         }
3139
3140         /* Reset the hw statistics */
3141         diag = fm10k_stats_reset(dev);
3142         if (diag != 0) {
3143                 PMD_INIT_LOG(ERR, "Stats reset failed: %d", diag);
3144                 return diag;
3145         }
3146
3147         /* Reset the hw */
3148         diag = fm10k_reset_hw(hw);
3149         if (diag != FM10K_SUCCESS) {
3150                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
3151                 return -EIO;
3152         }
3153
3154         /* Setup mailbox service */
3155         diag = fm10k_setup_mbx_service(hw);
3156         if (diag != FM10K_SUCCESS) {
3157                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
3158                 return -EIO;
3159         }
3160
3161         /*PF/VF has different interrupt handling mechanism */
3162         if (hw->mac.type == fm10k_mac_pf) {
3163                 /* register callback func to eal lib */
3164                 rte_intr_callback_register(intr_handle,
3165                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3166
3167                 /* enable MISC interrupt */
3168                 fm10k_dev_enable_intr_pf(dev);
3169         } else { /* VF */
3170                 rte_intr_callback_register(intr_handle,
3171                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3172
3173                 fm10k_dev_enable_intr_vf(dev);
3174         }
3175
3176         /* Enable intr after callback registered */
3177         rte_intr_enable(intr_handle);
3178
3179         hw->mac.ops.update_int_moderator(hw);
3180
3181         /* Make sure Switch Manager is ready before going forward. */
3182         if (hw->mac.type == fm10k_mac_pf) {
3183                 bool switch_ready = false;
3184
3185                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3186                         fm10k_mbx_lock(hw);
3187                         hw->mac.ops.get_host_state(hw, &switch_ready);
3188                         fm10k_mbx_unlock(hw);
3189                         if (switch_ready == true)
3190                                 break;
3191                         /* Delay some time to acquire async LPORT_MAP info. */
3192                         rte_delay_us(WAIT_SWITCH_MSG_US);
3193                 }
3194
3195                 if (switch_ready == false) {
3196                         PMD_INIT_LOG(ERR, "switch is not ready");
3197                         return -1;
3198                 }
3199         }
3200
3201         /*
3202          * Below function will trigger operations on mailbox, acquire lock to
3203          * avoid race condition from interrupt handler. Operations on mailbox
3204          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3205          * will handle and generate an interrupt to our side. Then,  FIFO in
3206          * mailbox will be touched.
3207          */
3208         fm10k_mbx_lock(hw);
3209         /* Enable port first */
3210         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3211                                         MAX_LPORT_NUM, 1);
3212
3213         /* Set unicast mode by default. App can change to other mode in other
3214          * API func.
3215          */
3216         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3217                                         FM10K_XCAST_MODE_NONE);
3218
3219         fm10k_mbx_unlock(hw);
3220
3221         /* Make sure default VID is ready before going forward. */
3222         if (hw->mac.type == fm10k_mac_pf) {
3223                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3224                         if (hw->mac.default_vid)
3225                                 break;
3226                         /* Delay some time to acquire async port VLAN info. */
3227                         rte_delay_us(WAIT_SWITCH_MSG_US);
3228                 }
3229
3230                 if (!hw->mac.default_vid) {
3231                         PMD_INIT_LOG(ERR, "default VID is not ready");
3232                         return -1;
3233                 }
3234         }
3235
3236         /* Add default mac address */
3237         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3238                 MAIN_VSI_POOL_NUMBER);
3239
3240         return 0;
3241 }
3242
3243 static int
3244 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3245 {
3246         PMD_INIT_FUNC_TRACE();
3247         fm10k_dev_close(dev);
3248         return 0;
3249 }
3250
3251 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3252         struct rte_pci_device *pci_dev)
3253 {
3254         return rte_eth_dev_pci_generic_probe(pci_dev,
3255                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3256 }
3257
3258 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3259 {
3260         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3261 }
3262
3263 /*
3264  * The set of PCI devices this driver supports. This driver will enable both PF
3265  * and SRIOV-VF devices.
3266  */
3267 static const struct rte_pci_id pci_id_fm10k_map[] = {
3268         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3269         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3270         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3271         { .vendor_id = 0, /* sentinel */ },
3272 };
3273
3274 static struct rte_pci_driver rte_pmd_fm10k = {
3275         .id_table = pci_id_fm10k_map,
3276         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
3277         .probe = eth_fm10k_pci_probe,
3278         .remove = eth_fm10k_pci_remove,
3279 };
3280
3281 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3282 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3283 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");
3284 RTE_LOG_REGISTER(fm10k_logtype_init, pmd.net.fm10k.init, NOTICE);
3285 RTE_LOG_REGISTER(fm10k_logtype_driver, pmd.net.fm10k.driver, NOTICE);
3286 #ifdef RTE_LIBRTE_FM10K_DEBUG_RX
3287 RTE_LOG_REGISTER(fm10k_logtype_rx, pmd.net.fm10k.rx, DEBUG);
3288 #endif
3289 #ifdef RTE_LIBRTE_FM10K_DEBUG_TX
3290 RTE_LOG_REGISTER(fm10k_logtype_tx, pmd.net.fm10k.tx, DEBUG);
3291 #endif
3292 #ifdef RTE_LIBRTE_FM10K_DEBUG_TX_FREE
3293 RTE_LOG_REGISTER(fm10k_logtype_tx_free, pmd.net.fm10k.tx_free, DEBUG);
3294 #endif